support remote access using fastapi; add double click/scroll; enable local run omniparser

2025-01-22 19:24:34 -08:00
parent e0a845d35c
commit ce199d52ab
9 changed files with 1120 additions and 14 deletions
--- a/demo/gradio/computer_use_demo/tools/computer.py
+++ b/demo/gradio/computer_use_demo/tools/computer.py
@@ -208,6 +208,12 @@ class ComputerTool(BaseAnthropicTool):
                    time.sleep(1)
                    self.send_to_vm("pyautogui.mouseUp()")
                return ToolResult(output=f"Performed {action}")
+        if action in ("scroll_up", "scroll_down"):
+            if action == "scroll_up":
+                self.send_to_vm("pyautogui.scroll(100)")
+            elif action == "scroll_down":
+                self.send_to_vm("pyautogui.scroll(-100)")
+            return ToolResult(output=f"Performed {action}")
            
        raise ToolError(f"Invalid action: {action}")