add drop down for omni output in the chat; add args for app.py;

2025-01-28 21:33:58 -08:00
parent 16570a9bf3
commit 7ea2239e10
10 changed files with 86 additions and 29 deletions
--- a/demo/gradio/app.py
+++ b/demo/gradio/app.py
@@ -1,5 +1,6 @@
 """
 Entrypoint for Gradio, see https://gradio.app/
+python app.py --windows_host_url xxxx:8006/ --omniparser_host_url localhost:8000
 """

 import platform
@@ -15,6 +16,7 @@ from pathlib import Path
 from typing import cast, Dict
 from PIL import Image
 import socket
+import argparse

 import gradio as gr
 from anthropic import APIResponse
@@ -39,6 +41,18 @@ Welcome to the OmniParser+X Demo! X = [GPT-4o/4o-mini, Claude, Phi, Llama]. Let
 Type a message and press submit to start OmniParser+X. Press the trash icon in the chat to clear the message history.
 '''

+def parse_arguments():
+    parser = argparse.ArgumentParser(description="Gradio App")
+    parser.add_argument("--windows_host_url", type=str, default='GCRSANDBOX336.redmond.corp.microsoft.com:8006/') # http://gcrsandbox336.redmond.corp.microsoft.com/
+    parser.add_argument("--omniparser_host_url", type=str, default="localhost:8000")
+    return parser.parse_args()
+args = parse_arguments()
+windows_host_url = args.windows_host_url
+omniparser_host_url = args.omniparser_host_url
+print(f"Windows host URL: {windows_host_url}")
+print(f"OmniParser host URL: {omniparser_host_url}")
+
+
 class Sender(StrEnum):
    USER = "user"
    BOT = "assistant"
@@ -68,8 +82,8 @@ def setup_state(state):
        state["only_n_most_recent_images"] = 2
    if 'chatbot_messages' not in state:
        state['chatbot_messages'] = []
-    if "omniparser_url" not in state:
-        state["omniparser_url"] = "localhost:8000"
+    # if "omniparser_url" not in state:
+    #     state["omniparser_url"] = "localhost:8000"

 async def main(state):
    """Render loop for Gradio"""
@@ -211,7 +225,7 @@ def process_input(user_input, state):
        api_response_callback=partial(_api_response_callback, response_state=state["responses"]),
        api_key=state["api_key"],
        only_n_most_recent_images=state["only_n_most_recent_images"],
-        omniparser_url=state["omniparser_url"]
+        omniparser_url=omniparser_host_url #state["omniparser_url"]
    ):  
        if loop_msg is None:
            yield state['chatbot_messages']
@@ -275,13 +289,13 @@ with gr.Blocks(theme=gr.themes.Default()) as demo:
                    placeholder="Paste your API key here",
                    interactive=True,
                )
-        with gr.Row():
-            omniparser_url = gr.Textbox(
-                label="OmniParser Base URL",
-                value="localhost:8000",
-                placeholder="Enter OmniParser base URL (e.g. localhost:8000)",
-                interactive=True
-            )
+        # with gr.Row():
+        #     omniparser_url = gr.Textbox(
+        #         label="OmniParser Base URL",
+        #         value="localhost:8000",
+        #         placeholder="Enter OmniParser base URL (e.g. localhost:8000)",
+        #         interactive=True
+        #     )
        # hide_images = gr.Checkbox(label="Hide screenshots", value=False)

    with gr.Row():
@@ -294,11 +308,20 @@ with gr.Blocks(theme=gr.themes.Default()) as demo:
        with gr.Column(scale=1):
            chatbot = gr.Chatbot(label="Chatbot History", autoscroll=True, height=580)
        with gr.Column(scale=3):
-            iframe = gr.HTML(
-                f'<iframe src="http://localhost:8006/vnc.html?view_only=1&autoconnect=1&resize=scale" width="100%" height="580" allow="fullscreen"></iframe>',
-                container=False,
-                elem_classes="no-padding"
-            )
+            if not windows_host_url:
+                iframe = gr.HTML(
+                    f'<iframe src="http://localhost:8006/vnc.html?view_only=1&autoconnect=1&resize=scale" width="100%" height="580" allow="fullscreen"></iframe>',
+                    container=False,
+                    elem_classes="no-padding"
+                )
+            else:
+                # machine_fqdn = socket.getfqdn()
+                # print('machine_fqdn:', machine_fqdn)
+                iframe = gr.HTML(
+                    f'<iframe src="http://{windows_host_url}/vnc.html?view_only=1&autoconnect=1&resize=scale" width="100%" height="580" allow="fullscreen"></iframe>',
+                    container=False,
+                    elem_classes="no-padding"
+                )

    def update_model(model_selection, state):
        state["model"] = model_selection
@@ -350,8 +373,8 @@ with gr.Blocks(theme=gr.themes.Default()) as demo:
        state["api_key"] = api_key_value
        state[f'{state["provider"]}_api_key'] = api_key_value

-    def update_omniparser_url(url_value, state):
-        state["omniparser_url"] = url_value
+    # def update_omniparser_url(url_value, state):
+    #     state["omniparser_url"] = url_value

    def clear_chat(state):
        # Reset message-related state
@@ -365,7 +388,7 @@ with gr.Blocks(theme=gr.themes.Default()) as demo:
    only_n_images.change(fn=update_only_n_images, inputs=[only_n_images, state], outputs=None)
    provider.change(fn=update_provider, inputs=[provider, state], outputs=api_key)
    api_key.change(fn=update_api_key, inputs=[api_key, state], outputs=None)
-    omniparser_url.change(fn=update_omniparser_url, inputs=[omniparser_url, state], outputs=None)
+    # omniparser_url.change(fn=update_omniparser_url, inputs=[omniparser_url, state], outputs=None)
    chatbot.clear(fn=clear_chat, inputs=[state], outputs=[chatbot])

    submit_button.click(process_input, [chat_input, state], chatbot)
--- a/demo/gradio/computer_use_demo/loop.py
+++ b/demo/gradio/computer_use_demo/loop.py
@@ -52,7 +52,7 @@ def sampling_loop_sync(
    Synchronous agentic sampling loop for the assistant/tool interaction of computer use.
    """
    print('in sampling_loop_sync, model:', model)
-    omniparser = OmniParser(url=f"http://{omniparser_url}/send_text/" if omniparser_url and omniparser_url != "localhost:8000" else None)
+    omniparser = OmniParser(url=f"http://{omniparser_url}/send_text/" if omniparser_url else None)
    if model == "claude-3-5-sonnet-20241022":
        # Register Actor and Executor
        actor = AnthropicActor(
--- a/demo/gradio/computer_use_demo/omniparser_agent/vlm_agent.py
+++ b/demo/gradio/computer_use_demo/omniparser_agent/vlm_agent.py
@@ -132,7 +132,15 @@ class VLMAgent:
                             sender="bot")
        self.output_callback(f'Set of Marks Screenshot for {colorful_text_vlm}:\n<img src="data:image/png;base64,{parsed_screen["som_image_base64"]}">', sender="bot")
        screen_info = str(parsed_screen['screen_info'])
-        self.output_callback(f'Screen Info for {colorful_text_vlm}:\n{screen_info}', sender="bot")
+        # self.output_callback(f'Screen Info for {colorful_text_vlm}:\n{screen_info}', sender="bot")
+        self.output_callback(
+                    f'<details>'
+                    f'  <summary>Screen Info for {colorful_text_vlm}</summary>'
+                    f'  <pre>{screen_info}</pre>'
+                    f'</details>',
+                    sender="bot"
+                )
+

        screenshot_uuid = parsed_screen['screenshot_uuid']
        screen_width, screen_height = parsed_screen['width'], parsed_screen['height']
@@ -155,7 +163,7 @@ class VLMAgent:
            planner_messages[-1]["content"].append(f"{OUTPUT_DIR}/screenshot_{screenshot_uuid}.png")
            planner_messages[-1]["content"].append(f"{OUTPUT_DIR}/screenshot_som_{screenshot_uuid}.png")

-        print(f"Sending messages to VLMPlanner : {planner_messages}")
+        # print(f"Sending messages to VLMPlanner : {planner_messages}")
        start = time.time()
        if "gpt" in self.model:
            vlm_response, token_usage = run_oai_interleaved(