readme

.gitignore

@@ -0,0 +1 @@
+weights/

README.md

@@ -6,6 +6,12 @@
 
 **OmniParser** is a comprehensive method for parsing user interface screenshots into structured and easy-to-understand elements, which significantly enhances the ability of GPT-4V to generate actions that can be accurately grounded in the corresponding regions of the interface. 
 
+## Install 
+```python
+conda create -n "omni" python==3.12
+pip install -r requirements.txt
+```
+
 ## Examples:
 We put together a few simple examples in the demo.ipynb. 
 

requirement.txt

@@ -1,14 +1,15 @@
-torch==2.2.2
+torch
-easyocr==1.7.1
+easyocr
-torchvision==0.17.2
+torchvision
 supervision==0.18.0
 openai==1.3.5
-transformers==4.40.2
+transformers
 ultralytics==8.1.24
 azure-identity
 numpy
-opencv-python==4.8.1.78
+opencv-python
-opencv-python-headless==4.8.0.74
+opencv-python-headless
-supervision==0.18.0
+gradio
-gradio==4.40.0
+dill
+accelerate
 

utils.py

@@ -74,11 +74,10 @@ def get_caption_model_processor(model_name="Salesforce/blip2-opt-2.7b", device=N
     return {'model': model.to(device), 'processor': processor}
 
 
-def get_yolo_model():
+def get_yolo_model(model_path):
     from ultralytics import YOLO
     # Load the model.
-    # model = YOLO('/home/yadonglu/sandbox/data/yolo/runs/detect/yolov8n_v8_xcyc/weights/best.pt')
+    model = YOLO(model_path)
-    model = YOLO('/home/yadonglu/sandbox/data/yolo/runs/detect/yolov8n_v8_seq_xcyc_b32_n4_office_ep20/weights/best.pt')
     return model