Dom/rpa_vision_v3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: replay E2E fonctionnel — 25/25 actions, 0 retries, SomEngine via serveur

Browse Source 
Validé sur PC Windows (DESKTOP-58D5CAC, 2560x1600) :
- 8 clics résolus visuellement (1 anchor_template, 1 som_text_match, 6 som_vlm)
- Score moyen 0.75, temps moyen 1.6s
- Texte tapé correctement (bonjour, test word, date, email)
- 0 retries, 2 actions non vérifiées (OK)

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Dom
2026-03-31 14:04:41 +02:00
parent 5e0b53cfd1
commit a7de6a488b
79542 changed files with 6091757 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
data/training/live_sessions/sess_20260314T213504_d41f20/live_events.jsonl Normal file
View File

@@ -0,0 +1,31 @@
{"session_id": "sess_20260314T213504_d41f20", "timestamp": 1773520504.6927588, "event": {"type": "window_focus_change", "from": null, "to": {"title": "RPA Vision V3 - Agent Assistant - Google Chrome", "app_name": "chrome.exe"}, "timestamp": 1773520504.3957508, "window": {"title": "RPA Vision V3 - Agent Assistant - Google Chrome", "app_name": "chrome.exe"}, "screenshot_context": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260314T213504_d41f20\\shots\\context_1773520504_focus_change.png"}}
{"session_id": "sess_20260314T213504_d41f20", "timestamp": 1773520505.0091257, "event": {"type": "heartbeat", "image": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260314T213504_d41f20\\shots\\context_1773520504_heartbeat.png", "timestamp": 1773520504.6286147}}
{"session_id": "sess_20260314T213504_d41f20", "timestamp": 1773520509.0775054, "event": {"type": "mouse_click", "button": "left", "pos": [825, 1567], "timestamp": 1773520508.7904565, "window": {"title": "RPA Vision V3 - Agent Assistant - Google Chrome", "app_name": "chrome.exe"}, "screenshot_id": "shot_0001", "vision_info": {"full": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260314T213504_d41f20\\shots\\shot_0001_full.png", "crop": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260314T213504_d41f20\\shots\\shot_0001_crop.png"}}}
{"session_id": "sess_20260314T213504_d41f20", "timestamp": 1773520510.162362, "event": {"type": "window_focus_change", "from": {"title": "RPA Vision V3 - Agent Assistant - Google Chrome", "app_name": "chrome.exe"}, "to": {"title": "Rechercher", "app_name": "SearchHost.exe"}, "timestamp": 1773520509.0712266, "window": {"title": "Rechercher", "app_name": "SearchHost.exe"}, "screenshot_context": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260314T213504_d41f20\\shots\\context_1773520509_focus_change.png"}}
{"session_id": "sess_20260314T213504_d41f20", "timestamp": 1773520510.3160949, "event": {"type": "heartbeat", "image": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260314T213504_d41f20\\shots\\context_1773520509_heartbeat.png", "timestamp": 1773520509.9135003}}
{"session_id": "sess_20260314T213504_d41f20", "timestamp": 1773520510.3280666, "event": {"type": "action_result", "base_shot_id": "shot_0001", "image": ""}}
{"session_id": "sess_20260314T213504_d41f20", "timestamp": 1773520513.7624075, "event": {"type": "mouse_click", "button": "left", "pos": [1423, 650], "timestamp": 1773520513.3366935, "window": {"title": "Rechercher", "app_name": "SearchHost.exe"}, "screenshot_id": "shot_0002", "vision_info": {"full": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260314T213504_d41f20\\shots\\shot_0002_full.png", "crop": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260314T213504_d41f20\\shots\\shot_0002_crop.png"}}}
{"session_id": "sess_20260314T213504_d41f20", "timestamp": 1773520514.2836018, "event": {"type": "window_focus_change", "from": {"title": "Rechercher", "app_name": "SearchHost.exe"}, "to": {"title": "unknown_window", "app_name": "explorer.exe"}, "timestamp": 1773520513.842688, "window": {"title": "unknown_window", "app_name": "explorer.exe"}, "screenshot_context": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260314T213504_d41f20\\shots\\context_1773520513_focus_change.png"}}
{"session_id": "sess_20260314T213504_d41f20", "timestamp": 1773520514.7917578, "event": {"type": "window_focus_change", "from": {"title": "unknown_window", "app_name": "explorer.exe"}, "to": {"title": "Nouvel onglet - Google Chrome", "app_name": "chrome.exe"}, "timestamp": 1773520514.6759045, "window": {"title": "Nouvel onglet - Google Chrome", "app_name": "chrome.exe"}, "screenshot_context": ""}}
{"session_id": "sess_20260314T213504_d41f20", "timestamp": 1773520515.211472, "event": {"type": "action_result", "base_shot_id": "shot_0002", "image": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260314T213504_d41f20\\shots\\context_1773520514_result_of_shot_0002.png"}}
{"session_id": "sess_20260314T213504_d41f20", "timestamp": 1773520516.2866433, "event": {"type": "mouse_click", "button": "left", "pos": [1268, 668], "timestamp": 1773520515.632417, "window": {"title": "Nouvel onglet - Google Chrome", "app_name": "chrome.exe"}, "screenshot_id": "shot_0003", "vision_info": {"full": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260314T213504_d41f20\\shots\\shot_0003_full.png", "crop": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260314T213504_d41f20\\shots\\shot_0003_crop.png"}}}
{"session_id": "sess_20260314T213504_d41f20", "timestamp": 1773520517.068889, "event": {"type": "action_result", "base_shot_id": "shot_0003", "image": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260314T213504_d41f20\\shots\\context_1773520516_result_of_shot_0003.png"}}
{"session_id": "sess_20260314T213504_d41f20", "timestamp": 1773520520.7182865, "event": {"type": "heartbeat", "image": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260314T213504_d41f20\\shots\\context_1773520520_heartbeat.png", "timestamp": 1773520520.1181622}}
{"session_id": "sess_20260314T213504_d41f20", "timestamp": 1773520527.683537, "event": {"type": "heartbeat", "image": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260314T213504_d41f20\\shots\\context_1773520525_heartbeat.png", "timestamp": 1773520525.26479}}
{"session_id": "sess_20260314T213504_d41f20", "timestamp": 1773520531.0313332, "event": {"type": "window_focus_change", "from": {"title": "Nouvel onglet - Google Chrome", "app_name": "chrome.exe"}, "to": {"title": "voiture de course - Recherche Google - Google Chrome", "app_name": "chrome.exe"}, "timestamp": 1773520529.8215058, "window": {"title": "voiture de course - Recherche Google - Google Chrome", "app_name": "chrome.exe"}, "screenshot_context": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260314T213504_d41f20\\shots\\context_1773520529_focus_change.png"}}
{"session_id": "sess_20260314T213504_d41f20", "timestamp": 1773520531.923085, "event": {"type": "heartbeat", "image": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260314T213504_d41f20\\shots\\context_1773520530_heartbeat.png", "timestamp": 1773520530.4694598}}
{"session_id": "sess_20260314T213504_d41f20", "timestamp": 1773520536.1984286, "event": {"type": "heartbeat", "image": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260314T213504_d41f20\\shots\\context_1773520535_heartbeat.png", "timestamp": 1773520535.6815}}
{"session_id": "sess_20260314T213504_d41f20", "timestamp": 1773520540.4496353, "event": {"type": "mouse_click", "button": "left", "pos": [2134, 1562], "timestamp": 1773520537.263285, "window": {"title": "voiture de course - Recherche Google - Google Chrome", "app_name": "chrome.exe"}, "screenshot_id": "shot_0004", "vision_info": {"full": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260314T213504_d41f20\\shots\\shot_0004_full.png", "crop": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260314T213504_d41f20\\shots\\shot_0004_crop.png"}}}
{"session_id": "sess_20260314T213504_d41f20", "timestamp": 1773520543.7158756, "event": {"type": "window_focus_change", "from": {"title": "voiture de course - Recherche Google - Google Chrome", "app_name": "chrome.exe"}, "to": {"title": "Fen\u00eatre de d\u00e9passement de capacit\u00e9 de la barre d\u2019\u00e9tat syst\u00e8me.", "app_name": "explorer.exe"}, "timestamp": 1773520537.5707417, "window": {"title": "Fen\u00eatre de d\u00e9passement de capacit\u00e9 de la barre d\u2019\u00e9tat syst\u00e8me.", "app_name": "explorer.exe"}, "screenshot_context": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260314T213504_d41f20\\shots\\context_1773520537_focus_change.png"}}
{"session_id": "sess_20260314T213504_d41f20", "timestamp": 1773520543.8901572, "event": {"type": "mouse_click", "button": "left", "pos": [2134, 1562], "timestamp": 1773520538.0876498, "window": {"title": "Fen\u00eatre de d\u00e9passement de capacit\u00e9 de la barre d\u2019\u00e9tat syst\u00e8me.", "app_name": "explorer.exe"}, "screenshot_id": "shot_0005", "vision_info": {"full": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260314T213504_d41f20\\shots\\shot_0005_full.png", "crop": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260314T213504_d41f20\\shots\\shot_0005_crop.png"}}}
{"session_id": "sess_20260314T213504_d41f20", "timestamp": 1773520544.0131264, "event": {"type": "action_result", "base_shot_id": "shot_0004", "image": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260314T213504_d41f20\\shots\\context_1773520538_result_of_shot_0004.png"}}
{"session_id": "sess_20260314T213504_d41f20", "timestamp": 1773520544.1580696, "event": {"type": "window_focus_change", "from": {"title": "Fen\u00eatre de d\u00e9passement de capacit\u00e9 de la barre d\u2019\u00e9tat syst\u00e8me.", "app_name": "explorer.exe"}, "to": {"title": "unknown_window", "app_name": "explorer.exe"}, "timestamp": 1773520538.785732, "window": {"title": "unknown_window", "app_name": "explorer.exe"}, "screenshot_context": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260314T213504_d41f20\\shots\\context_1773520538_focus_change.png"}}
{"session_id": "sess_20260314T213504_d41f20", "timestamp": 1773520544.3440008, "event": {"type": "mouse_click", "button": "left", "pos": [2134, 1562], "timestamp": 1773520539.173741, "window": {"title": "unknown_window", "app_name": "explorer.exe"}, "screenshot_id": "shot_0006", "vision_info": {"full": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260314T213504_d41f20\\shots\\shot_0006_full.png", "crop": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260314T213504_d41f20\\shots\\shot_0006_crop.png"}}}
{"session_id": "sess_20260314T213504_d41f20", "timestamp": 1773520544.3728378, "event": {"type": "action_result", "base_shot_id": "shot_0005", "image": ""}}
{"session_id": "sess_20260314T213504_d41f20", "timestamp": 1773520544.4986737, "event": {"type": "window_focus_change", "from": {"title": "unknown_window", "app_name": "explorer.exe"}, "to": {"title": "Fen\u00eatre de d\u00e9passement de capacit\u00e9 de la barre d\u2019\u00e9tat syst\u00e8me.", "app_name": "explorer.exe"}, "timestamp": 1773520539.495361, "window": {"title": "Fen\u00eatre de d\u00e9passement de capacit\u00e9 de la barre d\u2019\u00e9tat syst\u00e8me.", "app_name": "explorer.exe"}, "screenshot_context": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260314T213504_d41f20\\shots\\context_1773520539_focus_change.png"}}
{"session_id": "sess_20260314T213504_d41f20", "timestamp": 1773520544.613536, "event": {"type": "action_result", "base_shot_id": "shot_0006", "image": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260314T213504_d41f20\\shots\\context_1773520540_result_of_shot_0006.png"}}
{"session_id": "sess_20260314T213504_d41f20", "timestamp": 1773520544.7348192, "event": {"type": "heartbeat", "image": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260314T213504_d41f20\\shots\\context_1773520540_heartbeat.png", "timestamp": 1773520540.902443}}
{"session_id": "sess_20260314T213504_d41f20", "timestamp": 1773520544.9200122, "event": {"type": "mouse_click", "button": "right", "pos": [2020, 1343], "timestamp": 1773520542.392861, "window": {"title": "Fen\u00eatre de d\u00e9passement de capacit\u00e9 de la barre d\u2019\u00e9tat syst\u00e8me.", "app_name": "explorer.exe"}, "screenshot_id": "shot_0007", "vision_info": {"full": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260314T213504_d41f20\\shots\\shot_0007_full.png", "crop": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260314T213504_d41f20\\shots\\shot_0007_crop.png"}}}
{"session_id": "sess_20260314T213504_d41f20", "timestamp": 1773520545.0531945, "event": {"type": "window_focus_change", "from": {"title": "Fen\u00eatre de d\u00e9passement de capacit\u00e9 de la barre d\u2019\u00e9tat syst\u00e8me.", "app_name": "explorer.exe"}, "to": {"title": "unknown_window", "app_name": "python.exe"}, "timestamp": 1773520542.711964, "window": {"title": "unknown_window", "app_name": "python.exe"}, "screenshot_context": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260314T213504_d41f20\\shots\\context_1773520542_focus_change.png"}}
{"session_id": "sess_20260314T213504_d41f20", "timestamp": 1773520545.1746607, "event": {"type": "action_result", "base_shot_id": "shot_0007", "image": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260314T213504_d41f20\\shots\\context_1773520543_result_of_shot_0007.png"}}
{"session_id": "sess_20260314T213504_d41f20", "timestamp": 1773520545.354673, "event": {"type": "mouse_click", "button": "left", "pos": [1842, 1240], "timestamp": 1773520544.2551157, "window": {"title": "unknown_window", "app_name": "python.exe"}, "screenshot_id": "shot_0008", "vision_info": {"full": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260314T213504_d41f20\\shots\\shot_0008_full.png", "crop": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260314T213504_d41f20\\shots\\shot_0008_crop.png"}}}