Dom/rpa_vision_v3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: replay E2E fonctionnel — 25/25 actions, 0 retries, SomEngine via serveur

Browse Source 
Validé sur PC Windows (DESKTOP-58D5CAC, 2560x1600) :
- 8 clics résolus visuellement (1 anchor_template, 1 som_text_match, 6 som_vlm)
- Score moyen 0.75, temps moyen 1.6s
- Texte tapé correctement (bonjour, test word, date, email)
- 0 retries, 2 actions non vérifiées (OK)

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Dom
2026-03-31 14:04:41 +02:00
parent 5e0b53cfd1
commit a7de6a488b
79542 changed files with 6091757 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
data/training/live_sessions/sess_20260316T194221_f2f762/live_events.jsonl Normal file
View File

@@ -0,0 +1,33 @@
{"session_id": "sess_20260316T194221_f2f762", "timestamp": 1773686542.0778897, "event": {"type": "heartbeat", "image": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260316T194221_f2f762\\shots\\context_1773686541_heartbeat.png", "timestamp": 1773686541.9858012}}
{"session_id": "sess_20260316T194221_f2f762", "timestamp": 1773686542.192465, "event": {"type": "window_focus_change", "from": null, "to": {"title": "Lea - RPA Vision Agent V1", "app_name": "WindowsTerminal.exe"}, "timestamp": 1773686541.86071, "window": {"title": "Lea - RPA Vision Agent V1", "app_name": "WindowsTerminal.exe"}, "screenshot_context": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260316T194221_f2f762\\shots\\context_1773686541_focus_change.png"}}
{"session_id": "sess_20260316T194221_f2f762", "timestamp": 1773686543.7365494, "event": {"type": "mouse_click", "button": "left", "pos": [878, 1563], "timestamp": 1773686543.464017, "window": {"title": "Lea - RPA Vision Agent V1", "app_name": "WindowsTerminal.exe"}, "screenshot_id": "shot_0001", "vision_info": {"full": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260316T194221_f2f762\\shots\\shot_0001_full.png", "crop": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260316T194221_f2f762\\shots\\shot_0001_crop.png"}}}
{"session_id": "sess_20260316T194221_f2f762", "timestamp": 1773686544.417923, "event": {"type": "window_focus_change", "from": {"title": "Lea - RPA Vision Agent V1", "app_name": "WindowsTerminal.exe"}, "to": {"title": "Rechercher", "app_name": "SearchHost.exe"}, "timestamp": 1773686544.063269, "window": {"title": "Rechercher", "app_name": "SearchHost.exe"}, "screenshot_context": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260316T194221_f2f762\\shots\\context_1773686544_focus_change.png"}}
{"session_id": "sess_20260316T194221_f2f762", "timestamp": 1773686544.8226702, "event": {"type": "text_input", "text": "n", "pos": [878, 1563], "timestamp": 1773686544.8226702, "window": {"title": "Rechercher", "app_name": "SearchHost.exe"}}}
{"session_id": "sess_20260316T194221_f2f762", "timestamp": 1773686544.9376173, "event": {"type": "action_result", "base_shot_id": "shot_0001", "image": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260316T194221_f2f762\\shots\\context_1773686544_result_of_shot_0001.png"}}
{"session_id": "sess_20260316T194221_f2f762", "timestamp": 1773686545.9916105, "event": {"type": "text_input", "text": "ote", "pos": [878, 1563], "timestamp": 1773686545.990617, "window": {"title": "Rechercher", "app_name": "SearchHost.exe"}}}
{"session_id": "sess_20260316T194221_f2f762", "timestamp": 1773686547.2942367, "event": {"type": "heartbeat", "image": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260316T194221_f2f762\\shots\\context_1773686547_heartbeat.png", "timestamp": 1773686547.196}}
{"session_id": "sess_20260316T194221_f2f762", "timestamp": 1773686548.6447442, "event": {"type": "mouse_click", "button": "left", "pos": [1361, 646], "timestamp": 1773686548.310685, "window": {"title": "Rechercher", "app_name": "SearchHost.exe"}, "screenshot_id": "shot_0002", "vision_info": {"full": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260316T194221_f2f762\\shots\\shot_0002_full.png", "crop": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260316T194221_f2f762\\shots\\shot_0002_crop.png"}}}
{"session_id": "sess_20260316T194221_f2f762", "timestamp": 1773686549.0565379, "event": {"type": "window_focus_change", "from": {"title": "Rechercher", "app_name": "SearchHost.exe"}, "to": {"title": "unknown_window", "app_name": "explorer.exe"}, "timestamp": 1773686548.7945075, "window": {"title": "unknown_window", "app_name": "explorer.exe"}, "screenshot_context": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260316T194221_f2f762\\shots\\context_1773686548_focus_change.png"}}
{"session_id": "sess_20260316T194221_f2f762", "timestamp": 1773686549.7175174, "event": {"type": "window_focus_change", "from": {"title": "unknown_window", "app_name": "explorer.exe"}, "to": {"title": "test4.txt \u2013 Bloc-notes", "app_name": "Notepad.exe"}, "timestamp": 1773686549.467886, "window": {"title": "test4.txt \u2013 Bloc-notes", "app_name": "Notepad.exe"}, "screenshot_context": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260316T194221_f2f762\\shots\\context_1773686549_focus_change.png"}}
{"session_id": "sess_20260316T194221_f2f762", "timestamp": 1773686549.8275259, "event": {"type": "action_result", "base_shot_id": "shot_0002", "image": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260316T194221_f2f762\\shots\\context_1773686549_result_of_shot_0002.png"}}
{"session_id": "sess_20260316T194221_f2f762", "timestamp": 1773686551.4737146, "event": {"type": "mouse_click", "button": "left", "pos": [1421, 380], "timestamp": 1773686551.230842, "window": {"title": "test4.txt \u2013 Bloc-notes", "app_name": "Notepad.exe"}, "screenshot_id": "shot_0003", "vision_info": {"full": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260316T194221_f2f762\\shots\\shot_0003_full.png", "crop": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260316T194221_f2f762\\shots\\shot_0003_crop.png"}}}
{"session_id": "sess_20260316T194221_f2f762", "timestamp": 1773686551.8490205, "event": {"type": "window_focus_change", "from": {"title": "test4.txt \u2013 Bloc-notes", "app_name": "Notepad.exe"}, "to": {"title": "Sans titre \u2013 Bloc-notes", "app_name": "Notepad.exe"}, "timestamp": 1773686551.6307602, "window": {"title": "Sans titre \u2013 Bloc-notes", "app_name": "Notepad.exe"}, "screenshot_context": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260316T194221_f2f762\\shots\\context_1773686551_focus_change.png"}}
{"session_id": "sess_20260316T194221_f2f762", "timestamp": 1773686552.4138718, "event": {"type": "heartbeat", "image": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260316T194221_f2f762\\shots\\context_1773686552_heartbeat.png", "timestamp": 1773686552.3147247}}
{"session_id": "sess_20260316T194221_f2f762", "timestamp": 1773686552.5864449, "event": {"type": "mouse_click", "button": "left", "pos": [1328, 554], "timestamp": 1773686552.2827199, "window": {"title": "Sans titre \u2013 Bloc-notes", "app_name": "Notepad.exe"}, "screenshot_id": "shot_0004", "vision_info": {"full": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260316T194221_f2f762\\shots\\shot_0004_full.png", "crop": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260316T194221_f2f762\\shots\\shot_0004_crop.png"}}}
{"session_id": "sess_20260316T194221_f2f762", "timestamp": 1773686552.6115189, "event": {"type": "action_result", "base_shot_id": "shot_0003", "image": ""}}
{"session_id": "sess_20260316T194221_f2f762", "timestamp": 1773686553.4917054, "event": {"type": "action_result", "base_shot_id": "shot_0004", "image": ""}}
{"session_id": "sess_20260316T194221_f2f762", "timestamp": 1773686554.46861, "event": {"type": "window_focus_change", "from": {"title": "Sans titre \u2013 Bloc-notes", "app_name": "Notepad.exe"}, "to": {"title": "*he \u2013 Bloc-notes", "app_name": "Notepad.exe"}, "timestamp": 1773686554.25034, "window": {"title": "*he \u2013 Bloc-notes", "app_name": "Notepad.exe"}, "screenshot_context": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260316T194221_f2f762\\shots\\context_1773686554_focus_change.png"}}
{"session_id": "sess_20260316T194221_f2f762", "timestamp": 1773686555.0798025, "event": {"type": "window_focus_change", "from": {"title": "*he \u2013 Bloc-notes", "app_name": "Notepad.exe"}, "to": {"title": "*hello \u2013 Bloc-notes", "app_name": "Notepad.exe"}, "timestamp": 1773686554.8733265, "window": {"title": "*hello \u2013 Bloc-notes", "app_name": "Notepad.exe"}, "screenshot_context": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260316T194221_f2f762\\shots\\context_1773686554_focus_change.png"}}
{"session_id": "sess_20260316T194221_f2f762", "timestamp": 1773686555.2967846, "event": {"type": "text_input", "text": "hello", "pos": [1328, 554], "timestamp": 1773686555.2947943, "window": {"title": "*hello \u2013 Bloc-notes", "app_name": "Notepad.exe"}}}
{"session_id": "sess_20260316T194221_f2f762", "timestamp": 1773686557.2395787, "event": {"type": "mouse_click", "button": "left", "pos": [2460, 361], "timestamp": 1773686556.9942575, "window": {"title": "*hello \u2013 Bloc-notes", "app_name": "Notepad.exe"}, "screenshot_id": "shot_0005", "vision_info": {"full": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260316T194221_f2f762\\shots\\shot_0005_full.png", "crop": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260316T194221_f2f762\\shots\\shot_0005_crop.png"}}}
{"session_id": "sess_20260316T194221_f2f762", "timestamp": 1773686557.5307631, "event": {"type": "heartbeat", "image": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260316T194221_f2f762\\shots\\context_1773686557_heartbeat.png", "timestamp": 1773686557.442297}}
{"session_id": "sess_20260316T194221_f2f762", "timestamp": 1773686557.729783, "event": {"type": "window_focus_change", "from": {"title": "*hello \u2013 Bloc-notes", "app_name": "Notepad.exe"}, "to": {"title": "Lea - RPA Vision Agent V1", "app_name": "WindowsTerminal.exe"}, "timestamp": 1773686557.4915154, "window": {"title": "Lea - RPA Vision Agent V1", "app_name": "WindowsTerminal.exe"}, "screenshot_context": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260316T194221_f2f762\\shots\\context_1773686557_focus_change.png"}}
{"session_id": "sess_20260316T194221_f2f762", "timestamp": 1773686558.3505964, "event": {"type": "action_result", "base_shot_id": "shot_0005", "image": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260316T194221_f2f762\\shots\\context_1773686558_result_of_shot_0005.png"}}
{"session_id": "sess_20260316T194221_f2f762", "timestamp": 1773686559.7688613, "event": {"type": "mouse_click", "button": "left", "pos": [2120, 1562], "timestamp": 1773686559.5254219, "window": {"title": "Lea - RPA Vision Agent V1", "app_name": "WindowsTerminal.exe"}, "screenshot_id": "shot_0006", "vision_info": {"full": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260316T194221_f2f762\\shots\\shot_0006_full.png", "crop": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260316T194221_f2f762\\shots\\shot_0006_crop.png"}}}
{"session_id": "sess_20260316T194221_f2f762", "timestamp": 1773686560.334576, "event": {"type": "window_focus_change", "from": {"title": "Lea - RPA Vision Agent V1", "app_name": "WindowsTerminal.exe"}, "to": {"title": "Fen\u00eatre de d\u00e9passement de capacit\u00e9 de la barre d\u2019\u00e9tat syst\u00e8me.", "app_name": "explorer.exe"}, "timestamp": 1773686560.118596, "window": {"title": "Fen\u00eatre de d\u00e9passement de capacit\u00e9 de la barre d\u2019\u00e9tat syst\u00e8me.", "app_name": "explorer.exe"}, "screenshot_context": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260316T194221_f2f762\\shots\\context_1773686560_focus_change.png"}}
{"session_id": "sess_20260316T194221_f2f762", "timestamp": 1773686560.909771, "event": {"type": "action_result", "base_shot_id": "shot_0006", "image": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260316T194221_f2f762\\shots\\context_1773686560_result_of_shot_0006.png"}}
{"session_id": "sess_20260316T194221_f2f762", "timestamp": 1773686562.443376, "event": {"type": "mouse_click", "button": "right", "pos": [2021, 1340], "timestamp": 1773686562.153913, "window": {"title": "Fen\u00eatre de d\u00e9passement de capacit\u00e9 de la barre d\u2019\u00e9tat syst\u00e8me.", "app_name": "explorer.exe"}, "screenshot_id": "shot_0007", "vision_info": {"full": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260316T194221_f2f762\\shots\\shot_0007_full.png", "crop": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260316T194221_f2f762\\shots\\shot_0007_crop.png"}}}
{"session_id": "sess_20260316T194221_f2f762", "timestamp": 1773686562.682933, "event": {"type": "heartbeat", "image": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260316T194221_f2f762\\shots\\context_1773686562_heartbeat.png", "timestamp": 1773686562.5767071}}
{"session_id": "sess_20260316T194221_f2f762", "timestamp": 1773686562.976039, "event": {"type": "window_focus_change", "from": {"title": "Fen\u00eatre de d\u00e9passement de capacit\u00e9 de la barre d\u2019\u00e9tat syst\u00e8me.", "app_name": "explorer.exe"}, "to": {"title": "unknown_window", "app_name": "python.exe"}, "timestamp": 1773686562.7495286, "window": {"title": "unknown_window", "app_name": "python.exe"}, "screenshot_context": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260316T194221_f2f762\\shots\\context_1773686562_focus_change.png"}}
{"session_id": "sess_20260316T194221_f2f762", "timestamp": 1773686563.5476782, "event": {"type": "action_result", "base_shot_id": "shot_0007", "image": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260316T194221_f2f762\\shots\\context_1773686563_result_of_shot_0007.png"}}
{"session_id": "sess_20260316T194221_f2f762", "timestamp": 1773686563.9823236, "event": {"type": "mouse_click", "button": "left", "pos": [1870, 1117], "timestamp": 1773686563.7253947, "window": {"title": "unknown_window", "app_name": "python.exe"}, "screenshot_id": "shot_0008", "vision_info": {"full": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260316T194221_f2f762\\shots\\shot_0008_full.png", "crop": "C:\\rpa_vision\\agent_v1\\sessions\\sess_20260316T194221_f2f762\\shots\\shot_0008_crop.png"}}}