Dom/rpa_vision_v3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: replay E2E fonctionnel — 25/25 actions, 0 retries, SomEngine via serveur

Browse Source 
Validé sur PC Windows (DESKTOP-58D5CAC, 2560x1600) :
- 8 clics résolus visuellement (1 anchor_template, 1 som_text_match, 6 som_vlm)
- Score moyen 0.75, temps moyen 1.6s
- Texte tapé correctement (bonjour, test word, date, email)
- 0 retries, 2 actions non vérifiées (OK)

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Dom
2026-03-31 14:04:41 +02:00
parent 5e0b53cfd1
commit a7de6a488b
79542 changed files with 6091757 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

867
data/training/sessions/sess_20260105T172736_7dcebf54/sess_20260105T172736_7dcebf54.json Normal file
View File

@@ -0,0 +1,867 @@
{
"schema_version": "rawsession_v1",
"session_id": "sess_20260105T172736_7dcebf54",
"agent_version": "0.1.0",
"environment": {
"platform": "linux",
"hostname": "dom-X870-Riptide-WiFi",
"screen": {
"primary_resolution": [
1920,
1080
],
"display_scale": 1.0
}
},
"user": {
"id": "demo_user",
"label": "Démo agent_v0"
},
"context": {
"customer": "Clinique Demo",
"training_label": "Facturation_T2A_demo",
"notes": "Session réelle avec clics + screenshots + key combos."
},
"started_at": "2026-01-05T16:27:36Z",
"ended_at": "2026-01-05T16:28:26Z",
"events": [
{
"t": 3.112995555042289,
"type": "mouse_click",
"window": {
"title": ".env.local - rpa_vision_v3 - Kiro",
"app_name": "kiro"
},
"screenshot_id": "shot_0001",
"button": "left",
"pos": [
44,
1050
],
"keys": null,
"idle_ms": null,
"scroll_delta": null
},
{
"t": 4.58373133209534,
"type": "mouse_click",
"window": {
"title": ".env.local - rpa_vision_v3 - Kiro",
"app_name": "kiro"
},
"screenshot_id": "shot_0002",
"button": "left",
"pos": [
961,
559
],
"keys": null,
"idle_ms": null,
"scroll_delta": null
},
{
"t": 6.186219310038723,
"type": "mouse_click",
"window": {
"title": "Calculatrice",
"app_name": "gnome-calculato"
},
"screenshot_id": "shot_0003",
"button": "left",
"pos": [
366,
445
],
"keys": null,
"idle_ms": null,
"scroll_delta": null
},
{
"t": 7.20105162600521,
"type": "mouse_click",
"window": {
"title": "Calculatrice",
"app_name": "gnome-calculato"
},
"screenshot_id": "shot_0004",
"button": "left",
"pos": [
452,
456
],
"keys": null,
"idle_ms": null,
"scroll_delta": null
},
{
"t": 8.304793582996354,
"type": "mouse_click",
"window": {
"title": "Calculatrice",
"app_name": "gnome-calculato"
},
"screenshot_id": "shot_0005",
"button": "left",
"pos": [
371,
453
],
"keys": null,
"idle_ms": null,
"scroll_delta": null
},
{
"t": 9.630852095084265,
"type": "mouse_click",
"window": {
"title": "Calculatrice",
"app_name": "gnome-calculato"
},
"screenshot_id": "shot_0006",
"button": "left",
"pos": [
510,
545
],
"keys": null,
"idle_ms": null,
"scroll_delta": null
},
{
"t": 11.725292118033394,
"type": "mouse_click",
"window": {
"title": "Calculatrice",
"app_name": "gnome-calculato"
},
"screenshot_id": "shot_0007",
"button": "left",
"pos": [
530,
154
],
"keys": null,
"idle_ms": null,
"scroll_delta": null
},
{
"t": 12.896447239094414,
"type": "mouse_click",
"window": {
"title": ".env.local - rpa_vision_v3 - Kiro",
"app_name": "kiro"
},
"screenshot_id": "shot_0008",
"button": "left",
"pos": [
36,
1059
],
"keys": null,
"idle_ms": null,
"scroll_delta": null
},
{
"t": 13.983899859013036,
"type": "mouse_click",
"window": {
"title": ".env.local - rpa_vision_v3 - Kiro",
"app_name": "kiro"
},
"screenshot_id": "shot_0009",
"button": "left",
"pos": [
903,
557
],
"keys": null,
"idle_ms": null,
"scroll_delta": null
},
{
"t": 15.857324754004367,
"type": "mouse_click",
"window": {
"title": "Calculatrice",
"app_name": "gnome-calculato"
},
"screenshot_id": "shot_0010",
"button": "left",
"pos": [
356,
433
],
"keys": null,
"idle_ms": null,
"scroll_delta": null
},
{
"t": 16.703278140048496,
"type": "mouse_click",
"window": {
"title": "Calculatrice",
"app_name": "gnome-calculato"
},
"screenshot_id": "shot_0011",
"button": "left",
"pos": [
440,
445
],
"keys": null,
"idle_ms": null,
"scroll_delta": null
},
{
"t": 17.568454213091172,
"type": "mouse_click",
"window": {
"title": "Calculatrice",
"app_name": "gnome-calculato"
},
"screenshot_id": "shot_0012",
"button": "left",
"pos": [
378,
442
],
"keys": null,
"idle_ms": null,
"scroll_delta": null
},
{
"t": 18.6390130961081,
"type": "mouse_click",
"window": {
"title": "Calculatrice",
"app_name": "gnome-calculato"
},
"screenshot_id": "shot_0013",
"button": "left",
"pos": [
504,
557
],
"keys": null,
"idle_ms": null,
"scroll_delta": null
},
{
"t": 20.695704799029045,
"type": "mouse_click",
"window": {
"title": "Calculatrice",
"app_name": "gnome-calculato"
},
"screenshot_id": "shot_0014",
"button": "left",
"pos": [
530,
160
],
"keys": null,
"idle_ms": null,
"scroll_delta": null
},
{
"t": 22.754871695069596,
"type": "mouse_click",
"window": {
"title": ".env.local - rpa_vision_v3 - Kiro",
"app_name": "kiro"
},
"screenshot_id": "shot_0015",
"button": "left",
"pos": [
40,
1041
],
"keys": null,
"idle_ms": null,
"scroll_delta": null
},
{
"t": 24.42362946108915,
"type": "mouse_click",
"window": {
"title": ".env.local - rpa_vision_v3 - Kiro",
"app_name": "kiro"
},
"screenshot_id": "shot_0016",
"button": "left",
"pos": [
955,
583
],
"keys": null,
"idle_ms": null,
"scroll_delta": null
},
{
"t": 26.815814930014312,
"type": "mouse_click",
"window": {
"title": "Calculatrice",
"app_name": "gnome-calculato"
},
"screenshot_id": "shot_0017",
"button": "left",
"pos": [
346,
441
],
"keys": null,
"idle_ms": null,
"scroll_delta": null
},
{
"t": 27.504557627020404,
"type": "mouse_click",
"window": {
"title": "Calculatrice",
"app_name": "gnome-calculato"
},
"screenshot_id": "shot_0018",
"button": "left",
"pos": [
446,
442
],
"keys": null,
"idle_ms": null,
"scroll_delta": null
},
{
"t": 28.223480232059956,
"type": "mouse_click",
"window": {
"title": "Calculatrice",
"app_name": "gnome-calculato"
},
"screenshot_id": "shot_0019",
"button": "left",
"pos": [
360,
447
],
"keys": null,
"idle_ms": null,
"scroll_delta": null
},
{
"t": 29.223252651048824,
"type": "mouse_click",
"window": {
"title": "Calculatrice",
"app_name": "gnome-calculato"
},
"screenshot_id": "shot_0020",
"button": "left",
"pos": [
509,
543
],
"keys": null,
"idle_ms": null,
"scroll_delta": null
},
{
"t": 30.877979076001793,
"type": "mouse_click",
"window": {
"title": "Calculatrice",
"app_name": "gnome-calculato"
},
"screenshot_id": "shot_0021",
"button": "left",
"pos": [
528,
153
],
"keys": null,
"idle_ms": null,
"scroll_delta": null
},
{
"t": 32.47297390305903,
"type": "mouse_click",
"window": {
"title": ".env.local - rpa_vision_v3 - Kiro",
"app_name": "kiro"
},
"screenshot_id": "shot_0022",
"button": "left",
"pos": [
15,
1062
],
"keys": null,
"idle_ms": null,
"scroll_delta": null
},
{
"t": 33.83985221001785,
"type": "mouse_click",
"window": {
"title": ".env.local - rpa_vision_v3 - Kiro",
"app_name": "kiro"
},
"screenshot_id": "shot_0023",
"button": "left",
"pos": [
908,
572
],
"keys": null,
"idle_ms": null,
"scroll_delta": null
},
{
"t": 35.314242620021105,
"type": "mouse_click",
"window": {
"title": "Calculatrice",
"app_name": "gnome-calculato"
},
"screenshot_id": "shot_0024",
"button": "left",
"pos": [
358,
448
],
"keys": null,
"idle_ms": null,
"scroll_delta": null
},
{
"t": 36.235623224056326,
"type": "mouse_click",
"window": {
"title": "Calculatrice",
"app_name": "gnome-calculato"
},
"screenshot_id": "shot_0025",
"button": "left",
"pos": [
441,
441
],
"keys": null,
"idle_ms": null,
"scroll_delta": null
},
{
"t": 36.968684250023216,
"type": "mouse_click",
"window": {
"title": "Calculatrice",
"app_name": "gnome-calculato"
},
"screenshot_id": "shot_0026",
"button": "left",
"pos": [
379,
453
],
"keys": null,
"idle_ms": null,
"scroll_delta": null
},
{
"t": 37.799650871078484,
"type": "mouse_click",
"window": {
"title": "Calculatrice",
"app_name": "gnome-calculato"
},
"screenshot_id": "shot_0027",
"button": "left",
"pos": [
501,
572
],
"keys": null,
"idle_ms": null,
"scroll_delta": null
},
{
"t": 39.3597621290246,
"type": "mouse_click",
"window": {
"title": "Calculatrice",
"app_name": "gnome-calculato"
},
"screenshot_id": "shot_0028",
"button": "left",
"pos": [
528,
151
],
"keys": null,
"idle_ms": null,
"scroll_delta": null
},
{
"t": 39.959100433043204,
"type": "hover_idle",
"window": {
"title": ".env.local - rpa_vision_v3 - Kiro",
"app_name": "kiro"
},
"screenshot_id": "shot_0029",
"button": null,
"pos": [
528,
151
],
"keys": null,
"idle_ms": 791,
"scroll_delta": null
},
{
"t": 41.89569197606761,
"type": "mouse_click",
"window": {
"title": ".env.local - rpa_vision_v3 - Kiro",
"app_name": "kiro"
},
"screenshot_id": "shot_0030",
"button": "left",
"pos": [
47,
1063
],
"keys": null,
"idle_ms": null,
"scroll_delta": null
},
{
"t": 43.23150628909934,
"type": "mouse_click",
"window": {
"title": ".env.local - rpa_vision_v3 - Kiro",
"app_name": "kiro"
},
"screenshot_id": "shot_0031",
"button": "left",
"pos": [
875,
578
],
"keys": null,
"idle_ms": null,
"scroll_delta": null
},
{
"t": 44.81672757607885,
"type": "mouse_click",
"window": {
"title": "Calculatrice",
"app_name": "gnome-calculato"
},
"screenshot_id": "shot_0032",
"button": "left",
"pos": [
366,
434
],
"keys": null,
"idle_ms": null,
"scroll_delta": null
},
{
"t": 45.72128127410542,
"type": "mouse_click",
"window": {
"title": "Calculatrice",
"app_name": "gnome-calculato"
},
"screenshot_id": "shot_0033",
"button": "left",
"pos": [
407,
440
],
"keys": null,
"idle_ms": null,
"scroll_delta": null
},
{
"t": 46.4494580370374,
"type": "mouse_click",
"window": {
"title": "Calculatrice",
"app_name": "gnome-calculato"
},
"screenshot_id": "shot_0034",
"button": "left",
"pos": [
352,
447
],
"keys": null,
"idle_ms": null,
"scroll_delta": null
},
{
"t": 47.47228330303915,
"type": "mouse_click",
"window": {
"title": "Calculatrice",
"app_name": "gnome-calculato"
},
"screenshot_id": "shot_0035",
"button": "left",
"pos": [
482,
535
],
"keys": null,
"idle_ms": null,
"scroll_delta": null
},
{
"t": 47.99828573002014,
"type": "hover_idle",
"window": {
"title": "Calculatrice",
"app_name": "gnome-calculato"
},
"screenshot_id": "shot_0036",
"button": null,
"pos": [
482,
535
],
"keys": null,
"idle_ms": 787,
"scroll_delta": null
},
{
"t": 49.064281913102604,
"type": "mouse_click",
"window": {
"title": "Calculatrice",
"app_name": "gnome-calculato"
},
"screenshot_id": "shot_0037",
"button": "left",
"pos": [
524,
156
],
"keys": null,
"idle_ms": null,
"scroll_delta": null
},
{
"t": 50.44601065502502,
"type": "mouse_click",
"window": {
"title": ".env.local - rpa_vision_v3 - Kiro",
"app_name": "kiro"
},
"screenshot_id": "shot_0038",
"button": "left",
"pos": [
1657,
14
],
"keys": null,
"idle_ms": null,
"scroll_delta": null
}
],
"screenshots": [
{
"screenshot_id": "shot_0001",
"relative_path": "shots/shot_0001.png",
"captured_at": "2026-01-05T16:27:39Z"
},
{
"screenshot_id": "shot_0002",
"relative_path": "shots/shot_0002.png",
"captured_at": "2026-01-05T16:27:40Z"
},
{
"screenshot_id": "shot_0003",
"relative_path": "shots/shot_0003.png",
"captured_at": "2026-01-05T16:27:42Z"
},
{
"screenshot_id": "shot_0004",
"relative_path": "shots/shot_0004.png",
"captured_at": "2026-01-05T16:27:43Z"
},
{
"screenshot_id": "shot_0005",
"relative_path": "shots/shot_0005.png",
"captured_at": "2026-01-05T16:27:44Z"
},
{
"screenshot_id": "shot_0006",
"relative_path": "shots/shot_0006.png",
"captured_at": "2026-01-05T16:27:45Z"
},
{
"screenshot_id": "shot_0007",
"relative_path": "shots/shot_0007.png",
"captured_at": "2026-01-05T16:27:47Z"
},
{
"screenshot_id": "shot_0008",
"relative_path": "shots/shot_0008.png",
"captured_at": "2026-01-05T16:27:49Z"
},
{
"screenshot_id": "shot_0009",
"relative_path": "shots/shot_0009.png",
"captured_at": "2026-01-05T16:27:50Z"
},
{
"screenshot_id": "shot_0010",
"relative_path": "shots/shot_0010.png",
"captured_at": "2026-01-05T16:27:52Z"
},
{
"screenshot_id": "shot_0011",
"relative_path": "shots/shot_0011.png",
"captured_at": "2026-01-05T16:27:52Z"
},
{
"screenshot_id": "shot_0012",
"relative_path": "shots/shot_0012.png",
"captured_at": "2026-01-05T16:27:53Z"
},
{
"screenshot_id": "shot_0013",
"relative_path": "shots/shot_0013.png",
"captured_at": "2026-01-05T16:27:54Z"
},
{
"screenshot_id": "shot_0014",
"relative_path": "shots/shot_0014.png",
"captured_at": "2026-01-05T16:27:56Z"
},
{
"screenshot_id": "shot_0015",
"relative_path": "shots/shot_0015.png",
"captured_at": "2026-01-05T16:27:58Z"
},
{
"screenshot_id": "shot_0016",
"relative_path": "shots/shot_0016.png",
"captured_at": "2026-01-05T16:28:00Z"
},
{
"screenshot_id": "shot_0017",
"relative_path": "shots/shot_0017.png",
"captured_at": "2026-01-05T16:28:03Z"
},
{
"screenshot_id": "shot_0018",
"relative_path": "shots/shot_0018.png",
"captured_at": "2026-01-05T16:28:03Z"
},
{
"screenshot_id": "shot_0019",
"relative_path": "shots/shot_0019.png",
"captured_at": "2026-01-05T16:28:04Z"
},
{
"screenshot_id": "shot_0020",
"relative_path": "shots/shot_0020.png",
"captured_at": "2026-01-05T16:28:05Z"
},
{
"screenshot_id": "shot_0021",
"relative_path": "shots/shot_0021.png",
"captured_at": "2026-01-05T16:28:07Z"
},
{
"screenshot_id": "shot_0022",
"relative_path": "shots/shot_0022.png",
"captured_at": "2026-01-05T16:28:08Z"
},
{
"screenshot_id": "shot_0023",
"relative_path": "shots/shot_0023.png",
"captured_at": "2026-01-05T16:28:10Z"
},
{
"screenshot_id": "shot_0024",
"relative_path": "shots/shot_0024.png",
"captured_at": "2026-01-05T16:28:11Z"
},
{
"screenshot_id": "shot_0025",
"relative_path": "shots/shot_0025.png",
"captured_at": "2026-01-05T16:28:12Z"
},
{
"screenshot_id": "shot_0026",
"relative_path": "shots/shot_0026.png",
"captured_at": "2026-01-05T16:28:13Z"
},
{
"screenshot_id": "shot_0027",
"relative_path": "shots/shot_0027.png",
"captured_at": "2026-01-05T16:28:13Z"
},
{
"screenshot_id": "shot_0028",
"relative_path": "shots/shot_0028.png",
"captured_at": "2026-01-05T16:28:15Z"
},
{
"screenshot_id": "shot_0029",
"relative_path": "shots/shot_0029.png",
"captured_at": "2026-01-05T16:28:16Z"
},
{
"screenshot_id": "shot_0030",
"relative_path": "shots/shot_0030.png",
"captured_at": "2026-01-05T16:28:18Z"
},
{
"screenshot_id": "shot_0031",
"relative_path": "shots/shot_0031.png",
"captured_at": "2026-01-05T16:28:19Z"
},
{
"screenshot_id": "shot_0032",
"relative_path": "shots/shot_0032.png",
"captured_at": "2026-01-05T16:28:21Z"
},
{
"screenshot_id": "shot_0033",
"relative_path": "shots/shot_0033.png",
"captured_at": "2026-01-05T16:28:21Z"
},
{
"screenshot_id": "shot_0034",
"relative_path": "shots/shot_0034.png",
"captured_at": "2026-01-05T16:28:22Z"
},
{
"screenshot_id": "shot_0035",
"relative_path": "shots/shot_0035.png",
"captured_at": "2026-01-05T16:28:23Z"
},
{
"screenshot_id": "shot_0036",
"relative_path": "shots/shot_0036.png",
"captured_at": "2026-01-05T16:28:24Z"
},
{
"screenshot_id": "shot_0037",
"relative_path": "shots/shot_0037.png",
"captured_at": "2026-01-05T16:28:25Z"
},
{
"screenshot_id": "shot_0038",
"relative_path": "shots/shot_0038.png",
"captured_at": "2026-01-05T16:28:26Z"
}
]
}

57
data/training/sessions/sess_20260105T172736_7dcebf54/workflow_session1.json Normal file
View File

@@ -0,0 +1,57 @@
{
"workflow_id": "workflow_session1",
"name": "workflow_session1",
"description": "Auto-generated workflow",
"version": 1,
"learning_state": "OBSERVATION",
"created_at": "2026-03-10T17:36:05.585485",
"updated_at": "2026-03-10T17:36:05.585486",
"entry_nodes": [],
"end_nodes": [],
"nodes": [],
"edges": [],
"safety_rules": {
"require_confirmation_for": [],
"forbidden_windows": [],
"execution_timeout_minutes": 0
},
"stats": {
"total_executions": 0,
"success_count": 0,
"failure_count": 0,
"observed_runs": 0,
"assist_runs": 0,
"auto_candidate_runs": 0,
"auto_confirmed_runs": 0,
"avg_similarity": 0.0,
"recent_confidence": 0.0,
"last_executed": null
},
"learning": {
"observation_threshold": 5,
"coaching_threshold": 10,
"auto_candidate_threshold": 20,
"min_similarity_coaching": 0.9,
"min_success_rate_auto": 0.9,
"min_success_rate_confirmed": 0.95,
"rollback_confidence_threshold": 0.9
},
"metadata": {
"quality_report": {
"workflow_id": "workflow_session1",
"overall_score": 0.5,
"is_production_ready": false,
"cluster_count": 0,
"outlier_count": 0,
"validation_accuracy": 0.0,
"recommendations": [
"Pas assez de clusters pour calculer les m\u00e9triques",
"Pr\u00e9cision de validation crois\u00e9e insuffisante: 0.00% (minimum: 80%)"
],
"created_at": "2026-03-10T17:36:05.585742"
}
},
"loops": {},
"conditionals": {},
"references": []
}