Dom/rpa_vision_v3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: replay E2E fonctionnel — 25/25 actions, 0 retries, SomEngine via serveur

Browse Source 
Validé sur PC Windows (DESKTOP-58D5CAC, 2560x1600) :
- 8 clics résolus visuellement (1 anchor_template, 1 som_text_match, 6 som_vlm)
- Score moyen 0.75, temps moyen 1.6s
- Texte tapé correctement (bonjour, test word, date, email)
- 0 retries, 2 actions non vérifiées (OK)

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Dom
2026-03-31 14:04:41 +02:00
parent 5e0b53cfd1
commit a7de6a488b
79542 changed files with 6091757 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
data/training/screen_states/2026-01-13/state_20260113_221315_125031597064560.json Normal file
View File

@@ -0,0 +1,50 @@
{
"screen_state_id": "state_sess_20260113T221242_6fcadaaa_0009",
"timestamp": "2026-01-13T21:13:03.508097+00:00",
"session_id": "sess_20260113T221242_6fcadaaa",
"window": {
"app_name": "gnome-calculato",
"window_title": "Calculatrice",
"screen_resolution": [
1920,
1080
],
"workspace": "main"
},
"raw": {
"screenshot_path": "shots/shot_0010.png",
"capture_method": "agent_v0",
"file_size_bytes": 52530
},
"perception": {
"embedding": {
"provider": "pending",
"vector_id": "emb_shot_0010",
"dimensions": 512
},
"detected_text": [],
"text_detection_method": "none",
"confidence_avg": 1.0
},
"context": {
"current_workflow_candidate": null,
"workflow_step": null,
"user_id": "demo_user",
"tags": [
"Interaction_Calculatrice"
],
"business_variables": {
"customer": "Clinique Demo",
"training_label": "Interaction_Calculatrice",
"notes": "Session réelle avec clics + screenshots + key combos.",
"original_training_label": "Workflow_Calculatrice_20260113_2213"
}
},
"metadata": {},
"ui_elements": [],
"_metadata": {
"saved_at": "2026-01-13T22:13:15.821689",
"schema_version": "screenstate_v1",
"state_id": "20260113_221315_125031597064560"
}
}

50
data/training/screen_states/2026-01-13/state_20260113_221315_125031597065664.json Normal file
View File

@@ -0,0 +1,50 @@
{
"screen_state_id": "state_sess_20260113T221242_6fcadaaa_0008",
"timestamp": "2026-01-13T21:13:02.843990+00:00",
"session_id": "sess_20260113T221242_6fcadaaa",
"window": {
"app_name": "gnome-calculato",
"window_title": "Calculatrice",
"screen_resolution": [
1920,
1080
],
"workspace": "main"
},
"raw": {
"screenshot_path": "shots/shot_0009.png",
"capture_method": "agent_v0",
"file_size_bytes": 50848
},
"perception": {
"embedding": {
"provider": "pending",
"vector_id": "emb_shot_0009",
"dimensions": 512
},
"detected_text": [],
"text_detection_method": "none",
"confidence_avg": 1.0
},
"context": {
"current_workflow_candidate": null,
"workflow_step": null,
"user_id": "demo_user",
"tags": [
"Interaction_Calculatrice"
],
"business_variables": {
"customer": "Clinique Demo",
"training_label": "Interaction_Calculatrice",
"notes": "Session réelle avec clics + screenshots + key combos.",
"original_training_label": "Workflow_Calculatrice_20260113_2213"
}
},
"metadata": {},
"ui_elements": [],
"_metadata": {
"saved_at": "2026-01-13T22:13:15.821607",
"schema_version": "screenstate_v1",
"state_id": "20260113_221315_125031597065664"
}
}

50
data/training/screen_states/2026-01-13/state_20260113_221315_125031597065808.json Normal file
View File

@@ -0,0 +1,50 @@
{
"screen_state_id": "state_sess_20260113T221242_6fcadaaa_0002",
"timestamp": "2026-01-13T21:12:47.309900+00:00",
"session_id": "sess_20260113T221242_6fcadaaa",
"window": {
"app_name": "gnome-text-edit",
"window_title": "requirements.txt (~/ai/Geniusia_v2/geniusia2) - Éditeur de texte",
"screen_resolution": [
1920,
1080
],
"workspace": "main"
},
"raw": {
"screenshot_path": "shots/shot_0003.png",
"capture_method": "agent_v0",
"file_size_bytes": 100245
},
"perception": {
"embedding": {
"provider": "pending",
"vector_id": "emb_shot_0003",
"dimensions": 512
},
"detected_text": [],
"text_detection_method": "none",
"confidence_avg": 1.0
},
"context": {
"current_workflow_candidate": null,
"workflow_step": null,
"user_id": "demo_user",
"tags": [
"Interaction_Calculatrice"
],
"business_variables": {
"customer": "Clinique Demo",
"training_label": "Interaction_Calculatrice",
"notes": "Session réelle avec clics + screenshots + key combos.",
"original_training_label": "Workflow_Calculatrice_20260113_2213"
}
},
"metadata": {},
"ui_elements": [],
"_metadata": {
"saved_at": "2026-01-13T22:13:15.820650",
"schema_version": "screenstate_v1",
"state_id": "20260113_221315_125031597065808"
}
}

50
data/training/screen_states/2026-01-13/state_20260113_221315_125031597067728.json Normal file
View File

@@ -0,0 +1,50 @@
{
"screen_state_id": "state_sess_20260113T221242_6fcadaaa_0003",
"timestamp": "2026-01-13T21:12:49.635694+00:00",
"session_id": "sess_20260113T221242_6fcadaaa",
"window": {
"app_name": "gnome-text-edit",
"window_title": "ceci est un test (Brouillon) - Éditeur de texte",
"screen_resolution": [
1920,
1080
],
"workspace": "main"
},
"raw": {
"screenshot_path": "shots/shot_0004.png",
"capture_method": "agent_v0",
"file_size_bytes": 42046
},
"perception": {
"embedding": {
"provider": "pending",
"vector_id": "emb_shot_0004",
"dimensions": 512
},
"detected_text": [],
"text_detection_method": "none",
"confidence_avg": 1.0
},
"context": {
"current_workflow_candidate": null,
"workflow_step": null,
"user_id": "demo_user",
"tags": [
"Interaction_Calculatrice"
],
"business_variables": {
"customer": "Clinique Demo",
"training_label": "Interaction_Calculatrice",
"notes": "Session réelle avec clics + screenshots + key combos.",
"original_training_label": "Workflow_Calculatrice_20260113_2213"
}
},
"metadata": {},
"ui_elements": [],
"_metadata": {
"saved_at": "2026-01-13T22:13:15.820743",
"schema_version": "screenstate_v1",
"state_id": "20260113_221315_125031597067728"
}
}

50
data/training/screen_states/2026-01-13/state_20260113_221315_125031597069408.json Normal file
View File

@@ -0,0 +1,50 @@
{
"screen_state_id": "state_sess_20260113T221242_6fcadaaa_0004",
"timestamp": "2026-01-13T21:12:56.188738+00:00",
"session_id": "sess_20260113T221242_6fcadaaa",
"window": {
"app_name": "gnome-text-edit",
"window_title": "ceci est un test ceci (Brouillon) - Éditeur de texte",
"screen_resolution": [
1920,
1080
],
"workspace": "main"
},
"raw": {
"screenshot_path": "shots/shot_0005.png",
"capture_method": "agent_v0",
"file_size_bytes": 11498
},
"perception": {
"embedding": {
"provider": "pending",
"vector_id": "emb_shot_0005",
"dimensions": 512
},
"detected_text": [],
"text_detection_method": "none",
"confidence_avg": 1.0
},
"context": {
"current_workflow_candidate": null,
"workflow_step": null,
"user_id": "demo_user",
"tags": [
"Interaction_Calculatrice"
],
"business_variables": {
"customer": "Clinique Demo",
"training_label": "Interaction_Calculatrice",
"notes": "Session réelle avec clics + screenshots + key combos.",
"original_training_label": "Workflow_Calculatrice_20260113_2213"
}
},
"metadata": {},
"ui_elements": [],
"_metadata": {
"saved_at": "2026-01-13T22:13:15.820823",
"schema_version": "screenstate_v1",
"state_id": "20260113_221315_125031597069408"
}
}

50
data/training/screen_states/2026-01-13/state_20260113_221315_125031597069840.json Normal file
View File

@@ -0,0 +1,50 @@
{
"screen_state_id": "state_sess_20260113T221242_6fcadaaa_0007",
"timestamp": "2026-01-13T21:13:00.813658+00:00",
"session_id": "sess_20260113T221242_6fcadaaa",
"window": {
"app_name": "gnome-text-edit",
"window_title": "ceci est un test ceci (Brouillon) - Éditeur de texte",
"screen_resolution": [
1920,
1080
],
"workspace": "main"
},
"raw": {
"screenshot_path": "shots/shot_0008.png",
"capture_method": "agent_v0",
"file_size_bytes": 115090
},
"perception": {
"embedding": {
"provider": "pending",
"vector_id": "emb_shot_0008",
"dimensions": 512
},
"detected_text": [],
"text_detection_method": "none",
"confidence_avg": 1.0
},
"context": {
"current_workflow_candidate": null,
"workflow_step": null,
"user_id": "demo_user",
"tags": [
"Interaction_Calculatrice"
],
"business_variables": {
"customer": "Clinique Demo",
"training_label": "Interaction_Calculatrice",
"notes": "Session réelle avec clics + screenshots + key combos.",
"original_training_label": "Workflow_Calculatrice_20260113_2213"
}
},
"metadata": {},
"ui_elements": [],
"_metadata": {
"saved_at": "2026-01-13T22:13:15.821524",
"schema_version": "screenstate_v1",
"state_id": "20260113_221315_125031597069840"
}
}

50
data/training/screen_states/2026-01-13/state_20260113_221315_125031597071040.json Normal file
View File

@@ -0,0 +1,50 @@
{
"screen_state_id": "state_sess_20260113T221242_6fcadaaa_0005",
"timestamp": "2026-01-13T21:12:58.267036+00:00",
"session_id": "sess_20260113T221242_6fcadaaa",
"window": {
"app_name": "gnome-text-edit",
"window_title": "Enregistrer les modifications ?",
"screen_resolution": [
1920,
1080
],
"workspace": "main"
},
"raw": {
"screenshot_path": "shots/shot_0006.png",
"capture_method": "agent_v0",
"file_size_bytes": 24294
},
"perception": {
"embedding": {
"provider": "pending",
"vector_id": "emb_shot_0006",
"dimensions": 512
},
"detected_text": [],
"text_detection_method": "none",
"confidence_avg": 1.0
},
"context": {
"current_workflow_candidate": null,
"workflow_step": null,
"user_id": "demo_user",
"tags": [
"Interaction_Calculatrice"
],
"business_variables": {
"customer": "Clinique Demo",
"training_label": "Interaction_Calculatrice",
"notes": "Session réelle avec clics + screenshots + key combos.",
"original_training_label": "Workflow_Calculatrice_20260113_2213"
}
},
"metadata": {},
"ui_elements": [],
"_metadata": {
"saved_at": "2026-01-13T22:13:15.820899",
"schema_version": "screenstate_v1",
"state_id": "20260113_221315_125031597071040"
}
}

50
data/training/screen_states/2026-01-13/state_20260113_221315_125031597073584.json Normal file
View File

@@ -0,0 +1,50 @@
{
"screen_state_id": "state_sess_20260113T221242_6fcadaaa_0010",
"timestamp": "2026-01-13T21:13:04.010968+00:00",
"session_id": "sess_20260113T221242_6fcadaaa",
"window": {
"app_name": "gnome-calculato",
"window_title": "Calculatrice",
"screen_resolution": [
1920,
1080
],
"workspace": "main"
},
"raw": {
"screenshot_path": "shots/shot_0011.png",
"capture_method": "agent_v0",
"file_size_bytes": 42927
},
"perception": {
"embedding": {
"provider": "pending",
"vector_id": "emb_shot_0011",
"dimensions": 512
},
"detected_text": [],
"text_detection_method": "none",
"confidence_avg": 1.0
},
"context": {
"current_workflow_candidate": null,
"workflow_step": null,
"user_id": "demo_user",
"tags": [
"Interaction_Calculatrice"
],
"business_variables": {
"customer": "Clinique Demo",
"training_label": "Interaction_Calculatrice",
"notes": "Session réelle avec clics + screenshots + key combos.",
"original_training_label": "Workflow_Calculatrice_20260113_2213"
}
},
"metadata": {},
"ui_elements": [],
"_metadata": {
"saved_at": "2026-01-13T22:13:15.821769",
"schema_version": "screenstate_v1",
"state_id": "20260113_221315_125031597073584"
}
}

50
data/training/screen_states/2026-01-13/state_20260113_221315_125031597075168.json Normal file
View File

@@ -0,0 +1,50 @@
{
"screen_state_id": "state_sess_20260113T221242_6fcadaaa_0011",
"timestamp": "2026-01-13T21:13:04.731579+00:00",
"session_id": "sess_20260113T221242_6fcadaaa",
"window": {
"app_name": "gnome-calculato",
"window_title": "Calculatrice",
"screen_resolution": [
1920,
1080
],
"workspace": "main"
},
"raw": {
"screenshot_path": "shots/shot_0012.png",
"capture_method": "agent_v0",
"file_size_bytes": 40077
},
"perception": {
"embedding": {
"provider": "pending",
"vector_id": "emb_shot_0012",
"dimensions": 512
},
"detected_text": [],
"text_detection_method": "none",
"confidence_avg": 1.0
},
"context": {
"current_workflow_candidate": null,
"workflow_step": null,
"user_id": "demo_user",
"tags": [
"Interaction_Calculatrice"
],
"business_variables": {
"customer": "Clinique Demo",
"training_label": "Interaction_Calculatrice",
"notes": "Session réelle avec clics + screenshots + key combos.",
"original_training_label": "Workflow_Calculatrice_20260113_2213"
}
},
"metadata": {},
"ui_elements": [],
"_metadata": {
"saved_at": "2026-01-13T22:13:15.821845",
"schema_version": "screenstate_v1",
"state_id": "20260113_221315_125031597075168"
}
}

50
data/training/screen_states/2026-01-13/state_20260113_221315_125031597076848.json Normal file
View File

@@ -0,0 +1,50 @@
{
"screen_state_id": "state_sess_20260113T221242_6fcadaaa_0012",
"timestamp": "2026-01-13T21:13:08.930539+00:00",
"session_id": "sess_20260113T221242_6fcadaaa",
"window": {
"app_name": "gnome-calculato",
"window_title": "Calculatrice",
"screen_resolution": [
1920,
1080
],
"workspace": "main"
},
"raw": {
"screenshot_path": "shots/shot_0013.png",
"capture_method": "agent_v0",
"file_size_bytes": 11316
},
"perception": {
"embedding": {
"provider": "pending",
"vector_id": "emb_shot_0013",
"dimensions": 512
},
"detected_text": [],
"text_detection_method": "none",
"confidence_avg": 1.0
},
"context": {
"current_workflow_candidate": null,
"workflow_step": null,
"user_id": "demo_user",
"tags": [
"Interaction_Calculatrice"
],
"business_variables": {
"customer": "Clinique Demo",
"training_label": "Interaction_Calculatrice",
"notes": "Session réelle avec clics + screenshots + key combos.",
"original_training_label": "Workflow_Calculatrice_20260113_2213"
}
},
"metadata": {},
"ui_elements": [],
"_metadata": {
"saved_at": "2026-01-13T22:13:15.821920",
"schema_version": "screenstate_v1",
"state_id": "20260113_221315_125031597076848"
}
}

50
data/training/screen_states/2026-01-13/state_20260113_221315_125032610081024.json Normal file
View File

@@ -0,0 +1,50 @@
{
"screen_state_id": "state_sess_20260113T221242_6fcadaaa_0000",
"timestamp": "2026-01-13T21:12:44.695097+00:00",
"session_id": "sess_20260113T221242_6fcadaaa",
"window": {
"app_name": "kiro",
"window_title": "ARCHITECTURE_APPRENTISSAGE.md - rpa_vision_v3 - Kiro",
"screen_resolution": [
1920,
1080
],
"workspace": "main"
},
"raw": {
"screenshot_path": "shots/shot_0001.png",
"capture_method": "agent_v0",
"file_size_bytes": 152164
},
"perception": {
"embedding": {
"provider": "pending",
"vector_id": "emb_shot_0001",
"dimensions": 512
},
"detected_text": [],
"text_detection_method": "none",
"confidence_avg": 1.0
},
"context": {
"current_workflow_candidate": null,
"workflow_step": null,
"user_id": "demo_user",
"tags": [
"Interaction_Calculatrice"
],
"business_variables": {
"customer": "Clinique Demo",
"training_label": "Interaction_Calculatrice",
"notes": "Session réelle avec clics + screenshots + key combos.",
"original_training_label": "Workflow_Calculatrice_20260113_2213"
}
},
"metadata": {},
"ui_elements": [],
"_metadata": {
"saved_at": "2026-01-13T22:13:15.820376",
"schema_version": "screenstate_v1",
"state_id": "20260113_221315_125032610081024"
}
}

50
data/training/screen_states/2026-01-13/state_20260113_221315_125033411378544.json Normal file
View File

@@ -0,0 +1,50 @@
{
"screen_state_id": "state_sess_20260113T221242_6fcadaaa_0006",
"timestamp": "2026-01-13T21:12:59.322403+00:00",
"session_id": "sess_20260113T221242_6fcadaaa",
"window": {
"app_name": "gnome-text-edit",
"window_title": "ceci est un test ceci (Brouillon) - Éditeur de texte",
"screen_resolution": [
1920,
1080
],
"workspace": "main"
},
"raw": {
"screenshot_path": "shots/shot_0007.png",
"capture_method": "agent_v0",
"file_size_bytes": 24255
},
"perception": {
"embedding": {
"provider": "pending",
"vector_id": "emb_shot_0007",
"dimensions": 512
},
"detected_text": [],
"text_detection_method": "none",
"confidence_avg": 1.0
},
"context": {
"current_workflow_candidate": null,
"workflow_step": null,
"user_id": "demo_user",
"tags": [
"Interaction_Calculatrice"
],
"business_variables": {
"customer": "Clinique Demo",
"training_label": "Interaction_Calculatrice",
"notes": "Session réelle avec clics + screenshots + key combos.",
"original_training_label": "Workflow_Calculatrice_20260113_2213"
}
},
"metadata": {},
"ui_elements": [],
"_metadata": {
"saved_at": "2026-01-13T22:13:15.821442",
"schema_version": "screenstate_v1",
"state_id": "20260113_221315_125033411378544"
}
}

50
data/training/screen_states/2026-01-13/state_20260113_221315_125033411379024.json Normal file
View File

@@ -0,0 +1,50 @@
{
"screen_state_id": "state_sess_20260113T221242_6fcadaaa_0001",
"timestamp": "2026-01-13T21:12:46.854402+00:00",
"session_id": "sess_20260113T221242_6fcadaaa",
"window": {
"app_name": "kiro",
"window_title": "ARCHITECTURE_APPRENTISSAGE.md - rpa_vision_v3 - Kiro",
"screen_resolution": [
1920,
1080
],
"workspace": "main"
},
"raw": {
"screenshot_path": "shots/shot_0002.png",
"capture_method": "agent_v0",
"file_size_bytes": 187710
},
"perception": {
"embedding": {
"provider": "pending",
"vector_id": "emb_shot_0002",
"dimensions": 512
},
"detected_text": [],
"text_detection_method": "none",
"confidence_avg": 1.0
},
"context": {
"current_workflow_candidate": null,
"workflow_step": null,
"user_id": "demo_user",
"tags": [
"Interaction_Calculatrice"
],
"business_variables": {
"customer": "Clinique Demo",
"training_label": "Interaction_Calculatrice",
"notes": "Session réelle avec clics + screenshots + key combos.",
"original_training_label": "Workflow_Calculatrice_20260113_2213"
}
},
"metadata": {},
"ui_elements": [],
"_metadata": {
"saved_at": "2026-01-13T22:13:15.820548",
"schema_version": "screenstate_v1",
"state_id": "20260113_221315_125033411379024"
}
}