v1.0 - Version stable: multi-PC, détection UI-DETR-1, 3 modes exécution

- Frontend v4 accessible sur réseau local (192.168.1.40)
- Ports ouverts: 3002 (frontend), 5001 (backend), 5004 (dashboard)
- Ollama GPU fonctionnel
- Self-healing interactif
- Dashboard confiance

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Dom
2026-01-29 11:23:51 +01:00
parent 21bfa3b337
commit a27b74cf22
1595 changed files with 412691 additions and 400 deletions

View File

@@ -0,0 +1,4 @@
{
"element_id": "email_field",
"confidence": 0.93
}

View File

@@ -0,0 +1,8 @@
{
"description": "Test case for email field resolution with spatial constraints",
"category": "form_interaction",
"difficulty": "medium",
"created_by": "Dom, Alice Kiro",
"created_date": "2025-12-22",
"tags": ["email", "text_input", "form", "spatial_hints", "constraints"]
}

View File

@@ -0,0 +1,171 @@
{
"screen_state_id": "example_state_002",
"timestamp": "2025-12-22T10:35:00.000000",
"session_id": "example_session",
"window": {
"app_name": "ExampleApp",
"window_title": "Registration Form",
"screen_resolution": [1920, 1080],
"workspace": "main"
},
"raw": {
"screenshot_path": "/tmp/example_screenshot_002.png",
"capture_method": "mss",
"file_size_bytes": 3072
},
"perception": {
"embedding": {
"provider": "openclip_ViT-B-32",
"vector_id": "/tmp/example_embedding_002.npy",
"dimensions": 512
},
"detected_text": ["First Name", "Last Name", "Email", "Submit", "Cancel"],
"text_detection_method": "qwen_vl",
"confidence_avg": 0.88
},
"context": {
"current_workflow_candidate": null,
"workflow_step": null,
"user_id": "test_user",
"tags": ["registration", "form"],
"business_variables": {}
},
"metadata": {},
"ui_elements": [
{
"element_id": "first_name_field",
"type": "text_input",
"role": "form_input",
"bbox": {
"x": 300,
"y": 200,
"width": 180,
"height": 30
},
"center": [390, 215],
"label": "First Name",
"label_confidence": 0.92,
"embeddings": {
"image": null,
"text": null
},
"visual_features": {
"dominant_color": "#FFFFFF",
"has_icon": false,
"shape": "rectangle",
"size_category": "medium"
},
"tags": ["input", "required"],
"confidence": 0.89,
"metadata": {}
},
{
"element_id": "last_name_field",
"type": "text_input",
"role": "form_input",
"bbox": {
"x": 500,
"y": 200,
"width": 180,
"height": 30
},
"center": [590, 215],
"label": "Last Name",
"label_confidence": 0.91,
"embeddings": {
"image": null,
"text": null
},
"visual_features": {
"dominant_color": "#FFFFFF",
"has_icon": false,
"shape": "rectangle",
"size_category": "medium"
},
"tags": ["input", "required"],
"confidence": 0.87,
"metadata": {}
},
{
"element_id": "email_field",
"type": "text_input",
"role": "form_input",
"bbox": {
"x": 300,
"y": 250,
"width": 380,
"height": 30
},
"center": [490, 265],
"label": "Email",
"label_confidence": 0.94,
"embeddings": {
"image": null,
"text": null
},
"visual_features": {
"dominant_color": "#FFFFFF",
"has_icon": false,
"shape": "rectangle",
"size_category": "large"
},
"tags": ["input", "email", "required"],
"confidence": 0.93,
"metadata": {}
},
{
"element_id": "submit_button",
"type": "button",
"role": "primary_action",
"bbox": {
"x": 400,
"y": 320,
"width": 100,
"height": 35
},
"center": [450, 337],
"label": "Submit",
"label_confidence": 0.96,
"embeddings": {
"image": null,
"text": null
},
"visual_features": {
"dominant_color": "#28A745",
"has_icon": false,
"shape": "rounded_rectangle",
"size_category": "medium"
},
"tags": ["button", "primary"],
"confidence": 0.94,
"metadata": {}
},
{
"element_id": "cancel_button",
"type": "button",
"role": "secondary_action",
"bbox": {
"x": 520,
"y": 320,
"width": 100,
"height": 35
},
"center": [570, 337],
"label": "Cancel",
"label_confidence": 0.93,
"embeddings": {
"image": null,
"text": null
},
"visual_features": {
"dominant_color": "#6C757D",
"has_icon": false,
"shape": "rounded_rectangle",
"size_category": "medium"
},
"tags": ["button", "secondary"],
"confidence": 0.91,
"metadata": {}
}
]
}

View File

@@ -0,0 +1,19 @@
{
"by_role": "text_input",
"by_text": "Email",
"by_position": null,
"selection_policy": "first",
"fallback_strategy": "visual_similarity",
"context_hints": {
"below_text": "Last Name"
},
"hard_constraints": {
"min_area": 5000
},
"weights": {
"proximity": 0.4,
"alignment": 0.3,
"container": 0.2,
"roi_iou": 0.1
}
}