v1.0 - Version stable: multi-PC, détection UI-DETR-1, 3 modes exécution
- Frontend v4 accessible sur réseau local (192.168.1.40) - Ports ouverts: 3002 (frontend), 5001 (backend), 5004 (dashboard) - Ollama GPU fonctionnel - Self-healing interactif - Dashboard confiance Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
4
tests/dataset/example_form_002/expected.json
Normal file
4
tests/dataset/example_form_002/expected.json
Normal file
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"element_id": "email_field",
|
||||
"confidence": 0.93
|
||||
}
|
||||
8
tests/dataset/example_form_002/metadata.json
Normal file
8
tests/dataset/example_form_002/metadata.json
Normal file
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"description": "Test case for email field resolution with spatial constraints",
|
||||
"category": "form_interaction",
|
||||
"difficulty": "medium",
|
||||
"created_by": "Dom, Alice Kiro",
|
||||
"created_date": "2025-12-22",
|
||||
"tags": ["email", "text_input", "form", "spatial_hints", "constraints"]
|
||||
}
|
||||
171
tests/dataset/example_form_002/screen_state.json
Normal file
171
tests/dataset/example_form_002/screen_state.json
Normal file
@@ -0,0 +1,171 @@
|
||||
{
|
||||
"screen_state_id": "example_state_002",
|
||||
"timestamp": "2025-12-22T10:35:00.000000",
|
||||
"session_id": "example_session",
|
||||
"window": {
|
||||
"app_name": "ExampleApp",
|
||||
"window_title": "Registration Form",
|
||||
"screen_resolution": [1920, 1080],
|
||||
"workspace": "main"
|
||||
},
|
||||
"raw": {
|
||||
"screenshot_path": "/tmp/example_screenshot_002.png",
|
||||
"capture_method": "mss",
|
||||
"file_size_bytes": 3072
|
||||
},
|
||||
"perception": {
|
||||
"embedding": {
|
||||
"provider": "openclip_ViT-B-32",
|
||||
"vector_id": "/tmp/example_embedding_002.npy",
|
||||
"dimensions": 512
|
||||
},
|
||||
"detected_text": ["First Name", "Last Name", "Email", "Submit", "Cancel"],
|
||||
"text_detection_method": "qwen_vl",
|
||||
"confidence_avg": 0.88
|
||||
},
|
||||
"context": {
|
||||
"current_workflow_candidate": null,
|
||||
"workflow_step": null,
|
||||
"user_id": "test_user",
|
||||
"tags": ["registration", "form"],
|
||||
"business_variables": {}
|
||||
},
|
||||
"metadata": {},
|
||||
"ui_elements": [
|
||||
{
|
||||
"element_id": "first_name_field",
|
||||
"type": "text_input",
|
||||
"role": "form_input",
|
||||
"bbox": {
|
||||
"x": 300,
|
||||
"y": 200,
|
||||
"width": 180,
|
||||
"height": 30
|
||||
},
|
||||
"center": [390, 215],
|
||||
"label": "First Name",
|
||||
"label_confidence": 0.92,
|
||||
"embeddings": {
|
||||
"image": null,
|
||||
"text": null
|
||||
},
|
||||
"visual_features": {
|
||||
"dominant_color": "#FFFFFF",
|
||||
"has_icon": false,
|
||||
"shape": "rectangle",
|
||||
"size_category": "medium"
|
||||
},
|
||||
"tags": ["input", "required"],
|
||||
"confidence": 0.89,
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"element_id": "last_name_field",
|
||||
"type": "text_input",
|
||||
"role": "form_input",
|
||||
"bbox": {
|
||||
"x": 500,
|
||||
"y": 200,
|
||||
"width": 180,
|
||||
"height": 30
|
||||
},
|
||||
"center": [590, 215],
|
||||
"label": "Last Name",
|
||||
"label_confidence": 0.91,
|
||||
"embeddings": {
|
||||
"image": null,
|
||||
"text": null
|
||||
},
|
||||
"visual_features": {
|
||||
"dominant_color": "#FFFFFF",
|
||||
"has_icon": false,
|
||||
"shape": "rectangle",
|
||||
"size_category": "medium"
|
||||
},
|
||||
"tags": ["input", "required"],
|
||||
"confidence": 0.87,
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"element_id": "email_field",
|
||||
"type": "text_input",
|
||||
"role": "form_input",
|
||||
"bbox": {
|
||||
"x": 300,
|
||||
"y": 250,
|
||||
"width": 380,
|
||||
"height": 30
|
||||
},
|
||||
"center": [490, 265],
|
||||
"label": "Email",
|
||||
"label_confidence": 0.94,
|
||||
"embeddings": {
|
||||
"image": null,
|
||||
"text": null
|
||||
},
|
||||
"visual_features": {
|
||||
"dominant_color": "#FFFFFF",
|
||||
"has_icon": false,
|
||||
"shape": "rectangle",
|
||||
"size_category": "large"
|
||||
},
|
||||
"tags": ["input", "email", "required"],
|
||||
"confidence": 0.93,
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"element_id": "submit_button",
|
||||
"type": "button",
|
||||
"role": "primary_action",
|
||||
"bbox": {
|
||||
"x": 400,
|
||||
"y": 320,
|
||||
"width": 100,
|
||||
"height": 35
|
||||
},
|
||||
"center": [450, 337],
|
||||
"label": "Submit",
|
||||
"label_confidence": 0.96,
|
||||
"embeddings": {
|
||||
"image": null,
|
||||
"text": null
|
||||
},
|
||||
"visual_features": {
|
||||
"dominant_color": "#28A745",
|
||||
"has_icon": false,
|
||||
"shape": "rounded_rectangle",
|
||||
"size_category": "medium"
|
||||
},
|
||||
"tags": ["button", "primary"],
|
||||
"confidence": 0.94,
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"element_id": "cancel_button",
|
||||
"type": "button",
|
||||
"role": "secondary_action",
|
||||
"bbox": {
|
||||
"x": 520,
|
||||
"y": 320,
|
||||
"width": 100,
|
||||
"height": 35
|
||||
},
|
||||
"center": [570, 337],
|
||||
"label": "Cancel",
|
||||
"label_confidence": 0.93,
|
||||
"embeddings": {
|
||||
"image": null,
|
||||
"text": null
|
||||
},
|
||||
"visual_features": {
|
||||
"dominant_color": "#6C757D",
|
||||
"has_icon": false,
|
||||
"shape": "rounded_rectangle",
|
||||
"size_category": "medium"
|
||||
},
|
||||
"tags": ["button", "secondary"],
|
||||
"confidence": 0.91,
|
||||
"metadata": {}
|
||||
}
|
||||
]
|
||||
}
|
||||
19
tests/dataset/example_form_002/target_spec.json
Normal file
19
tests/dataset/example_form_002/target_spec.json
Normal file
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"by_role": "text_input",
|
||||
"by_text": "Email",
|
||||
"by_position": null,
|
||||
"selection_policy": "first",
|
||||
"fallback_strategy": "visual_similarity",
|
||||
"context_hints": {
|
||||
"below_text": "Last Name"
|
||||
},
|
||||
"hard_constraints": {
|
||||
"min_area": 5000
|
||||
},
|
||||
"weights": {
|
||||
"proximity": 0.4,
|
||||
"alignment": 0.3,
|
||||
"container": 0.2,
|
||||
"roi_iou": 0.1
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user