Dom/rpa_vision_v3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

v1.0 - Version stable: multi-PC, détection UI-DETR-1, 3 modes exécution

Browse Source 
- Frontend v4 accessible sur réseau local (192.168.1.40)
- Ports ouverts: 3002 (frontend), 5001 (backend), 5004 (dashboard)
- Ollama GPU fonctionnel
- Self-healing interactif
- Dashboard confiance

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Dom
2026-01-29 11:23:51 +01:00
parent 21bfa3b337
commit a27b74cf22
1595 changed files with 412691 additions and 400 deletions
Download Patch File Download Diff File Expand all files Collapse all files

661
tests/results/test_proprietes_etapes_vwb_10jan2026.json Normal file
View File

@@ -0,0 +1,661 @@
{
"backend_disponible": true,
"actions_disponibles": [
{
"category": "vision_ui",
"complexity": "simple",
"description": "Clique sur un élément UI identifié par une ancre visuelle avec reconnaissance automatique",
"examples": [
{
"description": "Clique sur le bouton 'Valider' d'un formulaire",
"name": "Clic sur bouton de validation",
"parameters": {
"click_type": "left",
"wait_before_click_ms": 200
}
},
{
"description": "Clic droit sur un élément pour ouvrir le menu",
"name": "Clic droit pour menu contextuel",
"parameters": {
"click_type": "right"
}
}
],
"icon": "🖱️",
"id": "click_anchor",
"metadata": {
"author": "Dom, Alice, Kiro",
"complexity": "simple",
"tags": [
"ui",
"interaction",
"vision",
"click"
],
"version": "1.0.0"
},
"name": "Clic sur Ancre Visuelle",
"parameters": {
"click_offset_x": {
"default": 0,
"description": "Décalage horizontal en pixels depuis le centre",
"required": false,
"type": "number"
},
"click_offset_y": {
"default": 0,
"description": "Décalage vertical en pixels depuis le centre",
"required": false,
"type": "number"
},
"click_type": {
"default": "left",
"description": "Type de clic à effectuer",
"options": [
"left",
"right",
"double",
"middle"
],
"required": false,
"type": "string"
},
"visual_anchor": {
"description": "Ancre visuelle pour localiser l'élément cible",
"properties": {
"anchor_type": {
"default": "screenshot",
"options": [
"screenshot",
"text",
"icon",
"composite"
],
"type": "string"
},
"confidence_threshold": {
"default": 0.8,
"max": 1.0,
"min": 0.0,
"type": "number"
},
"screenshot_base64": {
"description": "Image de référence en base64",
"type": "string"
},
"text_content": {
"description": "Texte à rechercher",
"type": "string"
}
},
"required": true,
"type": "VWBVisualAnchor"
},
"wait_before_click_ms": {
"default": 100,
"description": "Délai d'attente avant le clic en millisecondes",
"min": 0,
"required": false,
"type": "number"
}
}
},
{
"category": "vision_ui",
"complexity": "simple",
"description": "Saisit du texte dans un champ identifié par une ancre visuelle avec validation automatique",
"examples": [
{
"description": "Saisit une adresse email dans un champ de connexion",
"name": "Saisie d'email",
"parameters": {
"clear_field_first": true,
"press_enter_after": false,
"text_to_type": "user@example.com",
"validate_after_typing": true
}
},
{
"description": "Saisit un mot de passe avec validation",
"name": "Saisie de mot de passe",
"parameters": {
"press_enter_after": true,
"text_to_type": "********",
"typing_speed_ms": 30
}
}
],
"icon": "⌨️",
"id": "type_text",
"metadata": {
"author": "Dom, Alice, Kiro",
"complexity": "simple",
"tags": [
"ui",
"input",
"text",
"vision",
"typing"
],
"version": "1.0.0"
},
"name": "Saisie de Texte Visuelle",
"parameters": {
"clear_field_first": {
"default": true,
"description": "Vider le champ avant la saisie (Ctrl+A puis suppression)",
"required": false,
"type": "boolean"
},
"click_before_typing": {
"default": true,
"description": "Cliquer sur le champ avant la saisie pour le focus",
"required": false,
"type": "boolean"
},
"press_enter_after": {
"default": false,
"description": "Appuyer sur Entrée après la saisie",
"required": false,
"type": "boolean"
},
"text_to_type": {
"description": "Texte à saisir dans le champ",
"max_length": 1000,
"required": true,
"type": "string"
},
"typing_speed_ms": {
"default": 50,
"description": "Délai entre chaque caractère en millisecondes",
"max": 1000,
"min": 0,
"required": false,
"type": "number"
},
"validate_after_typing": {
"default": false,
"description": "Valider que le texte a été correctement saisi",
"required": false,
"type": "boolean"
},
"visual_anchor": {
"description": "Ancre visuelle pour localiser le champ de saisie",
"required": true,
"type": "VWBVisualAnchor"
}
}
},
{
"category": "control",
"complexity": "intermediate",
"description": "Attend qu'une ancre visuelle apparaisse ou disparaisse avec surveillance intelligente",
"examples": [
{
"description": "Attend qu'un indicateur de chargement disparaisse",
"name": "Attendre fin de chargement",
"parameters": {
"check_interval_ms": 250,
"max_wait_time_ms": 15000,
"wait_mode": "disappear"
}
},
{
"description": "Attend qu'une boîte de dialogue apparaisse",
"name": "Attendre apparition de dialogue",
"parameters": {
"max_wait_time_ms": 10000,
"stability_duration_ms": 500,
"wait_mode": "appear"
}
}
],
"icon": "⏳",
"id": "wait_for_anchor",
"metadata": {
"author": "Dom, Alice, Kiro",
"complexity": "intermediate",
"tags": [
"control",
"wait",
"vision",
"synchronization"
],
"version": "1.0.0"
},
"name": "Attente d'Ancre Visuelle",
"parameters": {
"check_interval_ms": {
"default": 500,
"description": "Intervalle de vérification en millisecondes",
"max": 5000,
"min": 100,
"required": false,
"type": "number"
},
"confidence_threshold": {
"default": 0.8,
"description": "Seuil de confiance pour la détection",
"max": 1.0,
"min": 0.0,
"required": false,
"type": "number"
},
"max_wait_time_ms": {
"default": 30000,
"description": "Délai d'attente maximum en millisecondes",
"max": 300000,
"min": 1000,
"required": false,
"type": "number"
},
"stability_duration_ms": {
"default": 1000,
"description": "Durée de stabilité requise pour le mode 'stable'",
"min": 100,
"required": false,
"type": "number"
},
"visual_anchor": {
"description": "Ancre visuelle à surveiller",
"required": true,
"type": "VWBVisualAnchor"
},
"wait_mode": {
"default": "appear",
"description": "Mode d'attente (apparition, disparition, ou stabilisation)",
"options": [
"appear",
"disappear",
"stable"
],
"required": false,
"type": "string"
}
}
},
{
"category": "navigation",
"complexity": "intermediate",
"description": "Fait défiler la page jusqu'à ce qu'une ancre visuelle soit visible",
"examples": [
{
"description": "Fait défiler vers le bas pour atteindre un bouton",
"name": "Défiler vers bouton en bas de page",
"parameters": {
"scroll_direction": "down",
"scroll_step_pixels": 150
}
}
],
"icon": "📜",
"id": "scroll_to_anchor",
"metadata": {
"author": "Dom, Alice, Kiro",
"complexity": "intermediate",
"tags": [
"navigation",
"scroll",
"vision"
],
"version": "1.0.0"
},
"name": "Défilement vers Ancre",
"parameters": {
"max_scroll_attempts": {
"default": 20,
"description": "Nombre maximum de tentatives de défilement",
"max": 100,
"min": 1,
"required": false,
"type": "number"
},
"pause_between_scrolls_ms": {
"default": 200,
"description": "Pause entre chaque défilement en millisecondes",
"min": 50,
"required": false,
"type": "number"
},
"scroll_direction": {
"default": "auto",
"description": "Direction de défilement",
"options": [
"auto",
"up",
"down",
"left",
"right"
],
"required": false,
"type": "string"
},
"scroll_step_pixels": {
"default": 100,
"description": "Nombre de pixels par étape de défilement",
"max": 500,
"min": 10,
"required": false,
"type": "number"
},
"visual_anchor": {
"description": "Ancre visuelle à atteindre par défilement",
"required": true,
"type": "VWBVisualAnchor"
}
}
},
{
"category": "data",
"complexity": "advanced",
"description": "Extrait le texte d'un élément identifié par une ancre visuelle",
"examples": [
{
"description": "Extrait le prix affiché sur une page produit",
"name": "Extraire prix d'un produit",
"parameters": {
"extraction_method": "ocr",
"output_variable": "product_price",
"text_processing": "clean"
}
}
],
"icon": "📤",
"id": "extract_text_from_anchor",
"metadata": {
"author": "Dom, Alice, Kiro",
"complexity": "advanced",
"tags": [
"data",
"extraction",
"ocr",
"vision"
],
"version": "1.0.0"
},
"name": "Extraction de Texte",
"parameters": {
"extraction_method": {
"default": "ocr",
"description": "Méthode d'extraction du texte",
"options": [
"ocr",
"clipboard",
"accessibility"
],
"required": false,
"type": "string"
},
"output_variable": {
"description": "Nom de la variable pour stocker le texte extrait",
"required": false,
"type": "string"
},
"text_processing": {
"default": "trim",
"description": "Traitement du texte extrait",
"options": [
"none",
"trim",
"normalize",
"clean"
],
"required": false,
"type": "string"
},
"visual_anchor": {
"description": "Ancre visuelle de l'élément contenant le texte",
"required": true,
"type": "VWBVisualAnchor"
}
}
},
{
"category": "validation",
"complexity": "simple",
"description": "Valide qu'une ancre visuelle est présente ou absente sur l'écran",
"examples": [
{
"description": "Vérifie qu'un message de succès est affiché",
"name": "Valider succès de connexion",
"parameters": {
"expected_presence": true,
"fail_on_mismatch": true
}
}
],
"icon": "✅",
"id": "validate_anchor_presence",
"metadata": {
"author": "Dom, Alice, Kiro",
"complexity": "simple",
"tags": [
"validation",
"verification",
"vision"
],
"version": "1.0.0"
},
"name": "Validation de Présence",
"parameters": {
"confidence_threshold": {
"default": 0.8,
"description": "Seuil de confiance pour la validation",
"max": 1.0,
"min": 0.0,
"required": false,
"type": "number"
},
"expected_presence": {
"default": true,
"description": "Présence attendue (true = doit être présent, false = doit être absent)",
"required": false,
"type": "boolean"
},
"fail_on_mismatch": {
"default": true,
"description": "Échouer si la validation ne correspond pas à l'attente",
"required": false,
"type": "boolean"
},
"visual_anchor": {
"description": "Ancre visuelle à valider",
"required": true,
"type": "VWBVisualAnchor"
}
}
},
{
"category": "vision_ui",
"complexity": "simple",
"description": "Met le focus sur un élément UI identifié par une ancre visuelle",
"examples": [
{
"description": "Met le focus sur un champ en le survolant",
"name": "Focus par survol",
"parameters": {
"focus_method": "hover",
"hover_duration_ms": 300
}
}
],
"icon": "🎯",
"id": "focus_anchor",
"metadata": {
"author": "Dom, Alice, Kiro",
"complexity": "simple",
"tags": [
"ui",
"focus",
"vision",
"interaction"
],
"version": "1.0.0"
},
"name": "Donner le Focus",
"parameters": {
"focus_method": {
"default": "hover",
"description": "Méthode pour donner le focus",
"options": [
"hover",
"tab",
"click_light"
],
"required": false,
"type": "string"
},
"hover_duration_ms": {
"default": 500,
"description": "Durée du survol en millisecondes",
"max": 3000,
"min": 100,
"required": false,
"type": "number"
},
"visual_anchor": {
"description": "Ancre visuelle pour localiser l'élément cible",
"required": true,
"type": "VWBVisualAnchor"
}
}
},
{
"category": "vision_ui",
"complexity": "intermediate",
"description": "Saisit un secret (mot de passe) de manière sécurisée dans un champ identifié par ancre visuelle",
"examples": [
{
"description": "Saisit un mot de passe de manière sécurisée",
"name": "Saisie mot de passe",
"parameters": {
"press_enter_after": true,
"secret_value": "********",
"typing_speed_ms": 50
}
}
],
"icon": "🔐",
"id": "type_secret",
"metadata": {
"author": "Dom, Alice, Kiro",
"complexity": "intermediate",
"tags": [
"ui",
"security",
"password",
"vision",
"secret"
],
"version": "1.0.0"
},
"name": "Saisie de Secret",
"parameters": {
"clear_field_first": {
"default": true,
"description": "Vider le champ avant la saisie",
"required": false,
"type": "boolean"
},
"press_enter_after": {
"default": false,
"description": "Appuyer sur Entrée après la saisie",
"required": false,
"type": "boolean"
},
"secret_ref": {
"description": "Référence vers un secret stocké de manière sécurisée",
"required": false,
"type": "string"
},
"secret_value": {
"description": "Valeur du secret à saisir (masquée dans les logs)",
"required": false,
"type": "string"
},
"typing_speed_ms": {
"default": 30,
"description": "Délai entre chaque caractère (plus lent pour sécurité)",
"max": 200,
"min": 10,
"required": false,
"type": "number"
},
"visual_anchor": {
"description": "Ancre visuelle pour localiser le champ de saisie",
"required": true,
"type": "VWBVisualAnchor"
}
}
},
{
"category": "control",
"complexity": "simple",
"description": "Exécute un raccourci clavier (combinaison de touches)",
"examples": [
{
"description": "Copie le contenu sélectionné",
"name": "Copier",
"parameters": {
"key_combination": "ctrl+c"
}
},
{
"description": "Actualise la page web",
"name": "Actualiser page",
"parameters": {
"key_combination": "f5"
}
}
],
"icon": "⌨️",
"id": "hotkey",
"metadata": {
"author": "Dom, Alice, Kiro",
"complexity": "simple",
"tags": [
"control",
"keyboard",
"shortcut",
"hotkey"
],
"version": "1.0.0"
},
"name": "Raccourci Clavier",
"parameters": {
"hold_duration_ms": {
"default": 100,
"description": "Durée de maintien des touches en millisecondes",
"max": 2000,
"min": 50,
"required": false,
"type": "number"
},
"key_combination": {
"description": "Combinaison de touches (ex: 'ctrl+c', 'alt+tab', 'f5')",
"required": true,
"type": "string"
},
"repeat_count": {
"default": 1,
"description": "Nombre de répétitions du raccourci",
"max": 10,
"min": 1,
"required": false,
"type": "number"
}
}
}
],
"frontend_disponible": true,
"composants_presents": {
"VWBActionProperties": true,
"PropertiesPanel": true,
"useVWBStepIntegration": true,
"catalogService": true,
"catalog_types": true
},
"integration_fonctionnelle": true,
"erreurs": []
}