feat(vwb): Ajouter SeeClick, Self-Healing interactif et Dashboard confiance
## Nouvelles fonctionnalités ### 1. SeeClick Adapter (visual grounding fallback) - Nouvel adapter pour le modèle SeeClick (HuggingFace) - Intégré dans la chaîne de fallback: CLIP → Template → SeeClick → Static - Localise les éléments GUI à partir de descriptions textuelles ### 2. Self-Healing Interactif - Dialogue qui propose des alternatives quand l'ancre n'est pas trouvée - L'utilisateur peut choisir: candidat alternatif, coords statiques, ou sauter - Nouveaux endpoints: /healing/status, /healing/choose, /healing/candidates - État "waiting_for_choice" pour mettre l'exécution en pause ### 3. Dashboard Confiance (temps réel) - Affiche les scores de confiance pendant l'exécution - Montre: méthode utilisée, distance, taux de succès - Interface pliable en bas à droite - Visible uniquement en mode intelligent/debug ## Fichiers ajoutés - core/detection/seeclick_adapter.py - frontend_v4/src/components/SelfHealingDialog.tsx - frontend_v4/src/components/ConfidenceDashboard.tsx ## Fichiers modifiés - core/detection/__init__.py - backend/services/intelligent_executor.py - backend/api_v3/execute.py - frontend_v4/src/App.tsx - frontend_v4/src/services/api.ts - docs/VISION_RPA_INTELLIGENT.md Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Dom
@@ -52,7 +52,13 @@ _execution_state = {
|
||||
'should_stop': False,
|
||||
'current_execution_id': None,
|
||||
'thread': None,
|
||||
'execution_mode': 'basic' # 'basic', 'intelligent', 'debug'
|
||||
'execution_mode': 'basic', # 'basic', 'intelligent', 'debug'
|
||||
# Self-healing interactif
|
||||
'waiting_for_choice': False,
|
||||
'pending_action': None, # Action en attente de choix utilisateur
|
||||
'candidates': [], # Candidats proposés
|
||||
'user_choice': None, # Choix de l'utilisateur (coordonnées ou 'skip' ou 'static')
|
||||
'current_step_info': None # Info sur l'étape en cours pour affichage
|
||||
}
|
||||
|
||||
|
||||
@@ -164,6 +170,85 @@ def execute_workflow_thread(execution_id: str, workflow_id: str, app):
|
||||
# === EXÉCUTION DE L'ACTION ===
|
||||
result = execute_action(step.action_type, params)
|
||||
|
||||
# === SELF-HEALING INTERACTIF ===
|
||||
# Si l'action nécessite un choix utilisateur, attendre
|
||||
if result.get('needs_user_choice'):
|
||||
print(f"🔄 [Self-Healing] Attente choix utilisateur pour étape {index + 1}")
|
||||
|
||||
# Stocker les informations pour le frontend
|
||||
_execution_state['waiting_for_choice'] = True
|
||||
_execution_state['pending_action'] = {
|
||||
'step_id': step.id,
|
||||
'step_index': index,
|
||||
'action_type': step.action_type,
|
||||
'params': params
|
||||
}
|
||||
_execution_state['candidates'] = result.get('candidates', [])
|
||||
_execution_state['current_step_info'] = {
|
||||
'index': index,
|
||||
'total': len(steps),
|
||||
'original_bbox': result.get('original_bbox'),
|
||||
'error': result.get('error')
|
||||
}
|
||||
_execution_state['user_choice'] = None
|
||||
|
||||
# Mettre à jour le status de l'exécution
|
||||
execution.status = 'waiting_user_choice'
|
||||
db.session.commit()
|
||||
|
||||
# Attendre le choix de l'utilisateur
|
||||
timeout_seconds = 120 # 2 minutes max
|
||||
waited = 0
|
||||
while _execution_state['waiting_for_choice'] and waited < timeout_seconds:
|
||||
if _execution_state['should_stop']:
|
||||
break
|
||||
time.sleep(0.5)
|
||||
waited += 0.5
|
||||
|
||||
# Vérifier si on doit arrêter
|
||||
if _execution_state['should_stop']:
|
||||
execution.status = 'cancelled'
|
||||
break
|
||||
|
||||
# Traiter le choix de l'utilisateur
|
||||
user_choice = _execution_state['user_choice']
|
||||
_execution_state['waiting_for_choice'] = False
|
||||
_execution_state['pending_action'] = None
|
||||
_execution_state['candidates'] = []
|
||||
|
||||
if user_choice is None:
|
||||
# Timeout - aucun choix
|
||||
step_result.status = 'error'
|
||||
step_result.error_message = "Timeout: aucun choix utilisateur"
|
||||
execution.failed_steps += 1
|
||||
execution.status = 'error'
|
||||
execution.error_message = f"Timeout à l'étape {index + 1}: aucun choix utilisateur"
|
||||
db.session.commit()
|
||||
break
|
||||
elif user_choice == 'skip':
|
||||
# Utilisateur a choisi de sauter l'étape
|
||||
step_result.status = 'skipped'
|
||||
step_result.output = {'skipped_by_user': True}
|
||||
execution.status = 'running'
|
||||
db.session.commit()
|
||||
print(f"⏭️ [Self-Healing] Étape {index + 1} sautée par l'utilisateur")
|
||||
continue
|
||||
elif user_choice == 'static':
|
||||
# Utiliser les coordonnées statiques
|
||||
result = execute_action_with_static_coords(step.action_type, params)
|
||||
execution.status = 'running'
|
||||
elif isinstance(user_choice, dict) and 'x' in user_choice:
|
||||
# Coordonnées choisies par l'utilisateur
|
||||
result = execute_action_with_coords(step.action_type, params, user_choice)
|
||||
execution.status = 'running'
|
||||
else:
|
||||
step_result.status = 'error'
|
||||
step_result.error_message = f"Choix invalide: {user_choice}"
|
||||
execution.failed_steps += 1
|
||||
execution.status = 'error'
|
||||
db.session.commit()
|
||||
break
|
||||
|
||||
step_result.ended_at = datetime.utcnow()
|
||||
step_result.duration_ms = int((step_result.ended_at - step_result.started_at).total_seconds() * 1000)
|
||||
|
||||
@@ -323,6 +408,75 @@ def execute_ai_analyze(params: dict) -> dict:
|
||||
return {'success': False, 'error': str(e)}
|
||||
|
||||
|
||||
def execute_action_with_coords(action_type: str, params: dict, coords: dict) -> dict:
|
||||
"""
|
||||
Exécute une action avec des coordonnées spécifiées par l'utilisateur (self-healing).
|
||||
"""
|
||||
import pyautogui
|
||||
import time
|
||||
|
||||
try:
|
||||
x, y = coords['x'], coords['y']
|
||||
print(f"🖱️ [Self-Healing] Clic aux coordonnées choisies: ({x}, {y})")
|
||||
|
||||
if action_type in ['double_click_anchor']:
|
||||
pyautogui.doubleClick(x, y)
|
||||
elif action_type in ['right_click_anchor']:
|
||||
pyautogui.rightClick(x, y)
|
||||
else:
|
||||
pyautogui.click(x, y)
|
||||
|
||||
time.sleep(2.0) # Délai après le clic
|
||||
|
||||
return {
|
||||
'success': True,
|
||||
'output': {
|
||||
'clicked_at': {'x': x, 'y': y},
|
||||
'method': 'user_choice',
|
||||
'self_healed': True
|
||||
}
|
||||
}
|
||||
except Exception as e:
|
||||
return {'success': False, 'error': str(e)}
|
||||
|
||||
|
||||
def execute_action_with_static_coords(action_type: str, params: dict) -> dict:
|
||||
"""
|
||||
Exécute une action avec les coordonnées statiques originales (fallback self-healing).
|
||||
"""
|
||||
import pyautogui
|
||||
import time
|
||||
|
||||
try:
|
||||
anchor = params.get('visual_anchor', {})
|
||||
bbox = anchor.get('bounding_box', {})
|
||||
|
||||
x = bbox.get('x', 0) + bbox.get('width', 0) / 2
|
||||
y = bbox.get('y', 0) + bbox.get('height', 0) / 2
|
||||
|
||||
print(f"🖱️ [Self-Healing] Clic aux coordonnées statiques: ({x}, {y})")
|
||||
|
||||
if action_type in ['double_click_anchor']:
|
||||
pyautogui.doubleClick(x, y)
|
||||
elif action_type in ['right_click_anchor']:
|
||||
pyautogui.rightClick(x, y)
|
||||
else:
|
||||
pyautogui.click(x, y)
|
||||
|
||||
time.sleep(2.0)
|
||||
|
||||
return {
|
||||
'success': True,
|
||||
'output': {
|
||||
'clicked_at': {'x': x, 'y': y},
|
||||
'method': 'static_fallback',
|
||||
'self_healed': True
|
||||
}
|
||||
}
|
||||
except Exception as e:
|
||||
return {'success': False, 'error': str(e)}
|
||||
|
||||
|
||||
def execute_action(action_type: str, params: dict) -> dict:
|
||||
"""
|
||||
Exécute une action RPA.
|
||||
@@ -403,11 +557,25 @@ def execute_action(action_type: str, params: dict) -> dict:
|
||||
}
|
||||
else:
|
||||
# En mode intelligent/debug, on refuse d'utiliser les coordonnées statiques
|
||||
# si l'ancre n'est pas trouvée - cela évite les clics au mauvais endroit
|
||||
# si l'ancre n'est pas trouvée - MAIS on peut proposer des alternatives
|
||||
reason = result.get('reason', 'Ancre non trouvée à l\'écran')
|
||||
confidence = result.get('confidence', 0)
|
||||
candidates = result.get('candidates', [])
|
||||
print(f"❌ [Vision] Ancre NON trouvée (confiance: {confidence:.2f})")
|
||||
print(f" Raison: {reason}")
|
||||
|
||||
# Si self-healing interactif activé, proposer des alternatives
|
||||
if _execution_state.get('execution_mode') == 'intelligent' and candidates:
|
||||
print(f"🔄 [Self-Healing] {len(candidates)} candidats disponibles - attente choix utilisateur")
|
||||
return {
|
||||
'success': False,
|
||||
'needs_user_choice': True, # Flag pour self-healing
|
||||
'error': f"Ancre non trouvée - veuillez choisir parmi les alternatives",
|
||||
'candidates': candidates,
|
||||
'original_bbox': bbox,
|
||||
'confidence': confidence
|
||||
}
|
||||
|
||||
return {
|
||||
'success': False,
|
||||
'error': f"Ancre non trouvée à l'écran (confiance: {confidence:.2f}). {reason}"
|
||||
@@ -707,7 +875,11 @@ def get_execution_status():
|
||||
'is_paused': _execution_state['is_paused'],
|
||||
'execution_mode': _execution_state.get('execution_mode', 'basic'),
|
||||
'execution': execution.to_dict() if execution else None,
|
||||
'session': session.to_dict()
|
||||
'session': session.to_dict(),
|
||||
# Self-healing interactif
|
||||
'waiting_for_choice': _execution_state.get('waiting_for_choice', False),
|
||||
'candidates': _execution_state.get('candidates', []) if _execution_state.get('waiting_for_choice') else [],
|
||||
'current_step_info': _execution_state.get('current_step_info') if _execution_state.get('waiting_for_choice') else None
|
||||
})
|
||||
|
||||
|
||||
@@ -734,3 +906,121 @@ def get_execution_history():
|
||||
'success': False,
|
||||
'error': str(e)
|
||||
}), 500
|
||||
|
||||
|
||||
# ==============================================================================
|
||||
# SELF-HEALING INTERACTIF - Endpoints
|
||||
# ==============================================================================
|
||||
|
||||
@api_v3_bp.route('/execute/healing/status', methods=['GET'])
|
||||
def get_healing_status():
|
||||
"""
|
||||
Retourne l'état du self-healing interactif.
|
||||
Si waiting_for_choice est True, retourne les candidats à afficher.
|
||||
"""
|
||||
global _execution_state
|
||||
|
||||
return jsonify({
|
||||
'success': True,
|
||||
'waiting_for_choice': _execution_state.get('waiting_for_choice', False),
|
||||
'candidates': _execution_state.get('candidates', []),
|
||||
'current_step_info': _execution_state.get('current_step_info'),
|
||||
'pending_action': _execution_state.get('pending_action')
|
||||
})
|
||||
|
||||
|
||||
@api_v3_bp.route('/execute/healing/choose', methods=['POST'])
|
||||
def submit_healing_choice():
|
||||
"""
|
||||
Soumet le choix de l'utilisateur pour le self-healing.
|
||||
|
||||
Request:
|
||||
{
|
||||
"choice": "skip" | "static" | {"x": 123, "y": 456}
|
||||
}
|
||||
|
||||
- "skip": Sauter cette étape et continuer
|
||||
- "static": Utiliser les coordonnées statiques originales
|
||||
- {"x": N, "y": N}: Utiliser ces coordonnées précises
|
||||
"""
|
||||
global _execution_state
|
||||
|
||||
if not _execution_state.get('waiting_for_choice'):
|
||||
return jsonify({
|
||||
'success': False,
|
||||
'error': "Aucune décision en attente"
|
||||
}), 400
|
||||
|
||||
data = request.get_json() or {}
|
||||
choice = data.get('choice')
|
||||
|
||||
if choice is None:
|
||||
return jsonify({
|
||||
'success': False,
|
||||
'error': "Choix non spécifié"
|
||||
}), 400
|
||||
|
||||
# Valider le choix
|
||||
valid_choices = ['skip', 'static']
|
||||
if isinstance(choice, str) and choice not in valid_choices:
|
||||
return jsonify({
|
||||
'success': False,
|
||||
'error': f"Choix invalide. Valeurs acceptées: {valid_choices} ou {{x, y}}"
|
||||
}), 400
|
||||
|
||||
if isinstance(choice, dict):
|
||||
if 'x' not in choice or 'y' not in choice:
|
||||
return jsonify({
|
||||
'success': False,
|
||||
'error': "Coordonnées invalides. Format attendu: {x: number, y: number}"
|
||||
}), 400
|
||||
|
||||
# Stocker le choix
|
||||
_execution_state['user_choice'] = choice
|
||||
_execution_state['waiting_for_choice'] = False
|
||||
|
||||
print(f"✅ [Self-Healing] Choix reçu: {choice}")
|
||||
|
||||
return jsonify({
|
||||
'success': True,
|
||||
'choice': choice
|
||||
})
|
||||
|
||||
|
||||
@api_v3_bp.route('/execute/healing/candidates', methods=['GET'])
|
||||
def get_healing_candidates():
|
||||
"""
|
||||
Retourne les candidats avec leurs screenshots pour affichage visuel.
|
||||
Permet à l'utilisateur de voir les alternatives possibles.
|
||||
"""
|
||||
global _execution_state
|
||||
|
||||
candidates = _execution_state.get('candidates', [])
|
||||
step_info = _execution_state.get('current_step_info', {})
|
||||
|
||||
# Enrichir les candidats avec plus d'informations si disponible
|
||||
enriched_candidates = []
|
||||
for i, candidate in enumerate(candidates):
|
||||
enriched = {
|
||||
'id': i,
|
||||
'element_id': candidate.get('element_id'),
|
||||
'score': candidate.get('score', candidate.get('combined_score', 0)),
|
||||
'bbox': candidate.get('bbox', {}),
|
||||
'distance': candidate.get('distance'),
|
||||
'method': candidate.get('method', 'unknown')
|
||||
}
|
||||
# Calculer le centre si bbox disponible
|
||||
bbox = candidate.get('bbox', {})
|
||||
if bbox:
|
||||
enriched['center'] = {
|
||||
'x': (bbox.get('x1', 0) + bbox.get('x2', 0)) // 2,
|
||||
'y': (bbox.get('y1', 0) + bbox.get('y2', 0)) // 2
|
||||
}
|
||||
enriched_candidates.append(enriched)
|
||||
|
||||
return jsonify({
|
||||
'success': True,
|
||||
'candidates': enriched_candidates,
|
||||
'step_info': step_info,
|
||||
'original_bbox': step_info.get('original_bbox')
|
||||
})
|
||||
|
||||
@@ -6,11 +6,18 @@ Utilise UI-DETR-1 pour la détection et le matching d'ancres visuelles
|
||||
import time
|
||||
import base64
|
||||
import io
|
||||
import sys
|
||||
import os
|
||||
from typing import Dict, Any, Optional, List, Tuple
|
||||
from dataclasses import dataclass
|
||||
from PIL import Image
|
||||
import numpy as np
|
||||
|
||||
# Ajouter le chemin racine pour les imports de core
|
||||
RPA_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
if RPA_ROOT not in sys.path:
|
||||
sys.path.insert(0, RPA_ROOT)
|
||||
|
||||
# Import du service de détection UI
|
||||
from .ui_detection_service import detect_ui_elements, DetectionResult, UIElement
|
||||
|
||||
@@ -764,7 +771,53 @@ def find_and_click(
|
||||
global_result['search_time_ms'] = (_time.time() - start_time) * 1000
|
||||
return global_result
|
||||
|
||||
# === STRATÉGIE 4: Coordonnées statiques (dernier recours) ===
|
||||
# === STRATÉGIE 4: SeeClick (visual grounding) ===
|
||||
# Essayer SeeClick si les autres méthodes ont échoué
|
||||
try:
|
||||
print("🎯 [Vision] Essai SeeClick (visual grounding)...")
|
||||
from core.detection.seeclick_adapter import get_seeclick
|
||||
|
||||
seeclick = get_seeclick()
|
||||
if seeclick.available:
|
||||
# Utiliser une description générique basée sur l'ancre
|
||||
# TODO: Améliorer avec une description plus précise
|
||||
description = "the clickable element or button"
|
||||
|
||||
grounding_result = seeclick.ground(screen_image, description, return_pixels=True)
|
||||
|
||||
if grounding_result.found:
|
||||
found_x = grounding_result.x_pixel
|
||||
found_y = grounding_result.y_pixel
|
||||
|
||||
# Vérifier la distance à la position originale si anchor_bbox existe
|
||||
accept_seeclick = True
|
||||
if anchor_bbox:
|
||||
orig_x = anchor_bbox.get('x', 0) + anchor_bbox.get('width', 0) // 2
|
||||
orig_y = anchor_bbox.get('y', 0) + anchor_bbox.get('height', 0) // 2
|
||||
distance = np.sqrt((found_x - orig_x)**2 + (found_y - orig_y)**2)
|
||||
|
||||
MAX_SEECLICK_DISTANCE = 200 # Plus permissif car c'est un fallback
|
||||
if distance > MAX_SEECLICK_DISTANCE:
|
||||
print(f"⛔ [Vision] SeeClick rejeté: distance {distance:.0f}px > {MAX_SEECLICK_DISTANCE}px max")
|
||||
accept_seeclick = False
|
||||
|
||||
if accept_seeclick:
|
||||
print(f"✅ [Vision] SeeClick réussi! Coordonnées: ({found_x}, {found_y})")
|
||||
return {
|
||||
'found': True,
|
||||
'confidence': grounding_result.confidence,
|
||||
'coordinates': {'x': found_x, 'y': found_y},
|
||||
'bbox': anchor_bbox,
|
||||
'method': 'seeclick_grounding',
|
||||
'search_time_ms': (_time.time() - start_time) * 1000,
|
||||
'raw_output': grounding_result.raw_output
|
||||
}
|
||||
except ImportError:
|
||||
print("ℹ️ [Vision] SeeClick non disponible (module non trouvé)")
|
||||
except Exception as seeclick_err:
|
||||
print(f"⚠️ [Vision] Erreur SeeClick: {seeclick_err}")
|
||||
|
||||
# === STRATÉGIE 5: Coordonnées statiques (dernier recours) ===
|
||||
if anchor_bbox:
|
||||
best_conf = max(global_result.get('confidence', 0), 0)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user