diff --git a/run.sh b/run.sh index effce06e1..124057153 100755 --- a/run.sh +++ b/run.sh @@ -36,7 +36,7 @@ show_help() { echo -e " ${BLUE}--server${NC} 🌐 API Server seul (port 8000)" echo -e " ${PURPLE}--dashboard${NC} 📊 Dashboard Web seul (port 5001)" echo -e " ${YELLOW}--monitoring${NC} 📈 Interface de monitoring (port 5003)" - echo -e " ${CYAN}--workflow${NC} 🔧 Visual Workflow Builder (port 3000)" + echo -e " ${CYAN}--workflow${NC} 🔧 Visual Workflow Builder (port 3002)" echo -e " ${GREEN}--agent${NC} đŸ“č Agent V0 (capture tool)" echo -e " ${BLUE}--chat${NC} 💬 Agent Chat (port 5002)" echo "" @@ -64,7 +64,7 @@ show_help() { echo -e " Dashboard: ${PURPLE}http://localhost:5001${NC}" echo -e " Agent Chat: ${BLUE}http://localhost:5002${NC}" echo -e " Monitoring: ${YELLOW}http://localhost:5003${NC}" - echo -e " Workflow Builder: ${CYAN}http://localhost:3000${NC}" + echo -e " Workflow Builder: ${CYAN}http://localhost:3002${NC}" echo "" } @@ -186,7 +186,7 @@ fi # Step 3: Check/Create Virtual Environment echo -e "${BLUE}[3/7]${NC} Setting up Python environment..." -VENV_DIR="venv_v3" +VENV_DIR=".venv" if [ ! -d "$VENV_DIR" ]; then echo " Creating virtual environment..." @@ -349,7 +349,8 @@ cleanup() { pkill -f "port 5001" 2>/dev/null || true pkill -f "port 5002" 2>/dev/null || true pkill -f "port 5003" 2>/dev/null || true - pkill -f "port 3000" 2>/dev/null || true + pkill -f "port 3002" 2>/dev/null || true + pkill -f "vite.*3002" 2>/dev/null || true deactivate 2>/dev/null || true echo -e "${GREEN}✓${NC} Cleanup complete" @@ -465,12 +466,12 @@ EOF workflow) echo "" - echo -e "${CYAN}🔧 Launching Visual Workflow Builder on port 3000...${NC}" + echo -e "${CYAN}🔧 Launching Visual Workflow Builder v4...${NC}" echo "" - echo "Access: http://localhost:3000" + echo "Access: http://localhost:3002 (frontend) / http://localhost:5001 (backend)" echo "" cd visual_workflow_builder - ./run.sh + ./run_v4.sh cd .. ;; @@ -581,10 +582,10 @@ if __name__ == '__main__': EOF MONITORING_PID=$(start_service "Monitoring" "$VENV_DIR/bin/python3 monitoring_server.py" "5003" "monitoring.log") - # Start Visual Workflow Builder (in background) - echo "Starting Visual Workflow Builder (port 3000)..." + # Start Visual Workflow Builder v4 (in background) + echo "Starting Visual Workflow Builder v4 (port 3002)..." cd visual_workflow_builder - ./run.sh > ../logs/workflow.log 2>&1 & + ./run_v4.sh > ../logs/workflow.log 2>&1 & WORKFLOW_PID=$! cd .. sleep 3 @@ -602,7 +603,7 @@ EOF echo -e " API Server: ${BLUE}http://localhost:8000${NC}" echo -e " Dashboard: ${PURPLE}http://localhost:5001${NC}" echo -e " Monitoring: ${YELLOW}http://localhost:5003${NC}" - echo -e " Workflow Builder: ${CYAN}http://localhost:3000${NC}" + echo -e " Workflow Builder: ${CYAN}http://localhost:3002${NC}" echo "" echo -e "${BOLD}📊 Logs:${NC}" echo " tail -f logs/api.log" @@ -697,7 +698,7 @@ EOF check_service_status "Dashboard" "5001" check_service_status "Agent Chat" "5002" check_service_status "Monitoring" "5003" - check_service_status "Workflow Builder" "3000" + check_service_status "Workflow Builder" "3002" echo "" ;; @@ -708,7 +709,8 @@ EOF pkill -f "port 5001" 2>/dev/null || true pkill -f "port 5002" 2>/dev/null || true pkill -f "port 5003" 2>/dev/null || true - pkill -f "port 3000" 2>/dev/null || true + pkill -f "port 3002" 2>/dev/null || true + pkill -f "vite.*3002" 2>/dev/null || true echo -e "${GREEN}✓${NC} All services stopped" ;; esac diff --git a/visual_workflow_builder/backend/actions/data/extraire_tableau.py b/visual_workflow_builder/backend/actions/data/extraire_tableau.py index d184e76fe..12ae1a6a9 100644 --- a/visual_workflow_builder/backend/actions/data/extraire_tableau.py +++ b/visual_workflow_builder/backend/actions/data/extraire_tableau.py @@ -14,6 +14,7 @@ Cas d'usage : from typing import Dict, Any, List, Optional, Tuple from datetime import datetime +import os import time import base64 import io @@ -26,9 +27,9 @@ from ...contracts.error import VWBErrorType, create_vwb_error from ...contracts.visual_anchor import VWBVisualAnchor -# Configuration par dĂ©faut -OLLAMA_DEFAULT_URL = "http://localhost:11434" -OLLAMA_DEFAULT_MODEL = "qwen2.5-vl:7b" +# Configuration par dĂ©faut (centralisĂ©e via variable d'environnement) +OLLAMA_DEFAULT_URL = os.environ.get("OLLAMA_URL", "http://localhost:11434") +OLLAMA_DEFAULT_MODEL = os.environ.get("VLM_MODEL", "qwen2.5-vl:7b") class VWBExtraireTableauAction(BaseVWBAction): diff --git a/visual_workflow_builder/backend/actions/intelligence/analyser_avec_ia.py b/visual_workflow_builder/backend/actions/intelligence/analyser_avec_ia.py index 5e42974e6..e718046e5 100644 --- a/visual_workflow_builder/backend/actions/intelligence/analyser_avec_ia.py +++ b/visual_workflow_builder/backend/actions/intelligence/analyser_avec_ia.py @@ -22,11 +22,12 @@ import requests from ..base_action import BaseVWBAction, VWBActionResult, VWBActionStatus from ...contracts.error import VWBErrorType, create_vwb_error from ...contracts.visual_anchor import VWBVisualAnchor +import os -# Configuration Ollama par dĂ©faut -OLLAMA_DEFAULT_URL = "http://localhost:11434" -OLLAMA_DEFAULT_MODEL = "qwen2.5-vl:7b" +# Configuration Ollama par dĂ©faut (configurable via variables d'environnement) +OLLAMA_DEFAULT_URL = os.environ.get("OLLAMA_URL", "http://localhost:11434") +OLLAMA_DEFAULT_MODEL = os.environ.get("VLM_MODEL", "qwen3-vl:8b") class VWBAnalyserAvecIAAction(BaseVWBAction): diff --git a/visual_workflow_builder/backend/actions/validation/verify_text_content.py b/visual_workflow_builder/backend/actions/validation/verify_text_content.py index dd7c45143..455636cea 100644 --- a/visual_workflow_builder/backend/actions/validation/verify_text_content.py +++ b/visual_workflow_builder/backend/actions/validation/verify_text_content.py @@ -12,6 +12,7 @@ Modes OCR disponibles: from typing import Dict, Any, List, Optional from datetime import datetime +import os import time import re import base64 @@ -36,9 +37,9 @@ class VWBVerifyTextContentAction(BaseVWBAction): - easyocr: OCR traditionnel (plus rapide, fallback) """ - # Configuration Ollama par dĂ©faut - OLLAMA_URL = "http://localhost:11434" - OLLAMA_MODEL = "qwen2.5-vl:7b" # ModĂšle de vision Qwen - excellent pour OCR + # Configuration Ollama par dĂ©faut (centralisĂ©e via variable d'environnement) + OLLAMA_URL = os.environ.get("OLLAMA_URL", "http://localhost:11434") + OLLAMA_MODEL = os.environ.get("VLM_MODEL", "qwen2.5-vl:7b") # ModĂšle de vision Qwen - excellent pour OCR def __init__( self, diff --git a/visual_workflow_builder/backend/api_v3/execute.py b/visual_workflow_builder/backend/api_v3/execute.py index 9db8cfff6..c8d413d76 100644 --- a/visual_workflow_builder/backend/api_v3/execute.py +++ b/visual_workflow_builder/backend/api_v3/execute.py @@ -16,9 +16,68 @@ import threading import time import base64 import os +import logging import subprocess from . import api_v3_bp +logger = logging.getLogger(__name__) + + +def safe_type_text(text): + """Saisie de texte compatible avec tous les claviers (AZERTY/QWERTY). + + pyautogui.write() envoie des codes de touches QWERTY qui produisent + des caractĂšres erronĂ©s sur AZERTY (* → ”, ( → 5, ) → °, etc.). + + PrioritĂ© : + 1. Presse-papier (xclip) + Ctrl+V → instantanĂ©, fiable pour tout texte + 2. xdotool type → respecte le layout clavier + 3. pyautogui.write() → dernier recours + """ + import shutil + import pyautogui + + # MĂ©thode 1 : Presse-papier (le plus fiable, gĂšre UTF-8/accents/CJK) + # xclip reste en vie comme daemon X11 pour servir le clipboard. + # On ne doit PAS attendre sa terminaison — juste envoyer les donnĂ©es + # et coller immĂ©diatement. + xclip = shutil.which('xclip') + if xclip: + try: + p = subprocess.Popen( + ['xclip', '-selection', 'clipboard'], + stdin=subprocess.PIPE, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL + ) + p.stdin.write(text.encode('utf-8')) + p.stdin.close() + # Ne PAS attendre xclip — il reste vivant comme owner du clipboard + time.sleep(0.2) + pyautogui.hotkey('ctrl', 'v') + time.sleep(0.3) + print(f" ✅ Saisie via presse-papier ({len(text)} car.)") + return + except Exception as e: + print(f" ⚠ xclip Ă©chouĂ©: {e}") + + # MĂ©thode 2 : xdotool type (respecte le layout clavier actif) + if shutil.which('xdotool'): + try: + subprocess.run( + ['xdotool', 'type', '--delay', '0', '--clearmodifiers', '--', text], + timeout=max(30, len(text) * 0.05), + check=True + ) + print(f" ✅ Saisie via xdotool type ({len(text)} car.)") + return + except Exception as e: + print(f" ⚠ xdotool type Ă©chouĂ©: {e}") + + # MĂ©thode 3 : pyautogui (dernier recours — problĂšmes AZERTY possibles) + print(" ⚠ Saisie via pyautogui.write() (AZERTY non garanti)") + pyautogui.write(text) + def minimize_active_window(): """Minimise la fenĂȘtre active (Linux avec xdotool)""" @@ -37,7 +96,7 @@ def minimize_active_window(): print(f"⚠ [Execute] Erreur minimisation: {e}") return False from db.models import db, Workflow, Step, Execution, ExecutionStep, VisualAnchor, get_session_state -from contracts.action_contracts import enforce_action_contract, ContractValidationError, get_required_params +from contracts.action_contracts import enforce_action_contract, ContractValidationError def generate_id(prefix: str) -> str: @@ -45,6 +104,14 @@ def generate_id(prefix: str) -> str: return f"{prefix}_{uuid.uuid4().hex[:12]}_{int(datetime.now().timestamp())}" +# Thread-safety : verrou pour protĂ©ger _execution_state (accĂšs R/W depuis le +# thread d'exĂ©cution ET les handlers HTTP Flask en parallĂšle). +_execution_lock = threading.RLock() + +# Event pour remplacer le polling actif du self-healing : +# le thread d'exĂ©cution attend (.wait), le handler HTTP signale (.set). +_healing_event = threading.Event() + # État de l'exĂ©cution en cours (en mĂ©moire) _execution_state = { 'is_running': False, @@ -53,6 +120,7 @@ _execution_state = { 'current_execution_id': None, 'thread': None, 'execution_mode': 'basic', # 'basic', 'intelligent', 'debug' + 'variables': {}, # Variables runtime du workflow (initialisĂ© ici, plus de crĂ©ation dynamique) # Self-healing interactif 'waiting_for_choice': False, 'pending_action': None, # Action en attente de choix utilisateur @@ -75,7 +143,7 @@ def execute_workflow_thread(execution_id: str, workflow_id: str, app): workflow = Workflow.query.get(workflow_id) if not execution or not workflow: - print(f"❌ [Execute] Workflow ou exĂ©cution non trouvĂ©") + logger.error("Workflow ou exĂ©cution non trouvĂ©") return steps = workflow.steps.order_by(Step.order).all() @@ -84,7 +152,7 @@ def execute_workflow_thread(execution_id: str, workflow_id: str, app): execution.started_at = datetime.utcnow() db.session.commit() - print(f"🚀 [Execute] DĂ©marrage workflow {workflow_id}: {len(steps)} Ă©tapes") + logger.info(f"DĂ©marrage workflow {workflow_id}: {len(steps)} Ă©tapes") for index, step in enumerate(steps): # VĂ©rifier si arrĂȘt demandĂ© @@ -155,7 +223,7 @@ def execute_workflow_thread(execution_id: str, workflow_id: str, app): try: enforce_action_contract(step.action_type, params) except ContractValidationError as e: - print(f"đŸš« [Execute] CONTRAT VIOLÉ pour Ă©tape {step.id}: {e}") + logger.warning(f"CONTRAT VIOLÉ pour Ă©tape {step.id}: {e}") step_result.status = 'error' step_result.error_message = f"Contrat violĂ©: {str(e)}" step_result.ended_at = datetime.utcnow() @@ -176,34 +244,31 @@ def execute_workflow_thread(execution_id: str, workflow_id: str, app): print(f"🔄 [Self-Healing] Attente choix utilisateur pour Ă©tape {index + 1}") # Stocker les informations pour le frontend - _execution_state['waiting_for_choice'] = True - _execution_state['pending_action'] = { - 'step_id': step.id, - 'step_index': index, - 'action_type': step.action_type, - 'params': params - } - _execution_state['candidates'] = result.get('candidates', []) - _execution_state['current_step_info'] = { - 'index': index, - 'total': len(steps), - 'original_bbox': result.get('original_bbox'), - 'error': result.get('error') - } - _execution_state['user_choice'] = None + with _execution_lock: + _healing_event.clear() + _execution_state['waiting_for_choice'] = True + _execution_state['pending_action'] = { + 'step_id': step.id, + 'step_index': index, + 'action_type': step.action_type, + 'params': params + } + _execution_state['candidates'] = result.get('candidates', []) + _execution_state['current_step_info'] = { + 'index': index, + 'total': len(steps), + 'original_bbox': result.get('original_bbox'), + 'error': result.get('error') + } + _execution_state['user_choice'] = None # Mettre Ă  jour le status de l'exĂ©cution execution.status = 'waiting_user_choice' db.session.commit() - # Attendre le choix de l'utilisateur + # Attendre le choix de l'utilisateur (Event au lieu de polling) timeout_seconds = 120 # 2 minutes max - waited = 0 - while _execution_state['waiting_for_choice'] and waited < timeout_seconds: - if _execution_state['should_stop']: - break - time.sleep(0.5) - waited += 0.5 + _healing_event.wait(timeout=timeout_seconds) # VĂ©rifier si on doit arrĂȘter if _execution_state['should_stop']: @@ -211,10 +276,11 @@ def execute_workflow_thread(execution_id: str, workflow_id: str, app): break # Traiter le choix de l'utilisateur - user_choice = _execution_state['user_choice'] - _execution_state['waiting_for_choice'] = False - _execution_state['pending_action'] = None - _execution_state['candidates'] = [] + with _execution_lock: + user_choice = _execution_state['user_choice'] + _execution_state['waiting_for_choice'] = False + _execution_state['pending_action'] = None + _execution_state['candidates'] = [] if user_choice is None: # Timeout - aucun choix @@ -261,7 +327,7 @@ def execute_workflow_thread(execution_id: str, workflow_id: str, app): step_result.status = 'error' step_result.error_message = result.get('error', 'Erreur inconnue') execution.failed_steps += 1 - print(f"❌ [Execute] Étape {index + 1} Ă©chouĂ©e: {step_result.error_message}") + logger.warning(f"Étape {index + 1} Ă©chouĂ©e: {step_result.error_message}") # ArrĂȘter sur erreur execution.status = 'error' @@ -272,7 +338,7 @@ def execute_workflow_thread(execution_id: str, workflow_id: str, app): db.session.commit() except Exception as e: - print(f"❌ [Execute] Exception Ă©tape {index + 1}: {e}") + logger.error(f"Exception Ă©tape {index + 1}: {e}", exc_info=True) step_result.status = 'error' step_result.error_message = str(e) step_result.ended_at = datetime.utcnow() @@ -289,11 +355,10 @@ def execute_workflow_thread(execution_id: str, workflow_id: str, app): execution.ended_at = datetime.utcnow() db.session.commit() - print(f"🏁 [Execute] Workflow terminĂ©: {execution.status}") - print(f" ComplĂ©tĂ©es: {execution.completed_steps}, ÉchouĂ©es: {execution.failed_steps}") + logger.info(f"Workflow terminĂ©: {execution.status} (complĂ©tĂ©es: {execution.completed_steps}, Ă©chouĂ©es: {execution.failed_steps})") except Exception as e: - print(f"❌ [Execute] Erreur fatale: {e}") + logger.error(f"Erreur fatale: {e}", exc_info=True) try: execution = Execution.query.get(execution_id) if execution: @@ -301,74 +366,130 @@ def execute_workflow_thread(execution_id: str, workflow_id: str, app): execution.error_message = f"Erreur fatale: {str(e)}" execution.ended_at = datetime.utcnow() db.session.commit() - except: - pass + except Exception as db_err: + print(f"⚠ [Execute] DB cleanup error: {db_err}") finally: - _execution_state['is_running'] = False - _execution_state['current_execution_id'] = None + with _execution_lock: + _execution_state['is_running'] = False + _execution_state['current_execution_id'] = None def execute_ai_analyze(params: dict) -> dict: """ ExĂ©cute une analyse IA avec Ollama. - Capture la zone de l'ancre et envoie Ă  l'IA pour analyse. + Deux modes : + - Mode texte : envoie le texte brut directement (meilleure qualitĂ©) + - Mode image : capture la zone de l'ancre et envoie le screenshot """ import requests + import re + global _execution_state try: - # RĂ©cupĂ©rer les paramĂštres - anchor = params.get('visual_anchor', {}) prompt = params.get('analysis_prompt', params.get('prompt', '')) - model = params.get('model', params.get('ollama_model', 'qwen2.5-vl:7b')) + model = params.get('model', params.get('ollama_model', 'qwen3-vl:8b')) output_variable = params.get('output_variable', 'resultat_analyse') - timeout_ms = params.get('timeout_ms', 60000) - temperature = params.get('temperature', 0.3) + timeout_ms = params.get('timeout_ms', 120000) # 2 minutes par dĂ©faut + temperature = params.get('temperature', 0.7) # MĂȘme dĂ©faut que CLI Ollama + max_tokens = params.get('max_tokens', -1) # -1 = illimitĂ© (dĂ©faut Ollama) + input_text = params.get('input_text', '') - # RĂ©cupĂ©rer l'image de l'ancre - screenshot_base64 = anchor.get('screenshot') + # RĂ©soudre les variables {{var}} dans input_text + variables = _execution_state.get('variables', {}) + if input_text and '{{' in input_text: + def replace_var(match): + var_name = match.group(1) + value = variables.get(var_name, match.group(0)) + print(f" 📌 Variable {{{{{var_name}}}}} → {str(value)[:50]}...") + return str(value) + input_text = re.sub(r'\{\{(\w+)\}\}', replace_var, input_text) - if not screenshot_base64: - # Capturer l'Ă©cran si pas d'image dans l'ancre - try: - from PIL import ImageGrab - import io + # DĂ©terminer le mode : texte ou image + use_text_mode = bool(input_text) + anchor = params.get('visual_anchor', {}) - bbox = anchor.get('bounding_box', {}) - if bbox: - # Capturer la zone spĂ©cifique - x, y = int(bbox.get('x', 0)), int(bbox.get('y', 0)) - w, h = int(bbox.get('width', 100)), int(bbox.get('height', 100)) - screenshot = ImageGrab.grab(bbox=(x, y, x + w, y + h)) - else: - # Capturer tout l'Ă©cran - screenshot = ImageGrab.grab() + print(f"đŸ€– [IA] Mode: {'TEXTE' if use_text_mode else 'IMAGE'}") - buffer = io.BytesIO() - screenshot.save(buffer, format='PNG') - screenshot_base64 = base64.b64encode(buffer.getvalue()).decode('utf-8') - except Exception as cap_err: - return {'success': False, 'error': f"Erreur capture: {cap_err}"} + if use_text_mode: + # ═══ MODE TEXTE : envoyer le texte directement (comme en CLI) ═══ + print(f"📝 [IA] Texte brut: {len(input_text)} caractĂšres") - if not prompt: - prompt = "DĂ©cris ce que tu vois dans cette image." + # Construire le prompt complet avec le texte en entrĂ©e + if prompt: + full_prompt = f"{prompt}\n\nVoici le texte :\n{input_text}" + else: + full_prompt = input_text - print(f"đŸ€– [IA] Analyse avec {model}...") - print(f" Prompt: {prompt[:80]}...") + # Pour les modĂšles Qwen, dĂ©sactiver le thinking Ă©tendu + if 'qwen' in model.lower() and not full_prompt.startswith('/no_think'): + full_prompt = f"/no_think\n{full_prompt}" - # Appeler Ollama - ollama_url = params.get('ollama_url', 'http://localhost:11434') + print(f"đŸ€– [IA] Analyse texte avec {model}...") - payload = { - "model": model, - "prompt": prompt, - "images": [screenshot_base64], - "stream": False, - "options": { - "temperature": temperature, - "num_predict": 1000 + ollama_url = params.get('ollama_url', 'http://localhost:11434') + options = {"temperature": temperature} + if max_tokens > 0: + options["num_predict"] = max_tokens + payload = { + "model": model, + "prompt": full_prompt, + "stream": False, + "options": options } - } + + else: + # ═══ MODE IMAGE : capturer l'Ă©cran et envoyer le screenshot ═══ + screenshot_base64 = anchor.get('screenshot') if anchor else None + + if not screenshot_base64: + try: + from PIL import ImageGrab + import io + + bbox = anchor.get('bounding_box', {}) if anchor else {} + + if bbox: + x, y = int(bbox.get('x', 0)), int(bbox.get('y', 0)) + w, h = int(bbox.get('width', 100)), int(bbox.get('height', 100)) + print(f"📾 [IA] Capture zone: ({x}, {y}) -> ({x+w}, {y+h})") + screenshot = ImageGrab.grab(bbox=(x, y, x + w, y + h)) + else: + print(f"📾 [IA] Capture Ă©cran complet") + screenshot = ImageGrab.grab() + + buffer = io.BytesIO() + screenshot.save(buffer, format='PNG') + screenshot_base64 = base64.b64encode(buffer.getvalue()).decode('utf-8') + except Exception as cap_err: + return {'success': False, 'error': f"Erreur capture: {cap_err}"} + + if not screenshot_base64: + return {'success': False, 'error': "Pas d'image Ă  analyser (ni ancre, ni capture)"} + + if not prompt: + prompt = "DĂ©cris ce que tu vois dans cette image." + + full_prompt = prompt + if 'qwen' in model.lower() and not full_prompt.startswith('/no_think'): + full_prompt = f"/no_think\n{full_prompt}" + + print(f"đŸ€– [IA] Analyse image avec {model}...") + + ollama_url = params.get('ollama_url', 'http://localhost:11434') + options = {"temperature": temperature} + if max_tokens > 0: + options["num_predict"] = max_tokens + payload = { + "model": model, + "prompt": full_prompt, + "images": [screenshot_base64], + "stream": False, + "options": options + } + + # ═══ APPEL OLLAMA ═══ + print(f" Prompt: {full_prompt[:100]}...") response = requests.post( f"{ollama_url}/api/generate", @@ -380,13 +501,15 @@ def execute_ai_analyze(params: dict) -> dict: result = response.json() analysis_text = result.get('response', '').strip() - print(f"✅ [IA] Analyse terminĂ©e ({len(analysis_text)} caractĂšres)") - print(f" RĂ©sultat: {analysis_text[:150]}...") + # Fallback : extraire du champ thinking si response vide + if not analysis_text and result.get('thinking'): + analysis_text = result.get('thinking', '').strip() - # Stocker le rĂ©sultat dans le contexte d'exĂ©cution pour les variables - global _execution_state - if 'variables' not in _execution_state: - _execution_state['variables'] = {} + print(f"✅ [IA] Analyse terminĂ©e ({len(analysis_text)} caractĂšres)") + if analysis_text: + print(f" RĂ©sultat: {analysis_text[:150]}...") + + # Stocker dans les variables d'exĂ©cution _execution_state['variables'][output_variable] = analysis_text return { @@ -394,7 +517,8 @@ def execute_ai_analyze(params: dict) -> dict: 'output': { 'analysis': analysis_text, 'variable': output_variable, - 'model': model + 'model': model, + 'mode': 'text' if use_text_mode else 'image' } } else: @@ -582,27 +706,34 @@ def execute_action(action_type: str, params: dict) -> dict: } except Exception as vision_err: - print(f"❌ [Vision] Erreur: {vision_err}") - return { - 'success': False, - 'error': f"Erreur vision: {str(vision_err)}" - } + print(f"⚠ [Vision] Erreur: {vision_err}") + if execution_mode in ['intelligent', 'debug']: + # En mode visuel, on NE fait PAS de fallback statique + return { + 'success': False, + 'error': f"Erreur vision: {vision_err}. Ancre introuvable Ă  l'Ă©cran." + } + else: + print(f"🔄 [Fallback] Mode basic: utilisation des coordonnĂ©es statiques...") - # === MODE BASIC (ou fallback) === - # Calculer le centre depuis les coordonnĂ©es statiques - x = bbox.get('x', 0) + bbox.get('width', 0) / 2 - y = bbox.get('y', 0) + bbox.get('height', 0) / 2 + # === MODE BASIC uniquement === + if execution_mode not in ['intelligent', 'debug']: + x = bbox.get('x', 0) + bbox.get('width', 0) / 2 + y = bbox.get('y', 0) + bbox.get('height', 0) / 2 - print(f"đŸ–±ïž [Action] Clic {click_type} Ă  ({x}, {y}) [mode: {execution_mode}]") + print(f"đŸ–±ïž [Action] Clic {click_type} Ă  ({x}, {y}) [mode: basic, coordonnĂ©es statiques]") - if click_type == 'double': - pyautogui.doubleClick(x, y) - elif click_type == 'right': - pyautogui.rightClick(x, y) - else: - pyautogui.click(x, y) + if click_type == 'double': + pyautogui.doubleClick(x, y) + elif click_type == 'right': + pyautogui.rightClick(x, y) + else: + pyautogui.click(x, y) - return {'success': True, 'output': {'clicked_at': {'x': x, 'y': y}, 'mode': execution_mode}} + return {'success': True, 'output': {'clicked_at': {'x': x, 'y': y}, 'mode': 'basic'}} + + # En mode intelligent/debug, si on arrive ici c'est que la vision n'a pas Ă©tĂ© tentĂ©e + return {'success': False, 'error': 'Ancre non trouvĂ©e (aucune mĂ©thode visuelle disponible)'} elif action_type in ['type_text', 'type']: text = params.get('text', '') @@ -631,8 +762,14 @@ def execute_action(action_type: str, params: dict) -> dict: # Petit dĂ©lai pour s'assurer que le focus est bon time.sleep(0.2) - # Utiliser write() pour supporter l'unicode (caractĂšres français, etc.) - pyautogui.write(text) + # Saisie compatible AZERTY/QWERTY (presse-papier > xdotool > pyautogui) + safe_type_text(text) + + # Stocker le texte dans une variable si output_variable est dĂ©fini + output_variable = params.get('output_variable') + if output_variable: + _execution_state['variables'][output_variable] = text + print(f" 📩 Texte stockĂ© dans variable '{output_variable}' ({len(text)} caractĂšres)") return {'success': True, 'output': {'typed': text[:100] + '...' if len(text) > 100 else text}} @@ -683,7 +820,7 @@ def start_execution(): data = request.get_json() or {} workflow_id = data.get('workflow_id') - execution_mode = data.get('execution_mode', 'basic') + execution_mode = data.get('execution_mode', 'intelligent') minimize_browser = data.get('minimize_browser', True) # ActivĂ© par dĂ©faut # Valider le mode @@ -727,12 +864,14 @@ def start_execution(): session = get_session_state() session.active_execution_id = execution.id - # RĂ©initialiser l'Ă©tat - _execution_state['is_running'] = True - _execution_state['is_paused'] = False - _execution_state['should_stop'] = False - _execution_state['current_execution_id'] = execution.id - _execution_state['execution_mode'] = execution_mode + # RĂ©initialiser l'Ă©tat (protĂ©gĂ© par lock) + with _execution_lock: + _execution_state['is_running'] = True + _execution_state['is_paused'] = False + _execution_state['should_stop'] = False + _execution_state['current_execution_id'] = execution.id + _execution_state['execution_mode'] = execution_mode + _execution_state['variables'] = {} # Reset des variables print(f"🎯 [API v3] Mode d'exĂ©cution: {execution_mode}") @@ -762,7 +901,8 @@ def start_execution(): except Exception as e: db.session.rollback() - _execution_state['is_running'] = False + with _execution_lock: + _execution_state['is_running'] = False return jsonify({ 'success': False, 'error': str(e) @@ -832,14 +972,17 @@ def stop_execution(): """ArrĂȘte l'exĂ©cution""" global _execution_state - if not _execution_state['is_running']: - return jsonify({ - 'success': False, - 'error': "Aucune exĂ©cution en cours" - }), 400 + with _execution_lock: + if not _execution_state['is_running']: + return jsonify({ + 'success': False, + 'error': "Aucune exĂ©cution en cours" + }), 400 - _execution_state['should_stop'] = True - _execution_state['is_paused'] = False + _execution_state['should_stop'] = True + _execution_state['is_paused'] = False + # DĂ©bloquer le thread s'il attend un choix self-healing + _healing_event.set() print(f"⛔ [API v3] ArrĂȘt demandĂ©") @@ -876,6 +1019,8 @@ def get_execution_status(): 'execution_mode': _execution_state.get('execution_mode', 'basic'), 'execution': execution.to_dict() if execution else None, 'session': session.to_dict(), + # Variables runtime du workflow + 'variables': _execution_state.get('variables', {}), # Self-healing interactif 'waiting_for_choice': _execution_state.get('waiting_for_choice', False), 'candidates': _execution_state.get('candidates', []) if _execution_state.get('waiting_for_choice') else [], @@ -975,9 +1120,11 @@ def submit_healing_choice(): 'error': "CoordonnĂ©es invalides. Format attendu: {x: number, y: number}" }), 400 - # Stocker le choix - _execution_state['user_choice'] = choice - _execution_state['waiting_for_choice'] = False + # Stocker le choix et rĂ©veiller le thread d'exĂ©cution + with _execution_lock: + _execution_state['user_choice'] = choice + _execution_state['waiting_for_choice'] = False + _healing_event.set() print(f"✅ [Self-Healing] Choix reçu: {choice}") diff --git a/visual_workflow_builder/backend/api_v3/workflow.py b/visual_workflow_builder/backend/api_v3/workflow.py index c23905db0..7736fe00e 100644 --- a/visual_workflow_builder/backend/api_v3/workflow.py +++ b/visual_workflow_builder/backend/api_v3/workflow.py @@ -7,14 +7,18 @@ GET /api/v3/workflow/{id} POST /api/v3/workflow/{id}/step PUT /api/v3/workflow/{id}/step/{step_id} DELETE /api/v3/workflow/{id}/step/{step_id} +POST /api/v3/workflow/{id}/validate +POST /api/v3/workflow/{id}/export-training """ from flask import jsonify, request from datetime import datetime import uuid +import json +import os from . import api_v3_bp from db.models import db, Workflow, Step, VisualAnchor, get_session_state -from contracts.action_contracts import enforce_action_contract, ContractValidationError, get_required_params +from contracts.action_contracts import enforce_action_contract, ContractValidationError, get_required_params, validate_action_contract def generate_id(prefix: str) -> str: @@ -461,3 +465,166 @@ def reorder_steps(workflow_id: str): 'success': False, 'error': str(e) }), 500 + + +def _validate_workflow_steps(workflow): + """ + Logique de validation partagĂ©e entre validate et export-training. + Retourne (errors, warnings, steps). + """ + errors = [] + warnings = [] + + steps = Step.query.filter_by(workflow_id=workflow.id).order_by(Step.order).all() + + if len(steps) == 0: + errors.append("Le workflow n'a aucune Ă©tape") + return errors, warnings, steps + + for i, step in enumerate(steps): + step_label = step.label or step.action_type + prefix = f"Étape {i+1} ({step_label})" + + # Valider le contrat d'action + params = step.parameters or {} + violations = validate_action_contract(step.action_type, params) + for v in violations: + errors.append(f"{prefix}: {v.message}") + + # VĂ©rifier l'ancre visuelle si requise + required = get_required_params(step.action_type) + if 'visual_anchor' in required and not step.anchor_id: + errors.append(f"{prefix}: ancre visuelle manquante") + + # Warnings + if not step.label or step.label == step.action_type: + warnings.append(f"{prefix}: pas de label personnalisĂ©") + + return errors, warnings, steps + + +@api_v3_bp.route('/workflow//validate', methods=['POST']) +def validate_workflow(workflow_id: str): + """ + Valide la structure d'un workflow. + + Response: + { + "success": true, + "is_valid": true/false, + "errors": [...], + "warnings": [...], + "step_count": 12 + } + """ + try: + workflow = Workflow.query.get(workflow_id) + if not workflow: + return jsonify({ + 'success': False, + 'error': f"Workflow '{workflow_id}' non trouvĂ©" + }), 404 + + errors, warnings, steps = _validate_workflow_steps(workflow) + + is_valid = len(errors) == 0 + + print(f"{'✅' if is_valid else '❌'} [API v3] Validation workflow {workflow_id}: " + f"{len(errors)} erreur(s), {len(warnings)} warning(s)") + + return jsonify({ + 'success': True, + 'is_valid': is_valid, + 'errors': errors, + 'warnings': warnings, + 'step_count': len(steps) + }) + + except Exception as e: + return jsonify({ + 'success': False, + 'error': str(e) + }), 500 + + +@api_v3_bp.route('/workflow//export-training', methods=['POST']) +def export_for_training(workflow_id: str): + """ + Exporte un workflow validĂ© au format JSON d'entraĂźnement. + + Response: + { + "success": true, + "export_path": "training_data/workflow_xxx_1234567890.json", + "training_entry": { ... } + } + """ + try: + workflow = Workflow.query.get(workflow_id) + if not workflow: + return jsonify({ + 'success': False, + 'error': f"Workflow '{workflow_id}' non trouvĂ©" + }), 404 + + # Valider d'abord + errors, warnings, steps = _validate_workflow_steps(workflow) + if len(errors) > 0: + return jsonify({ + 'success': False, + 'error': 'Le workflow contient des erreurs de validation', + 'errors': errors + }), 400 + + # Construire le training entry + training_entry = { + 'workflow_id': workflow.id, + 'workflow_name': workflow.name, + 'description': workflow.description or '', + 'tags': workflow.tags if hasattr(workflow, 'tags') and workflow.tags else [], + 'steps': [], + 'exported_at': datetime.utcnow().isoformat(), + 'metadata': { + 'step_count': len(steps), + 'action_types': list(set(s.action_type for s in steps)), + 'has_anchors': any(s.anchor_id for s in steps), + 'warnings': warnings + } + } + + for step in steps: + step_data = { + 'order': step.order, + 'action_type': step.action_type, + 'label': step.label, + 'parameters': step.parameters or {}, + 'has_anchor': bool(step.anchor_id) + } + training_entry['steps'].append(step_data) + + # Sauvegarder dans training_data/ + training_dir = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'training_data') + os.makedirs(training_dir, exist_ok=True) + + timestamp = int(datetime.now().timestamp()) + filename = f"workflow_{workflow_id}_{timestamp}.json" + filepath = os.path.join(training_dir, filename) + + with open(filepath, 'w', encoding='utf-8') as f: + json.dump(training_entry, f, ensure_ascii=False, indent=2) + + export_path = f"training_data/{filename}" + + print(f"📩 [API v3] Workflow exportĂ© pour entraĂźnement: {export_path}") + + return jsonify({ + 'success': True, + 'export_path': export_path, + 'training_entry': training_entry + }) + + except Exception as e: + return jsonify({ + 'success': False, + 'error': str(e) + }), 500 diff --git a/visual_workflow_builder/backend/app.py b/visual_workflow_builder/backend/app.py index 8f28329bd..6e1d2e443 100644 --- a/visual_workflow_builder/backend/app.py +++ b/visual_workflow_builder/backend/app.py @@ -12,6 +12,8 @@ from flask_socketio import SocketIO from flask_caching import Cache from flask_migrate import Migrate import os +import logging +from logging.handlers import RotatingFileHandler from dotenv import load_dotenv # Load environment variables @@ -20,6 +22,25 @@ load_dotenv() # Initialize Flask app app = Flask(__name__) +# ============================================================ +# Logging — fichier rotatif + console +# ============================================================ +_log_dir = os.path.join(os.path.dirname(__file__), 'logs') +os.makedirs(_log_dir, exist_ok=True) + +_file_handler = RotatingFileHandler( + os.path.join(_log_dir, 'vwb.log'), + maxBytes=5 * 1024 * 1024, # 5 MB + backupCount=3 +) +_file_handler.setLevel(logging.INFO) +_file_handler.setFormatter(logging.Formatter( + '%(asctime)s [%(levelname)s] %(name)s: %(message)s' +)) + +logging.getLogger().addHandler(_file_handler) +logging.getLogger().setLevel(logging.INFO) + # Configuration app.config['SECRET_KEY'] = os.getenv('SECRET_KEY', 'dev-secret-key-change-in-production') app.config['SQLALCHEMY_DATABASE_URI'] = os.getenv('DATABASE_URL', 'sqlite:///vwb_v3.db') @@ -134,18 +155,11 @@ except ImportError as e: # Catalogue VWB - actions VisionOnly # V2 avec VLM (Vision Language Model) pour dĂ©tection intelligente try: - from catalog_routes_v2_vlm import catalog_bp + from catalog_routes_v2_vlm import catalog_bp, VLM_MODEL app.register_blueprint(catalog_bp) - print("✅ Blueprint catalog V2 VLM (Ollama qwen2.5vl) enregistrĂ©") + print(f"✅ Blueprint catalog V2 VLM (Ollama {VLM_MODEL}) enregistrĂ©") except ImportError as e: print(f"⚠ Blueprint catalog V2 VLM dĂ©sactivĂ©: {e}") - # Fallback sur la version pyautogui - try: - from catalog_routes import catalog_bp - app.register_blueprint(catalog_bp) - print("✅ Blueprint catalog (fallback pyautogui) enregistrĂ©") - except ImportError as e2: - print(f"⚠ Blueprint catalog dĂ©sactivĂ©: {e2}") # API Images Ancres Visuelles - stockage serveur try: @@ -224,20 +238,6 @@ def execute_workflow_step(): step_type = data.get('stepType', 'click_anchor') parameters = data.get('parameters', {}) - # DEBUG: Écrire les donnĂ©es reçues dans un fichier - import json as json_module - with open('/tmp/vwb_debug.log', 'a') as debug_file: - debug_file.write(f"\n{'='*60}\n") - debug_file.write(f"[execute-step] stepType={step_type}, stepId={step_id}\n") - debug_file.write(f"[execute-step] parameters keys: {list(parameters.keys())}\n") - if 'visual_anchor' in parameters: - va = parameters['visual_anchor'] - debug_file.write(f"[execute-step] visual_anchor keys: {list(va.keys()) if va else 'None'}\n") - debug_file.write(f"[execute-step] visual_anchor.id: {va.get('id')}\n") - debug_file.write(f"[execute-step] visual_anchor.thumbnail_url: {va.get('thumbnail_url') or (va.get('metadata', {}) or {}).get('thumbnail_url')}\n") - debug_file.write(f"[execute-step] FULL visual_anchor: {json_module.dumps(va, default=str)[:500]}\n") - debug_file.flush() - # Convert to catalog execute format catalog_request = { 'type': step_type, @@ -245,13 +245,13 @@ def execute_workflow_step(): 'parameters': parameters } - # Call the internal catalog execute endpoint - from catalog_routes import catalog_bp + # Call the internal catalog execute endpoint (v2 VLM) + from catalog_routes_v2_vlm import catalog_bp # Direct execution via catalog try: # Import the execute function directly - from catalog_routes import execute_action as catalog_execute + from catalog_routes_v2_vlm import execute_action as catalog_execute # We need to simulate Flask request context - use internal call from flask import current_app with current_app.test_request_context( diff --git a/visual_workflow_builder/backend/app_lightweight.py b/visual_workflow_builder/backend/app_lightweight.py index fcd6e8c38..0c3659b35 100644 --- a/visual_workflow_builder/backend/app_lightweight.py +++ b/visual_workflow_builder/backend/app_lightweight.py @@ -50,7 +50,7 @@ except ImportError as e: # Import des routes du catalogue VWB try: - from catalog_routes import register_catalog_routes + from catalog_routes_v2_vlm import register_catalog_routes CATALOG_ROUTES_AVAILABLE = True print("✅ Routes du catalogue VWB disponibles") except ImportError as e: diff --git a/visual_workflow_builder/backend/catalog_routes.py b/visual_workflow_builder/backend/catalog_routes.py deleted file mode 100644 index de1f42478..000000000 --- a/visual_workflow_builder/backend/catalog_routes.py +++ /dev/null @@ -1,1832 +0,0 @@ -""" -Routes API Catalogue VWB - Endpoints pour Actions VisionOnly - -Auteur : Dom, Alice, Kiro - 09 janvier 2026 - -Ce module dĂ©finit les routes Flask pour l'API du catalogue d'actions VisionOnly -du Visual Workflow Builder. - -Endpoints : -- GET /api/vwb/catalog/actions : Liste des actions disponibles -- POST /api/vwb/catalog/execute : ExĂ©cution d'une action -- GET /api/vwb/catalog/actions/{action_id} : DĂ©tails d'une action -- POST /api/vwb/catalog/validate : Validation d'une action -""" - -from flask import Blueprint, request, jsonify -from typing import Dict, Any, List, Optional -from datetime import datetime -import traceback - -# Import des actions et contrats VWB -try: - from visual_workflow_builder.backend.actions import ( - BaseVWBAction, VWBActionResult, VWBActionStatus, - VWBClickAnchorAction, VWBTypeTextAction, VWBWaitForAnchorAction, - VWBFocusAnchorAction, VWBTypeSecretAction, VWBScrollToAnchorAction, - VWBExtractTextAction - ) - from visual_workflow_builder.backend.contracts.error import VWBErrorType, VWBErrorSeverity, create_vwb_error - from visual_workflow_builder.backend.contracts.evidence import VWBEvidenceType - from visual_workflow_builder.backend.contracts.visual_anchor import VWBVisualAnchor, VWBVisualAnchorType - from visual_workflow_builder.backend.contracts.action_contracts import ( - enforce_action_contract, ContractValidationError, get_required_params - ) - ACTIONS_AVAILABLE = True - CONTRACTS_AVAILABLE = True - print("✅ Actions VWB importĂ©es avec succĂšs") -except ImportError as e: - print(f"⚠ Actions VWB non disponibles: {e}") - try: - # Essayer import relatif - from .actions import ( - BaseVWBAction, VWBActionResult, VWBActionStatus, - VWBClickAnchorAction, VWBTypeTextAction, VWBWaitForAnchorAction, - VWBFocusAnchorAction, VWBTypeSecretAction, VWBScrollToAnchorAction, - VWBExtractTextAction - ) - from .contracts.error import VWBErrorType, VWBErrorSeverity, create_vwb_error - from .contracts.evidence import VWBEvidenceType - from .contracts.visual_anchor import VWBVisualAnchor, VWBVisualAnchorType - from .contracts.action_contracts import ( - enforce_action_contract, ContractValidationError, get_required_params - ) - ACTIONS_AVAILABLE = True - CONTRACTS_AVAILABLE = True - print("✅ Actions VWB importĂ©es avec import relatif") - except ImportError as e2: - print(f"⚠ Import relatif Ă©chouĂ© aussi: {e2}") - ACTIONS_AVAILABLE = False - CONTRACTS_AVAILABLE = False - BaseVWBAction = None - -# Import du ScreenCapturer (Option A thread-safe) -try: - import sys - from pathlib import Path - sys.path.insert(0, str(Path(__file__).parent.parent.parent)) - from core.capture import ScreenCapturer - SCREEN_CAPTURER_AVAILABLE = True -except ImportError: - SCREEN_CAPTURER_AVAILABLE = False - ScreenCapturer = None - -# Import pour la recherche visuelle dynamique -import os -try: - import cv2 - import numpy as np - import base64 - from io import BytesIO - from PIL import Image - CV2_AVAILABLE = True -except ImportError: - CV2_AVAILABLE = False - cv2 = None - np = None - - -def find_visual_anchor_on_screen(anchor_image_base64: str, confidence_threshold: float = 0.7, bounding_box: Dict = None, expected_position: Dict = None) -> Optional[Dict[str, Any]]: - """ - Recherche dynamiquement une ancre visuelle sur l'Ă©cran actuel. - - Utilise le template matching OpenCV pour trouver l'image de l'ancre - sur la capture d'Ă©cran actuelle. - - Args: - anchor_image_base64: Image de l'ancre encodĂ©e en base64 - confidence_threshold: Seuil de confiance minimum (0.0-1.0) - bounding_box: Si fourni, extrait cette rĂ©gion de l'image de l'ancre avant recherche - expected_position: Position attendue (x, y) pour validation - si la position trouvĂ©e - est trop Ă©loignĂ©e (>300px), on retourne None - - Returns: - Dict avec les coordonnĂ©es trouvĂ©es ou None si non trouvĂ© - { - "found": True, - "x": 100, - "y": 200, - "width": 50, - "height": 30, - "confidence": 0.95, - "center_x": 125, - "center_y": 215 - } - """ - if not CV2_AVAILABLE: - print("⚠ OpenCV non disponible pour la recherche visuelle") - return None - - try: - import pyautogui - - # 1. Capturer l'Ă©cran actuel - print("📾 [Visual Search] Capture de l'Ă©cran...") - screenshot = pyautogui.screenshot() - screen_array = np.array(screenshot) - screen_bgr = cv2.cvtColor(screen_array, cv2.COLOR_RGB2BGR) - screen_gray = cv2.cvtColor(screen_bgr, cv2.COLOR_BGR2GRAY) - screen_h, screen_w = screen_gray.shape[:2] - - # 2. DĂ©coder l'image de l'ancre depuis base64 - print("🔍 [Visual Search] DĂ©codage de l'image de l'ancre...") - - # GĂ©rer les diffĂ©rents formats base64 - if ',' in anchor_image_base64: - # Format "data:image/png;base64,XXXX" - anchor_image_base64 = anchor_image_base64.split(',')[1] - - anchor_bytes = base64.b64decode(anchor_image_base64) - anchor_pil = Image.open(BytesIO(anchor_bytes)) - anchor_array = np.array(anchor_pil.convert('RGB')) - - # 3. Si bounding_box fourni et que l'image est grande, extraire la rĂ©gion - ref_h, ref_w = anchor_array.shape[:2] - print(f"📐 [Visual Search] Image rĂ©fĂ©rence: {ref_w}x{ref_h}") - - if bounding_box and ref_w > 200 and ref_h > 200: - # L'image de rĂ©fĂ©rence est l'Ă©cran capturĂ© (ex: 1200x675) - # Le bounding_box contient les coordonnĂ©es de sĂ©lection EN PIXELS de cette image - # Donc on utilise directement ces coordonnĂ©es pour l'extraction (pas de scale!) - bbox_x = bounding_box.get('x', 0) - bbox_y = bounding_box.get('y', 0) - bbox_w = bounding_box.get('width', 100) - bbox_h = bounding_box.get('height', 100) - - print(f"📊 [Visual Search] Image rĂ©fĂ©rence: {ref_w}x{ref_h}, Écran actuel: {screen_w}x{screen_h}") - print(f"📊 [Visual Search] BBox (pixels image): x={bbox_x:.1f}, y={bbox_y:.1f}, w={bbox_w:.1f}, h={bbox_h:.1f}") - - # Les coordonnĂ©es du bounding_box sont DÉJÀ en pixels de l'image de rĂ©fĂ©rence - # Pas besoin de scale pour l'extraction! - crop_x = int(bbox_x) - crop_y = int(bbox_y) - crop_w = int(bbox_w) - crop_h = int(bbox_h) - - # S'assurer que les coordonnĂ©es sont valides - crop_x = max(0, min(crop_x, ref_w - 10)) - crop_y = max(0, min(crop_y, ref_h - 10)) - crop_w = max(20, min(crop_w, ref_w - crop_x)) - crop_h = max(20, min(crop_h, ref_h - crop_y)) - - print(f"✂ [Visual Search] Extraction rĂ©gion: ({crop_x}, {crop_y}) - {crop_w}x{crop_h}") - - # Extraire la rĂ©gion de l'image de rĂ©fĂ©rence - anchor_array = anchor_array[crop_y:crop_y+crop_h, crop_x:crop_x+crop_w] - - anchor_bgr = cv2.cvtColor(anchor_array, cv2.COLOR_RGB2BGR) - anchor_gray = cv2.cvtColor(anchor_bgr, cv2.COLOR_BGR2GRAY) - - anchor_h, anchor_w = anchor_gray.shape[:2] - print(f"📐 [Visual Search] Taille ancre finale: {anchor_w}x{anchor_h}") - - # 3. Template matching avec plusieurs mĂ©thodes - methods = [ - (cv2.TM_CCOEFF_NORMED, "TM_CCOEFF_NORMED"), - (cv2.TM_CCORR_NORMED, "TM_CCORR_NORMED"), - ] - - best_result = None - best_confidence = 0 - - for method, method_name in methods: - result = cv2.matchTemplate(screen_gray, anchor_gray, method) - min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(result) - - confidence = max_val - - if confidence > best_confidence: - best_confidence = confidence - best_result = { - "found": True, - "x": max_loc[0], - "y": max_loc[1], - "width": anchor_w, - "height": anchor_h, - "confidence": confidence, - "center_x": max_loc[0] + anchor_w // 2, - "center_y": max_loc[1] + anchor_h // 2, - "method": method_name - } - - if best_result and best_result["confidence"] >= confidence_threshold: - print(f"✅ [Visual Search] Ancre trouvĂ©e!") - print(f" Position: ({best_result['center_x']}, {best_result['center_y']})") - print(f" Confiance: {best_result['confidence']:.2%}") - print(f" MĂ©thode: {best_result['method']}") - - # Validation de la position attendue - # Évite de cliquer sur une copie de l'image (ex: dans l'interface VWB) - if expected_position: - exp_x = expected_position.get('x', 0) - exp_y = expected_position.get('y', 0) - found_x = best_result['center_x'] - found_y = best_result['center_y'] - distance = ((found_x - exp_x) ** 2 + (found_y - exp_y) ** 2) ** 0.5 - - print(f"📍 [Visual Search] Validation position:") - print(f" Attendue: ({exp_x}, {exp_y})") - print(f" TrouvĂ©e: ({found_x}, {found_y})") - print(f" Distance: {distance:.0f}px") - - MAX_DISTANCE = 300 # TolĂ©rance en pixels - HIGH_CONFIDENCE_THRESHOLD = 0.75 # Seuil de haute confiance (abaissĂ© pour ĂȘtre plus permissif) - - if distance > MAX_DISTANCE: - # Si confiance suffisante, on fait confiance Ă  Visual Search mĂȘme si position Ă©loignĂ©e - # Car les coordonnĂ©es statiques peuvent ĂȘtre incorrectes (capture faite Ă  un autre moment) - if best_result['confidence'] >= HIGH_CONFIDENCE_THRESHOLD: - print(f"⚠ [Visual Search] Position Ă©loignĂ©e ({distance:.0f}px > {MAX_DISTANCE}px)") - print(f" MAIS confiance suffisante ({best_result['confidence']:.2%} >= {HIGH_CONFIDENCE_THRESHOLD:.0%})") - print(f" → Utilisation de la position Visual Search (coordonnĂ©es statiques ignorĂ©es)") - else: - print(f"⚠ [Visual Search] Position trop Ă©loignĂ©e ({distance:.0f}px > {MAX_DISTANCE}px)") - print(f" Confiance insuffisante ({best_result['confidence']:.2%} < {HIGH_CONFIDENCE_THRESHOLD:.0%})") - print(f" → Utilisation des coordonnĂ©es statiques") - return None - else: - print(f"✅ [Visual Search] Position validĂ©e (distance < {MAX_DISTANCE}px)") - - return best_result - else: - conf_found = best_confidence if best_result else 0 - print(f"❌ [Visual Search] Ancre non trouvĂ©e (confiance: {conf_found:.2%} < {confidence_threshold:.2%})") - return None - - except Exception as e: - print(f"❌ [Visual Search] Erreur: {e}") - traceback.print_exc() - return None - - -def find_anchor_multiscale(anchor_image_base64: str, scales: List[float] = None, confidence_threshold: float = 0.7) -> Optional[Dict[str, Any]]: - """ - Recherche visuelle multi-Ă©chelle pour trouver l'ancre mĂȘme si elle a changĂ© de taille. - - Args: - anchor_image_base64: Image de l'ancre encodĂ©e en base64 - scales: Liste des facteurs d'Ă©chelle Ă  tester (ex: [0.8, 0.9, 1.0, 1.1, 1.2]) - confidence_threshold: Seuil de confiance minimum - - Returns: - Dict avec les coordonnĂ©es trouvĂ©es ou None - """ - if scales is None: - scales = [0.7, 0.8, 0.9, 1.0, 1.1, 1.2, 1.3] - - if not CV2_AVAILABLE: - return None - - try: - import pyautogui - - # Capturer l'Ă©cran - screenshot = pyautogui.screenshot() - screen_array = np.array(screenshot) - screen_gray = cv2.cvtColor(screen_array, cv2.COLOR_RGB2GRAY) - - # DĂ©coder l'ancre - if ',' in anchor_image_base64: - anchor_image_base64 = anchor_image_base64.split(',')[1] - - anchor_bytes = base64.b64decode(anchor_image_base64) - anchor_pil = Image.open(BytesIO(anchor_bytes)) - anchor_array = np.array(anchor_pil.convert('RGB')) - anchor_gray = cv2.cvtColor(anchor_array, cv2.COLOR_RGB2GRAY) - - best_result = None - best_confidence = 0 - - for scale in scales: - # Redimensionner l'ancre - new_w = int(anchor_gray.shape[1] * scale) - new_h = int(anchor_gray.shape[0] * scale) - - if new_w < 10 or new_h < 10: - continue - if new_w > screen_gray.shape[1] or new_h > screen_gray.shape[0]: - continue - - resized = cv2.resize(anchor_gray, (new_w, new_h)) - - result = cv2.matchTemplate(screen_gray, resized, cv2.TM_CCOEFF_NORMED) - min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(result) - - if max_val > best_confidence: - best_confidence = max_val - best_result = { - "found": True, - "x": max_loc[0], - "y": max_loc[1], - "width": new_w, - "height": new_h, - "confidence": max_val, - "center_x": max_loc[0] + new_w // 2, - "center_y": max_loc[1] + new_h // 2, - "scale": scale - } - - if best_result and best_result["confidence"] >= confidence_threshold: - print(f"✅ [MultiScale] TrouvĂ© Ă  l'Ă©chelle {best_result['scale']:.1f}x") - print(f" Position: ({best_result['center_x']}, {best_result['center_y']})") - print(f" Confiance: {best_result['confidence']:.2%}") - return best_result - - return None - - except Exception as e: - print(f"❌ [MultiScale] Erreur: {e}") - return None - -# CrĂ©er le blueprint pour les routes catalogue -catalog_bp = Blueprint('catalog', __name__, url_prefix='/api/vwb/catalog') - -# Instance globale du ScreenCapturer (initialisĂ©e Ă  la demande) -_screen_capturer_instance = None - - -def get_screen_capturer(): - """ - Obtient l'instance du ScreenCapturer (initialisation paresseuse). - - Returns: - ScreenCapturer ou None si non disponible - """ - global _screen_capturer_instance - if _screen_capturer_instance is None and SCREEN_CAPTURER_AVAILABLE: - try: - _screen_capturer_instance = ScreenCapturer( - buffer_size=5, - detect_changes=False - ) - print("✅ ScreenCapturer initialisĂ© pour le catalogue VWB") - except Exception as e: - print(f"⚠ Erreur initialisation ScreenCapturer: {e}") - return None - return _screen_capturer_instance - - -def create_action_from_config(action_config: Dict[str, Any]) -> Optional[BaseVWBAction]: - """ - CrĂ©e une instance d'action Ă  partir de la configuration. - - Args: - action_config: Configuration de l'action - - Returns: - Instance d'action ou None si erreur - """ - if not ACTIONS_AVAILABLE: - print("⚠ Actions VWB non disponibles") - return None - - try: - action_type = action_config.get('type') - action_id = action_config.get('action_id', f"action_{datetime.now().strftime('%Y%m%d_%H%M%S')}") - parameters = action_config.get('parameters', {}) - - # Obtenir le ScreenCapturer - screen_capturer = get_screen_capturer() - - # CrĂ©er l'ancre visuelle si fournie - if 'visual_anchor' in parameters and isinstance(parameters['visual_anchor'], dict): - anchor_data = parameters['visual_anchor'] - visual_anchor = VWBVisualAnchor.from_dict(anchor_data) - parameters['visual_anchor'] = visual_anchor - - # CrĂ©er l'action selon le type - if action_type == 'click_anchor': - return VWBClickAnchorAction( - action_id=action_id, - parameters=parameters, - screen_capturer=screen_capturer - ) - elif action_type == 'type_text': - return VWBTypeTextAction( - action_id=action_id, - parameters=parameters, - screen_capturer=screen_capturer - ) - elif action_type == 'wait_for_anchor': - return VWBWaitForAnchorAction( - action_id=action_id, - parameters=parameters, - screen_capturer=screen_capturer - ) - elif action_type == 'focus_anchor': - return VWBFocusAnchorAction( - action_id=action_id, - parameters=parameters, - screen_capturer=screen_capturer - ) - elif action_type == 'type_secret': - return VWBTypeSecretAction( - action_id=action_id, - parameters=parameters, - screen_capturer=screen_capturer - ) - elif action_type == 'scroll_to_anchor': - return VWBScrollToAnchorAction( - action_id=action_id, - parameters=parameters, - screen_capturer=screen_capturer - ) - elif action_type == 'extract_text': - return VWBExtractTextAction( - action_id=action_id, - parameters=parameters, - screen_capturer=screen_capturer - ) - else: - print(f"⚠ Type d'action non supportĂ©: {action_type}") - return None - - except Exception as e: - print(f"❌ Erreur crĂ©ation action: {e}") - return None - - -@catalog_bp.route('/actions', methods=['GET']) -def list_actions(): - """ - Liste toutes les actions disponibles dans le catalogue VWB. - - Query Parameters: - - category: Filtrer par catĂ©gorie (vision_ui, control, data) - - search: Recherche textuelle dans nom/description - - Response: - { - "success": true, - "actions": [ - { - "id": "click_anchor", - "name": "Clic sur Ancre Visuelle", - "description": "...", - "category": "vision_ui", - "parameters": {...}, - "examples": [...] - } - ], - "total": 3, - "categories": ["vision_ui"] - } - """ - try: - # ParamĂštres de requĂȘte - category_filter = request.args.get('category') - search_query = request.args.get('search', '').lower() - - # DĂ©finir les actions disponibles - available_actions = [ - { - "id": "click_anchor", - "name": "Clic sur Ancre Visuelle", - "description": "Clique sur un Ă©lĂ©ment UI identifiĂ© par une ancre visuelle", - "category": "vision_ui", - "icon": "đŸ–±ïž", - "parameters": { - "visual_anchor": { - "type": "VWBVisualAnchor", - "required": True, - "description": "Ancre visuelle pour localiser l'Ă©lĂ©ment" - }, - "click_type": { - "type": "string", - "required": False, - "default": "left", - "options": ["left", "right", "double"], - "description": "Type de clic Ă  effectuer" - }, - "click_offset_x": { - "type": "number", - "required": False, - "default": 0, - "description": "DĂ©calage horizontal en pixels" - }, - "click_offset_y": { - "type": "number", - "required": False, - "default": 0, - "description": "DĂ©calage vertical en pixels" - }, - "confidence_threshold": { - "type": "number", - "required": False, - "default": 0.8, - "min": 0.0, - "max": 1.0, - "description": "Seuil de confiance pour la dĂ©tection" - } - }, - "examples": [ - { - "name": "Clic sur bouton de validation", - "description": "Clique sur le bouton 'Valider' d'un formulaire", - "parameters": { - "click_type": "left", - "confidence_threshold": 0.9 - } - } - ] - }, - { - "id": "type_text", - "name": "Saisie de Texte", - "description": "Saisit du texte dans un champ identifiĂ© par une ancre visuelle", - "category": "vision_ui", - "icon": "⌚", - "parameters": { - "visual_anchor": { - "type": "VWBVisualAnchor", - "required": True, - "description": "Ancre visuelle pour localiser le champ" - }, - "text_to_type": { - "type": "string", - "required": True, - "description": "Texte Ă  saisir" - }, - "clear_field_first": { - "type": "boolean", - "required": False, - "default": True, - "description": "Vider le champ avant la saisie" - }, - "click_before_typing": { - "type": "boolean", - "required": False, - "default": True, - "description": "Cliquer sur le champ avant la saisie" - }, - "press_enter_after": { - "type": "boolean", - "required": False, - "default": False, - "description": "Appuyer sur EntrĂ©e aprĂšs la saisie" - }, - "typing_speed_ms": { - "type": "number", - "required": False, - "default": 50, - "min": 0, - "description": "DĂ©lai entre caractĂšres en millisecondes" - } - }, - "examples": [ - { - "name": "Saisie d'email", - "description": "Saisit une adresse email dans un champ", - "parameters": { - "text_to_type": "user@example.com", - "clear_field_first": True, - "press_enter_after": False - } - } - ] - }, - { - "id": "wait_for_anchor", - "name": "Attente d'Ancre Visuelle", - "description": "Attend qu'une ancre visuelle apparaisse ou disparaisse", - "category": "control", - "icon": "⏳", - "parameters": { - "visual_anchor": { - "type": "VWBVisualAnchor", - "required": True, - "description": "Ancre visuelle Ă  surveiller" - }, - "wait_mode": { - "type": "string", - "required": False, - "default": "appear", - "options": ["appear", "disappear"], - "description": "Mode d'attente (apparition ou disparition)" - }, - "max_wait_time_ms": { - "type": "number", - "required": False, - "default": 30000, - "min": 1000, - "description": "DĂ©lai d'attente maximum en millisecondes" - }, - "check_interval_ms": { - "type": "number", - "required": False, - "default": 500, - "min": 100, - "description": "Intervalle de vĂ©rification en millisecondes" - } - }, - "examples": [ - { - "name": "Attendre chargement", - "description": "Attend qu'un indicateur de chargement disparaisse", - "parameters": { - "wait_mode": "disappear", - "max_wait_time_ms": 10000 - } - } - ] - }, - { - "id": "focus_anchor", - "name": "Donner le Focus Ă  un ÉlĂ©ment", - "description": "Donne le focus Ă  un Ă©lĂ©ment UI identifiĂ© par une ancre visuelle", - "category": "vision_ui", - "icon": "🎯", - "parameters": { - "visual_anchor": { - "type": "VWBVisualAnchor", - "required": True, - "description": "Ancre visuelle pour localiser l'Ă©lĂ©ment" - }, - "focus_method": { - "type": "string", - "required": False, - "default": "click", - "options": ["click", "tab", "keyboard"], - "description": "MĂ©thode pour donner le focus" - }, - "verify_focus": { - "type": "boolean", - "required": False, - "default": True, - "description": "VĂ©rifier que le focus a Ă©tĂ© donnĂ©" - }, - "confidence_threshold": { - "type": "number", - "required": False, - "default": 0.8, - "min": 0.0, - "max": 1.0, - "description": "Seuil de confiance pour la dĂ©tection" - } - }, - "examples": [ - { - "name": "Focus sur champ de saisie", - "description": "Donne le focus Ă  un champ pour la saisie clavier", - "parameters": { - "focus_method": "click", - "verify_focus": True - } - } - ] - }, - { - "id": "type_secret", - "name": "Saisie SĂ©curisĂ©e", - "description": "Saisit de maniĂšre sĂ©curisĂ©e des donnĂ©es sensibles (mots de passe, codes)", - "category": "vision_ui", - "icon": "🔐", - "parameters": { - "visual_anchor": { - "type": "VWBVisualAnchor", - "required": True, - "description": "Ancre visuelle pour localiser le champ de saisie" - }, - "secret_text": { - "type": "string", - "required": True, - "sensitive": True, - "description": "Texte secret Ă  saisir (masquĂ© dans les logs)" - }, - "clear_field_first": { - "type": "boolean", - "required": False, - "default": True, - "description": "Vider le champ avant la saisie" - }, - "mask_in_evidence": { - "type": "boolean", - "required": False, - "default": True, - "description": "Masquer le texte dans les Evidence" - }, - "typing_speed_ms": { - "type": "number", - "required": False, - "default": 30, - "min": 0, - "description": "DĂ©lai entre caractĂšres (plus rapide pour sĂ©curitĂ©)" - } - }, - "examples": [ - { - "name": "Saisie mot de passe", - "description": "Saisit un mot de passe dans un champ de connexion", - "parameters": { - "clear_field_first": True, - "mask_in_evidence": True - } - } - ] - }, - { - "id": "scroll_to_anchor", - "name": "DĂ©filer vers un ÉlĂ©ment", - "description": "Fait dĂ©filer la page jusqu'Ă  ce qu'un Ă©lĂ©ment soit visible", - "category": "vision_ui", - "icon": "📜", - "parameters": { - "visual_anchor": { - "type": "VWBVisualAnchor", - "required": True, - "description": "Ancre visuelle pour localiser l'Ă©lĂ©ment cible" - }, - "scroll_direction": { - "type": "string", - "required": False, - "default": "vertical", - "options": ["vertical", "horizontal", "both"], - "description": "Direction du dĂ©filement" - }, - "scroll_speed": { - "type": "string", - "required": False, - "default": "medium", - "options": ["slow", "medium", "fast"], - "description": "Vitesse du dĂ©filement" - }, - "max_scroll_attempts": { - "type": "number", - "required": False, - "default": 10, - "min": 1, - "description": "Nombre maximum de tentatives de dĂ©filement" - }, - "target_position": { - "type": "string", - "required": False, - "default": "center", - "options": ["top", "center", "bottom"], - "description": "Position cible de l'Ă©lĂ©ment dans la vue" - } - }, - "examples": [ - { - "name": "DĂ©filer vers un bouton", - "description": "Fait dĂ©filer verticalement pour trouver un bouton", - "parameters": { - "scroll_direction": "vertical", - "target_position": "center" - } - } - ] - }, - { - "id": "extract_text", - "name": "Extraire du Texte", - "description": "Extrait du texte d'une zone identifiĂ©e par une ancre visuelle", - "category": "data", - "icon": "📝", - "parameters": { - "visual_anchor": { - "type": "VWBVisualAnchor", - "required": True, - "description": "Ancre visuelle pour localiser la zone de texte" - }, - "extraction_mode": { - "type": "string", - "required": False, - "default": "full", - "options": ["full", "lines", "words", "numbers", "custom"], - "description": "Mode d'extraction du texte" - }, - "text_filters": { - "type": "array", - "required": False, - "default": [], - "description": "Filtres Ă  appliquer au texte extrait" - }, - "output_format": { - "type": "string", - "required": False, - "default": "text", - "options": ["text", "json", "structured"], - "description": "Format de sortie du texte extrait" - }, - "confidence_threshold": { - "type": "number", - "required": False, - "default": 0.8, - "min": 0.0, - "max": 1.0, - "description": "Seuil de confiance pour la dĂ©tection" - } - }, - "examples": [ - { - "name": "Extraire un numĂ©ro de facture", - "description": "Extrait un numĂ©ro de facture d'un document", - "parameters": { - "extraction_mode": "numbers", - "text_filters": ["digits_only"] - } - } - ] - } - ] - - # Filtrer par catĂ©gorie - if category_filter: - available_actions = [ - action for action in available_actions - if action['category'] == category_filter - ] - - # Filtrer par recherche - if search_query: - available_actions = [ - action for action in available_actions - if (search_query in action['name'].lower() or - search_query in action['description'].lower()) - ] - - # Obtenir les catĂ©gories uniques - categories = list(set(action['category'] for action in available_actions)) - - return jsonify({ - "success": True, - "actions": available_actions, - "total": len(available_actions), - "categories": sorted(categories), - "screen_capturer_available": SCREEN_CAPTURER_AVAILABLE - }) - - except Exception as e: - return jsonify({ - "success": False, - "error": f"Erreur lors de la rĂ©cupĂ©ration des actions: {str(e)}", - "traceback": traceback.format_exc() - }), 500 - - -@catalog_bp.route('/actions/', methods=['GET']) -def get_action_details(action_id: str): - """ - Obtient les dĂ©tails d'une action spĂ©cifique. - - Path Parameters: - - action_id: Identifiant de l'action - - Response: - { - "success": true, - "action": { - "id": "click_anchor", - "name": "...", - "description": "...", - "parameters": {...}, - "examples": [...], - "documentation": "..." - } - } - """ - try: - # RĂ©cupĂ©rer toutes les actions - all_actions_response = list_actions() - all_actions_data = all_actions_response.get_json() - - if not all_actions_data.get('success'): - return all_actions_response - - # Trouver l'action demandĂ©e - action = None - for act in all_actions_data['actions']: - if act['id'] == action_id: - action = act - break - - if not action: - return jsonify({ - "success": False, - "error": f"Action '{action_id}' non trouvĂ©e" - }), 404 - - # Ajouter de la documentation dĂ©taillĂ©e - documentation = { - "click_anchor": """ -# Clic sur Ancre Visuelle - -Cette action permet de cliquer sur un Ă©lĂ©ment de l'interface utilisateur en utilisant une ancre visuelle pour le localiser. - -## Fonctionnement -1. Capture l'Ă©cran actuel -2. Recherche l'ancre visuelle dans l'image -3. Calcule les coordonnĂ©es de clic -4. Effectue le clic au type spĂ©cifiĂ© -5. GĂ©nĂšre des preuves d'exĂ©cution (Evidence) - -## Cas d'usage -- Cliquer sur des boutons -- SĂ©lectionner des Ă©lĂ©ments de menu -- Activer des contrĂŽles UI -- Naviguer dans des interfaces - -## Bonnes pratiques -- Utilisez des ancres visuelles distinctives -- Ajustez le seuil de confiance selon la stabilitĂ© de l'UI -- Testez avec diffĂ©rentes rĂ©solutions d'Ă©cran - """, - "type_text": """ -# Saisie de Texte - -Cette action permet de saisir du texte dans un champ de saisie identifiĂ© par une ancre visuelle. - -## Fonctionnement -1. Localise le champ de saisie via l'ancre visuelle -2. Clique sur le champ (optionnel) -3. Vide le champ existant (optionnel) -4. Saisit le texte caractĂšre par caractĂšre -5. Appuie sur EntrĂ©e (optionnel) - -## Cas d'usage -- Remplir des formulaires -- Saisir des identifiants de connexion -- Entrer des donnĂ©es dans des champs -- Effectuer des recherches - -## Bonnes pratiques -- VĂ©rifiez que le champ est actif avant la saisie -- Utilisez un dĂ©lai de saisie appropriĂ© -- Testez avec diffĂ©rents types de champs - """, - "wait_for_anchor": """ -# Attente d'Ancre Visuelle - -Cette action permet d'attendre qu'un Ă©lĂ©ment apparaisse ou disparaisse de l'Ă©cran. - -## Fonctionnement -1. Surveille l'Ă©cran Ă  intervalles rĂ©guliers -2. Recherche l'ancre visuelle Ă  chaque vĂ©rification -3. Attend que la condition soit remplie (apparition/disparition) -4. Retourne le rĂ©sultat ou timeout - -## Cas d'usage -- Attendre la fin d'un chargement -- Synchroniser avec des animations -- Attendre l'apparition de dialogues -- GĂ©rer les Ă©tats transitoires - -## Bonnes pratiques -- DĂ©finissez des dĂ©lais d'attente raisonnables -- Utilisez des intervalles de vĂ©rification appropriĂ©s -- PrĂ©voyez des conditions de sortie - """ - } - - action['documentation'] = documentation.get(action_id, "Documentation non disponible") - - return jsonify({ - "success": True, - "action": action - }) - - except Exception as e: - return jsonify({ - "success": False, - "error": f"Erreur lors de la rĂ©cupĂ©ration de l'action: {str(e)}" - }), 500 - - -@catalog_bp.route('/execute', methods=['POST']) -def execute_action(): - """ - ExĂ©cute une action du catalogue VWB. - - Request Body: - { - "type": "click_anchor", - "action_id": "action_001", - "step_id": "step_001", - "parameters": { - "visual_anchor": {...}, - "click_type": "left" - }, - "workflow_id": "workflow_001", - "user_id": "user_001" - } - - Response: - { - "success": true, - "result": { - "action_id": "action_001", - "step_id": "step_001", - "status": "success", - "execution_time_ms": 1250.5, - "output_data": {...}, - "evidence_list": [...], - "error": null - } - } - """ - try: - # Valider le corps de la requĂȘte - data = request.get_json() - if not data: - return jsonify({ - "success": False, - "error": "Corps de requĂȘte JSON requis" - }), 400 - - # Extraire les paramĂštres (supporter les deux formats: type/stepType, step_id/stepId) - action_type = data.get('type') or data.get('stepType') - step_id = data.get('step_id') or data.get('stepId', f"step_{datetime.now().strftime('%Y%m%d_%H%M%S')}") - workflow_id = data.get('workflow_id') or data.get('workflowId') - user_id = data.get('user_id') or data.get('userId') - - if not action_type: - return jsonify({ - "success": False, - "error": "ParamĂštre 'type' ou 'stepType' requis" - }), 400 - - parameters = data.get('parameters', {}) - - # LOG DEBUG - Voir ce qui arrive du frontend - print(f"\n{'='*60}") - print(f"đŸ”„ REQUÊTE EXECUTE REÇUE:") - print(f" stepType: {action_type}") - print(f" stepId: {step_id}") - print(f" parameters keys: {list(parameters.keys())}") - print(f"{'='*60}\n") - - # === VALIDATION CONTRAT STRICT === - # BLOQUE l'exĂ©cution si le contrat n'est pas respectĂ© - if CONTRACTS_AVAILABLE: - try: - enforce_action_contract(action_type, parameters) - except ContractValidationError as e: - print(f"đŸš« [CONTRAT VIOLÉ] Action '{action_type}' bloquĂ©e!") - print(f" ParamĂštres requis: {get_required_params(action_type)}") - print(f" ParamĂštres reçus: {list(parameters.keys())}") - return jsonify({ - "success": False, - "status": "error", - "should_stop": True, - "error": e.to_dict(), - "message": str(e) - }), 400 - else: - print(f"⚠ [Contract] Validation de contrat non disponible") - - # === MODE EXÉCUTION DIRECTE AVEC PYAUTOGUI === - # Si on a des coordonnĂ©es bounding_box, on exĂ©cute directement sans passer par les classes VWB - direct_execution = data.get('direct_execution', True) # Par dĂ©faut, exĂ©cution directe - if direct_execution: - try: - import pyautogui - import time - - # RĂ©cupĂ©rer la rĂ©solution de l'Ă©cran - screen_width, screen_height = pyautogui.size() - print(f"📐 [Direct] RĂ©solution: {screen_width}x{screen_height}") - - result_message = '' - execution_success = True - start_time = datetime.now() - - if action_type in ['click', 'click_anchor']: - # IMPORTANT: Minimiser la fenĂȘtre active (navigateur VWB) pour Ă©viter - # que la miniature de l'ancre soit dĂ©tectĂ©e Ă  la place de l'Ă©lĂ©ment rĂ©el - try: - import subprocess - # Utiliser xdotool pour minimiser la fenĂȘtre active - subprocess.run(['xdotool', 'getactivewindow', 'windowminimize'], - timeout=2, capture_output=True) - print("📉 [Prepare] FenĂȘtre active minimisĂ©e") - time.sleep(0.5) # Attendre que la fenĂȘtre soit minimisĂ©e - except Exception as minimize_error: - print(f"⚠ [Prepare] Impossible de minimiser la fenĂȘtre: {minimize_error}") - - # RĂ©cupĂ©rer l'ancre visuelle - visual_anchor = parameters.get('visual_anchor', {}) - target = parameters.get('target', {}) - - # DEBUG: Écrire dans fichier de log - with open('/tmp/vwb_debug.log', 'a') as df: - df.write(f"\n--- catalog_routes execute ---\n") - df.write(f"parameters keys: {list(parameters.keys())}\n") - df.write(f"visual_anchor keys: {list(visual_anchor.keys()) if visual_anchor else 'None'}\n") - df.write(f"visual_anchor.anchor_id: {visual_anchor.get('anchor_id')}\n") - df.write(f"visual_anchor.id: {visual_anchor.get('id')}\n") - df.flush() - - # === RECHERCHE VISUELLE DYNAMIQUE === - # Si l'ancre contient une image (screenshot), on la cherche sur l'Ă©cran actuel - # Chercher dans tous les formats possibles - anchor_screenshot = ( - visual_anchor.get('screenshot') or - visual_anchor.get('image') or - visual_anchor.get('reference_image_base64') or - visual_anchor.get('referenceImage') or - target.get('screenshot') or - target.get('image') or - target.get('reference_image_base64') or - target.get('referenceImage') or - # Chercher dans metadata aussi - (visual_anchor.get('metadata', {}) or {}).get('reference_image') or - (target.get('metadata', {}) or {}).get('reference_image') - ) - - # Si pas d'image base64, essayer de charger depuis l'URL du serveur - if not anchor_screenshot: - original_url = ( - visual_anchor.get('reference_image_url') or - (visual_anchor.get('metadata', {}) or {}).get('reference_image_url') - ) - if original_url: - try: - # Extraire l'anchor_id depuis l'URL - # Format: /api/anchor-images/{anchor_id}/original - import re - match = re.search(r'/api/anchor-images/([^/]+)/original', original_url) - if match: - anchor_id = match.group(1) - # Charger l'image depuis le disque - from services.anchor_image_service import get_original_path - original_path = get_original_path(anchor_id) - if original_path and os.path.exists(original_path): - with open(original_path, 'rb') as f: - anchor_screenshot = base64.b64encode(f.read()).decode('utf-8') - print(f"✅ [Debug] Image chargĂ©e depuis le serveur: {anchor_id}") - except Exception as e: - print(f"⚠ [Debug] Erreur chargement image serveur: {e}") - - # FALLBACK: Charger depuis anchor_id directement si prĂ©sent - if not anchor_screenshot: - anchor_id = ( - visual_anchor.get('id') or - visual_anchor.get('anchor_id') or - target.get('id') or - target.get('anchor_id') - ) - - # Essayer aussi d'extraire anchor_id depuis thumbnail_url ou metadata.thumbnail_url - if not anchor_id: - thumbnail_url = ( - visual_anchor.get('thumbnail_url') or - (visual_anchor.get('metadata', {}) or {}).get('thumbnail_url') or - target.get('thumbnail_url') or - (target.get('metadata', {}) or {}).get('thumbnail_url') - ) - if thumbnail_url: - import re - match = re.search(r'/api/anchor-images/([^/]+)/', thumbnail_url) - if match: - anchor_id = match.group(1) - print(f"📎 [Debug] anchor_id extrait de thumbnail_url: {anchor_id}") - - with open('/tmp/vwb_debug.log', 'a') as df: - df.write(f"Fallback anchor_id trouvĂ©: {anchor_id}\n") - df.flush() - - if anchor_id and anchor_id.startswith('anchor_'): - try: - import os as os_module - from services.anchor_image_service import get_original_path - original_path = get_original_path(anchor_id) - with open('/tmp/vwb_debug.log', 'a') as df: - df.write(f"original_path: {original_path}, exists: {os_module.path.exists(original_path) if original_path else False}\n") - df.flush() - if original_path and os_module.path.exists(original_path): - with open(original_path, 'rb') as f: - anchor_screenshot = base64.b64encode(f.read()).decode('utf-8') - with open('/tmp/vwb_debug.log', 'a') as df: - df.write(f"✅ Image chargĂ©e via anchor_id: {anchor_id}, len={len(anchor_screenshot)}\n") - df.flush() - except Exception as e: - with open('/tmp/vwb_debug.log', 'a') as df: - df.write(f"⚠ Erreur chargement via anchor_id: {e}\n") - df.flush() - - print(f"🔍 [Debug] visual_anchor keys: {list(visual_anchor.keys()) if visual_anchor else 'None'}") - print(f"🔍 [Debug] target keys: {list(target.keys()) if target else 'None'}") - print(f"🔍 [Debug] anchor_screenshot trouvĂ©: {bool(anchor_screenshot)} (longueur: {len(anchor_screenshot) if anchor_screenshot else 0})") - - # Mode de recherche visuelle DÉSACTIVÉ par dĂ©faut (retour Ă  l'approche du 14 janvier) - # La Visual Search avec template matching trouve trop de faux positifs (miniatures, icĂŽnes similaires) - # On utilise les coordonnĂ©es statiques du bounding_box qui sont plus fiables - use_visual_search = data.get('use_visual_search', False) # DĂ©sactivĂ© par dĂ©faut - print(f"🎯 [Mode Simple] Utilisation des coordonnĂ©es bounding_box directement (Visual Search: {use_visual_search})") - confidence_threshold = parameters.get('confidence_threshold', 0.7) - - # Extraire le bounding_box pour le cropping de l'image de rĂ©fĂ©rence - bounding_box = ( - visual_anchor.get('bounding_box') or - visual_anchor.get('boundingBox') or - target.get('boundingBox') or - target.get('bounding_box') or - {} - ) - print(f"🔍 [Debug] bounding_box: {bounding_box}") - - x, y = 0, 0 - search_method = "static" - - if use_visual_search: - print(f"🔎 [Visual Search] Recherche dynamique de l'ancre sur l'Ă©cran...") - - # Calculer la position attendue pour validation - # Évite les fausses dĂ©tections (ex: copie dans l'interface VWB) - expected_position = None - if bounding_box: - bbox_x = bounding_box.get('x', 0) - bbox_y = bounding_box.get('y', 0) - bbox_w = bounding_box.get('width', 0) - bbox_h = bounding_box.get('height', 0) - - # RĂ©cupĂ©rer la rĂ©solution de l'image originale depuis les mĂ©tadonnĂ©es - metadata = ( - visual_anchor.get('metadata') or - target.get('metadata') or - {} - ) - screen_resolution = metadata.get('screen_resolution') or {} - ref_width = screen_resolution.get('width') or screen_width - ref_height = screen_resolution.get('height') or screen_height - - # Calculer le scale pour convertir vers l'Ă©cran rĂ©el - if ref_width == screen_width and ref_height == screen_height: - scale_to_screen_x = 1.0 - scale_to_screen_y = 1.0 - else: - scale_to_screen_x = screen_width / ref_width - scale_to_screen_y = screen_height / ref_height - - exp_x = int((bbox_x + bbox_w / 2) * scale_to_screen_x) - exp_y = int((bbox_y + bbox_h / 2) * scale_to_screen_y) - expected_position = {'x': exp_x, 'y': exp_y} - print(f"📍 [Visual Search] Position attendue: ({exp_x}, {exp_y}) [ref: {ref_width}x{ref_height}]") - - # Essayer d'abord la recherche normale avec le bounding_box pour cropper - visual_result = find_visual_anchor_on_screen(anchor_screenshot, confidence_threshold, bounding_box, expected_position) - - # Note: On ne fait PAS de fallback multi-Ă©chelle quand on a une position attendue - # car si la validation a Ă©chouĂ©, le multi-Ă©chelle trouvera probablement la mĂȘme fausse copie - # Les coordonnĂ©es statiques sont plus fiables dans ce cas - if not visual_result and not expected_position: - # Fallback: recherche multi-Ă©chelle (seulement si pas de position attendue) - print(f"🔄 [Visual Search] Tentative multi-Ă©chelle...") - visual_result = find_anchor_multiscale(anchor_screenshot, confidence_threshold=confidence_threshold * 0.9) - - if visual_result: - x = visual_result['center_x'] - y = visual_result['center_y'] - search_method = f"visual ({visual_result.get('method', 'multiscale')}, conf={visual_result['confidence']:.2%})" - print(f"✅ [Visual Search] ÉlĂ©ment trouvĂ© Ă  ({x}, {y})") - else: - print(f"⚠ [Visual Search] Ancre non trouvĂ©e, fallback sur coordonnĂ©es statiques") - - # === MÉTHODE PYAUTOGUI AMÉLIORÉE === - # PrioritĂ© sur pyautogui car les coordonnĂ©es bounding_box du frontend sont incorrectes - with open('/tmp/vwb_debug.log', 'a') as df: - df.write(f"=== PYAUTOGUI CHECK ===\n") - df.write(f"x={x}, y={y}, anchor_screenshot={bool(anchor_screenshot)}, len={len(anchor_screenshot) if anchor_screenshot else 0}\n") - df.flush() - - if x == 0 and y == 0 and anchor_screenshot: - try: - import pyautogui - import tempfile - import os - - with open('/tmp/vwb_debug.log', 'a') as df: - df.write(f"🔍 PyAutoGUI: DĂ©marrage recherche...\n") - df.flush() - - # Sauvegarder l'image de l'ancre temporairement - anchor_img_data = base64.b64decode(anchor_screenshot) - with tempfile.NamedTemporaryFile(suffix='.png', delete=False) as tmp_file: - tmp_file.write(anchor_img_data) - tmp_path = tmp_file.name - - try: - location = None - found_conf = 0 - found_region = "unknown" - - # DĂ©finir les rĂ©gions de recherche prioritaires - # PRIORITÉ 1: Dock en bas (car c'est lĂ  que l'utilisateur a son dock Ubuntu) - # PRIORITÉ 2: Autres rĂ©gions si pas trouvĂ© en bas - dock_regions = [ - # Dock en bas (100 derniers pixels) - PRIORITAIRE - ("dock-bas", (0, screen_height - 100, screen_width, 100)), - # Dock Ă  gauche (80 premiers pixels) - ("dock-gauche", (0, 0, 80, screen_height)), - # Dock Ă  droite (80 derniers pixels) - ("dock-droite", (screen_width - 80, 0, 80, screen_height)), - ] - - # StratĂ©gie: tester CHAQUE rĂ©gion avec TOUTES les confidences - # avant de passer Ă  la suivante (prioritĂ© au dock-bas) - # Note: certaines images nĂ©cessitent conf=0.4 pour ĂȘtre trouvĂ©es - confidence_levels = [0.8, 0.7, 0.6, 0.5, 0.4] - - # Pour chaque rĂ©gion (en commençant par dock-bas) - for region_name, region in dock_regions: - if location: - break - # Tester toutes les confidences pour cette rĂ©gion - for conf in confidence_levels: - try: - loc = pyautogui.locateOnScreen( - tmp_path, - confidence=conf, - region=region, - grayscale=True # RĂ©duire les faux positifs couleur - ) - if loc: - location = loc - found_conf = conf - found_region = region_name - print(f"🎯 [PyAutoGUI] TrouvĂ© dans {region_name} avec confidence={conf}") - break - except Exception as e: - pass - - # Si pas trouvĂ© dans les docks, chercher sur l'Ă©cran complet - if not location: - for conf in confidence_levels: - try: - loc = pyautogui.locateOnScreen( - tmp_path, - confidence=conf, - grayscale=True - ) - if loc: - location = loc - found_conf = conf - found_region = "Ă©cran complet" - print(f"🔍 [PyAutoGUI] TrouvĂ© sur Ă©cran complet avec confidence={conf}") - break - except Exception as e: - pass - - if location: - # Obtenir le centre de la zone trouvĂ©e - center = pyautogui.center(location) - x, y = center.x, center.y - search_method = f"pyautogui ({found_region}, conf={found_conf})" - with open('/tmp/vwb_debug.log', 'a') as df: - df.write(f"✅ PyAutoGUI: TrouvĂ© Ă  ({x}, {y}) - rĂ©gion: {found_region}, conf={found_conf}\n") - df.flush() - else: - with open('/tmp/vwb_debug.log', 'a') as df: - df.write(f"⚠ PyAutoGUI: Ancre NON TROUVÉE aprĂšs toutes les tentatives\n") - df.flush() - finally: - # Nettoyer le fichier temporaire - import os as os_cleanup - os_cleanup.unlink(tmp_path) - - except Exception as e: - print(f"⚠ [PyAutoGUI] Erreur: {e}") - - # === FALLBACK: COORDONNÉES STATIQUES === - # Note: Les coordonnĂ©es bounding_box du frontend peuvent ĂȘtre incorrectes - # (bug connu: x nĂ©gatif, y dĂ©calĂ©). Ce fallback n'est utilisĂ© que si pyautogui Ă©choue. - if x == 0 and y == 0: - bounding_box = ( - visual_anchor.get('bounding_box') or - visual_anchor.get('boundingBox') or - target.get('boundingBox') or - target.get('bounding_box') or - {} - ) - - if bounding_box: - bbox_x = bounding_box.get('x', 0) - bbox_y = bounding_box.get('y', 0) - bbox_w = bounding_box.get('width', 0) - bbox_h = bounding_box.get('height', 0) - - print(f"📩 [Static] BBox brute: x={bbox_x}, y={bbox_y}, w={bbox_w}, h={bbox_h}") - - # VALIDATION: Rejeter les coordonnĂ©es invalides - if bbox_x < 0 or bbox_y < 0: - print(f"⚠ [Static] CoordonnĂ©es invalides (nĂ©gatives): x={bbox_x}, y={bbox_y} - IGNORÉ") - elif bbox_w <= 0 or bbox_h <= 0: - print(f"⚠ [Static] Dimensions invalides: w={bbox_w}, h={bbox_h} - IGNORÉ") - else: - # RĂ©cupĂ©rer la rĂ©solution de l'image originale depuis les mĂ©tadonnĂ©es - # Les coordonnĂ©es du bounding_box sont en pixels de l'image ORIGINALE - # (pas une image redimensionnĂ©e) - metadata = ( - visual_anchor.get('metadata') or - target.get('metadata') or - {} - ) - screen_resolution = metadata.get('screen_resolution') or {} - ref_width = screen_resolution.get('width') or screen_width # Utiliser la rĂ©solution Ă©cran si non spĂ©cifiĂ© - ref_height = screen_resolution.get('height') or screen_height - - # Si la rĂ©solution originale correspond Ă  l'Ă©cran actuel, pas besoin de scale - if ref_width == screen_width and ref_height == screen_height: - scale_to_screen_x = 1.0 - scale_to_screen_y = 1.0 - print(f"📐 [Static] Image originale = Écran actuel ({screen_width}x{screen_height}), pas de conversion") - else: - scale_to_screen_x = screen_width / ref_width - scale_to_screen_y = screen_height / ref_height - print(f"📐 [Static] Image originale: {ref_width}x{ref_height}, Écran: {screen_width}x{screen_height}") - print(f"📐 [Static] Scale: x={scale_to_screen_x:.2f}, y={scale_to_screen_y:.2f}") - - # Convertir vers coordonnĂ©es Ă©cran - screen_x = (bbox_x + bbox_w / 2) * scale_to_screen_x - screen_y = (bbox_y + bbox_h / 2) * scale_to_screen_y - - # Validation finale: les coordonnĂ©es doivent ĂȘtre dans l'Ă©cran - if 0 < screen_x < screen_width and 0 < screen_y < screen_height: - x = int(screen_x) - y = int(screen_y) - search_method = "static (coordonnĂ©es enregistrĂ©es)" - print(f"🎯 [Static] Centre image: ({bbox_x + bbox_w/2:.0f}, {bbox_y + bbox_h/2:.0f}) -> Écran: ({x}, {y})") - else: - print(f"⚠ [Static] CoordonnĂ©es hors Ă©cran: ({screen_x:.0f}, {screen_y:.0f}) - IGNORÉ") - - # === EXÉCUTER LE CLIC === - with open('/tmp/vwb_debug.log', 'a') as df: - df.write(f"=== CLIC FINAL ===\n") - df.write(f"x={x}, y={y}, search_method={search_method}\n") - df.flush() - - if x > 0 and y > 0: - click_type = parameters.get('click_type', 'left') - with open('/tmp/vwb_debug.log', 'a') as df: - df.write(f"đŸ–±ïž Clic {click_type} Ă  ({x}, {y}) via {search_method}\n") - df.flush() - - if click_type == 'right': - pyautogui.rightClick(x, y) - elif click_type == 'double': - pyautogui.doubleClick(x, y) - else: - pyautogui.click(x, y) - result_message = f'Clic {click_type} effectuĂ© Ă  ({x}, {y}) [{search_method}]' - else: - result_message = 'CoordonnĂ©es invalides - ancre non trouvĂ©e' - execution_success = False - - elif action_type in ['type', 'type_text', 'saisir_texte']: - text = parameters.get('text', parameters.get('texte', '')) - if text: - print(f"⌚ [Direct] Saisie: {text[:30]}...") - if text.isascii(): - pyautogui.typewrite(text, interval=0.05) - else: - pyautogui.write(text) - result_message = f'Texte saisi: {text[:50]}...' - else: - result_message = 'Aucun texte Ă  saisir' - - elif action_type in ['wait', 'attendre', 'wait_for_anchor']: - wait_time = parameters.get('duration', parameters.get('timeout_ms', 1000)) / 1000.0 - print(f"⏳ [Direct] Attente {wait_time}s") - time.sleep(wait_time) - result_message = f'Attente de {wait_time}s terminĂ©e' - - elif action_type in ['hotkey', 'raccourci']: - keys = parameters.get('keys', parameters.get('touches', [])) - if keys: - print(f"⌚ [Direct] Raccourci: {'+'.join(keys)}") - pyautogui.hotkey(*keys) - result_message = f'Raccourci {"+".join(keys)} exĂ©cutĂ©' - else: - result_message = 'Aucune touche dĂ©finie' - - elif action_type in ['scroll', 'defiler']: - direction = parameters.get('direction', 'down') - amount = parameters.get('amount', 3) - clicks = amount if direction == 'down' else -amount - print(f"📜 [Direct] Scroll {direction} de {amount}") - pyautogui.scroll(clicks) - result_message = f'Scroll {direction} effectuĂ©' - - else: - result_message = f'Type {action_type} non gĂ©rĂ© en mode direct' - execution_success = False - - end_time = datetime.now() - execution_time_ms = (end_time - start_time).total_seconds() * 1000 - - return jsonify({ - "success": True, - "result": { - "action_id": f"direct_{action_type}_{step_id}", - "step_id": step_id, - "status": "success" if execution_success else "failed", - "execution_time_ms": execution_time_ms, - "output_data": {"message": result_message}, - "evidence_list": [], - "error": None if execution_success else {"message": result_message} - } - }) - - except Exception as e: - print(f"❌ [Direct] Erreur: {e}") - traceback.print_exc() - return jsonify({ - "success": False, - "error": f"Erreur exĂ©cution directe: {str(e)}" - }), 500 - - # === MODE VWB CLASSIQUE (si direct_execution=False) === - # CrĂ©er la configuration de l'action - action_config = { - "type": action_type, - "action_id": data.get('action_id', f"action_{action_type}_{datetime.now().strftime('%Y%m%d_%H%M%S')}"), - "parameters": parameters - } - - # CrĂ©er l'instance d'action - action = create_action_from_config(action_config) - if not action: - return jsonify({ - "success": False, - "error": { - "message": f"Impossible de crĂ©er l'action de type '{action_type}'", - "type": "action_creation_failed" - } - }), 400 - - print(f"🚀 ExĂ©cution de l'action {action_type} (ID: {action.action_id})") - - # ExĂ©cuter l'action - result = action.execute( - step_id=step_id, - workflow_id=workflow_id, - user_id=user_id - ) - - # Convertir le rĂ©sultat en dictionnaire - result_dict = { - "action_id": result.action_id, - "step_id": result.step_id, - "status": result.status.value, - "start_time": result.start_time.isoformat(), - "end_time": result.end_time.isoformat(), - "execution_time_ms": result.execution_time_ms, - "output_data": result.output_data, - "evidence_list": [evidence.to_dict() for evidence in result.evidence_list], - "error": result.error.to_dict() if result.error else None, - "retry_count": result.retry_count, - "workflow_id": result.workflow_id, - "user_id": result.user_id, - "session_id": result.session_id - } - - status_emoji = "✅" if result.is_success() else "❌" - print(f"{status_emoji} Action {action_type} terminĂ©e en {result.execution_time_ms:.1f}ms") - - return jsonify({ - "success": True, - "result": result_dict - }) - - except Exception as e: - error_msg = f"Erreur lors de l'exĂ©cution de l'action: {str(e)}" - print(f"❌ {error_msg}") - - return jsonify({ - "success": False, - "error": error_msg, - "traceback": traceback.format_exc() - }), 500 - - -@catalog_bp.route('/validate', methods=['POST']) -def validate_action(): - """ - Valide la configuration d'une action sans l'exĂ©cuter. - - Request Body: - { - "type": "click_anchor", - "parameters": { - "visual_anchor": {...}, - "click_type": "left" - } - } - - Response: - { - "success": true, - "validation": { - "is_valid": true, - "errors": [], - "warnings": [], - "suggestions": [] - } - } - """ - try: - # Valider le corps de la requĂȘte - data = request.get_json() - if not data: - return jsonify({ - "success": False, - "error": "Corps de requĂȘte JSON requis" - }), 400 - - action_type = data.get('type') - if not action_type: - return jsonify({ - "success": False, - "error": "ParamĂštre 'type' requis" - }), 400 - - # CrĂ©er la configuration de l'action - action_config = { - "type": action_type, - "action_id": "validation_test", - "parameters": data.get('parameters', {}) - } - - # CrĂ©er l'instance d'action pour validation - action = create_action_from_config(action_config) - if not action: - return jsonify({ - "is_valid": False, - "errors": [{"parameter": "type", "message": f"Type d'action '{action_type}' non supportĂ©", "severity": "error"}], - "warnings": [], - "suggestions": [{"type": "best_practice", "message": "VĂ©rifiez que le type d'action est correct", "priority": "medium"}] - }) - - # Valider les paramĂštres - validation_errors = action.validate_parameters() - - # GĂ©nĂ©rer des avertissements et suggestions - warnings = [] - suggestions = [] - - # VĂ©rifications spĂ©cifiques selon le type d'action - if action_type == 'click_anchor': - if hasattr(action, 'confidence_threshold') and action.confidence_threshold < 0.7: - warnings.append("Seuil de confiance faible (< 0.7) - risque de faux positifs") - suggestions.append("ConsidĂ©rez augmenter le seuil de confiance Ă  0.8 ou plus") - - elif action_type == 'type_text': - if hasattr(action, 'text_to_type') and len(action.text_to_type) > 100: - warnings.append("Texte trĂšs long (> 100 caractĂšres) - saisie lente") - suggestions.append("ConsidĂ©rez diviser en plusieurs actions de saisie") - - elif action_type == 'wait_for_anchor': - if hasattr(action, 'max_wait_time_ms') and action.max_wait_time_ms > 60000: - warnings.append("DĂ©lai d'attente trĂšs long (> 60s)") - suggestions.append("ConsidĂ©rez rĂ©duire le dĂ©lai d'attente") - - # VĂ©rifier la disponibilitĂ© du ScreenCapturer - if not SCREEN_CAPTURER_AVAILABLE: - warnings.append("ScreenCapturer non disponible - actions simulĂ©es uniquement") - suggestions.append("Installez les dĂ©pendances de capture d'Ă©cran pour un fonctionnement complet") - - validation_result = { - "is_valid": len(validation_errors) == 0, - "errors": [{"parameter": "validation", "message": error, "severity": "error"} for error in validation_errors], - "warnings": [{"parameter": "validation", "message": warning, "impact": "medium"} for warning in warnings], - "suggestions": [{"type": "best_practice", "message": suggestion, "priority": "medium"} for suggestion in suggestions] - } - - return jsonify(validation_result) - - except Exception as e: - return jsonify({ - "success": False, - "error": f"Erreur lors de la validation: {str(e)}" - }), 500 - - -@catalog_bp.route('/categories', methods=['GET']) -def list_categories(): - """ - Liste toutes les catĂ©gories d'actions disponibles dans le catalogue VWB. - - Response: - { - "success": true, - "categories": [ - { - "id": "vision_ui", - "name": "Interface Utilisateur", - "description": "Actions d'interaction avec les Ă©lĂ©ments visuels", - "icon": "đŸ–±ïž", - "actionCount": 5 - } - ], - "total": 3 - } - """ - try: - # DĂ©finir les catĂ©gories disponibles avec mĂ©tadonnĂ©es - available_categories = [ - { - "id": "vision_ui", - "name": "Interface Utilisateur", - "description": "Actions d'interaction avec les Ă©lĂ©ments visuels de l'interface", - "icon": "đŸ–±ïž", - "actionCount": 5, - "color": "#2196f3", - "isEnabled": True - }, - { - "id": "control", - "name": "ContrĂŽle de Flux", - "description": "Actions de contrĂŽle et synchronisation des workflows", - "icon": "⏳", - "actionCount": 1, - "color": "#ff9800", - "isEnabled": True - }, - { - "id": "data", - "name": "DonnĂ©es", - "description": "Actions de manipulation et extraction de donnĂ©es", - "icon": "📊", - "actionCount": 1, - "color": "#4caf50", - "isEnabled": True - } - ] - - return jsonify({ - "success": True, - "categories": available_categories, - "total": len(available_categories), - "screen_capturer_available": SCREEN_CAPTURER_AVAILABLE - }) - - except Exception as e: - return jsonify({ - "success": False, - "error": f"Erreur lors de la rĂ©cupĂ©ration des catĂ©gories: {str(e)}", - "traceback": traceback.format_exc() - }), 500 - - -@catalog_bp.route('/health', methods=['GET']) -def catalog_health(): - """ - VĂ©rifie la santĂ© du service catalogue VWB. - - Response: - { - "success": true, - "status": "healthy", - "services": { - "screen_capturer": true, - "actions": 3 - }, - "timestamp": "2026-01-09T23:30:00" - } - """ - try: - # VĂ©rifier les services - screen_capturer = get_screen_capturer() - - services_status = { - "screen_capturer": screen_capturer is not None, - "actions": 7, # Nombre d'actions disponibles - "screen_capturer_method": getattr(screen_capturer, 'method', 'unavailable') if screen_capturer else 'unavailable' - } - - # DĂ©terminer le statut global - overall_status = "healthy" if services_status["screen_capturer"] else "degraded" - - return jsonify({ - "success": True, - "status": overall_status, - "services": services_status, - "timestamp": datetime.now().isoformat(), - "version": "1.0.0-catalog" - }) - - except Exception as e: - return jsonify({ - "success": False, - "status": "unhealthy", - "error": str(e), - "timestamp": datetime.now().isoformat() - }), 500 - - -# Fonction pour enregistrer les routes dans l'application Flask -def register_catalog_routes(app): - """ - Enregistre les routes du catalogue dans l'application Flask. - - Args: - app: Instance Flask - """ - app.register_blueprint(catalog_bp) - print("📋 Routes du catalogue VWB enregistrĂ©es") - print(" - GET /api/vwb/catalog/actions") - print(" - GET /api/vwb/catalog/categories") - print(" - GET /api/vwb/catalog/actions/") - print(" - POST /api/vwb/catalog/execute") - print(" - POST /api/vwb/catalog/validate") - print(" - GET /api/vwb/catalog/health") \ No newline at end of file diff --git a/visual_workflow_builder/backend/catalog_routes_v2_vlm.py b/visual_workflow_builder/backend/catalog_routes_v2_vlm.py index 9d1d7f37e..36a5e10a0 100644 --- a/visual_workflow_builder/backend/catalog_routes_v2_vlm.py +++ b/visual_workflow_builder/backend/catalog_routes_v2_vlm.py @@ -25,14 +25,12 @@ import traceback # Import des actions et contrats VWB try: from visual_workflow_builder.backend.actions import ( - BaseVWBAction, VWBActionResult, VWBActionStatus, + BaseVWBAction, VWBClickAnchorAction, VWBTypeTextAction, VWBWaitForAnchorAction, VWBFocusAnchorAction, VWBTypeSecretAction, VWBScrollToAnchorAction, VWBExtractTextAction ) - from visual_workflow_builder.backend.contracts.error import VWBErrorType, VWBErrorSeverity, create_vwb_error - from visual_workflow_builder.backend.contracts.evidence import VWBEvidenceType - from visual_workflow_builder.backend.contracts.visual_anchor import VWBVisualAnchor, VWBVisualAnchorType + from visual_workflow_builder.backend.contracts.visual_anchor import VWBVisualAnchor ACTIONS_AVAILABLE = True print("✅ Actions VWB importĂ©es avec succĂšs") except ImportError as e: @@ -40,14 +38,12 @@ except ImportError as e: try: # Essayer import relatif from .actions import ( - BaseVWBAction, VWBActionResult, VWBActionStatus, + BaseVWBAction, VWBClickAnchorAction, VWBTypeTextAction, VWBWaitForAnchorAction, VWBFocusAnchorAction, VWBTypeSecretAction, VWBScrollToAnchorAction, VWBExtractTextAction ) - from .contracts.error import VWBErrorType, VWBErrorSeverity, create_vwb_error - from .contracts.evidence import VWBEvidenceType - from .contracts.visual_anchor import VWBVisualAnchor, VWBVisualAnchorType + from .contracts.visual_anchor import VWBVisualAnchor ACTIONS_AVAILABLE = True print("✅ Actions VWB importĂ©es avec import relatif") except ImportError as e2: @@ -97,7 +93,7 @@ try: sys.path.insert(0, '/home/dom/ai/rpa_vision_v3') if '/home/dom/ai/OmniParser' not in sys.path: sys.path.insert(0, '/home/dom/ai/OmniParser') - from core.detection.omniparser_adapter import get_omniparser, find_element as omniparser_find + from core.detection.omniparser_adapter import get_omniparser omniparser_adapter = get_omniparser() OMNIPARSER_AVAILABLE = omniparser_adapter.available print(f"✅ OmniParser disponible: {OMNIPARSER_AVAILABLE}") @@ -108,19 +104,18 @@ except Exception as e: OMNIPARSER_AVAILABLE = False # ============================================================================ -# VLM (Vision Language Model) - Ollama qwen2.5vl (fallback si OmniParser Ă©choue) +# VLM (Vision Language Model) - Ollama (fallback si OmniParser Ă©choue) +# Configurable via variable d'environnement VLM_MODEL # ============================================================================ -OLLAMA_URL = "http://localhost:11434" -VLM_MODEL = "qwen2.5vl:7b" # ModĂšle vision local - bon Ă©quilibre prĂ©cision/vitesse +OLLAMA_URL = os.environ.get("OLLAMA_URL", "http://localhost:11434") +VLM_MODEL = os.environ.get("VLM_MODEL", "qwen3-vl:8b") # qwen3-vl offre une meilleure qualitĂ© OCR # ============================================================================ # Pipeline VLM Coarse → Refine → Refine++ (Template Matching) # ============================================================================ from pydantic import BaseModel, Field -from typing import Literal, Optional, List -from dataclasses import dataclass # Ollama est optionnel - le template matching fonctionnera sans try: diff --git a/visual_workflow_builder/backend/contracts/action_contracts.py b/visual_workflow_builder/backend/contracts/action_contracts.py index 471d557c5..359e6ab25 100644 --- a/visual_workflow_builder/backend/contracts/action_contracts.py +++ b/visual_workflow_builder/backend/contracts/action_contracts.py @@ -279,10 +279,9 @@ VWB_ACTION_CONTRACTS: Dict[str, ActionContract] = { "ai_analyze_text": ActionContract( action_type="ai_analyze_text", - description="Analyser du texte avec IA", - required_params=["visual_anchor", "analysis_prompt"], - optional_params=["model", "output_variable"], - param_validators={"visual_anchor": lambda p: has_visual_anchor({"visual_anchor": p})} + description="Analyser du texte ou une image avec IA", + required_params=[], # prompt est vĂ©rifiĂ© manuellement (accept prompt ou analysis_prompt) + optional_params=["prompt", "analysis_prompt", "visual_anchor", "input_text", "model", "output_variable", "temperature", "timeout_ms"], ), "db_save_data": ActionContract( diff --git a/visual_workflow_builder/backend/services/intelligent_executor.py b/visual_workflow_builder/backend/services/intelligent_executor.py index 2b617b1b8..5a4df5d47 100644 --- a/visual_workflow_builder/backend/services/intelligent_executor.py +++ b/visual_workflow_builder/backend/services/intelligent_executor.py @@ -817,36 +817,20 @@ def find_and_click( except Exception as seeclick_err: print(f"⚠ [Vision] Erreur SeeClick: {seeclick_err}") - # === STRATÉGIE 5: CoordonnĂ©es statiques (dernier recours) === + # === Toutes les mĂ©thodes visuelles ont Ă©chouĂ© === if anchor_bbox: best_conf = max(global_result.get('confidence', 0), 0) - - # Utiliser coordonnĂ©es statiques seulement si confiance > 0.5 - if best_conf >= 0.5: - print(f"⚠ [Vision] Fallback: coordonnĂ©es statiques (confiance: {best_conf:.2f})") - center_x = anchor_bbox.get('x', 0) + anchor_bbox.get('width', 0) // 2 - center_y = anchor_bbox.get('y', 0) + anchor_bbox.get('height', 0) // 2 - return { - 'found': True, - 'coordinates': {'x': int(center_x), 'y': int(center_y)}, - 'bbox': anchor_bbox, - 'confidence': best_conf, - 'method': 'static_fallback', - 'search_time_ms': (_time.time() - start_time) * 1000, - 'candidates': [] - } - else: - print(f"❌ [Vision] Ancre non trouvĂ©e (confiance: {best_conf:.2f})") - return { - 'found': False, - 'coordinates': None, - 'bbox': anchor_bbox, - 'confidence': best_conf, - 'method': 'not_found', - 'search_time_ms': (_time.time() - start_time) * 1000, - 'candidates': [], - 'reason': 'Ancre non trouvĂ©e Ă  l\'Ă©cran' - } + print(f"❌ [Vision] Ancre non trouvĂ©e Ă  l'Ă©cran (meilleure confiance: {best_conf:.2f})") + return { + 'found': False, + 'coordinates': None, + 'bbox': anchor_bbox, + 'confidence': best_conf, + 'method': 'not_found', + 'search_time_ms': (_time.time() - start_time) * 1000, + 'candidates': [], + 'reason': 'Aucune mĂ©thode visuelle n\'a trouvĂ© l\'ancre Ă  l\'Ă©cran' + } # Pas de bbox, impossible de chercher return { diff --git a/visual_workflow_builder/backend/services/real_screen_capture.py b/visual_workflow_builder/backend/services/real_screen_capture.py index 1f35d15ab..05ae6387a 100644 --- a/visual_workflow_builder/backend/services/real_screen_capture.py +++ b/visual_workflow_builder/backend/services/real_screen_capture.py @@ -36,7 +36,8 @@ except ImportError as e: UIDetector = None try: - from core.models.screen_state import ScreenState, UIElement + from core.models.screen_state import ScreenState + from core.models.ui_element import UIElement SCREEN_STATE_AVAILABLE = True except ImportError as e: print(f"Warning: ScreenState non disponible: {e}") diff --git a/visual_workflow_builder/backend/services/ui_detection_service.py b/visual_workflow_builder/backend/services/ui_detection_service.py index f8a001b86..1c296df17 100644 --- a/visual_workflow_builder/backend/services/ui_detection_service.py +++ b/visual_workflow_builder/backend/services/ui_detection_service.py @@ -18,8 +18,12 @@ from dataclasses import dataclass import numpy as np from PIL import Image -# Configuration -MODEL_PATH = "/home/dom/ai/rpa_vision_v3/models/ui-detr-1/model.pth" +# Configuration — chemin relatif Ă  la racine du projet, ou variable d'environnement +_PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..')) +MODEL_PATH = os.environ.get( + "UI_DETR_MODEL_PATH", + os.path.join(_PROJECT_ROOT, "models", "ui-detr-1", "model.pth") +) CONFIDENCE_THRESHOLD = 0.35 RESOLUTION = 1600 diff --git a/visual_workflow_builder/frontend_v4/src/App.tsx b/visual_workflow_builder/frontend_v4/src/App.tsx index c39e92d32..e74d9678b 100644 --- a/visual_workflow_builder/frontend_v4/src/App.tsx +++ b/visual_workflow_builder/frontend_v4/src/App.tsx @@ -1,13 +1,15 @@ -import { useState, useEffect, useCallback } from 'react'; +import { useState, useEffect, useCallback, useRef } from 'react'; import { ReactFlow, Controls, Background, useNodesState, useEdgesState, + useReactFlow, + addEdge, ReactFlowProvider, } from '@xyflow/react'; -import type { Node, Edge, NodeTypes } from '@xyflow/react'; +import type { Node, Edge, NodeTypes, Connection } from '@xyflow/react'; import '@xyflow/react/dist/style.css'; import * as api from './services/api'; @@ -27,6 +29,7 @@ import type { Variable } from './components/VariableManager'; import CaptureLibrary from './components/CaptureLibrary'; import SelfHealingDialog from './components/SelfHealingDialog'; import ConfidenceDashboard from './components/ConfidenceDashboard'; +import WorkflowValidation from './components/WorkflowValidation'; const nodeTypes: NodeTypes = { step: StepNode, @@ -43,9 +46,16 @@ function App() { const [isExecutionRunning, setIsExecutionRunning] = useState(false); const [detectionZone, setDetectionZone] = useState<{x: number; y: number; width: number; height: number} | null>(null); const [variables, setVariables] = useState([]); + const [runtimeVariables, setRuntimeVariables] = useState>({}); const [showWorkflowManager, setShowWorkflowManager] = useState(false); const [currentCapture, setCurrentCapture] = useState(null); + // React Flow instance pour screenToFlowPosition + const reactFlowInstance = useReactFlow(); + + // Tracker le workflow chargĂ© pour ne pas Ă©craser les edges manuelles + const loadedWorkflowIdRef = useRef(null); + // Self-healing interactif const [showSelfHealing, setShowSelfHealing] = useState(false); const [healingCandidates, setHealingCandidates] = useState([]); @@ -56,7 +66,10 @@ function App() { try { const state = await api.getState(); setAppState(state); - updateNodesFromWorkflow(state.workflow?.steps || []); + updateNodesFromWorkflow( + state.workflow?.steps || [], + state.workflow?.id + ); } catch (err) { setError((err as Error).message); } @@ -75,6 +88,11 @@ function App() { const status = await api.getExecutionStatus(); setIsExecutionRunning(status.is_running); + // Extraire les variables runtime du status d'exĂ©cution + if (status.variables && typeof status.variables === 'object') { + setRuntimeVariables(status.variables as Record); + } + // Self-healing interactif: detecter si on attend un choix utilisateur if (status.waiting_for_choice && status.candidates) { setHealingCandidates(status.candidates); @@ -100,7 +118,9 @@ function App() { }, [isExecutionRunning, loadState]); // Convertir les Ă©tapes en nƓuds React Flow - const updateNodesFromWorkflow = (steps: Step[]) => { + // Les edges ne sont gĂ©nĂ©rĂ©es automatiquement que lors du premier chargement + // d'un workflow. Ensuite, les connexions manuelles de l'utilisateur sont prĂ©servĂ©es. + const updateNodesFromWorkflow = (steps: Step[], workflowId?: string) => { const newNodes: Node[] = steps.map((step, index) => ({ id: step.id, type: 'step', @@ -108,22 +128,29 @@ function App() { data: { step }, })); - const newEdges: Edge[] = []; - for (let i = 0; i < steps.length - 1; i++) { - newEdges.push({ - id: `e-${steps[i].id}-${steps[i + 1].id}`, - source: steps[i].id, - sourceHandle: 'bottom', - target: steps[i + 1].id, - targetHandle: 'top', - type: 'smoothstep', - animated: false, - style: { strokeWidth: 2 }, - }); - } - setNodes(newNodes); - setEdges(newEdges); + + // Ne rĂ©gĂ©nĂ©rer les edges QUE si on charge un workflow diffĂ©rent + const isNewWorkflow = workflowId && workflowId !== loadedWorkflowIdRef.current; + if (isNewWorkflow) { + loadedWorkflowIdRef.current = workflowId; + + const newEdges: Edge[] = []; + for (let i = 0; i < steps.length - 1; i++) { + newEdges.push({ + id: `e-${steps[i].id}-${steps[i + 1].id}`, + source: steps[i].id, + sourceHandle: 'bottom', + target: steps[i + 1].id, + targetHandle: 'top', + type: 'smoothstep', + animated: false, + style: { strokeWidth: 2 }, + }); + } + setEdges(newEdges); + } + // Sinon : les edges existantes sont conservĂ©es (connexions manuelles prĂ©servĂ©es) }; // Actions @@ -316,22 +343,48 @@ function App() { } }; - // Drop d'un outil sur le canvas + // Connexion entre deux nƓuds (drag d'un handle Ă  un autre) + const onConnect = useCallback( + (connection: Connection) => { + setEdges((eds) => + addEdge( + { + ...connection, + type: 'smoothstep', + animated: false, + style: { strokeWidth: 2 }, + }, + eds + ) + ); + }, + [setEdges] + ); + + // Suppression d'edges (touche Suppr/Backspace) + const onEdgesDelete = useCallback( + (deletedEdges: Edge[]) => { + console.log(`đŸ—‘ïž ${deletedEdges.length} liaison(s) supprimĂ©e(s)`); + }, + [] + ); + + // Drop d'un outil sur le canvas (position corrigĂ©e avec zoom/pan) const onDrop = useCallback( (event: React.DragEvent) => { event.preventDefault(); const actionType = event.dataTransfer.getData('actionType') as ActionType; if (!actionType) return; - const reactFlowBounds = event.currentTarget.getBoundingClientRect(); - const position = { - x: event.clientX - reactFlowBounds.left, - y: event.clientY - reactFlowBounds.top, - }; + // Utiliser screenToFlowPosition pour tenir compte du zoom et du pan + const position = reactFlowInstance.screenToFlowPosition({ + x: event.clientX, + y: event.clientY, + }); handleAddStep(actionType, position); }, - [appState] + [appState, reactFlowInstance] ); const onDragOver = useCallback((event: React.DragEvent) => { @@ -356,6 +409,9 @@ function App() { onOpenManager={() => setShowWorkflowManager(true)} onRename={handleRenameWorkflow} /> + + {/* Erreur */} @@ -389,9 +449,12 @@ function App() { edges={edges} onNodesChange={onNodesChange} onEdgesChange={onEdgesChange} + onConnect={onConnect} + onEdgesDelete={onEdgesDelete} onNodeClick={(_, node) => handleSelectStep(node.id)} onNodeDragStop={handleNodeDragStop} nodeTypes={nodeTypes} + deleteKeyCode="Delete" fitView > @@ -430,6 +493,8 @@ function App() { onVariableCreate={handleVariableCreate} onVariableUpdate={handleVariableUpdate} onVariableDelete={handleVariableDelete} + steps={appState?.workflow?.steps || []} + runtimeVariables={runtimeVariables} /> @@ -473,11 +538,7 @@ function App() { }} /> - {/* Confidence Dashboard - scores en temps reel */} - + {/* ConfidenceDashboard dĂ©placĂ© dans le header */} ); } diff --git a/visual_workflow_builder/frontend_v4/src/components/AIModelSelector.tsx b/visual_workflow_builder/frontend_v4/src/components/AIModelSelector.tsx new file mode 100644 index 000000000..c4fb3552d --- /dev/null +++ b/visual_workflow_builder/frontend_v4/src/components/AIModelSelector.tsx @@ -0,0 +1,238 @@ +/** + * Composant de sĂ©lection de modĂšle IA avec listing Ollama dynamique + * Propose des modĂšles recommandĂ©s selon le type de tĂąche + */ + +import { useState, useEffect, useCallback, useRef } from 'react'; +import { + listModels, + getRecommendedModels, + checkOllamaStatus, + AI_TASK_TYPES, + MODEL_RECOMMENDATIONS, + type OllamaModelInfo, + type AITaskType, +} from '../services/ollamaService'; + +interface Props { + taskType: AITaskType; + selectedModel: string; + onModelChange: (model: string) => void; + needsVision?: boolean; +} + +export default function AIModelSelector({ + taskType, + selectedModel, + onModelChange, + needsVision = false, +}: Props) { + const [models, setModels] = useState([]); + const [loading, setLoading] = useState(true); + const [ollamaStatus, setOllamaStatus] = useState<{ available: boolean; version?: string }>({ available: false }); + const [showAllModels, setShowAllModels] = useState(false); + const [recommendedModels, setRecommendedModels] = useState<{ + visionModels: OllamaModelInfo[]; + textModels: OllamaModelInfo[]; + }>({ visionModels: [], textModels: [] }); + + // Ref stable pour onModelChange (Ă©vite les boucles infinies dans les effets) + const onModelChangeRef = useRef(onModelChange); + useEffect(() => { onModelChangeRef.current = onModelChange; }); + + // Charger le statut Ollama et les modĂšles (donnĂ©es uniquement, pas d'auto-sĂ©lection) + const loadModels = useCallback(async () => { + setLoading(true); + try { + const status = await checkOllamaStatus(); + setOllamaStatus(status); + + if (status.available) { + const allModels = await listModels(); + setModels(allModels); + + const recommended = await getRecommendedModels(taskType); + setRecommendedModels({ + visionModels: recommended.visionModels, + textModels: recommended.textModels, + }); + } + } catch (err) { + console.error('Erreur chargement modĂšles:', err); + } finally { + setLoading(false); + } + }, [taskType]); + + useEffect(() => { + loadModels(); + }, [loadModels]); + + // Auto-correction synchrone : quand le modĂšle sĂ©lectionnĂ© est incompatible + // avec le mode actuel (vision vs texte), on sĂ©lectionne automatiquement + // le premier modĂšle recommandĂ© compatible + useEffect(() => { + if (loading || models.length === 0) return; + + const currentInfo = models.find(m => m.name === selectedModel); + const isIncompatible = currentInfo && + (needsVision ? !currentInfo.isVision : currentInfo.isVision); + + if (!selectedModel || isIncompatible) { + const recs = MODEL_RECOMMENDATIONS[taskType]; + const defaultName = needsVision ? recs?.vision?.[0] : recs?.text?.[0]; + + if (defaultName) { + const found = models.find(m => m.name.includes(defaultName.split(':')[0])); + if (found) { onModelChangeRef.current(found.name); return; } + } + + // Fallback : premier modĂšle compatible + const fallback = models.find(m => needsVision ? m.isVision : !m.isVision); + if (fallback) onModelChangeRef.current(fallback.name); + else if (models.length > 0) onModelChangeRef.current(models[0].name); + } + }, [loading, models, needsVision, taskType, selectedModel]); + + const taskInfo = AI_TASK_TYPES.find(t => t.id === taskType); + const recommendations = MODEL_RECOMMENDATIONS[taskType]; + + // Filtrer les modĂšles Ă  afficher + const filteredModels = needsVision + ? recommendedModels.visionModels + : showAllModels + ? models + : recommendedModels.textModels; + + // VĂ©rifier si le modĂšle sĂ©lectionnĂ© est dans les options visibles du dropdown + const selectedModelInOptions = selectedModel && ( + filteredModels.some(m => m.name === selectedModel) || + (showAllModels && models.some(m => m.name === selectedModel)) + ); + + if (loading) { + return ( +
+
Chargement des modĂšles...
+
+ ); + } + + if (!ollamaStatus.available) { + return ( +
+
+ ⚠ + Ollama non disponible +
+ +
+ ); + } + + return ( +
+ {/* Info sur le type de tĂąche */} +
+ {taskInfo?.icon} + {recommendations?.description} +
+ + {/* SĂ©lecteur de modĂšle */} +
+ + HTML fonctionne correctement */} + {selectedModel && !selectedModelInOptions && ( + + )} + + {/* Groupe des modÚles recommandés */} + {recommendedModels.visionModels.length > 0 && needsVision && ( + + {recommendedModels.visionModels.slice(0, 3).map(m => ( + + ))} + + )} + + {recommendedModels.textModels.length > 0 && !needsVision && ( + + {recommendedModels.textModels.slice(0, 3).map(m => ( + + ))} + + )} + + {/* Tous les modÚles si demandé */} + {showAllModels && ( + + {models + .filter(m => needsVision ? m.isVision : !m.isVision) + .filter(m => !recommendedModels.textModels.slice(0, 3).some(r => r.name === m.name)) + .filter(m => !recommendedModels.visionModels.slice(0, 3).some(r => r.name === m.name)) + .map(m => ( + + ))} + + )} + + )} + +
+ + {/* Toggle pour voir tous les modĂšles */} +
+ + + {needsVision && ( + đŸ‘ïž Vision requise + )} +
+ + {/* ModÚle sélectionné */} + {selectedModel && ( +
+ Sélectionné: + {selectedModel} +
+ )} + + {/* Statut Ollama */} +
+ + Ollama v{ollamaStatus.version} + +
+
+ ); +} diff --git a/visual_workflow_builder/frontend_v4/src/components/ConfidenceDashboard.tsx b/visual_workflow_builder/frontend_v4/src/components/ConfidenceDashboard.tsx index c2bc95056..7bab752fc 100644 --- a/visual_workflow_builder/frontend_v4/src/components/ConfidenceDashboard.tsx +++ b/visual_workflow_builder/frontend_v4/src/components/ConfidenceDashboard.tsx @@ -1,11 +1,11 @@ /** * Confidence Dashboard Component * - * Affiche les scores de confiance en temps rĂ©el pendant l'exĂ©cution. - * Montre CLIP score, template score, distance et mĂ©thode utilisĂ©e. + * Badge compact dans le header avec dropdown pour les scores de confiance. + * S'affiche uniquement en mode intelligent/debug. */ -import { useState, useEffect } from 'react'; +import { useState, useEffect, useRef } from 'react'; interface StepScore { stepIndex: number; @@ -27,7 +27,19 @@ interface Props { export default function ConfidenceDashboard({ isExecutionRunning, executionMode }: Props) { const [scores, setScores] = useState([]); const [currentStep, setCurrentStep] = useState(0); - const [isExpanded, setIsExpanded] = useState(true); + const [isOpen, setIsOpen] = useState(false); + const dropdownRef = useRef(null); + + // Fermer le dropdown au clic extĂ©rieur + useEffect(() => { + const handleClickOutside = (e: MouseEvent) => { + if (dropdownRef.current && !dropdownRef.current.contains(e.target as Node)) { + setIsOpen(false); + } + }; + document.addEventListener('mousedown', handleClickOutside); + return () => document.removeEventListener('mousedown', handleClickOutside); + }, []); // Polling pour les scores en temps rĂ©el useEffect(() => { @@ -41,7 +53,6 @@ export default function ConfidenceDashboard({ isExecutionRunning, executionMode if (data.success && data.execution) { setCurrentStep(data.execution.current_step_index || 0); - // Si on a des resultats d'etapes, les ajouter if (data.execution.step_results) { const newScores: StepScore[] = data.execution.step_results.map((result: any, index: number) => ({ stepIndex: index, @@ -66,27 +77,19 @@ export default function ConfidenceDashboard({ isExecutionRunning, executionMode return () => clearInterval(interval); }, [isExecutionRunning]); - // Reset quand l'execution s'arrete - useEffect(() => { - if (!isExecutionRunning) { - // Garder les scores pour review - } - }, [isExecutionRunning]); - if (executionMode === 'basic') { - return null; // Pas de dashboard en mode basic + return null; } const getConfidenceColor = (confidence: number): string => { - if (confidence >= 0.8) return '#a6e3a1'; // Vert - if (confidence >= 0.5) return '#f9e2af'; // Jaune - return '#f38ba8'; // Rouge + if (confidence >= 0.8) return '#a6e3a1'; + if (confidence >= 0.5) return '#f9e2af'; + return '#f38ba8'; }; const getMethodIcon = (method: string): string => { switch (method) { - case 'clip': return '🧠'; - case 'clip_embedding': return '🧠'; + case 'clip': case 'clip_embedding': return '🧠'; case 'zoned_template': return '📍'; case 'direct_template': return '🔍'; case 'seeclick_grounding': return '🎯'; @@ -104,278 +107,238 @@ export default function ConfidenceDashboard({ isExecutionRunning, executionMode ? (scores.filter(s => s.success).length / scores.length) * 100 : 0; - return ( -
-
setIsExpanded(!isExpanded)}> -
- 📊 - Scores de confiance - {isExecutionRunning && ( - LIVE - )} -
-
- {isExpanded ? 'â–Œ' : '▶'} -
-
+ const avgPct = (averageConfidence * 100).toFixed(0); - {isExpanded && ( -
- {/* Metriques globales */} -
-
- Etape actuelle - {currentStep + 1} + return ( +
+ {/* Badge compact dans le header */} + + + {/* Dropdown avec les détails */} + {isOpen && ( +
+ {/* MĂ©triques globales */} +
+
+ Étape + {currentStep + 1}
-
- Confiance moy. - - {(averageConfidence * 100).toFixed(0)}% +
+ Confiance + + {avgPct}%
-
- Taux succes - +
+ SuccĂšs + {successRate.toFixed(0)}%
- {/* Liste des scores par etape */} -
+ {/* Liste des scores */} +
{scores.length === 0 ? ( -
- {isExecutionRunning - ? "En attente de resultats..." - : "Aucune execution en cours"} +
+ {isExecutionRunning ? "En attente..." : "Aucune exécution"}
) : ( scores.map((score) => (
-
- #{score.stepIndex + 1} - {getMethodIcon(score.method)} -
-
- {score.method} - {score.distance !== undefined && ( - {score.distance.toFixed(0)}px - )} -
+ #{score.stepIndex + 1} + {getMethodIcon(score.method)} + {score.method} + {score.distance !== undefined && ( + {score.distance.toFixed(0)}px + )}
- - {(score.confidence * 100).toFixed(0)}% - + {(score.confidence * 100).toFixed(0)}%
)) )}
- {/* Legende */} -
- 🧠 CLIP - 📍 Template zone - 🎯 SeeClick - 📌 Static + {/* LĂ©gende */} +
+ 🧠 CLIP + 📍 Template + 🎯 SeeClick + 📌 Static
)}