feat(vwb): Ajouter SeeClick, Self-Healing interactif et Dashboard confiance
## Nouvelles fonctionnalités ### 1. SeeClick Adapter (visual grounding fallback) - Nouvel adapter pour le modèle SeeClick (HuggingFace) - Intégré dans la chaîne de fallback: CLIP → Template → SeeClick → Static - Localise les éléments GUI à partir de descriptions textuelles ### 2. Self-Healing Interactif - Dialogue qui propose des alternatives quand l'ancre n'est pas trouvée - L'utilisateur peut choisir: candidat alternatif, coords statiques, ou sauter - Nouveaux endpoints: /healing/status, /healing/choose, /healing/candidates - État "waiting_for_choice" pour mettre l'exécution en pause ### 3. Dashboard Confiance (temps réel) - Affiche les scores de confiance pendant l'exécution - Montre: méthode utilisée, distance, taux de succès - Interface pliable en bas à droite - Visible uniquement en mode intelligent/debug ## Fichiers ajoutés - core/detection/seeclick_adapter.py - frontend_v4/src/components/SelfHealingDialog.tsx - frontend_v4/src/components/ConfidenceDashboard.tsx ## Fichiers modifiés - core/detection/__init__.py - backend/services/intelligent_executor.py - backend/api_v3/execute.py - frontend_v4/src/App.tsx - frontend_v4/src/services/api.ts - docs/VISION_RPA_INTELLIGENT.md Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Dom
@@ -52,7 +52,13 @@ _execution_state = {
|
||||
'should_stop': False,
|
||||
'current_execution_id': None,
|
||||
'thread': None,
|
||||
'execution_mode': 'basic' # 'basic', 'intelligent', 'debug'
|
||||
'execution_mode': 'basic', # 'basic', 'intelligent', 'debug'
|
||||
# Self-healing interactif
|
||||
'waiting_for_choice': False,
|
||||
'pending_action': None, # Action en attente de choix utilisateur
|
||||
'candidates': [], # Candidats proposés
|
||||
'user_choice': None, # Choix de l'utilisateur (coordonnées ou 'skip' ou 'static')
|
||||
'current_step_info': None # Info sur l'étape en cours pour affichage
|
||||
}
|
||||
|
||||
|
||||
@@ -164,6 +170,85 @@ def execute_workflow_thread(execution_id: str, workflow_id: str, app):
|
||||
# === EXÉCUTION DE L'ACTION ===
|
||||
result = execute_action(step.action_type, params)
|
||||
|
||||
# === SELF-HEALING INTERACTIF ===
|
||||
# Si l'action nécessite un choix utilisateur, attendre
|
||||
if result.get('needs_user_choice'):
|
||||
print(f"🔄 [Self-Healing] Attente choix utilisateur pour étape {index + 1}")
|
||||
|
||||
# Stocker les informations pour le frontend
|
||||
_execution_state['waiting_for_choice'] = True
|
||||
_execution_state['pending_action'] = {
|
||||
'step_id': step.id,
|
||||
'step_index': index,
|
||||
'action_type': step.action_type,
|
||||
'params': params
|
||||
}
|
||||
_execution_state['candidates'] = result.get('candidates', [])
|
||||
_execution_state['current_step_info'] = {
|
||||
'index': index,
|
||||
'total': len(steps),
|
||||
'original_bbox': result.get('original_bbox'),
|
||||
'error': result.get('error')
|
||||
}
|
||||
_execution_state['user_choice'] = None
|
||||
|
||||
# Mettre à jour le status de l'exécution
|
||||
execution.status = 'waiting_user_choice'
|
||||
db.session.commit()
|
||||
|
||||
# Attendre le choix de l'utilisateur
|
||||
timeout_seconds = 120 # 2 minutes max
|
||||
waited = 0
|
||||
while _execution_state['waiting_for_choice'] and waited < timeout_seconds:
|
||||
if _execution_state['should_stop']:
|
||||
break
|
||||
time.sleep(0.5)
|
||||
waited += 0.5
|
||||
|
||||
# Vérifier si on doit arrêter
|
||||
if _execution_state['should_stop']:
|
||||
execution.status = 'cancelled'
|
||||
break
|
||||
|
||||
# Traiter le choix de l'utilisateur
|
||||
user_choice = _execution_state['user_choice']
|
||||
_execution_state['waiting_for_choice'] = False
|
||||
_execution_state['pending_action'] = None
|
||||
_execution_state['candidates'] = []
|
||||
|
||||
if user_choice is None:
|
||||
# Timeout - aucun choix
|
||||
step_result.status = 'error'
|
||||
step_result.error_message = "Timeout: aucun choix utilisateur"
|
||||
execution.failed_steps += 1
|
||||
execution.status = 'error'
|
||||
execution.error_message = f"Timeout à l'étape {index + 1}: aucun choix utilisateur"
|
||||
db.session.commit()
|
||||
break
|
||||
elif user_choice == 'skip':
|
||||
# Utilisateur a choisi de sauter l'étape
|
||||
step_result.status = 'skipped'
|
||||
step_result.output = {'skipped_by_user': True}
|
||||
execution.status = 'running'
|
||||
db.session.commit()
|
||||
print(f"⏭️ [Self-Healing] Étape {index + 1} sautée par l'utilisateur")
|
||||
continue
|
||||
elif user_choice == 'static':
|
||||
# Utiliser les coordonnées statiques
|
||||
result = execute_action_with_static_coords(step.action_type, params)
|
||||
execution.status = 'running'
|
||||
elif isinstance(user_choice, dict) and 'x' in user_choice:
|
||||
# Coordonnées choisies par l'utilisateur
|
||||
result = execute_action_with_coords(step.action_type, params, user_choice)
|
||||
execution.status = 'running'
|
||||
else:
|
||||
step_result.status = 'error'
|
||||
step_result.error_message = f"Choix invalide: {user_choice}"
|
||||
execution.failed_steps += 1
|
||||
execution.status = 'error'
|
||||
db.session.commit()
|
||||
break
|
||||
|
||||
step_result.ended_at = datetime.utcnow()
|
||||
step_result.duration_ms = int((step_result.ended_at - step_result.started_at).total_seconds() * 1000)
|
||||
|
||||
@@ -323,6 +408,75 @@ def execute_ai_analyze(params: dict) -> dict:
|
||||
return {'success': False, 'error': str(e)}
|
||||
|
||||
|
||||
def execute_action_with_coords(action_type: str, params: dict, coords: dict) -> dict:
|
||||
"""
|
||||
Exécute une action avec des coordonnées spécifiées par l'utilisateur (self-healing).
|
||||
"""
|
||||
import pyautogui
|
||||
import time
|
||||
|
||||
try:
|
||||
x, y = coords['x'], coords['y']
|
||||
print(f"🖱️ [Self-Healing] Clic aux coordonnées choisies: ({x}, {y})")
|
||||
|
||||
if action_type in ['double_click_anchor']:
|
||||
pyautogui.doubleClick(x, y)
|
||||
elif action_type in ['right_click_anchor']:
|
||||
pyautogui.rightClick(x, y)
|
||||
else:
|
||||
pyautogui.click(x, y)
|
||||
|
||||
time.sleep(2.0) # Délai après le clic
|
||||
|
||||
return {
|
||||
'success': True,
|
||||
'output': {
|
||||
'clicked_at': {'x': x, 'y': y},
|
||||
'method': 'user_choice',
|
||||
'self_healed': True
|
||||
}
|
||||
}
|
||||
except Exception as e:
|
||||
return {'success': False, 'error': str(e)}
|
||||
|
||||
|
||||
def execute_action_with_static_coords(action_type: str, params: dict) -> dict:
|
||||
"""
|
||||
Exécute une action avec les coordonnées statiques originales (fallback self-healing).
|
||||
"""
|
||||
import pyautogui
|
||||
import time
|
||||
|
||||
try:
|
||||
anchor = params.get('visual_anchor', {})
|
||||
bbox = anchor.get('bounding_box', {})
|
||||
|
||||
x = bbox.get('x', 0) + bbox.get('width', 0) / 2
|
||||
y = bbox.get('y', 0) + bbox.get('height', 0) / 2
|
||||
|
||||
print(f"🖱️ [Self-Healing] Clic aux coordonnées statiques: ({x}, {y})")
|
||||
|
||||
if action_type in ['double_click_anchor']:
|
||||
pyautogui.doubleClick(x, y)
|
||||
elif action_type in ['right_click_anchor']:
|
||||
pyautogui.rightClick(x, y)
|
||||
else:
|
||||
pyautogui.click(x, y)
|
||||
|
||||
time.sleep(2.0)
|
||||
|
||||
return {
|
||||
'success': True,
|
||||
'output': {
|
||||
'clicked_at': {'x': x, 'y': y},
|
||||
'method': 'static_fallback',
|
||||
'self_healed': True
|
||||
}
|
||||
}
|
||||
except Exception as e:
|
||||
return {'success': False, 'error': str(e)}
|
||||
|
||||
|
||||
def execute_action(action_type: str, params: dict) -> dict:
|
||||
"""
|
||||
Exécute une action RPA.
|
||||
@@ -403,11 +557,25 @@ def execute_action(action_type: str, params: dict) -> dict:
|
||||
}
|
||||
else:
|
||||
# En mode intelligent/debug, on refuse d'utiliser les coordonnées statiques
|
||||
# si l'ancre n'est pas trouvée - cela évite les clics au mauvais endroit
|
||||
# si l'ancre n'est pas trouvée - MAIS on peut proposer des alternatives
|
||||
reason = result.get('reason', 'Ancre non trouvée à l\'écran')
|
||||
confidence = result.get('confidence', 0)
|
||||
candidates = result.get('candidates', [])
|
||||
print(f"❌ [Vision] Ancre NON trouvée (confiance: {confidence:.2f})")
|
||||
print(f" Raison: {reason}")
|
||||
|
||||
# Si self-healing interactif activé, proposer des alternatives
|
||||
if _execution_state.get('execution_mode') == 'intelligent' and candidates:
|
||||
print(f"🔄 [Self-Healing] {len(candidates)} candidats disponibles - attente choix utilisateur")
|
||||
return {
|
||||
'success': False,
|
||||
'needs_user_choice': True, # Flag pour self-healing
|
||||
'error': f"Ancre non trouvée - veuillez choisir parmi les alternatives",
|
||||
'candidates': candidates,
|
||||
'original_bbox': bbox,
|
||||
'confidence': confidence
|
||||
}
|
||||
|
||||
return {
|
||||
'success': False,
|
||||
'error': f"Ancre non trouvée à l'écran (confiance: {confidence:.2f}). {reason}"
|
||||
@@ -707,7 +875,11 @@ def get_execution_status():
|
||||
'is_paused': _execution_state['is_paused'],
|
||||
'execution_mode': _execution_state.get('execution_mode', 'basic'),
|
||||
'execution': execution.to_dict() if execution else None,
|
||||
'session': session.to_dict()
|
||||
'session': session.to_dict(),
|
||||
# Self-healing interactif
|
||||
'waiting_for_choice': _execution_state.get('waiting_for_choice', False),
|
||||
'candidates': _execution_state.get('candidates', []) if _execution_state.get('waiting_for_choice') else [],
|
||||
'current_step_info': _execution_state.get('current_step_info') if _execution_state.get('waiting_for_choice') else None
|
||||
})
|
||||
|
||||
|
||||
@@ -734,3 +906,121 @@ def get_execution_history():
|
||||
'success': False,
|
||||
'error': str(e)
|
||||
}), 500
|
||||
|
||||
|
||||
# ==============================================================================
|
||||
# SELF-HEALING INTERACTIF - Endpoints
|
||||
# ==============================================================================
|
||||
|
||||
@api_v3_bp.route('/execute/healing/status', methods=['GET'])
|
||||
def get_healing_status():
|
||||
"""
|
||||
Retourne l'état du self-healing interactif.
|
||||
Si waiting_for_choice est True, retourne les candidats à afficher.
|
||||
"""
|
||||
global _execution_state
|
||||
|
||||
return jsonify({
|
||||
'success': True,
|
||||
'waiting_for_choice': _execution_state.get('waiting_for_choice', False),
|
||||
'candidates': _execution_state.get('candidates', []),
|
||||
'current_step_info': _execution_state.get('current_step_info'),
|
||||
'pending_action': _execution_state.get('pending_action')
|
||||
})
|
||||
|
||||
|
||||
@api_v3_bp.route('/execute/healing/choose', methods=['POST'])
|
||||
def submit_healing_choice():
|
||||
"""
|
||||
Soumet le choix de l'utilisateur pour le self-healing.
|
||||
|
||||
Request:
|
||||
{
|
||||
"choice": "skip" | "static" | {"x": 123, "y": 456}
|
||||
}
|
||||
|
||||
- "skip": Sauter cette étape et continuer
|
||||
- "static": Utiliser les coordonnées statiques originales
|
||||
- {"x": N, "y": N}: Utiliser ces coordonnées précises
|
||||
"""
|
||||
global _execution_state
|
||||
|
||||
if not _execution_state.get('waiting_for_choice'):
|
||||
return jsonify({
|
||||
'success': False,
|
||||
'error': "Aucune décision en attente"
|
||||
}), 400
|
||||
|
||||
data = request.get_json() or {}
|
||||
choice = data.get('choice')
|
||||
|
||||
if choice is None:
|
||||
return jsonify({
|
||||
'success': False,
|
||||
'error': "Choix non spécifié"
|
||||
}), 400
|
||||
|
||||
# Valider le choix
|
||||
valid_choices = ['skip', 'static']
|
||||
if isinstance(choice, str) and choice not in valid_choices:
|
||||
return jsonify({
|
||||
'success': False,
|
||||
'error': f"Choix invalide. Valeurs acceptées: {valid_choices} ou {{x, y}}"
|
||||
}), 400
|
||||
|
||||
if isinstance(choice, dict):
|
||||
if 'x' not in choice or 'y' not in choice:
|
||||
return jsonify({
|
||||
'success': False,
|
||||
'error': "Coordonnées invalides. Format attendu: {x: number, y: number}"
|
||||
}), 400
|
||||
|
||||
# Stocker le choix
|
||||
_execution_state['user_choice'] = choice
|
||||
_execution_state['waiting_for_choice'] = False
|
||||
|
||||
print(f"✅ [Self-Healing] Choix reçu: {choice}")
|
||||
|
||||
return jsonify({
|
||||
'success': True,
|
||||
'choice': choice
|
||||
})
|
||||
|
||||
|
||||
@api_v3_bp.route('/execute/healing/candidates', methods=['GET'])
|
||||
def get_healing_candidates():
|
||||
"""
|
||||
Retourne les candidats avec leurs screenshots pour affichage visuel.
|
||||
Permet à l'utilisateur de voir les alternatives possibles.
|
||||
"""
|
||||
global _execution_state
|
||||
|
||||
candidates = _execution_state.get('candidates', [])
|
||||
step_info = _execution_state.get('current_step_info', {})
|
||||
|
||||
# Enrichir les candidats avec plus d'informations si disponible
|
||||
enriched_candidates = []
|
||||
for i, candidate in enumerate(candidates):
|
||||
enriched = {
|
||||
'id': i,
|
||||
'element_id': candidate.get('element_id'),
|
||||
'score': candidate.get('score', candidate.get('combined_score', 0)),
|
||||
'bbox': candidate.get('bbox', {}),
|
||||
'distance': candidate.get('distance'),
|
||||
'method': candidate.get('method', 'unknown')
|
||||
}
|
||||
# Calculer le centre si bbox disponible
|
||||
bbox = candidate.get('bbox', {})
|
||||
if bbox:
|
||||
enriched['center'] = {
|
||||
'x': (bbox.get('x1', 0) + bbox.get('x2', 0)) // 2,
|
||||
'y': (bbox.get('y1', 0) + bbox.get('y2', 0)) // 2
|
||||
}
|
||||
enriched_candidates.append(enriched)
|
||||
|
||||
return jsonify({
|
||||
'success': True,
|
||||
'candidates': enriched_candidates,
|
||||
'step_info': step_info,
|
||||
'original_bbox': step_info.get('original_bbox')
|
||||
})
|
||||
|
||||
@@ -6,11 +6,18 @@ Utilise UI-DETR-1 pour la détection et le matching d'ancres visuelles
|
||||
import time
|
||||
import base64
|
||||
import io
|
||||
import sys
|
||||
import os
|
||||
from typing import Dict, Any, Optional, List, Tuple
|
||||
from dataclasses import dataclass
|
||||
from PIL import Image
|
||||
import numpy as np
|
||||
|
||||
# Ajouter le chemin racine pour les imports de core
|
||||
RPA_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
if RPA_ROOT not in sys.path:
|
||||
sys.path.insert(0, RPA_ROOT)
|
||||
|
||||
# Import du service de détection UI
|
||||
from .ui_detection_service import detect_ui_elements, DetectionResult, UIElement
|
||||
|
||||
@@ -764,7 +771,53 @@ def find_and_click(
|
||||
global_result['search_time_ms'] = (_time.time() - start_time) * 1000
|
||||
return global_result
|
||||
|
||||
# === STRATÉGIE 4: Coordonnées statiques (dernier recours) ===
|
||||
# === STRATÉGIE 4: SeeClick (visual grounding) ===
|
||||
# Essayer SeeClick si les autres méthodes ont échoué
|
||||
try:
|
||||
print("🎯 [Vision] Essai SeeClick (visual grounding)...")
|
||||
from core.detection.seeclick_adapter import get_seeclick
|
||||
|
||||
seeclick = get_seeclick()
|
||||
if seeclick.available:
|
||||
# Utiliser une description générique basée sur l'ancre
|
||||
# TODO: Améliorer avec une description plus précise
|
||||
description = "the clickable element or button"
|
||||
|
||||
grounding_result = seeclick.ground(screen_image, description, return_pixels=True)
|
||||
|
||||
if grounding_result.found:
|
||||
found_x = grounding_result.x_pixel
|
||||
found_y = grounding_result.y_pixel
|
||||
|
||||
# Vérifier la distance à la position originale si anchor_bbox existe
|
||||
accept_seeclick = True
|
||||
if anchor_bbox:
|
||||
orig_x = anchor_bbox.get('x', 0) + anchor_bbox.get('width', 0) // 2
|
||||
orig_y = anchor_bbox.get('y', 0) + anchor_bbox.get('height', 0) // 2
|
||||
distance = np.sqrt((found_x - orig_x)**2 + (found_y - orig_y)**2)
|
||||
|
||||
MAX_SEECLICK_DISTANCE = 200 # Plus permissif car c'est un fallback
|
||||
if distance > MAX_SEECLICK_DISTANCE:
|
||||
print(f"⛔ [Vision] SeeClick rejeté: distance {distance:.0f}px > {MAX_SEECLICK_DISTANCE}px max")
|
||||
accept_seeclick = False
|
||||
|
||||
if accept_seeclick:
|
||||
print(f"✅ [Vision] SeeClick réussi! Coordonnées: ({found_x}, {found_y})")
|
||||
return {
|
||||
'found': True,
|
||||
'confidence': grounding_result.confidence,
|
||||
'coordinates': {'x': found_x, 'y': found_y},
|
||||
'bbox': anchor_bbox,
|
||||
'method': 'seeclick_grounding',
|
||||
'search_time_ms': (_time.time() - start_time) * 1000,
|
||||
'raw_output': grounding_result.raw_output
|
||||
}
|
||||
except ImportError:
|
||||
print("ℹ️ [Vision] SeeClick non disponible (module non trouvé)")
|
||||
except Exception as seeclick_err:
|
||||
print(f"⚠️ [Vision] Erreur SeeClick: {seeclick_err}")
|
||||
|
||||
# === STRATÉGIE 5: Coordonnées statiques (dernier recours) ===
|
||||
if anchor_bbox:
|
||||
best_conf = max(global_result.get('confidence', 0), 0)
|
||||
|
||||
|
||||
@@ -25,6 +25,8 @@ import ExecutionOverlay from './components/ExecutionOverlay';
|
||||
import VariableManager from './components/VariableManager';
|
||||
import type { Variable } from './components/VariableManager';
|
||||
import CaptureLibrary from './components/CaptureLibrary';
|
||||
import SelfHealingDialog from './components/SelfHealingDialog';
|
||||
import ConfidenceDashboard from './components/ConfidenceDashboard';
|
||||
|
||||
const nodeTypes: NodeTypes = {
|
||||
step: StepNode,
|
||||
@@ -44,6 +46,11 @@ function App() {
|
||||
const [showWorkflowManager, setShowWorkflowManager] = useState(false);
|
||||
const [currentCapture, setCurrentCapture] = useState<Capture | null>(null);
|
||||
|
||||
// Self-healing interactif
|
||||
const [showSelfHealing, setShowSelfHealing] = useState(false);
|
||||
const [healingCandidates, setHealingCandidates] = useState<any[]>([]);
|
||||
const [healingStepInfo, setHealingStepInfo] = useState<any>(null);
|
||||
|
||||
// Charger l'état initial
|
||||
const loadState = useCallback(async () => {
|
||||
try {
|
||||
@@ -68,11 +75,19 @@ function App() {
|
||||
const status = await api.getExecutionStatus();
|
||||
setIsExecutionRunning(status.is_running);
|
||||
|
||||
// Self-healing interactif: detecter si on attend un choix utilisateur
|
||||
if (status.waiting_for_choice && status.candidates) {
|
||||
setHealingCandidates(status.candidates);
|
||||
setHealingStepInfo(status.current_step_info);
|
||||
setShowSelfHealing(true);
|
||||
}
|
||||
|
||||
// Mettre à jour l'état si l'exécution est terminée
|
||||
// Note: Ne PAS fermer l'overlay automatiquement pour permettre
|
||||
// à l'utilisateur de voir les résultats de détection
|
||||
if (!status.is_running) {
|
||||
await loadState();
|
||||
setShowSelfHealing(false);
|
||||
// L'overlay reste visible, l'utilisateur peut le fermer manuellement
|
||||
}
|
||||
} catch (err) {
|
||||
@@ -289,6 +304,18 @@ function App() {
|
||||
setVariables(prev => prev.filter(v => v.id !== id));
|
||||
};
|
||||
|
||||
// Self-healing: soumettre le choix de l'utilisateur
|
||||
const handleSelfHealingChoice = async (choice: 'skip' | 'static' | { x: number; y: number }) => {
|
||||
try {
|
||||
await api.submitHealingChoice(choice);
|
||||
setShowSelfHealing(false);
|
||||
setHealingCandidates([]);
|
||||
setHealingStepInfo(null);
|
||||
} catch (err) {
|
||||
setError((err as Error).message);
|
||||
}
|
||||
};
|
||||
|
||||
// Drop d'un outil sur le canvas
|
||||
const onDrop = useCallback(
|
||||
(event: React.DragEvent) => {
|
||||
@@ -433,6 +460,24 @@ function App() {
|
||||
onClose={() => setShowWorkflowManager(false)}
|
||||
/>
|
||||
)}
|
||||
|
||||
{/* Self-Healing Dialog */}
|
||||
<SelfHealingDialog
|
||||
isOpen={showSelfHealing}
|
||||
candidates={healingCandidates}
|
||||
stepInfo={healingStepInfo}
|
||||
onChoose={handleSelfHealingChoice}
|
||||
onClose={() => {
|
||||
setShowSelfHealing(false);
|
||||
handleStopExecution();
|
||||
}}
|
||||
/>
|
||||
|
||||
{/* Confidence Dashboard - scores en temps reel */}
|
||||
<ConfidenceDashboard
|
||||
isExecutionRunning={isExecutionRunning}
|
||||
executionMode={executionMode}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,406 @@
|
||||
/**
|
||||
* Confidence Dashboard Component
|
||||
*
|
||||
* Affiche les scores de confiance en temps réel pendant l'exécution.
|
||||
* Montre CLIP score, template score, distance et méthode utilisée.
|
||||
*/
|
||||
|
||||
import { useState, useEffect } from 'react';
|
||||
|
||||
interface StepScore {
|
||||
stepIndex: number;
|
||||
stepType: string;
|
||||
method: string;
|
||||
confidence: number;
|
||||
distance?: number;
|
||||
clipScore?: number;
|
||||
templateScore?: number;
|
||||
timestamp: number;
|
||||
success: boolean;
|
||||
}
|
||||
|
||||
interface Props {
|
||||
isExecutionRunning: boolean;
|
||||
executionMode: 'basic' | 'intelligent' | 'debug';
|
||||
}
|
||||
|
||||
export default function ConfidenceDashboard({ isExecutionRunning, executionMode }: Props) {
|
||||
const [scores, setScores] = useState<StepScore[]>([]);
|
||||
const [currentStep, setCurrentStep] = useState<number>(0);
|
||||
const [isExpanded, setIsExpanded] = useState(true);
|
||||
|
||||
// Polling pour les scores en temps réel
|
||||
useEffect(() => {
|
||||
if (!isExecutionRunning) return;
|
||||
|
||||
const pollScores = async () => {
|
||||
try {
|
||||
const response = await fetch('/api/v3/execute/status');
|
||||
const data = await response.json();
|
||||
|
||||
if (data.success && data.execution) {
|
||||
setCurrentStep(data.execution.current_step_index || 0);
|
||||
|
||||
// Si on a des resultats d'etapes, les ajouter
|
||||
if (data.execution.step_results) {
|
||||
const newScores: StepScore[] = data.execution.step_results.map((result: any, index: number) => ({
|
||||
stepIndex: index,
|
||||
stepType: result.action_type || 'unknown',
|
||||
method: result.output?.method || 'static',
|
||||
confidence: result.output?.confidence || 1.0,
|
||||
distance: result.output?.distance,
|
||||
clipScore: result.output?.clip_score,
|
||||
templateScore: result.output?.template_score,
|
||||
timestamp: new Date(result.ended_at).getTime(),
|
||||
success: result.status === 'success'
|
||||
}));
|
||||
setScores(newScores);
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('Erreur polling scores:', err);
|
||||
}
|
||||
};
|
||||
|
||||
const interval = setInterval(pollScores, 1000);
|
||||
return () => clearInterval(interval);
|
||||
}, [isExecutionRunning]);
|
||||
|
||||
// Reset quand l'execution s'arrete
|
||||
useEffect(() => {
|
||||
if (!isExecutionRunning) {
|
||||
// Garder les scores pour review
|
||||
}
|
||||
}, [isExecutionRunning]);
|
||||
|
||||
if (executionMode === 'basic') {
|
||||
return null; // Pas de dashboard en mode basic
|
||||
}
|
||||
|
||||
const getConfidenceColor = (confidence: number): string => {
|
||||
if (confidence >= 0.8) return '#a6e3a1'; // Vert
|
||||
if (confidence >= 0.5) return '#f9e2af'; // Jaune
|
||||
return '#f38ba8'; // Rouge
|
||||
};
|
||||
|
||||
const getMethodIcon = (method: string): string => {
|
||||
switch (method) {
|
||||
case 'clip': return '🧠';
|
||||
case 'clip_embedding': return '🧠';
|
||||
case 'zoned_template': return '📍';
|
||||
case 'direct_template': return '🔍';
|
||||
case 'seeclick_grounding': return '🎯';
|
||||
case 'static_fallback': return '📌';
|
||||
case 'user_choice': return '👆';
|
||||
default: return '⚡';
|
||||
}
|
||||
};
|
||||
|
||||
const averageConfidence = scores.length > 0
|
||||
? scores.reduce((acc, s) => acc + s.confidence, 0) / scores.length
|
||||
: 0;
|
||||
|
||||
const successRate = scores.length > 0
|
||||
? (scores.filter(s => s.success).length / scores.length) * 100
|
||||
: 0;
|
||||
|
||||
return (
|
||||
<div className="confidence-dashboard">
|
||||
<div className="dashboard-header" onClick={() => setIsExpanded(!isExpanded)}>
|
||||
<div className="header-left">
|
||||
<span className="dashboard-icon">📊</span>
|
||||
<span className="dashboard-title">Scores de confiance</span>
|
||||
{isExecutionRunning && (
|
||||
<span className="live-indicator">LIVE</span>
|
||||
)}
|
||||
</div>
|
||||
<div className="header-right">
|
||||
<span className="toggle-icon">{isExpanded ? '▼' : '▶'}</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{isExpanded && (
|
||||
<div className="dashboard-content">
|
||||
{/* Metriques globales */}
|
||||
<div className="metrics-row">
|
||||
<div className="metric">
|
||||
<span className="metric-label">Etape actuelle</span>
|
||||
<span className="metric-value">{currentStep + 1}</span>
|
||||
</div>
|
||||
<div className="metric">
|
||||
<span className="metric-label">Confiance moy.</span>
|
||||
<span
|
||||
className="metric-value"
|
||||
style={{ color: getConfidenceColor(averageConfidence) }}
|
||||
>
|
||||
{(averageConfidence * 100).toFixed(0)}%
|
||||
</span>
|
||||
</div>
|
||||
<div className="metric">
|
||||
<span className="metric-label">Taux succes</span>
|
||||
<span
|
||||
className="metric-value"
|
||||
style={{ color: getConfidenceColor(successRate / 100) }}
|
||||
>
|
||||
{successRate.toFixed(0)}%
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Liste des scores par etape */}
|
||||
<div className="scores-list">
|
||||
{scores.length === 0 ? (
|
||||
<div className="no-scores">
|
||||
{isExecutionRunning
|
||||
? "En attente de resultats..."
|
||||
: "Aucune execution en cours"}
|
||||
</div>
|
||||
) : (
|
||||
scores.map((score) => (
|
||||
<div
|
||||
key={score.stepIndex}
|
||||
className={`score-item ${score.success ? 'success' : 'error'} ${score.stepIndex === currentStep ? 'current' : ''}`}
|
||||
>
|
||||
<div className="score-step">
|
||||
<span className="step-number">#{score.stepIndex + 1}</span>
|
||||
<span className="method-icon">{getMethodIcon(score.method)}</span>
|
||||
</div>
|
||||
<div className="score-details">
|
||||
<span className="method-name">{score.method}</span>
|
||||
{score.distance !== undefined && (
|
||||
<span className="distance">{score.distance.toFixed(0)}px</span>
|
||||
)}
|
||||
</div>
|
||||
<div
|
||||
className="confidence-bar"
|
||||
style={{
|
||||
'--confidence': `${score.confidence * 100}%`,
|
||||
'--confidence-color': getConfidenceColor(score.confidence)
|
||||
} as React.CSSProperties}
|
||||
>
|
||||
<span className="confidence-value">
|
||||
{(score.confidence * 100).toFixed(0)}%
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
))
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Legende */}
|
||||
<div className="legend">
|
||||
<span className="legend-item">🧠 CLIP</span>
|
||||
<span className="legend-item">📍 Template zone</span>
|
||||
<span className="legend-item">🎯 SeeClick</span>
|
||||
<span className="legend-item">📌 Static</span>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<style>{`
|
||||
.confidence-dashboard {
|
||||
position: fixed;
|
||||
bottom: 20px;
|
||||
right: 20px;
|
||||
width: 320px;
|
||||
background: #1e1e2e;
|
||||
border-radius: 12px;
|
||||
box-shadow: 0 8px 32px rgba(0, 0, 0, 0.4);
|
||||
border: 1px solid #313244;
|
||||
overflow: hidden;
|
||||
z-index: 1000;
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
.dashboard-header {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
padding: 12px 16px;
|
||||
background: #313244;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.header-left {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
.dashboard-icon {
|
||||
font-size: 16px;
|
||||
}
|
||||
|
||||
.dashboard-title {
|
||||
font-weight: 600;
|
||||
color: #cdd6f4;
|
||||
}
|
||||
|
||||
.live-indicator {
|
||||
padding: 2px 6px;
|
||||
background: #f38ba8;
|
||||
color: #1e1e2e;
|
||||
border-radius: 4px;
|
||||
font-size: 10px;
|
||||
font-weight: bold;
|
||||
animation: pulse 1.5s infinite;
|
||||
}
|
||||
|
||||
@keyframes pulse {
|
||||
0%, 100% { opacity: 1; }
|
||||
50% { opacity: 0.6; }
|
||||
}
|
||||
|
||||
.toggle-icon {
|
||||
color: #a6adc8;
|
||||
font-size: 10px;
|
||||
}
|
||||
|
||||
.dashboard-content {
|
||||
padding: 12px;
|
||||
}
|
||||
|
||||
.metrics-row {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
margin-bottom: 12px;
|
||||
padding-bottom: 12px;
|
||||
border-bottom: 1px solid #313244;
|
||||
}
|
||||
|
||||
.metric {
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
.metric-label {
|
||||
display: block;
|
||||
font-size: 10px;
|
||||
color: #a6adc8;
|
||||
margin-bottom: 4px;
|
||||
text-transform: uppercase;
|
||||
}
|
||||
|
||||
.metric-value {
|
||||
font-size: 18px;
|
||||
font-weight: bold;
|
||||
color: #cdd6f4;
|
||||
}
|
||||
|
||||
.scores-list {
|
||||
max-height: 200px;
|
||||
overflow-y: auto;
|
||||
}
|
||||
|
||||
.no-scores {
|
||||
text-align: center;
|
||||
color: #a6adc8;
|
||||
padding: 20px;
|
||||
font-style: italic;
|
||||
}
|
||||
|
||||
.score-item {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
padding: 8px;
|
||||
margin-bottom: 4px;
|
||||
background: #313244;
|
||||
border-radius: 6px;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
.score-item.current {
|
||||
border: 1px solid #89b4fa;
|
||||
background: rgba(137, 180, 250, 0.1);
|
||||
}
|
||||
|
||||
.score-item.error {
|
||||
border-left: 3px solid #f38ba8;
|
||||
}
|
||||
|
||||
.score-item.success {
|
||||
border-left: 3px solid #a6e3a1;
|
||||
}
|
||||
|
||||
.score-step {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 4px;
|
||||
min-width: 50px;
|
||||
}
|
||||
|
||||
.step-number {
|
||||
color: #89b4fa;
|
||||
font-weight: bold;
|
||||
}
|
||||
|
||||
.method-icon {
|
||||
font-size: 14px;
|
||||
}
|
||||
|
||||
.score-details {
|
||||
flex: 1;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 2px;
|
||||
}
|
||||
|
||||
.method-name {
|
||||
color: #cdd6f4;
|
||||
font-size: 11px;
|
||||
}
|
||||
|
||||
.distance {
|
||||
color: #fab387;
|
||||
font-size: 10px;
|
||||
}
|
||||
|
||||
.confidence-bar {
|
||||
width: 60px;
|
||||
height: 20px;
|
||||
background: #45475a;
|
||||
border-radius: 4px;
|
||||
position: relative;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.confidence-bar::before {
|
||||
content: '';
|
||||
position: absolute;
|
||||
left: 0;
|
||||
top: 0;
|
||||
bottom: 0;
|
||||
width: var(--confidence);
|
||||
background: var(--confidence-color);
|
||||
border-radius: 4px;
|
||||
transition: width 0.3s;
|
||||
}
|
||||
|
||||
.confidence-value {
|
||||
position: relative;
|
||||
z-index: 1;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
height: 100%;
|
||||
font-size: 10px;
|
||||
font-weight: bold;
|
||||
color: #1e1e2e;
|
||||
}
|
||||
|
||||
.legend {
|
||||
display: flex;
|
||||
gap: 12px;
|
||||
justify-content: center;
|
||||
margin-top: 12px;
|
||||
padding-top: 8px;
|
||||
border-top: 1px solid #313244;
|
||||
}
|
||||
|
||||
.legend-item {
|
||||
font-size: 10px;
|
||||
color: #a6adc8;
|
||||
}
|
||||
`}</style>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,422 @@
|
||||
/**
|
||||
* Self-Healing Dialog Component
|
||||
*
|
||||
* Affiche les candidats alternatifs quand l'ancre n'est pas trouvée
|
||||
* et permet à l'utilisateur de choisir une action.
|
||||
*/
|
||||
|
||||
import { useState, useEffect } from 'react';
|
||||
|
||||
interface Candidate {
|
||||
id: number;
|
||||
element_id: number;
|
||||
score: number;
|
||||
bbox: {
|
||||
x1: number;
|
||||
y1: number;
|
||||
x2: number;
|
||||
y2: number;
|
||||
};
|
||||
center?: {
|
||||
x: number;
|
||||
y: number;
|
||||
};
|
||||
distance?: number;
|
||||
method?: string;
|
||||
}
|
||||
|
||||
interface StepInfo {
|
||||
index: number;
|
||||
total: number;
|
||||
original_bbox?: {
|
||||
x: number;
|
||||
y: number;
|
||||
width: number;
|
||||
height: number;
|
||||
};
|
||||
error?: string;
|
||||
}
|
||||
|
||||
interface Props {
|
||||
isOpen: boolean;
|
||||
candidates: Candidate[];
|
||||
stepInfo: StepInfo | null;
|
||||
onChoose: (choice: 'skip' | 'static' | { x: number; y: number }) => void;
|
||||
onClose: () => void;
|
||||
}
|
||||
|
||||
export default function SelfHealingDialog({ isOpen, candidates, stepInfo, onChoose, onClose }: Props) {
|
||||
const [selectedCandidate, setSelectedCandidate] = useState<number | null>(null);
|
||||
const [customCoords, setCustomCoords] = useState({ x: '', y: '' });
|
||||
|
||||
// Reset quand le dialog s'ouvre
|
||||
useEffect(() => {
|
||||
if (isOpen) {
|
||||
setSelectedCandidate(null);
|
||||
setCustomCoords({ x: '', y: '' });
|
||||
}
|
||||
}, [isOpen]);
|
||||
|
||||
if (!isOpen) return null;
|
||||
|
||||
const handleCandidateClick = (candidate: Candidate) => {
|
||||
setSelectedCandidate(candidate.id);
|
||||
if (candidate.center) {
|
||||
setCustomCoords({
|
||||
x: candidate.center.x.toString(),
|
||||
y: candidate.center.y.toString()
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
const handleSubmit = () => {
|
||||
if (customCoords.x && customCoords.y) {
|
||||
onChoose({
|
||||
x: parseInt(customCoords.x),
|
||||
y: parseInt(customCoords.y)
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="self-healing-overlay">
|
||||
<div className="self-healing-dialog">
|
||||
<div className="dialog-header">
|
||||
<h2>Self-Healing Required</h2>
|
||||
<button className="close-btn" onClick={onClose}>×</button>
|
||||
</div>
|
||||
|
||||
<div className="dialog-content">
|
||||
{stepInfo && (
|
||||
<div className="step-info">
|
||||
<span className="step-badge">
|
||||
Etape {stepInfo.index + 1}/{stepInfo.total}
|
||||
</span>
|
||||
<p className="error-message">{stepInfo.error || "L'ancre visuelle n'a pas ete trouvee"}</p>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="options-section">
|
||||
<h3>Alternatives possibles</h3>
|
||||
|
||||
{candidates.length > 0 ? (
|
||||
<div className="candidates-list">
|
||||
{candidates.map((candidate) => (
|
||||
<div
|
||||
key={candidate.id}
|
||||
className={`candidate-item ${selectedCandidate === candidate.id ? 'selected' : ''}`}
|
||||
onClick={() => handleCandidateClick(candidate)}
|
||||
>
|
||||
<div className="candidate-info">
|
||||
<span className="candidate-id">#{candidate.element_id}</span>
|
||||
<span className="candidate-score">
|
||||
Confiance: {(candidate.score * 100).toFixed(0)}%
|
||||
</span>
|
||||
{candidate.distance !== undefined && (
|
||||
<span className="candidate-distance">
|
||||
Distance: {candidate.distance.toFixed(0)}px
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
{candidate.center && (
|
||||
<div className="candidate-coords">
|
||||
({candidate.center.x}, {candidate.center.y})
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
) : (
|
||||
<p className="no-candidates">Aucun candidat similaire trouve</p>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className="manual-section">
|
||||
<h3>Coordonnees manuelles</h3>
|
||||
<div className="coords-input">
|
||||
<label>
|
||||
X:
|
||||
<input
|
||||
type="number"
|
||||
value={customCoords.x}
|
||||
onChange={(e) => setCustomCoords(prev => ({ ...prev, x: e.target.value }))}
|
||||
placeholder="X"
|
||||
/>
|
||||
</label>
|
||||
<label>
|
||||
Y:
|
||||
<input
|
||||
type="number"
|
||||
value={customCoords.y}
|
||||
onChange={(e) => setCustomCoords(prev => ({ ...prev, y: e.target.value }))}
|
||||
placeholder="Y"
|
||||
/>
|
||||
</label>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="dialog-actions">
|
||||
<button
|
||||
className="action-btn primary"
|
||||
onClick={handleSubmit}
|
||||
disabled={!customCoords.x || !customCoords.y}
|
||||
>
|
||||
Utiliser ces coordonnees
|
||||
</button>
|
||||
<button
|
||||
className="action-btn secondary"
|
||||
onClick={() => onChoose('static')}
|
||||
>
|
||||
Utiliser position originale
|
||||
</button>
|
||||
<button
|
||||
className="action-btn warning"
|
||||
onClick={() => onChoose('skip')}
|
||||
>
|
||||
Sauter cette etape
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<style>{`
|
||||
.self-healing-overlay {
|
||||
position: fixed;
|
||||
top: 0;
|
||||
left: 0;
|
||||
right: 0;
|
||||
bottom: 0;
|
||||
background: rgba(0, 0, 0, 0.7);
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
z-index: 10000;
|
||||
}
|
||||
|
||||
.self-healing-dialog {
|
||||
background: #1e1e2e;
|
||||
border-radius: 12px;
|
||||
width: 500px;
|
||||
max-width: 90vw;
|
||||
max-height: 80vh;
|
||||
overflow: hidden;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
box-shadow: 0 20px 60px rgba(0, 0, 0, 0.5);
|
||||
border: 1px solid #313244;
|
||||
}
|
||||
|
||||
.dialog-header {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
padding: 16px 20px;
|
||||
background: #313244;
|
||||
border-bottom: 1px solid #45475a;
|
||||
}
|
||||
|
||||
.dialog-header h2 {
|
||||
margin: 0;
|
||||
font-size: 18px;
|
||||
color: #f5c2e7;
|
||||
}
|
||||
|
||||
.close-btn {
|
||||
background: none;
|
||||
border: none;
|
||||
color: #a6adc8;
|
||||
font-size: 24px;
|
||||
cursor: pointer;
|
||||
padding: 0;
|
||||
line-height: 1;
|
||||
}
|
||||
|
||||
.close-btn:hover {
|
||||
color: #f38ba8;
|
||||
}
|
||||
|
||||
.dialog-content {
|
||||
padding: 20px;
|
||||
overflow-y: auto;
|
||||
flex: 1;
|
||||
}
|
||||
|
||||
.step-info {
|
||||
margin-bottom: 20px;
|
||||
padding: 12px;
|
||||
background: #313244;
|
||||
border-radius: 8px;
|
||||
}
|
||||
|
||||
.step-badge {
|
||||
display: inline-block;
|
||||
padding: 4px 8px;
|
||||
background: #89b4fa;
|
||||
color: #1e1e2e;
|
||||
border-radius: 4px;
|
||||
font-size: 12px;
|
||||
font-weight: bold;
|
||||
margin-bottom: 8px;
|
||||
}
|
||||
|
||||
.error-message {
|
||||
margin: 0;
|
||||
color: #f9e2af;
|
||||
font-size: 14px;
|
||||
}
|
||||
|
||||
.options-section, .manual-section {
|
||||
margin-bottom: 20px;
|
||||
}
|
||||
|
||||
h3 {
|
||||
color: #cdd6f4;
|
||||
font-size: 14px;
|
||||
margin: 0 0 12px 0;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.5px;
|
||||
}
|
||||
|
||||
.candidates-list {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
.candidate-item {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
padding: 12px;
|
||||
background: #313244;
|
||||
border-radius: 8px;
|
||||
cursor: pointer;
|
||||
border: 2px solid transparent;
|
||||
transition: all 0.2s;
|
||||
}
|
||||
|
||||
.candidate-item:hover {
|
||||
background: #45475a;
|
||||
}
|
||||
|
||||
.candidate-item.selected {
|
||||
border-color: #89b4fa;
|
||||
background: rgba(137, 180, 250, 0.1);
|
||||
}
|
||||
|
||||
.candidate-info {
|
||||
display: flex;
|
||||
gap: 12px;
|
||||
align-items: center;
|
||||
}
|
||||
|
||||
.candidate-id {
|
||||
color: #89b4fa;
|
||||
font-weight: bold;
|
||||
}
|
||||
|
||||
.candidate-score {
|
||||
color: #a6e3a1;
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
.candidate-distance {
|
||||
color: #fab387;
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
.candidate-coords {
|
||||
color: #a6adc8;
|
||||
font-family: monospace;
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
.no-candidates {
|
||||
color: #a6adc8;
|
||||
font-style: italic;
|
||||
text-align: center;
|
||||
padding: 20px;
|
||||
}
|
||||
|
||||
.coords-input {
|
||||
display: flex;
|
||||
gap: 16px;
|
||||
}
|
||||
|
||||
.coords-input label {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
color: #a6adc8;
|
||||
font-size: 14px;
|
||||
}
|
||||
|
||||
.coords-input input {
|
||||
width: 100px;
|
||||
padding: 8px 12px;
|
||||
background: #313244;
|
||||
border: 1px solid #45475a;
|
||||
border-radius: 6px;
|
||||
color: #cdd6f4;
|
||||
font-size: 14px;
|
||||
}
|
||||
|
||||
.coords-input input:focus {
|
||||
outline: none;
|
||||
border-color: #89b4fa;
|
||||
}
|
||||
|
||||
.dialog-actions {
|
||||
display: flex;
|
||||
gap: 12px;
|
||||
padding: 16px 20px;
|
||||
background: #313244;
|
||||
border-top: 1px solid #45475a;
|
||||
}
|
||||
|
||||
.action-btn {
|
||||
flex: 1;
|
||||
padding: 12px 16px;
|
||||
border: none;
|
||||
border-radius: 8px;
|
||||
font-size: 14px;
|
||||
font-weight: 500;
|
||||
cursor: pointer;
|
||||
transition: all 0.2s;
|
||||
}
|
||||
|
||||
.action-btn:disabled {
|
||||
opacity: 0.5;
|
||||
cursor: not-allowed;
|
||||
}
|
||||
|
||||
.action-btn.primary {
|
||||
background: #89b4fa;
|
||||
color: #1e1e2e;
|
||||
}
|
||||
|
||||
.action-btn.primary:hover:not(:disabled) {
|
||||
background: #b4befe;
|
||||
}
|
||||
|
||||
.action-btn.secondary {
|
||||
background: #45475a;
|
||||
color: #cdd6f4;
|
||||
}
|
||||
|
||||
.action-btn.secondary:hover {
|
||||
background: #585b70;
|
||||
}
|
||||
|
||||
.action-btn.warning {
|
||||
background: #f38ba8;
|
||||
color: #1e1e2e;
|
||||
}
|
||||
|
||||
.action-btn.warning:hover {
|
||||
background: #eba0ac;
|
||||
}
|
||||
`}</style>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -2,7 +2,7 @@
|
||||
* API Client - Toutes les interactions avec le backend
|
||||
*/
|
||||
|
||||
import type { AppState, Workflow, Step, Execution, Capture, ActionType } from '../types';
|
||||
import type { AppState, Workflow, Step, Execution, Capture, ActionType, ExecutionMode } from '../types';
|
||||
|
||||
const API_BASE = '/api/v3';
|
||||
|
||||
@@ -61,6 +61,18 @@ export async function deleteWorkflow(workflowId: string): Promise<{ deleted_id:
|
||||
return request('DELETE', `/workflow/${workflowId}`);
|
||||
}
|
||||
|
||||
export async function updateWorkflow(
|
||||
workflowId: string,
|
||||
updates: {
|
||||
name?: string;
|
||||
description?: string;
|
||||
tags?: string[];
|
||||
triggerExamples?: string[];
|
||||
}
|
||||
): Promise<{ workflow: Workflow }> {
|
||||
return request('PUT', `/workflow/${workflowId}`, updates);
|
||||
}
|
||||
|
||||
// Steps
|
||||
export async function addStep(
|
||||
workflowId: string,
|
||||
@@ -126,8 +138,14 @@ export function getAnchorThumbnailUrl(anchorId: string): string {
|
||||
}
|
||||
|
||||
// Execution
|
||||
export async function startExecution(workflowId?: string): Promise<{ execution: Execution; session: AppState['session'] }> {
|
||||
return request('POST', '/execute/start', workflowId ? { workflow_id: workflowId } : {});
|
||||
export async function startExecution(
|
||||
workflowId?: string,
|
||||
mode?: ExecutionMode
|
||||
): Promise<{ execution: Execution; session: AppState['session'] }> {
|
||||
return request('POST', '/execute/start', {
|
||||
workflow_id: workflowId,
|
||||
execution_mode: mode || 'basic'
|
||||
});
|
||||
}
|
||||
|
||||
export async function pauseExecution(): Promise<{ execution: Execution }> {
|
||||
@@ -147,6 +165,29 @@ export async function getExecutionStatus(): Promise<{
|
||||
is_paused: boolean;
|
||||
execution: Execution | null;
|
||||
session: AppState['session'];
|
||||
// Self-healing interactif
|
||||
waiting_for_choice?: boolean;
|
||||
candidates?: Array<{
|
||||
id: number;
|
||||
element_id: number;
|
||||
score: number;
|
||||
bbox: { x1: number; y1: number; x2: number; y2: number };
|
||||
center?: { x: number; y: number };
|
||||
distance?: number;
|
||||
}>;
|
||||
current_step_info?: {
|
||||
index: number;
|
||||
total: number;
|
||||
original_bbox?: { x: number; y: number; width: number; height: number };
|
||||
error?: string;
|
||||
};
|
||||
}> {
|
||||
return request('GET', '/execute/status');
|
||||
}
|
||||
|
||||
// Self-Healing Interactif
|
||||
export async function submitHealingChoice(
|
||||
choice: 'skip' | 'static' | { x: number; y: number }
|
||||
): Promise<{ success: boolean; choice: unknown }> {
|
||||
return request('POST', '/execute/healing/choose', { choice });
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user