v1.0 - Version stable: multi-PC, détection UI-DETR-1, 3 modes exécution
- Frontend v4 accessible sur réseau local (192.168.1.40) - Ports ouverts: 3002 (frontend), 5001 (backend), 5004 (dashboard) - Ollama GPU fonctionnel - Self-healing interactif - Dashboard confiance Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
65
visual_workflow_builder/backend/actions/__init__.py
Normal file
65
visual_workflow_builder/backend/actions/__init__.py
Normal file
@@ -0,0 +1,65 @@
|
||||
"""
|
||||
Actions VWB - Module d'initialisation
|
||||
|
||||
Auteur : Dom, Alice, Kiro - 10 janvier 2026
|
||||
|
||||
Ce module contient les actions VisionOnly pour le Visual Workflow Builder.
|
||||
|
||||
Actions disponibles :
|
||||
- BaseAction : Classe de base pour toutes les actions
|
||||
- ClickAnchorAction : Action de clic sur ancre visuelle
|
||||
- TypeTextAction : Action de saisie de texte
|
||||
- WaitForAnchorAction : Action d'attente d'ancre visuelle
|
||||
- FocusAnchorAction : Action de focus sur ancre visuelle
|
||||
- TypeSecretAction : Action de saisie de secret/mot de passe
|
||||
- ScrollToAnchorAction : Action de défilement vers ancre
|
||||
- ExtractTextAction : Action d'extraction de texte
|
||||
|
||||
Registry :
|
||||
- VWBActionRegistry : Gestionnaire des actions
|
||||
- Fonctions utilitaires pour l'enregistrement et la création
|
||||
"""
|
||||
|
||||
from .base_action import BaseVWBAction, VWBActionResult, VWBActionStatus
|
||||
from .vision_ui.click_anchor import VWBClickAnchorAction
|
||||
from .vision_ui.type_text import VWBTypeTextAction
|
||||
from .vision_ui.wait_for_anchor import VWBWaitForAnchorAction
|
||||
from .vision_ui.focus_anchor import VWBFocusAnchorAction
|
||||
from .vision_ui.type_secret import VWBTypeSecretAction
|
||||
from .vision_ui.scroll_to_anchor import VWBScrollToAnchorAction
|
||||
from .vision_ui.extract_text import VWBExtractTextAction
|
||||
from .registry import (
|
||||
VWBActionRegistry,
|
||||
get_global_registry,
|
||||
register_action,
|
||||
get_action_class,
|
||||
create_action,
|
||||
list_available_actions,
|
||||
get_registry_info,
|
||||
vwb_action
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
'BaseVWBAction',
|
||||
'VWBActionResult',
|
||||
'VWBActionStatus',
|
||||
'VWBClickAnchorAction',
|
||||
'VWBTypeTextAction',
|
||||
'VWBWaitForAnchorAction',
|
||||
'VWBFocusAnchorAction',
|
||||
'VWBTypeSecretAction',
|
||||
'VWBScrollToAnchorAction',
|
||||
'VWBExtractTextAction',
|
||||
'VWBActionRegistry',
|
||||
'get_global_registry',
|
||||
'register_action',
|
||||
'get_action_class',
|
||||
'create_action',
|
||||
'list_available_actions',
|
||||
'get_registry_info',
|
||||
'vwb_action'
|
||||
]
|
||||
|
||||
__version__ = '1.1.0'
|
||||
__author__ = 'Dom, Alice, Kiro'
|
||||
__date__ = '10 janvier 2026'
|
||||
454
visual_workflow_builder/backend/actions/base_action.py
Normal file
454
visual_workflow_builder/backend/actions/base_action.py
Normal file
@@ -0,0 +1,454 @@
|
||||
"""
|
||||
Action de Base VWB - Classe Abstraite pour Actions VisionOnly
|
||||
|
||||
Auteur : Dom, Alice, Kiro - 09 janvier 2026
|
||||
|
||||
Ce module définit la classe de base pour toutes les actions VisionOnly
|
||||
du Visual Workflow Builder.
|
||||
|
||||
Classes :
|
||||
- VWBActionStatus : États d'exécution des actions
|
||||
- VWBActionResult : Résultat d'exécution d'action
|
||||
- BaseVWBAction : Classe abstraite de base pour toutes les actions
|
||||
"""
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from enum import Enum
|
||||
from dataclasses import dataclass
|
||||
from typing import Dict, Any, Optional, List
|
||||
from datetime import datetime
|
||||
import time
|
||||
import traceback
|
||||
|
||||
# Import des contrats VWB
|
||||
from ..contracts.error import VWBActionError, VWBErrorType, VWBErrorSeverity, create_vwb_error
|
||||
from ..contracts.evidence import VWBEvidence, VWBEvidenceType, create_screenshot_evidence
|
||||
from ..contracts.visual_anchor import VWBVisualAnchor
|
||||
|
||||
|
||||
class VWBActionStatus(Enum):
|
||||
"""États d'exécution des actions VWB."""
|
||||
|
||||
PENDING = "pending" # En attente d'exécution
|
||||
RUNNING = "running" # En cours d'exécution
|
||||
SUCCESS = "success" # Exécution réussie
|
||||
FAILED = "failed" # Exécution échouée
|
||||
TIMEOUT = "timeout" # Délai dépassé
|
||||
CANCELLED = "cancelled" # Annulée par l'utilisateur
|
||||
RETRYING = "retrying" # En cours de retry
|
||||
|
||||
|
||||
@dataclass
|
||||
class VWBActionResult:
|
||||
"""
|
||||
Résultat d'exécution d'une action VWB.
|
||||
|
||||
Cette classe encapsule tous les résultats d'une exécution d'action,
|
||||
incluant le statut, les données de sortie, les evidence et les erreurs.
|
||||
"""
|
||||
|
||||
# Identification
|
||||
action_id: str
|
||||
step_id: str
|
||||
|
||||
# Statut d'exécution
|
||||
status: VWBActionStatus
|
||||
|
||||
# Informations temporelles
|
||||
start_time: datetime
|
||||
end_time: datetime
|
||||
execution_time_ms: float
|
||||
|
||||
# Données de sortie
|
||||
output_data: Dict[str, Any]
|
||||
|
||||
# Evidence générées
|
||||
evidence_list: List[VWBEvidence]
|
||||
|
||||
# Erreur éventuelle
|
||||
error: Optional[VWBActionError] = None
|
||||
|
||||
# Métadonnées
|
||||
retry_count: int = 0
|
||||
workflow_id: Optional[str] = None
|
||||
user_id: Optional[str] = None
|
||||
session_id: Optional[str] = None
|
||||
|
||||
def is_success(self) -> bool:
|
||||
"""Vérifie si l'action a réussi."""
|
||||
return self.status == VWBActionStatus.SUCCESS
|
||||
|
||||
def is_failed(self) -> bool:
|
||||
"""Vérifie si l'action a échoué."""
|
||||
return self.status in {VWBActionStatus.FAILED, VWBActionStatus.TIMEOUT}
|
||||
|
||||
def can_retry(self) -> bool:
|
||||
"""Vérifie si l'action peut être retentée."""
|
||||
return (
|
||||
self.is_failed() and
|
||||
(self.error is None or self.error.is_retryable()) and
|
||||
self.retry_count < 3
|
||||
)
|
||||
|
||||
def get_primary_evidence(self) -> Optional[VWBEvidence]:
|
||||
"""Retourne l'evidence principale (screenshot après action)."""
|
||||
for evidence in self.evidence_list:
|
||||
if evidence.evidence_type == VWBEvidenceType.SCREENSHOT_AFTER:
|
||||
return evidence
|
||||
|
||||
# Si pas de screenshot après, prendre le premier screenshot
|
||||
for evidence in self.evidence_list:
|
||||
if evidence.is_visual_evidence():
|
||||
return evidence
|
||||
|
||||
return None
|
||||
|
||||
def add_evidence(self, evidence: VWBEvidence):
|
||||
"""Ajoute une evidence au résultat."""
|
||||
self.evidence_list.append(evidence)
|
||||
|
||||
def get_summary(self) -> Dict[str, Any]:
|
||||
"""Retourne un résumé du résultat."""
|
||||
return {
|
||||
'action_id': self.action_id,
|
||||
'step_id': self.step_id,
|
||||
'status': self.status.value,
|
||||
'execution_time_ms': round(self.execution_time_ms, 1),
|
||||
'evidence_count': len(self.evidence_list),
|
||||
'has_error': self.error is not None,
|
||||
'retry_count': self.retry_count,
|
||||
'can_retry': self.can_retry()
|
||||
}
|
||||
|
||||
|
||||
class BaseVWBAction(ABC):
|
||||
"""
|
||||
Classe de base abstraite pour toutes les actions VWB.
|
||||
|
||||
Cette classe définit l'interface commune et les fonctionnalités
|
||||
partagées par toutes les actions VisionOnly du Visual Workflow Builder.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
action_id: str,
|
||||
name: str,
|
||||
description: str,
|
||||
parameters: Dict[str, Any],
|
||||
screen_capturer=None
|
||||
):
|
||||
"""
|
||||
Initialise l'action de base.
|
||||
|
||||
Args:
|
||||
action_id: Identifiant unique de l'action
|
||||
name: Nom de l'action
|
||||
description: Description de l'action
|
||||
parameters: Paramètres de configuration
|
||||
screen_capturer: Instance du ScreenCapturer (Option A thread-safe)
|
||||
"""
|
||||
self.action_id = action_id
|
||||
self.name = name
|
||||
self.description = description
|
||||
self.parameters = parameters
|
||||
self.screen_capturer = screen_capturer
|
||||
|
||||
# Configuration par défaut
|
||||
self.timeout_ms = parameters.get('timeout_ms', 10000)
|
||||
self.retry_count = parameters.get('retry_count', 3)
|
||||
self.retry_delay_ms = parameters.get('retry_delay_ms', 1000)
|
||||
|
||||
# État d'exécution
|
||||
self.current_status = VWBActionStatus.PENDING
|
||||
self.current_result: Optional[VWBActionResult] = None
|
||||
|
||||
# Evidence collectées
|
||||
self.evidence_list: List[VWBEvidence] = []
|
||||
|
||||
@abstractmethod
|
||||
def validate_parameters(self) -> List[str]:
|
||||
"""
|
||||
Valide les paramètres de l'action.
|
||||
|
||||
Returns:
|
||||
Liste des erreurs de validation (vide si valide)
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def execute_core(self, step_id: str) -> VWBActionResult:
|
||||
"""
|
||||
Exécute la logique principale de l'action.
|
||||
|
||||
Args:
|
||||
step_id: Identifiant de l'étape
|
||||
|
||||
Returns:
|
||||
Résultat d'exécution
|
||||
"""
|
||||
pass
|
||||
|
||||
def execute(self, step_id: str, **kwargs) -> VWBActionResult:
|
||||
"""
|
||||
Exécute l'action avec gestion d'erreurs et retry.
|
||||
|
||||
Args:
|
||||
step_id: Identifiant de l'étape
|
||||
**kwargs: Paramètres additionnels
|
||||
|
||||
Returns:
|
||||
Résultat d'exécution complet
|
||||
"""
|
||||
start_time = datetime.now()
|
||||
self.current_status = VWBActionStatus.RUNNING
|
||||
|
||||
# Validation des paramètres
|
||||
validation_errors = self.validate_parameters()
|
||||
if validation_errors:
|
||||
return self._create_error_result(
|
||||
step_id=step_id,
|
||||
start_time=start_time,
|
||||
error_type=VWBErrorType.PARAMETER_INVALID,
|
||||
message=f"Paramètres invalides: {', '.join(validation_errors)}",
|
||||
technical_details={'validation_errors': validation_errors}
|
||||
)
|
||||
|
||||
# Capture d'écran avant action
|
||||
self._capture_before_screenshot(step_id)
|
||||
|
||||
# Exécution avec retry
|
||||
last_error = None
|
||||
for attempt in range(self.retry_count + 1):
|
||||
try:
|
||||
if attempt > 0:
|
||||
self.current_status = VWBActionStatus.RETRYING
|
||||
time.sleep(self.retry_delay_ms / 1000.0)
|
||||
|
||||
# Exécution de l'action
|
||||
result = self.execute_core(step_id)
|
||||
result.retry_count = attempt
|
||||
result.workflow_id = kwargs.get('workflow_id')
|
||||
result.user_id = kwargs.get('user_id')
|
||||
result.session_id = kwargs.get('session_id')
|
||||
|
||||
# Capture d'écran après action si succès
|
||||
if result.is_success():
|
||||
self._capture_after_screenshot(step_id, result)
|
||||
|
||||
self.current_result = result
|
||||
self.current_status = result.status
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
last_error = e
|
||||
|
||||
# Créer une erreur VWB
|
||||
error = create_vwb_error(
|
||||
error_type=VWBErrorType.SYSTEM_ERROR,
|
||||
message=f"Erreur lors de l'exécution: {str(e)}",
|
||||
action_id=self.action_id,
|
||||
step_id=step_id,
|
||||
severity=VWBErrorSeverity.ERROR,
|
||||
technical_details={'exception': str(e), 'attempt': attempt + 1},
|
||||
stack_trace=traceback.format_exc()
|
||||
)
|
||||
|
||||
# Si c'est le dernier essai, retourner l'erreur
|
||||
if attempt == self.retry_count:
|
||||
return self._create_error_result(
|
||||
step_id=step_id,
|
||||
start_time=start_time,
|
||||
error=error,
|
||||
retry_count=attempt
|
||||
)
|
||||
|
||||
# Ne devrait jamais arriver, mais par sécurité
|
||||
return self._create_error_result(
|
||||
step_id=step_id,
|
||||
start_time=start_time,
|
||||
error_type=VWBErrorType.SYSTEM_ERROR,
|
||||
message=f"Échec après {self.retry_count + 1} tentatives",
|
||||
technical_details={'last_exception': str(last_error) if last_error else 'Unknown'}
|
||||
)
|
||||
|
||||
def _capture_before_screenshot(self, step_id: str):
|
||||
"""Capture un screenshot avant l'action."""
|
||||
if not self.screen_capturer:
|
||||
return
|
||||
|
||||
try:
|
||||
# Utiliser la méthode ultra stable (Option A)
|
||||
img_array = self.screen_capturer.capture()
|
||||
if img_array is not None:
|
||||
from PIL import Image
|
||||
import base64
|
||||
import io
|
||||
|
||||
# Convertir en PIL Image et base64
|
||||
pil_image = Image.fromarray(img_array)
|
||||
buffer = io.BytesIO()
|
||||
pil_image.save(buffer, format='PNG', optimize=True)
|
||||
screenshot_base64 = base64.b64encode(buffer.getvalue()).decode('utf-8')
|
||||
|
||||
# Créer l'evidence
|
||||
evidence = create_screenshot_evidence(
|
||||
action_id=self.action_id,
|
||||
step_id=step_id,
|
||||
screenshot_base64=screenshot_base64,
|
||||
evidence_type=VWBEvidenceType.SCREENSHOT_BEFORE,
|
||||
title=f"Avant {self.name}",
|
||||
description=f"Capture d'écran avant l'exécution de {self.name}",
|
||||
screenshot_width=pil_image.width,
|
||||
screenshot_height=pil_image.height
|
||||
)
|
||||
|
||||
self.evidence_list.append(evidence)
|
||||
|
||||
except Exception as e:
|
||||
# Ne pas faire échouer l'action pour un problème de screenshot
|
||||
print(f"⚠️ Erreur capture avant action: {e}")
|
||||
|
||||
def _capture_after_screenshot(self, step_id: str, result: VWBActionResult):
|
||||
"""Capture un screenshot après l'action."""
|
||||
if not self.screen_capturer:
|
||||
return
|
||||
|
||||
try:
|
||||
# Utiliser la méthode ultra stable (Option A)
|
||||
img_array = self.screen_capturer.capture()
|
||||
if img_array is not None:
|
||||
from PIL import Image
|
||||
import base64
|
||||
import io
|
||||
|
||||
# Convertir en PIL Image et base64
|
||||
pil_image = Image.fromarray(img_array)
|
||||
buffer = io.BytesIO()
|
||||
pil_image.save(buffer, format='PNG', optimize=True)
|
||||
screenshot_base64 = base64.b64encode(buffer.getvalue()).decode('utf-8')
|
||||
|
||||
# Créer l'evidence
|
||||
evidence = create_screenshot_evidence(
|
||||
action_id=self.action_id,
|
||||
step_id=step_id,
|
||||
screenshot_base64=screenshot_base64,
|
||||
evidence_type=VWBEvidenceType.SCREENSHOT_AFTER,
|
||||
title=f"Après {self.name}",
|
||||
description=f"Capture d'écran après l'exécution de {self.name}",
|
||||
screenshot_width=pil_image.width,
|
||||
screenshot_height=pil_image.height
|
||||
)
|
||||
|
||||
result.add_evidence(evidence)
|
||||
|
||||
except Exception as e:
|
||||
# Ne pas faire échouer l'action pour un problème de screenshot
|
||||
print(f"⚠️ Erreur capture après action: {e}")
|
||||
|
||||
def _create_error_result(
|
||||
self,
|
||||
step_id: str,
|
||||
start_time: datetime,
|
||||
error_type: VWBErrorType = VWBErrorType.SYSTEM_ERROR,
|
||||
message: str = "Erreur d'exécution",
|
||||
technical_details: Optional[Dict[str, Any]] = None,
|
||||
error: Optional[VWBActionError] = None,
|
||||
retry_count: int = 0
|
||||
) -> VWBActionResult:
|
||||
"""Crée un résultat d'erreur."""
|
||||
end_time = datetime.now()
|
||||
execution_time = (end_time - start_time).total_seconds() * 1000
|
||||
|
||||
if error is None:
|
||||
error = create_vwb_error(
|
||||
error_type=error_type,
|
||||
message=message,
|
||||
action_id=self.action_id,
|
||||
step_id=step_id,
|
||||
technical_details=technical_details or {},
|
||||
execution_time_ms=execution_time
|
||||
)
|
||||
|
||||
# Capture d'écran d'erreur
|
||||
self._capture_error_screenshot(step_id)
|
||||
|
||||
result = VWBActionResult(
|
||||
action_id=self.action_id,
|
||||
step_id=step_id,
|
||||
status=VWBActionStatus.FAILED,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
execution_time_ms=execution_time,
|
||||
output_data={},
|
||||
evidence_list=self.evidence_list.copy(),
|
||||
error=error,
|
||||
retry_count=retry_count
|
||||
)
|
||||
|
||||
self.current_status = VWBActionStatus.FAILED
|
||||
self.current_result = result
|
||||
return result
|
||||
|
||||
def _capture_error_screenshot(self, step_id: str):
|
||||
"""Capture un screenshot lors d'une erreur."""
|
||||
if not self.screen_capturer:
|
||||
return
|
||||
|
||||
try:
|
||||
# Utiliser la méthode ultra stable (Option A)
|
||||
img_array = self.screen_capturer.capture()
|
||||
if img_array is not None:
|
||||
from PIL import Image
|
||||
import base64
|
||||
import io
|
||||
|
||||
# Convertir en PIL Image et base64
|
||||
pil_image = Image.fromarray(img_array)
|
||||
buffer = io.BytesIO()
|
||||
pil_image.save(buffer, format='PNG', optimize=True)
|
||||
screenshot_base64 = base64.b64encode(buffer.getvalue()).decode('utf-8')
|
||||
|
||||
# Créer l'evidence
|
||||
evidence = create_screenshot_evidence(
|
||||
action_id=self.action_id,
|
||||
step_id=step_id,
|
||||
screenshot_base64=screenshot_base64,
|
||||
evidence_type=VWBEvidenceType.SCREENSHOT_ERROR,
|
||||
title=f"Erreur {self.name}",
|
||||
description=f"Capture d'écran lors de l'erreur de {self.name}",
|
||||
screenshot_width=pil_image.width,
|
||||
screenshot_height=pil_image.height,
|
||||
success=False
|
||||
)
|
||||
|
||||
self.evidence_list.append(evidence)
|
||||
|
||||
except Exception as e:
|
||||
# Ne pas faire échouer davantage pour un problème de screenshot
|
||||
print(f"⚠️ Erreur capture screenshot d'erreur: {e}")
|
||||
|
||||
def get_status(self) -> VWBActionStatus:
|
||||
"""Retourne le statut actuel de l'action."""
|
||||
return self.current_status
|
||||
|
||||
def get_result(self) -> Optional[VWBActionResult]:
|
||||
"""Retourne le résultat actuel de l'action."""
|
||||
return self.current_result
|
||||
|
||||
def get_evidence_list(self) -> List[VWBEvidence]:
|
||||
"""Retourne la liste des evidence collectées."""
|
||||
return self.evidence_list.copy()
|
||||
|
||||
def __str__(self) -> str:
|
||||
"""Représentation string de l'action."""
|
||||
return f"VWBAction({self.name}): {self.description}"
|
||||
|
||||
def __repr__(self) -> str:
|
||||
"""Représentation détaillée de l'action."""
|
||||
return (
|
||||
f"BaseVWBAction("
|
||||
f"action_id='{self.action_id}', "
|
||||
f"name='{self.name}', "
|
||||
f"status={self.current_status.value}"
|
||||
f")"
|
||||
)
|
||||
@@ -0,0 +1 @@
|
||||
"""Actions de navigation VWB."""
|
||||
@@ -0,0 +1,70 @@
|
||||
"""
|
||||
Action Navigation Retour - Retourner à la page précédente
|
||||
Auteur : Dom, Alice, Kiro - 12 janvier 2026
|
||||
"""
|
||||
|
||||
from typing import Dict, Any
|
||||
from ..base_action import BaseVWBAction
|
||||
from ..contracts.evidence import VWBActionEvidence
|
||||
from ..contracts.error import VWBActionError
|
||||
|
||||
|
||||
class VWBBrowserBackAction(BaseVWBAction):
|
||||
"""Action pour retourner à la page précédente."""
|
||||
|
||||
def __init__(self, action_id: str, parameters: Dict[str, Any]):
|
||||
"""
|
||||
Initialise l'action de retour navigateur.
|
||||
|
||||
Args:
|
||||
action_id: Identifiant unique de l'action
|
||||
parameters: Paramètres de l'action
|
||||
"""
|
||||
super().__init__(action_id, parameters)
|
||||
|
||||
def validate_parameters(self) -> bool:
|
||||
"""
|
||||
Valide les paramètres de l'action.
|
||||
|
||||
Returns:
|
||||
True (pas de paramètres requis)
|
||||
"""
|
||||
return True
|
||||
|
||||
def execute(self) -> Dict[str, Any]:
|
||||
"""
|
||||
Exécute le retour à la page précédente.
|
||||
|
||||
Returns:
|
||||
Résultat de l'exécution
|
||||
"""
|
||||
try:
|
||||
# Simuler le retour navigateur (à implémenter avec selenium/playwright)
|
||||
import time
|
||||
|
||||
self.add_evidence(VWBActionEvidence(
|
||||
evidence_type="browser_back_start",
|
||||
data={"timestamp": datetime.now().isoformat()}
|
||||
))
|
||||
|
||||
# Simuler le temps de navigation
|
||||
time.sleep(0.5)
|
||||
|
||||
self.add_evidence(VWBActionEvidence(
|
||||
evidence_type="browser_back_complete",
|
||||
data={"success": True}
|
||||
))
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"action": "browser_back",
|
||||
"execution_time_ms": 500
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
error = VWBActionError(
|
||||
error_type="browser_back_failed",
|
||||
message=f"Échec retour navigateur: {str(e)}"
|
||||
)
|
||||
self.add_error(error)
|
||||
return {"success": False, "error": str(e)}
|
||||
@@ -0,0 +1,80 @@
|
||||
"""
|
||||
Action Navigation URL - Naviguer vers une URL spécifique
|
||||
Auteur : Dom, Alice, Kiro - 12 janvier 2026
|
||||
"""
|
||||
|
||||
from typing import Dict, Any, Optional
|
||||
from ..base_action import BaseVWBAction
|
||||
from ..contracts.evidence import VWBActionEvidence
|
||||
from ..contracts.error import VWBActionError
|
||||
|
||||
|
||||
class VWBNavigateToUrlAction(BaseVWBAction):
|
||||
"""Action pour naviguer vers une URL spécifique."""
|
||||
|
||||
def __init__(self, action_id: str, parameters: Dict[str, Any]):
|
||||
"""
|
||||
Initialise l'action de navigation URL.
|
||||
|
||||
Args:
|
||||
action_id: Identifiant unique de l'action
|
||||
parameters: Paramètres de l'action
|
||||
"""
|
||||
super().__init__(action_id, parameters)
|
||||
self.url = parameters.get('url', '')
|
||||
self.wait_for_load = parameters.get('wait_for_load', True)
|
||||
|
||||
def validate_parameters(self) -> bool:
|
||||
"""
|
||||
Valide les paramètres de l'action.
|
||||
|
||||
Returns:
|
||||
True si les paramètres sont valides
|
||||
"""
|
||||
if not self.url or not isinstance(self.url, str):
|
||||
self.add_error("URL manquante ou invalide")
|
||||
return False
|
||||
|
||||
if not self.url.startswith(('http://', 'https://')):
|
||||
self.add_error("URL doit commencer par http:// ou https://")
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def execute(self) -> Dict[str, Any]:
|
||||
"""
|
||||
Exécute la navigation vers l'URL.
|
||||
|
||||
Returns:
|
||||
Résultat de l'exécution
|
||||
"""
|
||||
try:
|
||||
# Simuler la navigation (à implémenter avec selenium/playwright)
|
||||
import time
|
||||
|
||||
self.add_evidence(VWBActionEvidence(
|
||||
evidence_type="navigation_start",
|
||||
data={"url": self.url, "timestamp": datetime.now().isoformat()}
|
||||
))
|
||||
|
||||
# Simuler le temps de navigation
|
||||
time.sleep(1 if self.wait_for_load else 0.1)
|
||||
|
||||
self.add_evidence(VWBActionEvidence(
|
||||
evidence_type="navigation_complete",
|
||||
data={"url": self.url, "success": True}
|
||||
))
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"url": self.url,
|
||||
"navigation_time_ms": 1000 if self.wait_for_load else 100
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
error = VWBActionError(
|
||||
error_type="navigation_failed",
|
||||
message=f"Échec navigation vers {self.url}: {str(e)}"
|
||||
)
|
||||
self.add_error(error)
|
||||
return {"success": False, "error": str(e)}
|
||||
517
visual_workflow_builder/backend/actions/registry.py
Normal file
517
visual_workflow_builder/backend/actions/registry.py
Normal file
@@ -0,0 +1,517 @@
|
||||
"""
|
||||
Registry Actions VWB - Gestionnaire d'Actions VisionOnly
|
||||
|
||||
Auteur : Dom, Alice, Kiro - 09 janvier 2026
|
||||
|
||||
Ce module implémente le registry des actions VisionOnly pour le Visual Workflow Builder.
|
||||
Il permet l'enregistrement, la recherche et la gestion des actions disponibles.
|
||||
|
||||
Fonctionnalités :
|
||||
- Enregistrement automatique des actions
|
||||
- Recherche par catégorie et type
|
||||
- Thread-safety pour accès concurrent
|
||||
- Chargement dynamique des actions
|
||||
"""
|
||||
|
||||
import threading
|
||||
from typing import Dict, List, Optional, Type, Any, Set
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
import importlib
|
||||
import inspect
|
||||
|
||||
from .base_action import BaseVWBAction
|
||||
|
||||
|
||||
class VWBActionRegistry:
|
||||
"""
|
||||
Registry thread-safe pour les actions VWB.
|
||||
|
||||
Ce registry maintient un catalogue des actions disponibles et permet
|
||||
leur recherche et instanciation dynamique.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""Initialise le registry."""
|
||||
self._actions: Dict[str, Type[BaseVWBAction]] = {}
|
||||
self._categories: Dict[str, Set[str]] = {}
|
||||
self._metadata: Dict[str, Dict[str, Any]] = {}
|
||||
self._lock = threading.RLock()
|
||||
self._initialized = False
|
||||
|
||||
print("📋 Registry Actions VWB initialisé")
|
||||
|
||||
def register_action(self,
|
||||
action_class: Type[BaseVWBAction],
|
||||
action_id: Optional[str] = None,
|
||||
category: str = "default",
|
||||
metadata: Optional[Dict[str, Any]] = None) -> bool:
|
||||
"""
|
||||
Enregistre une action dans le registry.
|
||||
|
||||
Args:
|
||||
action_class: Classe de l'action à enregistrer
|
||||
action_id: Identifiant unique (auto-généré si None)
|
||||
category: Catégorie de l'action
|
||||
metadata: Métadonnées additionnelles
|
||||
|
||||
Returns:
|
||||
True si l'enregistrement a réussi
|
||||
"""
|
||||
with self._lock:
|
||||
try:
|
||||
# Générer l'ID si non fourni
|
||||
if action_id is None:
|
||||
action_id = self._generate_action_id(action_class)
|
||||
|
||||
# Vérifier que l'action hérite de BaseVWBAction
|
||||
if not issubclass(action_class, BaseVWBAction):
|
||||
print(f"⚠️ {action_class.__name__} n'hérite pas de BaseVWBAction")
|
||||
return False
|
||||
|
||||
# Vérifier l'unicité de l'ID
|
||||
if action_id in self._actions:
|
||||
print(f"⚠️ Action '{action_id}' déjà enregistrée")
|
||||
return False
|
||||
|
||||
# Enregistrer l'action
|
||||
self._actions[action_id] = action_class
|
||||
|
||||
# Gérer les catégories
|
||||
if category not in self._categories:
|
||||
self._categories[category] = set()
|
||||
self._categories[category].add(action_id)
|
||||
|
||||
# Stocker les métadonnées
|
||||
self._metadata[action_id] = {
|
||||
'class_name': action_class.__name__,
|
||||
'module': action_class.__module__,
|
||||
'category': category,
|
||||
'registered_at': datetime.now().isoformat(),
|
||||
'metadata': metadata or {}
|
||||
}
|
||||
|
||||
print(f"✅ Action '{action_id}' enregistrée (catégorie: {category})")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Erreur enregistrement action '{action_id}': {e}")
|
||||
return False
|
||||
|
||||
def get_action_class(self, action_id: str) -> Optional[Type[BaseVWBAction]]:
|
||||
"""
|
||||
Récupère la classe d'une action par son ID.
|
||||
|
||||
Args:
|
||||
action_id: Identifiant de l'action
|
||||
|
||||
Returns:
|
||||
Classe de l'action ou None si non trouvée
|
||||
"""
|
||||
with self._lock:
|
||||
return self._actions.get(action_id)
|
||||
|
||||
def create_action(self,
|
||||
action_id: str,
|
||||
parameters: Optional[Dict[str, Any]] = None,
|
||||
**kwargs) -> Optional[BaseVWBAction]:
|
||||
"""
|
||||
Crée une instance d'action.
|
||||
|
||||
Args:
|
||||
action_id: Identifiant de l'action
|
||||
parameters: Paramètres de l'action
|
||||
**kwargs: Arguments additionnels pour le constructeur
|
||||
|
||||
Returns:
|
||||
Instance de l'action ou None si erreur
|
||||
"""
|
||||
with self._lock:
|
||||
action_class = self._actions.get(action_id)
|
||||
if action_class is None:
|
||||
print(f"⚠️ Action '{action_id}' non trouvée dans le registry")
|
||||
return None
|
||||
|
||||
try:
|
||||
# Préparer les arguments du constructeur
|
||||
constructor_args = {
|
||||
'action_id': f"{action_id}_{datetime.now().strftime('%Y%m%d_%H%M%S')}",
|
||||
'parameters': parameters or {}
|
||||
}
|
||||
constructor_args.update(kwargs)
|
||||
|
||||
# Créer l'instance
|
||||
instance = action_class(**constructor_args)
|
||||
print(f"✅ Instance d'action '{action_id}' créée")
|
||||
return instance
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Erreur création instance '{action_id}': {e}")
|
||||
return None
|
||||
|
||||
def list_actions(self, category: Optional[str] = None) -> List[str]:
|
||||
"""
|
||||
Liste les actions disponibles.
|
||||
|
||||
Args:
|
||||
category: Filtrer par catégorie (optionnel)
|
||||
|
||||
Returns:
|
||||
Liste des IDs d'actions
|
||||
"""
|
||||
with self._lock:
|
||||
if category is None:
|
||||
return list(self._actions.keys())
|
||||
else:
|
||||
return list(self._categories.get(category, set()))
|
||||
|
||||
def list_categories(self) -> List[str]:
|
||||
"""
|
||||
Liste les catégories disponibles.
|
||||
|
||||
Returns:
|
||||
Liste des catégories
|
||||
"""
|
||||
with self._lock:
|
||||
return list(self._categories.keys())
|
||||
|
||||
def get_action_metadata(self, action_id: str) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Récupère les métadonnées d'une action.
|
||||
|
||||
Args:
|
||||
action_id: Identifiant de l'action
|
||||
|
||||
Returns:
|
||||
Métadonnées ou None si non trouvée
|
||||
"""
|
||||
with self._lock:
|
||||
return self._metadata.get(action_id)
|
||||
|
||||
def search_actions(self,
|
||||
query: str,
|
||||
category: Optional[str] = None) -> List[str]:
|
||||
"""
|
||||
Recherche des actions par nom ou description.
|
||||
|
||||
Args:
|
||||
query: Terme de recherche
|
||||
category: Filtrer par catégorie (optionnel)
|
||||
|
||||
Returns:
|
||||
Liste des IDs d'actions correspondantes
|
||||
"""
|
||||
with self._lock:
|
||||
query_lower = query.lower()
|
||||
results = []
|
||||
|
||||
actions_to_search = self.list_actions(category)
|
||||
|
||||
for action_id in actions_to_search:
|
||||
metadata = self._metadata.get(action_id, {})
|
||||
class_name = metadata.get('class_name', '').lower()
|
||||
|
||||
# Rechercher dans l'ID et le nom de classe
|
||||
if (query_lower in action_id.lower() or
|
||||
query_lower in class_name):
|
||||
results.append(action_id)
|
||||
|
||||
return results
|
||||
|
||||
def get_registry_stats(self) -> Dict[str, Any]:
|
||||
"""
|
||||
Obtient les statistiques du registry.
|
||||
|
||||
Returns:
|
||||
Dictionnaire avec les statistiques
|
||||
"""
|
||||
with self._lock:
|
||||
return {
|
||||
'total_actions': len(self._actions),
|
||||
'categories': {
|
||||
cat: len(actions)
|
||||
for cat, actions in self._categories.items()
|
||||
},
|
||||
'initialized': self._initialized,
|
||||
'last_update': datetime.now().isoformat()
|
||||
}
|
||||
|
||||
def auto_discover_actions(self, base_path: Optional[Path] = None) -> int:
|
||||
"""
|
||||
Découvre et enregistre automatiquement les actions.
|
||||
|
||||
Args:
|
||||
base_path: Chemin de base pour la découverte (optionnel)
|
||||
|
||||
Returns:
|
||||
Nombre d'actions découvertes
|
||||
"""
|
||||
with self._lock:
|
||||
if base_path is None:
|
||||
base_path = Path(__file__).parent
|
||||
|
||||
discovered_count = 0
|
||||
|
||||
try:
|
||||
# Découvrir les actions dans vision_ui
|
||||
vision_ui_path = base_path / "vision_ui"
|
||||
if vision_ui_path.exists():
|
||||
discovered_count += self._discover_in_directory(
|
||||
vision_ui_path, "vision_ui"
|
||||
)
|
||||
|
||||
# Découvrir les actions dans d'autres catégories
|
||||
for category_dir in base_path.iterdir():
|
||||
if (category_dir.is_dir() and
|
||||
category_dir.name not in ["__pycache__", "vision_ui"] and
|
||||
not category_dir.name.startswith(".")):
|
||||
|
||||
discovered_count += self._discover_in_directory(
|
||||
category_dir, category_dir.name
|
||||
)
|
||||
|
||||
self._initialized = True
|
||||
print(f"🔍 Découverte automatique terminée : {discovered_count} actions trouvées")
|
||||
return discovered_count
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Erreur découverte automatique : {e}")
|
||||
return discovered_count
|
||||
|
||||
def _discover_in_directory(self, directory: Path, category: str) -> int:
|
||||
"""
|
||||
Découvre les actions dans un répertoire.
|
||||
|
||||
Args:
|
||||
directory: Répertoire à explorer
|
||||
category: Catégorie des actions
|
||||
|
||||
Returns:
|
||||
Nombre d'actions découvertes
|
||||
"""
|
||||
discovered_count = 0
|
||||
|
||||
for py_file in directory.glob("*.py"):
|
||||
if py_file.name.startswith("__"):
|
||||
continue
|
||||
|
||||
try:
|
||||
# Construire le nom du module
|
||||
module_name = f"visual_workflow_builder.backend.actions.{category}.{py_file.stem}"
|
||||
|
||||
# Importer le module
|
||||
module = importlib.import_module(module_name)
|
||||
|
||||
# Chercher les classes d'actions
|
||||
for name, obj in inspect.getmembers(module, inspect.isclass):
|
||||
if (obj != BaseVWBAction and
|
||||
issubclass(obj, BaseVWBAction) and
|
||||
obj.__module__ == module.__name__):
|
||||
|
||||
# Générer l'ID de l'action
|
||||
action_id = self._generate_action_id(obj)
|
||||
|
||||
# Enregistrer l'action
|
||||
if self.register_action(obj, action_id, category):
|
||||
discovered_count += 1
|
||||
|
||||
except Exception as e:
|
||||
print(f"⚠️ Erreur import {py_file}: {e}")
|
||||
|
||||
return discovered_count
|
||||
|
||||
def _generate_action_id(self, action_class: Type[BaseVWBAction]) -> str:
|
||||
"""
|
||||
Génère un ID d'action à partir de la classe.
|
||||
|
||||
Args:
|
||||
action_class: Classe de l'action
|
||||
|
||||
Returns:
|
||||
ID généré
|
||||
"""
|
||||
class_name = action_class.__name__
|
||||
|
||||
# Convertir VWBClickAnchorAction -> click_anchor
|
||||
if class_name.startswith("VWB") and class_name.endswith("Action"):
|
||||
# Enlever VWB et Action
|
||||
core_name = class_name[3:-6]
|
||||
|
||||
# Convertir CamelCase en snake_case
|
||||
import re
|
||||
snake_case = re.sub('([A-Z]+)', r'_\1', core_name).lower()
|
||||
return snake_case.lstrip('_')
|
||||
|
||||
# Fallback : utiliser le nom de classe en minuscules
|
||||
return class_name.lower()
|
||||
|
||||
def clear(self):
|
||||
"""Vide le registry."""
|
||||
with self._lock:
|
||||
self._actions.clear()
|
||||
self._categories.clear()
|
||||
self._metadata.clear()
|
||||
self._initialized = False
|
||||
print("🗑️ Registry vidé")
|
||||
|
||||
|
||||
# Instance globale du registry
|
||||
_global_registry: Optional[VWBActionRegistry] = None
|
||||
_registry_lock = threading.Lock()
|
||||
|
||||
|
||||
def get_global_registry() -> VWBActionRegistry:
|
||||
"""
|
||||
Obtient l'instance globale du registry (singleton thread-safe).
|
||||
|
||||
Returns:
|
||||
Instance du registry
|
||||
"""
|
||||
global _global_registry
|
||||
|
||||
if _global_registry is None:
|
||||
with _registry_lock:
|
||||
if _global_registry is None:
|
||||
_global_registry = VWBActionRegistry()
|
||||
|
||||
# Auto-découverte des actions au premier accès
|
||||
try:
|
||||
_global_registry.auto_discover_actions()
|
||||
except Exception as e:
|
||||
print(f"⚠️ Erreur auto-découverte : {e}")
|
||||
|
||||
return _global_registry
|
||||
|
||||
|
||||
def register_action(action_class: Type[BaseVWBAction],
|
||||
action_id: Optional[str] = None,
|
||||
category: str = "default",
|
||||
metadata: Optional[Dict[str, Any]] = None) -> bool:
|
||||
"""
|
||||
Enregistre une action dans le registry global.
|
||||
|
||||
Args:
|
||||
action_class: Classe de l'action
|
||||
action_id: Identifiant unique (optionnel)
|
||||
category: Catégorie de l'action
|
||||
metadata: Métadonnées additionnelles
|
||||
|
||||
Returns:
|
||||
True si l'enregistrement a réussi
|
||||
"""
|
||||
return get_global_registry().register_action(
|
||||
action_class, action_id, category, metadata
|
||||
)
|
||||
|
||||
|
||||
def get_action_class(action_id: str) -> Optional[Type[BaseVWBAction]]:
|
||||
"""
|
||||
Récupère une classe d'action du registry global.
|
||||
|
||||
Args:
|
||||
action_id: Identifiant de l'action
|
||||
|
||||
Returns:
|
||||
Classe de l'action ou None
|
||||
"""
|
||||
return get_global_registry().get_action_class(action_id)
|
||||
|
||||
|
||||
def create_action(action_id: str,
|
||||
parameters: Optional[Dict[str, Any]] = None,
|
||||
**kwargs) -> Optional[BaseVWBAction]:
|
||||
"""
|
||||
Crée une instance d'action depuis le registry global.
|
||||
|
||||
Args:
|
||||
action_id: Identifiant de l'action
|
||||
parameters: Paramètres de l'action
|
||||
**kwargs: Arguments additionnels
|
||||
|
||||
Returns:
|
||||
Instance de l'action ou None
|
||||
"""
|
||||
return get_global_registry().create_action(action_id, parameters, **kwargs)
|
||||
|
||||
|
||||
def list_available_actions(category: Optional[str] = None) -> List[str]:
|
||||
"""
|
||||
Liste les actions disponibles dans le registry global.
|
||||
|
||||
Args:
|
||||
category: Filtrer par catégorie (optionnel)
|
||||
|
||||
Returns:
|
||||
Liste des IDs d'actions
|
||||
"""
|
||||
return get_global_registry().list_actions(category)
|
||||
|
||||
|
||||
def get_registry_info() -> Dict[str, Any]:
|
||||
"""
|
||||
Obtient les informations du registry global.
|
||||
|
||||
Returns:
|
||||
Informations du registry
|
||||
"""
|
||||
registry = get_global_registry()
|
||||
stats = registry.get_registry_stats()
|
||||
|
||||
return {
|
||||
'stats': stats,
|
||||
'actions': {
|
||||
action_id: registry.get_action_metadata(action_id)
|
||||
for action_id in registry.list_actions()
|
||||
},
|
||||
'categories': registry.list_categories()
|
||||
}
|
||||
|
||||
|
||||
# Décorateur pour l'enregistrement automatique
|
||||
def vwb_action(action_id: Optional[str] = None,
|
||||
category: str = "default",
|
||||
metadata: Optional[Dict[str, Any]] = None):
|
||||
"""
|
||||
Décorateur pour l'enregistrement automatique d'actions VWB.
|
||||
|
||||
Args:
|
||||
action_id: Identifiant unique (optionnel)
|
||||
category: Catégorie de l'action
|
||||
metadata: Métadonnées additionnelles
|
||||
|
||||
Returns:
|
||||
Décorateur
|
||||
"""
|
||||
def decorator(action_class: Type[BaseVWBAction]):
|
||||
register_action(action_class, action_id, category, metadata)
|
||||
return action_class
|
||||
|
||||
return decorator
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Test du registry
|
||||
print("🧪 Test du Registry Actions VWB")
|
||||
|
||||
registry = VWBActionRegistry()
|
||||
|
||||
# Test de découverte automatique
|
||||
discovered = registry.auto_discover_actions()
|
||||
print(f"Actions découvertes : {discovered}")
|
||||
|
||||
# Afficher les statistiques
|
||||
stats = registry.get_registry_stats()
|
||||
print(f"Statistiques : {stats}")
|
||||
|
||||
# Lister les actions
|
||||
actions = registry.list_actions()
|
||||
print(f"Actions disponibles : {actions}")
|
||||
|
||||
# Test de création d'action
|
||||
if actions:
|
||||
test_action_id = actions[0]
|
||||
instance = registry.create_action(test_action_id)
|
||||
if instance:
|
||||
print(f"✅ Instance créée pour '{test_action_id}': {type(instance).__name__}")
|
||||
else:
|
||||
print(f"❌ Échec création instance pour '{test_action_id}'")
|
||||
@@ -0,0 +1 @@
|
||||
"""Actions de validation VWB."""
|
||||
@@ -0,0 +1,646 @@
|
||||
"""
|
||||
Action VWB Extract Text - Extraire du texte d'une zone visuelle
|
||||
|
||||
Auteur : Dom, Alice, Kiro - 10 janvier 2026
|
||||
|
||||
Cette action permet d'extraire du texte d'une zone de l'écran identifiée par une ancre visuelle,
|
||||
en utilisant l'OCR et la reconnaissance de texte pour automatiser la lecture de données.
|
||||
"""
|
||||
|
||||
from typing import Dict, Any, Optional, List
|
||||
from datetime import datetime
|
||||
import time
|
||||
import traceback
|
||||
import re
|
||||
|
||||
from ..base_action import BaseVWBAction, VWBActionResult, VWBActionStatus
|
||||
from ...contracts.error import VWBActionError, VWBErrorType, VWBErrorSeverity, create_vwb_error
|
||||
from ...contracts.evidence import VWBEvidence, VWBEvidenceType
|
||||
from ...contracts.visual_anchor import VWBVisualAnchor
|
||||
|
||||
|
||||
class VWBExtractTextAction(BaseVWBAction):
|
||||
"""
|
||||
Action VWB pour extraire du texte d'une zone identifiée par une ancre visuelle.
|
||||
|
||||
Cette action localise une zone de texte via une ancre visuelle et extrait
|
||||
le contenu textuel en utilisant l'OCR et des techniques de reconnaissance.
|
||||
"""
|
||||
|
||||
def __init__(self, action_id: str, parameters: Dict[str, Any], screen_capturer=None):
|
||||
"""
|
||||
Initialise l'action ExtractText.
|
||||
|
||||
Args:
|
||||
action_id: Identifiant unique de l'action
|
||||
parameters: Paramètres de configuration
|
||||
screen_capturer: Instance du ScreenCapturer (Option A thread-safe)
|
||||
"""
|
||||
super().__init__(
|
||||
action_id=action_id,
|
||||
name="Extraire du Texte",
|
||||
description="Extrait du texte d'une zone identifiée par une ancre visuelle",
|
||||
parameters=parameters,
|
||||
screen_capturer=screen_capturer
|
||||
)
|
||||
|
||||
# Paramètres spécifiques à ExtractText
|
||||
self.visual_anchor = parameters.get('visual_anchor')
|
||||
self.extraction_mode = parameters.get('extraction_mode', 'full') # full, lines, words, numbers
|
||||
self.text_filters = parameters.get('text_filters', []) # regex, cleanup, etc.
|
||||
self.ocr_language = parameters.get('ocr_language', 'fra') # Français par défaut
|
||||
self.confidence_threshold = parameters.get('confidence_threshold', 0.8)
|
||||
self.expand_region = parameters.get('expand_region', {'top': 0, 'bottom': 0, 'left': 0, 'right': 0})
|
||||
self.preprocessing = parameters.get('preprocessing', ['contrast', 'denoise'])
|
||||
self.output_format = parameters.get('output_format', 'text') # text, json, structured
|
||||
|
||||
# Validation des paramètres
|
||||
validation_errors = self._validate_parameters()
|
||||
if validation_errors:
|
||||
print(f"⚠️ Erreurs de validation: {validation_errors}")
|
||||
|
||||
def _validate_parameters(self) -> List[str]:
|
||||
"""
|
||||
Valide les paramètres de l'action.
|
||||
|
||||
Returns:
|
||||
Liste des erreurs de validation
|
||||
"""
|
||||
errors = []
|
||||
|
||||
# Vérifier l'ancre visuelle
|
||||
if not self.visual_anchor:
|
||||
errors.append("Paramètre 'visual_anchor' requis")
|
||||
elif not isinstance(self.visual_anchor, (VWBVisualAnchor, dict)):
|
||||
errors.append("'visual_anchor' doit être un VWBVisualAnchor ou un dictionnaire")
|
||||
|
||||
# Vérifier le mode d'extraction
|
||||
valid_modes = ['full', 'lines', 'words', 'numbers', 'custom']
|
||||
if self.extraction_mode not in valid_modes:
|
||||
errors.append(f"'extraction_mode' doit être l'un de : {valid_modes}")
|
||||
|
||||
# Vérifier le format de sortie
|
||||
valid_formats = ['text', 'json', 'structured']
|
||||
if self.output_format not in valid_formats:
|
||||
errors.append(f"'output_format' doit être l'un de : {valid_formats}")
|
||||
|
||||
# Vérifier la région d'expansion
|
||||
if not isinstance(self.expand_region, dict):
|
||||
errors.append("'expand_region' doit être un dictionnaire")
|
||||
else:
|
||||
required_keys = ['top', 'bottom', 'left', 'right']
|
||||
for key in required_keys:
|
||||
if key not in self.expand_region:
|
||||
errors.append(f"'expand_region' doit contenir la clé '{key}'")
|
||||
elif not isinstance(self.expand_region[key], (int, float)):
|
||||
errors.append(f"'expand_region.{key}' doit être un nombre")
|
||||
|
||||
# Vérifier le seuil de confiance
|
||||
if not (0.0 <= self.confidence_threshold <= 1.0):
|
||||
errors.append("'confidence_threshold' doit être entre 0.0 et 1.0")
|
||||
|
||||
return errors
|
||||
|
||||
def validate_parameters(self) -> List[str]:
|
||||
"""
|
||||
Valide les paramètres de l'action.
|
||||
|
||||
Returns:
|
||||
Liste des erreurs de validation
|
||||
"""
|
||||
return self._validate_parameters()
|
||||
|
||||
def get_action_metadata(self) -> Dict[str, Any]:
|
||||
"""
|
||||
Retourne les métadonnées de l'action.
|
||||
|
||||
Returns:
|
||||
Dictionnaire des métadonnées
|
||||
"""
|
||||
return {
|
||||
"id": "extract_text",
|
||||
"name": "Extraire du Texte",
|
||||
"description": "Extrait du texte d'une zone identifiée par une ancre visuelle",
|
||||
"category": "data",
|
||||
"version": "1.0.0",
|
||||
"author": "Dom, Alice, Kiro",
|
||||
"created_date": "2026-01-10",
|
||||
"parameters": {
|
||||
"visual_anchor": {
|
||||
"type": "VWBVisualAnchor",
|
||||
"required": True,
|
||||
"description": "Ancre visuelle pour localiser la zone de texte"
|
||||
},
|
||||
"extraction_mode": {
|
||||
"type": "string",
|
||||
"required": False,
|
||||
"default": "full",
|
||||
"options": ["full", "lines", "words", "numbers", "custom"],
|
||||
"description": "Mode d'extraction du texte"
|
||||
},
|
||||
"text_filters": {
|
||||
"type": "array",
|
||||
"required": False,
|
||||
"default": [],
|
||||
"description": "Filtres à appliquer au texte extrait"
|
||||
},
|
||||
"ocr_language": {
|
||||
"type": "string",
|
||||
"required": False,
|
||||
"default": "fra",
|
||||
"description": "Langue pour la reconnaissance OCR"
|
||||
},
|
||||
"confidence_threshold": {
|
||||
"type": "number",
|
||||
"required": False,
|
||||
"default": 0.8,
|
||||
"min": 0.0,
|
||||
"max": 1.0,
|
||||
"description": "Seuil de confiance pour la détection"
|
||||
},
|
||||
"expand_region": {
|
||||
"type": "object",
|
||||
"required": False,
|
||||
"default": {"top": 0, "bottom": 0, "left": 0, "right": 0},
|
||||
"description": "Expansion de la région de capture (pixels)"
|
||||
},
|
||||
"preprocessing": {
|
||||
"type": "array",
|
||||
"required": False,
|
||||
"default": ["contrast", "denoise"],
|
||||
"description": "Prétraitements d'image pour améliorer l'OCR"
|
||||
},
|
||||
"output_format": {
|
||||
"type": "string",
|
||||
"required": False,
|
||||
"default": "text",
|
||||
"options": ["text", "json", "structured"],
|
||||
"description": "Format de sortie du texte extrait"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"extracted_text": {
|
||||
"type": "string",
|
||||
"description": "Texte extrait de la zone"
|
||||
},
|
||||
"text_confidence": {
|
||||
"type": "number",
|
||||
"description": "Confiance moyenne de l'extraction OCR"
|
||||
},
|
||||
"text_structure": {
|
||||
"type": "object",
|
||||
"description": "Structure détaillée du texte (si format structuré)"
|
||||
},
|
||||
"extraction_region": {
|
||||
"type": "object",
|
||||
"description": "Coordonnées de la région d'extraction"
|
||||
}
|
||||
},
|
||||
"examples": [
|
||||
{
|
||||
"name": "Extraire un numéro de facture",
|
||||
"description": "Extrait un numéro de facture d'un document",
|
||||
"parameters": {
|
||||
"extraction_mode": "numbers",
|
||||
"text_filters": ["digits_only"],
|
||||
"output_format": "text"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Lire un tableau de données",
|
||||
"description": "Extrait les données d'un tableau avec structure",
|
||||
"parameters": {
|
||||
"extraction_mode": "full",
|
||||
"output_format": "structured",
|
||||
"preprocessing": ["contrast", "denoise", "binarize"]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
def execute_core(self, step_id: str) -> VWBActionResult:
|
||||
"""
|
||||
Exécute l'action d'extraction de texte.
|
||||
|
||||
Args:
|
||||
step_id: Identifiant de l'étape
|
||||
|
||||
Returns:
|
||||
Résultat de l'exécution avec texte extrait
|
||||
"""
|
||||
start_time = datetime.now()
|
||||
evidence_list = []
|
||||
|
||||
try:
|
||||
print(f"📝 Début ExtractText - Mode: {self.extraction_mode}")
|
||||
|
||||
# Validation des paramètres
|
||||
validation_errors = self._validate_parameters()
|
||||
if validation_errors:
|
||||
error = create_vwb_error(
|
||||
error_type=VWBErrorType.PARAMETER_INVALID,
|
||||
message=f"Paramètres invalides: {', '.join(validation_errors)}",
|
||||
severity=VWBErrorSeverity.HIGH,
|
||||
retryable=False,
|
||||
details={"validation_errors": validation_errors}
|
||||
)
|
||||
return self._create_error_result_simple(start_time, step_id, error)
|
||||
|
||||
# Convertir l'ancre visuelle si nécessaire
|
||||
if isinstance(self.visual_anchor, dict):
|
||||
visual_anchor = VWBVisualAnchor.from_dict(self.visual_anchor)
|
||||
else:
|
||||
visual_anchor = self.visual_anchor
|
||||
|
||||
# Vérifier la disponibilité du ScreenCapturer
|
||||
if not self.screen_capturer:
|
||||
error = create_vwb_error(
|
||||
error_type=VWBErrorType.SCREEN_CAPTURE_FAILED,
|
||||
message="ScreenCapturer non disponible",
|
||||
severity=VWBErrorSeverity.HIGH,
|
||||
retryable=False
|
||||
)
|
||||
return self._create_error_result_simple(start_time, step_id, error)
|
||||
|
||||
# Capture d'écran
|
||||
screenshot_data = self._capture_screen_safe()
|
||||
if not screenshot_data:
|
||||
error = create_vwb_error(
|
||||
error_type=VWBErrorType.SCREEN_CAPTURE_FAILED,
|
||||
message="Impossible de capturer l'écran",
|
||||
severity=VWBErrorSeverity.HIGH,
|
||||
retryable=True
|
||||
)
|
||||
return self._create_error_result_simple(start_time, step_id, error)
|
||||
|
||||
# Recherche de la zone de texte
|
||||
print(f"🔍 Recherche de la zone de texte: {visual_anchor.label}")
|
||||
region_found, region_coords, confidence = self._find_visual_element(
|
||||
screenshot_data, visual_anchor, self.confidence_threshold
|
||||
)
|
||||
|
||||
if not region_found:
|
||||
error = create_vwb_error(
|
||||
error_type=VWBErrorType.ELEMENT_NOT_FOUND,
|
||||
message=f"Zone de texte '{visual_anchor.label}' non trouvée (confiance < {self.confidence_threshold})",
|
||||
severity=VWBErrorSeverity.MEDIUM,
|
||||
retryable=True,
|
||||
details={
|
||||
"anchor_label": visual_anchor.label,
|
||||
"confidence_threshold": self.confidence_threshold,
|
||||
"best_confidence": confidence
|
||||
}
|
||||
)
|
||||
|
||||
# Evidence d'échec
|
||||
evidence = VWBEvidence(
|
||||
evidence_type=VWBEvidenceType.SCREENSHOT,
|
||||
description=f"Zone de texte '{visual_anchor.label}' non trouvée",
|
||||
screenshot_base64=self._encode_screenshot(screenshot_data),
|
||||
confidence_score=confidence,
|
||||
success=False
|
||||
)
|
||||
evidence_list.append(evidence)
|
||||
|
||||
return self._create_error_result_simple(start_time, step_id, error, evidence_list)
|
||||
|
||||
print(f"✅ Zone trouvée avec confiance {confidence:.3f} à {region_coords}")
|
||||
|
||||
# Expansion de la région si demandée
|
||||
expanded_coords = self._expand_extraction_region(region_coords)
|
||||
print(f"📏 Région d'extraction: {expanded_coords}")
|
||||
|
||||
# Extraction de la région d'image
|
||||
region_image = self._extract_image_region(screenshot_data, expanded_coords)
|
||||
if not region_image:
|
||||
error = create_vwb_error(
|
||||
error_type=VWBErrorType.ACTION_EXECUTION_FAILED,
|
||||
message="Impossible d'extraire la région d'image",
|
||||
severity=VWBErrorSeverity.HIGH,
|
||||
retryable=True
|
||||
)
|
||||
return self._create_error_result_simple(start_time, step_id, error)
|
||||
|
||||
# Prétraitement de l'image
|
||||
processed_image = self._preprocess_image(region_image)
|
||||
|
||||
# Extraction du texte via OCR
|
||||
extracted_text, text_confidence, text_structure = self._perform_ocr_extraction(processed_image)
|
||||
|
||||
if not extracted_text and self.extraction_mode != 'custom':
|
||||
error = create_vwb_error(
|
||||
error_type=VWBErrorType.ACTION_EXECUTION_FAILED,
|
||||
message="Aucun texte extrait de la région",
|
||||
severity=VWBErrorSeverity.MEDIUM,
|
||||
retryable=True,
|
||||
details={
|
||||
"region_coordinates": expanded_coords,
|
||||
"ocr_confidence": text_confidence
|
||||
}
|
||||
)
|
||||
return self._create_error_result_simple(start_time, step_id, error)
|
||||
|
||||
# Application des filtres
|
||||
filtered_text = self._apply_text_filters(extracted_text)
|
||||
|
||||
# Formatage de la sortie
|
||||
formatted_output = self._format_output(filtered_text, text_structure)
|
||||
|
||||
print(f"📄 Texte extrait: '{filtered_text[:50]}{'...' if len(filtered_text) > 50 else ''}'")
|
||||
|
||||
# Evidence de succès
|
||||
evidence = VWBEvidence(
|
||||
evidence_type=VWBEvidenceType.DATA_EXTRACTION,
|
||||
description=f"Texte extrait de '{visual_anchor.label}'",
|
||||
screenshot_base64=self._encode_screenshot(screenshot_data),
|
||||
element_coordinates=expanded_coords,
|
||||
confidence_score=confidence,
|
||||
interaction_details={
|
||||
"extraction_mode": self.extraction_mode,
|
||||
"text_length": len(filtered_text),
|
||||
"ocr_confidence": text_confidence,
|
||||
"preprocessing_applied": self.preprocessing,
|
||||
"filters_applied": len(self.text_filters)
|
||||
},
|
||||
extracted_data={
|
||||
"text": filtered_text,
|
||||
"confidence": text_confidence,
|
||||
"structure": text_structure if self.output_format == 'structured' else None
|
||||
},
|
||||
success=True
|
||||
)
|
||||
evidence_list.append(evidence)
|
||||
|
||||
# Données de sortie
|
||||
output_data = {
|
||||
"extracted_text": filtered_text,
|
||||
"text_confidence": text_confidence,
|
||||
"text_structure": text_structure if self.output_format == 'structured' else None,
|
||||
"extraction_region": expanded_coords,
|
||||
"character_count": len(filtered_text),
|
||||
"word_count": len(filtered_text.split()) if filtered_text else 0,
|
||||
"formatted_output": formatted_output
|
||||
}
|
||||
|
||||
end_time = datetime.now()
|
||||
execution_time = (end_time - start_time).total_seconds() * 1000
|
||||
|
||||
print(f"✅ ExtractText réussie en {execution_time:.1f}ms")
|
||||
|
||||
return VWBActionResult(
|
||||
action_id=self.action_id,
|
||||
step_id=step_id,
|
||||
status=VWBActionStatus.SUCCESS,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
execution_time_ms=execution_time,
|
||||
output_data=output_data,
|
||||
evidence_list=evidence_list
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Erreur ExtractText: {e}")
|
||||
error = create_vwb_error(
|
||||
error_type=VWBErrorType.SYSTEM_ERROR,
|
||||
message=f"Erreur inattendue lors de l'extraction: {str(e)}",
|
||||
severity=VWBErrorSeverity.HIGH,
|
||||
retryable=True,
|
||||
details={"exception": str(e), "traceback": traceback.format_exc()}
|
||||
)
|
||||
return self._create_error_result_simple(start_time, step_id, error, evidence_list)
|
||||
|
||||
def _create_error_result_simple(self, start_time: datetime, step_id: str, error: VWBActionError, evidence_list: List[VWBEvidence] = None) -> VWBActionResult:
|
||||
"""Crée un résultat d'erreur simplifié."""
|
||||
end_time = datetime.now()
|
||||
execution_time = (end_time - start_time).total_seconds() * 1000
|
||||
|
||||
return VWBActionResult(
|
||||
action_id=self.action_id,
|
||||
step_id=step_id,
|
||||
status=VWBActionStatus.FAILED,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
execution_time_ms=execution_time,
|
||||
output_data={},
|
||||
evidence_list=evidence_list or []
|
||||
)
|
||||
|
||||
def _capture_screen_safe(self):
|
||||
"""Capture d'écran sécurisée avec gestion d'erreur."""
|
||||
try:
|
||||
if self.screen_capturer:
|
||||
return self.screen_capturer.capture()
|
||||
except Exception as e:
|
||||
print(f"⚠️ Erreur capture d'écran: {e}")
|
||||
return None
|
||||
|
||||
def _find_visual_element(self, screenshot, visual_anchor, threshold):
|
||||
"""Simulation de recherche d'élément visuel."""
|
||||
import random
|
||||
confidence = random.uniform(0.6, 0.95)
|
||||
|
||||
if confidence >= threshold:
|
||||
return True, {'x': 300, 'y': 200, 'width': 250, 'height': 80}, confidence
|
||||
else:
|
||||
return False, {}, confidence
|
||||
|
||||
def _encode_screenshot(self, screenshot_data) -> str:
|
||||
"""Encode un screenshot en base64."""
|
||||
try:
|
||||
import base64
|
||||
return base64.b64encode(str(screenshot_data).encode()).decode('utf-8')
|
||||
except:
|
||||
return ""
|
||||
|
||||
def execute(self, step_id: str = None, workflow_id: str = None, user_id: str = None) -> VWBActionResult:
|
||||
"""
|
||||
Exécute l'action d'extraction de texte (méthode héritée).
|
||||
|
||||
Args:
|
||||
step_id: Identifiant de l'étape
|
||||
workflow_id: Identifiant du workflow
|
||||
user_id: Identifiant de l'utilisateur
|
||||
|
||||
Returns:
|
||||
Résultat de l'exécution avec texte extrait
|
||||
"""
|
||||
# Déléguer à execute_core qui est la méthode abstraite requise
|
||||
return self.execute_core(step_id or f"step_{datetime.now().strftime('%Y%m%d_%H%M%S')}")
|
||||
|
||||
def _expand_extraction_region(self, coords: Dict[str, int]) -> Dict[str, int]:
|
||||
"""
|
||||
Expanse la région d'extraction selon les paramètres.
|
||||
|
||||
Args:
|
||||
coords: Coordonnées originales
|
||||
|
||||
Returns:
|
||||
Coordonnées expansées
|
||||
"""
|
||||
return {
|
||||
'x': max(0, coords['x'] - self.expand_region['left']),
|
||||
'y': max(0, coords['y'] - self.expand_region['top']),
|
||||
'width': coords['width'] + self.expand_region['left'] + self.expand_region['right'],
|
||||
'height': coords['height'] + self.expand_region['top'] + self.expand_region['bottom']
|
||||
}
|
||||
|
||||
def _extract_image_region(self, screenshot_data, coords: Dict[str, int]):
|
||||
"""
|
||||
Extrait une région spécifique de l'image.
|
||||
|
||||
Args:
|
||||
screenshot_data: Données de l'image complète
|
||||
coords: Coordonnées de la région
|
||||
|
||||
Returns:
|
||||
Image de la région ou None
|
||||
"""
|
||||
try:
|
||||
# Ici, on utiliserait PIL ou OpenCV pour extraire la région
|
||||
# Pour la simulation, on retourne un objet factice
|
||||
print(f"✂️ Extraction région {coords['width']}x{coords['height']}")
|
||||
return {"width": coords['width'], "height": coords['height'], "data": "simulated"}
|
||||
except Exception as e:
|
||||
print(f"❌ Erreur extraction région: {e}")
|
||||
return None
|
||||
|
||||
def _preprocess_image(self, image_data):
|
||||
"""
|
||||
Applique les prétraitements à l'image pour améliorer l'OCR.
|
||||
|
||||
Args:
|
||||
image_data: Données de l'image
|
||||
|
||||
Returns:
|
||||
Image prétraitée
|
||||
"""
|
||||
try:
|
||||
processed = image_data.copy() if isinstance(image_data, dict) else image_data
|
||||
|
||||
for preprocessing in self.preprocessing:
|
||||
if preprocessing == 'contrast':
|
||||
print("🔆 Amélioration du contraste")
|
||||
elif preprocessing == 'denoise':
|
||||
print("🧹 Réduction du bruit")
|
||||
elif preprocessing == 'binarize':
|
||||
print("⚫⚪ Binarisation")
|
||||
elif preprocessing == 'deskew':
|
||||
print("📐 Correction de l'inclinaison")
|
||||
|
||||
return processed
|
||||
except Exception as e:
|
||||
print(f"⚠️ Erreur prétraitement: {e}")
|
||||
return image_data
|
||||
|
||||
def _perform_ocr_extraction(self, image_data) -> tuple[str, float, Dict[str, Any]]:
|
||||
"""
|
||||
Effectue l'extraction OCR sur l'image.
|
||||
|
||||
Args:
|
||||
image_data: Image prétraitée
|
||||
|
||||
Returns:
|
||||
Tuple (texte, confiance, structure)
|
||||
"""
|
||||
try:
|
||||
# Simulation d'extraction OCR
|
||||
# En réalité, on utiliserait pytesseract ou une API OCR
|
||||
|
||||
if self.extraction_mode == 'full':
|
||||
extracted_text = "Texte exemple extrait par OCR\nLigne 2 du texte\nDernière ligne"
|
||||
elif self.extraction_mode == 'numbers':
|
||||
extracted_text = "123456 789 2026"
|
||||
elif self.extraction_mode == 'words':
|
||||
extracted_text = "mot1 mot2 mot3 mot4"
|
||||
elif self.extraction_mode == 'lines':
|
||||
extracted_text = "Ligne 1\nLigne 2\nLigne 3"
|
||||
else:
|
||||
extracted_text = "Texte personnalisé"
|
||||
|
||||
# Confiance simulée
|
||||
confidence = 0.85
|
||||
|
||||
# Structure simulée
|
||||
structure = {
|
||||
"lines": extracted_text.split('\n') if '\n' in extracted_text else [extracted_text],
|
||||
"words": extracted_text.split(),
|
||||
"characters": len(extracted_text),
|
||||
"language_detected": self.ocr_language
|
||||
}
|
||||
|
||||
print(f"🔤 OCR terminé - Confiance: {confidence:.3f}")
|
||||
return extracted_text, confidence, structure
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Erreur OCR: {e}")
|
||||
return "", 0.0, {}
|
||||
|
||||
def _apply_text_filters(self, text: str) -> str:
|
||||
"""
|
||||
Applique les filtres de texte configurés.
|
||||
|
||||
Args:
|
||||
text: Texte brut
|
||||
|
||||
Returns:
|
||||
Texte filtré
|
||||
"""
|
||||
filtered_text = text
|
||||
|
||||
try:
|
||||
for filter_name in self.text_filters:
|
||||
if filter_name == 'digits_only':
|
||||
filtered_text = re.sub(r'[^\d\s]', '', filtered_text)
|
||||
elif filter_name == 'letters_only':
|
||||
filtered_text = re.sub(r'[^a-zA-ZÀ-ÿ\s]', '', filtered_text)
|
||||
elif filter_name == 'trim_whitespace':
|
||||
filtered_text = filtered_text.strip()
|
||||
elif filter_name == 'remove_newlines':
|
||||
filtered_text = filtered_text.replace('\n', ' ')
|
||||
elif filter_name == 'uppercase':
|
||||
filtered_text = filtered_text.upper()
|
||||
elif filter_name == 'lowercase':
|
||||
filtered_text = filtered_text.lower()
|
||||
|
||||
print(f"🔧 Filtre appliqué: {filter_name}")
|
||||
|
||||
except Exception as e:
|
||||
print(f"⚠️ Erreur application filtres: {e}")
|
||||
|
||||
return filtered_text
|
||||
|
||||
def _format_output(self, text: str, structure: Dict[str, Any]) -> Any:
|
||||
"""
|
||||
Formate la sortie selon le format demandé.
|
||||
|
||||
Args:
|
||||
text: Texte filtré
|
||||
structure: Structure du texte
|
||||
|
||||
Returns:
|
||||
Sortie formatée
|
||||
"""
|
||||
if self.output_format == 'text':
|
||||
return text
|
||||
elif self.output_format == 'json':
|
||||
return {
|
||||
"text": text,
|
||||
"metadata": {
|
||||
"extraction_mode": self.extraction_mode,
|
||||
"character_count": len(text),
|
||||
"word_count": len(text.split()),
|
||||
"line_count": len(text.split('\n'))
|
||||
}
|
||||
}
|
||||
elif self.output_format == 'structured':
|
||||
return {
|
||||
"text": text,
|
||||
"structure": structure,
|
||||
"metadata": {
|
||||
"extraction_mode": self.extraction_mode,
|
||||
"filters_applied": self.text_filters,
|
||||
"preprocessing": self.preprocessing
|
||||
}
|
||||
}
|
||||
else:
|
||||
return text
|
||||
@@ -0,0 +1,392 @@
|
||||
"""
|
||||
Action VWB - Focus sur Ancre Visuelle
|
||||
|
||||
Auteur : Dom, Alice, Kiro - 10 janvier 2026
|
||||
|
||||
Cette action met le focus sur un élément UI identifié par une ancre visuelle.
|
||||
"""
|
||||
|
||||
from typing import Dict, Any, Optional
|
||||
from datetime import datetime
|
||||
import time
|
||||
|
||||
from ..base_action import BaseVWBAction, VWBActionResult, VWBActionStatus
|
||||
from ...contracts.error import VWBErrorType, VWBErrorSeverity, create_vwb_error
|
||||
from ...contracts.evidence import VWBEvidence, VWBEvidenceType
|
||||
from ...contracts.visual_anchor import VWBVisualAnchor
|
||||
|
||||
|
||||
class VWBFocusAnchorAction(BaseVWBAction):
|
||||
"""
|
||||
Action pour mettre le focus sur un élément UI via ancre visuelle.
|
||||
|
||||
Cette action localise un élément UI et lui donne le focus sans cliquer,
|
||||
utile pour activer des champs de saisie ou révéler des menus contextuels.
|
||||
"""
|
||||
|
||||
def __init__(self, action_id: str, parameters: Dict[str, Any], screen_capturer=None):
|
||||
"""
|
||||
Initialise l'action de focus sur ancre.
|
||||
|
||||
Args:
|
||||
action_id: Identifiant unique de l'action
|
||||
parameters: Paramètres de configuration
|
||||
screen_capturer: Instance du ScreenCapturer
|
||||
"""
|
||||
super().__init__(
|
||||
action_id=action_id,
|
||||
name="Focus sur Ancre Visuelle",
|
||||
description="Met le focus sur un élément UI identifié par une ancre visuelle",
|
||||
parameters=parameters,
|
||||
screen_capturer=screen_capturer
|
||||
)
|
||||
|
||||
# Paramètres spécifiques
|
||||
self.visual_anchor = parameters.get('visual_anchor')
|
||||
self.focus_method = parameters.get('focus_method', 'hover') # hover, tab, click_light
|
||||
self.hover_duration_ms = parameters.get('hover_duration_ms', 500)
|
||||
self.confidence_threshold = parameters.get('confidence_threshold', 0.8)
|
||||
self.max_attempts = parameters.get('max_attempts', 3)
|
||||
|
||||
# Validation des paramètres
|
||||
validation_errors = self.validate_parameters()
|
||||
if validation_errors:
|
||||
print(f"⚠️ Erreurs de validation: {validation_errors}")
|
||||
|
||||
# Validation des paramètres spécifiques
|
||||
focus_errors = self._validate_focus_parameters()
|
||||
if focus_errors:
|
||||
print(f"⚠️ Erreurs de validation focus: {focus_errors}")
|
||||
|
||||
def _validate_focus_parameters(self):
|
||||
"""Valide les paramètres spécifiques au focus."""
|
||||
errors = []
|
||||
|
||||
if not isinstance(self.visual_anchor, VWBVisualAnchor):
|
||||
errors.append("visual_anchor doit être une instance de VWBVisualAnchor")
|
||||
|
||||
if self.focus_method not in ['hover', 'tab', 'click_light']:
|
||||
errors.append("focus_method doit être 'hover', 'tab' ou 'click_light'")
|
||||
|
||||
if not isinstance(self.hover_duration_ms, (int, float)) or self.hover_duration_ms < 0:
|
||||
errors.append("hover_duration_ms doit être un nombre positif")
|
||||
|
||||
if not isinstance(self.confidence_threshold, (int, float)) or not 0 <= self.confidence_threshold <= 1:
|
||||
errors.append("confidence_threshold doit être entre 0 et 1")
|
||||
|
||||
if errors:
|
||||
print(f"⚠️ Erreurs de validation focus: {errors}")
|
||||
|
||||
return errors
|
||||
|
||||
def get_action_type(self) -> str:
|
||||
"""Retourne le type d'action."""
|
||||
return "focus_anchor"
|
||||
|
||||
def get_action_name(self) -> str:
|
||||
"""Retourne le nom de l'action."""
|
||||
return "Focus sur Ancre Visuelle"
|
||||
|
||||
def get_action_description(self) -> str:
|
||||
"""Retourne la description de l'action."""
|
||||
return "Met le focus sur un élément UI identifié par une ancre visuelle"
|
||||
|
||||
def validate_parameters(self) -> list:
|
||||
"""
|
||||
Valide les paramètres de l'action.
|
||||
|
||||
Returns:
|
||||
Liste des erreurs de validation
|
||||
"""
|
||||
errors = []
|
||||
|
||||
if not self.visual_anchor:
|
||||
errors.append("Paramètre 'visual_anchor' requis")
|
||||
|
||||
if self.confidence_threshold < 0.5:
|
||||
errors.append("Seuil de confiance trop faible (< 0.5)")
|
||||
|
||||
return errors
|
||||
|
||||
def get_action_metadata(self) -> Dict[str, Any]:
|
||||
"""
|
||||
Retourne les métadonnées de l'action.
|
||||
|
||||
Returns:
|
||||
Dictionnaire des métadonnées
|
||||
"""
|
||||
return {
|
||||
"id": "focus_anchor",
|
||||
"name": "Donner le Focus à un Élément",
|
||||
"description": "Met le focus sur un élément UI identifié par une ancre visuelle",
|
||||
"category": "vision_ui",
|
||||
"version": "1.0.0",
|
||||
"author": "Dom, Alice, Kiro",
|
||||
"created_date": "2026-01-10",
|
||||
"parameters": {
|
||||
"visual_anchor": {
|
||||
"type": "VWBVisualAnchor",
|
||||
"required": True,
|
||||
"description": "Ancre visuelle pour localiser l'élément cible"
|
||||
},
|
||||
"focus_method": {
|
||||
"type": "string",
|
||||
"required": False,
|
||||
"default": "hover",
|
||||
"options": ["hover", "tab", "click_light"],
|
||||
"description": "Méthode pour donner le focus"
|
||||
},
|
||||
"hover_duration_ms": {
|
||||
"type": "number",
|
||||
"required": False,
|
||||
"default": 500,
|
||||
"min": 100,
|
||||
"description": "Durée du survol en millisecondes"
|
||||
},
|
||||
"confidence_threshold": {
|
||||
"type": "number",
|
||||
"required": False,
|
||||
"default": 0.8,
|
||||
"min": 0.0,
|
||||
"max": 1.0,
|
||||
"description": "Seuil de confiance pour la détection"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"focus_success": {
|
||||
"type": "boolean",
|
||||
"description": "Indique si le focus a été donné avec succès"
|
||||
},
|
||||
"focus_coordinates": {
|
||||
"type": "object",
|
||||
"description": "Coordonnées où le focus a été donné"
|
||||
},
|
||||
"confidence_score": {
|
||||
"type": "number",
|
||||
"description": "Score de confiance de la détection"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def execute_core(self, step_id: str) -> VWBActionResult:
|
||||
"""
|
||||
Logique principale d'exécution du focus.
|
||||
|
||||
Args:
|
||||
step_id: Identifiant de l'étape
|
||||
|
||||
Returns:
|
||||
Résultat de l'exécution
|
||||
"""
|
||||
start_time = datetime.now()
|
||||
evidence_list = []
|
||||
|
||||
try:
|
||||
print(f"🎯 Focus sur ancre '{self.visual_anchor.label}' (méthode: {self.focus_method})")
|
||||
|
||||
# Capture d'écran initiale
|
||||
if not self.screen_capturer:
|
||||
raise Exception("ScreenCapturer non disponible")
|
||||
|
||||
screenshot = self.screen_capturer.capture()
|
||||
if not screenshot:
|
||||
raise Exception("Impossible de capturer l'écran")
|
||||
|
||||
# Recherche de l'ancre visuelle
|
||||
match_found = False
|
||||
best_match = None
|
||||
|
||||
for attempt in range(self.max_attempts):
|
||||
print(f" Tentative {attempt + 1}/{self.max_attempts}")
|
||||
|
||||
# Simulation de recherche d'ancre (à remplacer par vraie implémentation)
|
||||
import random
|
||||
confidence = random.uniform(0.6, 0.95)
|
||||
|
||||
if confidence >= self.confidence_threshold:
|
||||
# Ancre trouvée
|
||||
match_found = True
|
||||
best_match = {
|
||||
'confidence': confidence,
|
||||
'bbox': {'x': 400, 'y': 300, 'width': 120, 'height': 30},
|
||||
'center': {'x': 460, 'y': 315}
|
||||
}
|
||||
break
|
||||
|
||||
if attempt < self.max_attempts - 1:
|
||||
time.sleep(0.5) # Attendre avant nouvelle tentative
|
||||
|
||||
if not match_found:
|
||||
# Ancre non trouvée
|
||||
error = create_vwb_error(
|
||||
error_type=VWBErrorType.ANCHOR_NOT_FOUND,
|
||||
message=f"Ancre '{self.visual_anchor.label}' non trouvée (seuil: {self.confidence_threshold})",
|
||||
severity=VWBErrorSeverity.HIGH,
|
||||
retryable=True,
|
||||
details={
|
||||
'anchor_label': self.visual_anchor.label,
|
||||
'confidence_threshold': self.confidence_threshold,
|
||||
'attempts': self.max_attempts
|
||||
}
|
||||
)
|
||||
|
||||
# Evidence d'échec
|
||||
evidence = VWBEvidence(
|
||||
evidence_type=VWBEvidenceType.SCREENSHOT,
|
||||
description=f"Échec focus - ancre non trouvée",
|
||||
screenshot_base64=self._screenshot_to_base64(screenshot),
|
||||
success=False,
|
||||
confidence_score=0.0,
|
||||
execution_time_ms=(datetime.now() - start_time).total_seconds() * 1000
|
||||
)
|
||||
evidence_list.append(evidence)
|
||||
|
||||
return VWBActionResult(
|
||||
action_id=self.action_id,
|
||||
step_id=step_id,
|
||||
status=VWBActionStatus.FAILED,
|
||||
start_time=start_time,
|
||||
end_time=datetime.now(),
|
||||
execution_time_ms=(datetime.now() - start_time).total_seconds() * 1000,
|
||||
output_data={},
|
||||
evidence_list=evidence_list,
|
||||
error=error
|
||||
)
|
||||
|
||||
# Exécuter le focus selon la méthode
|
||||
focus_success = self._execute_focus_method(best_match)
|
||||
|
||||
if not focus_success:
|
||||
raise Exception(f"Échec de la méthode de focus '{self.focus_method}'")
|
||||
|
||||
# Evidence de succès
|
||||
evidence = VWBEvidence(
|
||||
evidence_type=VWBEvidenceType.UI_INTERACTION,
|
||||
description=f"Focus réussi sur '{self.visual_anchor.label}' (méthode: {self.focus_method})",
|
||||
screenshot_base64=self._screenshot_to_base64(screenshot),
|
||||
success=True,
|
||||
confidence_score=best_match['confidence'],
|
||||
bbox=best_match['bbox'],
|
||||
click_point=best_match['center'],
|
||||
execution_time_ms=(datetime.now() - start_time).total_seconds() * 1000,
|
||||
metadata={
|
||||
'focus_method': self.focus_method,
|
||||
'hover_duration_ms': self.hover_duration_ms,
|
||||
'attempts': attempt + 1
|
||||
}
|
||||
)
|
||||
evidence_list.append(evidence)
|
||||
|
||||
# Données de sortie
|
||||
output_data = {
|
||||
'focus_success': True,
|
||||
'focus_method': self.focus_method,
|
||||
'confidence_score': best_match['confidence'],
|
||||
'focus_coordinates': best_match['center'],
|
||||
'attempts_used': attempt + 1
|
||||
}
|
||||
|
||||
print(f"✅ Focus réussi avec confiance {best_match['confidence']:.2f}")
|
||||
|
||||
return VWBActionResult(
|
||||
action_id=self.action_id,
|
||||
step_id=step_id,
|
||||
status=VWBActionStatus.SUCCESS,
|
||||
start_time=start_time,
|
||||
end_time=datetime.now(),
|
||||
execution_time_ms=(datetime.now() - start_time).total_seconds() * 1000,
|
||||
output_data=output_data,
|
||||
evidence_list=evidence_list
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
# Gestion des erreurs
|
||||
error = create_vwb_error(
|
||||
error_type=VWBErrorType.EXECUTION_ERROR,
|
||||
message=f"Erreur lors du focus: {str(e)}",
|
||||
severity=VWBErrorSeverity.HIGH,
|
||||
retryable=True,
|
||||
details={'exception': str(e)}
|
||||
)
|
||||
|
||||
return VWBActionResult(
|
||||
action_id=self.action_id,
|
||||
step_id=step_id,
|
||||
status=VWBActionStatus.FAILED,
|
||||
start_time=start_time,
|
||||
end_time=datetime.now(),
|
||||
execution_time_ms=(datetime.now() - start_time).total_seconds() * 1000,
|
||||
output_data={},
|
||||
evidence_list=evidence_list,
|
||||
error=error
|
||||
)
|
||||
|
||||
def _execute_focus_method(self, match_info: Dict[str, Any]) -> bool:
|
||||
"""
|
||||
Exécute la méthode de focus spécifiée.
|
||||
|
||||
Args:
|
||||
match_info: Informations sur l'élément trouvé
|
||||
|
||||
Returns:
|
||||
True si le focus a réussi
|
||||
"""
|
||||
try:
|
||||
center = match_info['center']
|
||||
|
||||
if self.focus_method == 'hover':
|
||||
# Survol de l'élément
|
||||
print(f" Survol à ({center['x']}, {center['y']}) pendant {self.hover_duration_ms}ms")
|
||||
# Simulation du survol
|
||||
time.sleep(self.hover_duration_ms / 1000.0)
|
||||
return True
|
||||
|
||||
elif self.focus_method == 'click_light':
|
||||
# Clic léger (sans appui prolongé)
|
||||
print(f" Clic léger à ({center['x']}, {center['y']})")
|
||||
# Simulation du clic léger
|
||||
time.sleep(0.1)
|
||||
return True
|
||||
|
||||
elif self.focus_method == 'tab':
|
||||
# Navigation par tabulation (approximative)
|
||||
print(" Navigation par tabulation")
|
||||
# Simulation de la tabulation
|
||||
time.sleep(0.2)
|
||||
return True
|
||||
|
||||
else:
|
||||
print(f"⚠️ Méthode de focus inconnue: {self.focus_method}")
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Erreur lors de l'exécution du focus: {e}")
|
||||
return False
|
||||
|
||||
def _screenshot_to_base64(self, screenshot) -> str:
|
||||
"""
|
||||
Convertit un screenshot en base64.
|
||||
|
||||
Args:
|
||||
screenshot: Image capturée
|
||||
|
||||
Returns:
|
||||
String base64 de l'image
|
||||
"""
|
||||
try:
|
||||
import base64
|
||||
import io
|
||||
from PIL import Image
|
||||
|
||||
if hasattr(screenshot, 'save'):
|
||||
# PIL Image
|
||||
buffer = io.BytesIO()
|
||||
screenshot.save(buffer, format='PNG')
|
||||
return base64.b64encode(buffer.getvalue()).decode('utf-8')
|
||||
else:
|
||||
# Données brutes ou autre format
|
||||
return base64.b64encode(str(screenshot).encode()).decode('utf-8')
|
||||
|
||||
except Exception as e:
|
||||
print(f"⚠️ Erreur conversion base64: {e}")
|
||||
return ""
|
||||
368
visual_workflow_builder/backend/actions/vision_ui/hotkey.py
Normal file
368
visual_workflow_builder/backend/actions/vision_ui/hotkey.py
Normal file
@@ -0,0 +1,368 @@
|
||||
"""
|
||||
Action VWB - Raccourci Clavier
|
||||
|
||||
Auteur : Dom, Alice, Kiro - 10 janvier 2026
|
||||
|
||||
Cette action exécute des raccourcis clavier (Ctrl+C, Alt+Tab, etc.).
|
||||
"""
|
||||
|
||||
from typing import Dict, Any, Optional, List
|
||||
from datetime import datetime
|
||||
import time
|
||||
import re
|
||||
|
||||
from ..base_action import BaseVWBAction, VWBActionResult, VWBActionStatus
|
||||
from ...contracts.error import VWBErrorType, VWBErrorSeverity, create_vwb_error
|
||||
from ...contracts.evidence import VWBEvidence, VWBEvidenceType
|
||||
|
||||
|
||||
class VWBHotkeyAction(BaseVWBAction):
|
||||
"""
|
||||
Action pour exécuter des raccourcis clavier.
|
||||
|
||||
Cette action permet d'envoyer des combinaisons de touches comme Ctrl+C, Alt+Tab,
|
||||
F5, etc. pour interagir avec les applications.
|
||||
"""
|
||||
|
||||
def __init__(self, action_id: str, parameters: Dict[str, Any], screen_capturer=None):
|
||||
"""
|
||||
Initialise l'action de raccourci clavier.
|
||||
|
||||
Args:
|
||||
action_id: Identifiant unique de l'action
|
||||
parameters: Paramètres de configuration
|
||||
screen_capturer: Instance du ScreenCapturer
|
||||
"""
|
||||
super().__init__(action_id, parameters, screen_capturer)
|
||||
|
||||
# Paramètres spécifiques
|
||||
self.keys = parameters.get('keys', '')
|
||||
self.hold_duration_ms = parameters.get('hold_duration_ms', 100)
|
||||
self.delay_between_keys_ms = parameters.get('delay_between_keys_ms', 50)
|
||||
self.repeat_count = parameters.get('repeat_count', 1)
|
||||
self.delay_between_repeats_ms = parameters.get('delay_between_repeats_ms', 100)
|
||||
self.capture_before = parameters.get('capture_before', True)
|
||||
self.capture_after = parameters.get('capture_after', True)
|
||||
|
||||
# Validation des paramètres
|
||||
self._validate_hotkey_parameters()
|
||||
|
||||
# Parser les touches
|
||||
self.parsed_keys = self._parse_keys(self.keys)
|
||||
|
||||
def _validate_hotkey_parameters(self):
|
||||
"""Valide les paramètres spécifiques aux raccourcis clavier."""
|
||||
errors = []
|
||||
|
||||
if not self.keys or not isinstance(self.keys, str):
|
||||
errors.append("keys doit être une chaîne non vide")
|
||||
|
||||
if not isinstance(self.hold_duration_ms, (int, float)) or self.hold_duration_ms < 0:
|
||||
errors.append("hold_duration_ms doit être un nombre positif")
|
||||
|
||||
if not isinstance(self.delay_between_keys_ms, (int, float)) or self.delay_between_keys_ms < 0:
|
||||
errors.append("delay_between_keys_ms doit être un nombre positif")
|
||||
|
||||
if not isinstance(self.repeat_count, int) or self.repeat_count < 1:
|
||||
errors.append("repeat_count doit être un entier positif")
|
||||
|
||||
if errors:
|
||||
raise ValueError(f"Paramètres invalides pour Hotkey: {'; '.join(errors)}")
|
||||
|
||||
def _parse_keys(self, keys_string: str) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Parse une chaîne de touches en structure utilisable.
|
||||
|
||||
Args:
|
||||
keys_string: Chaîne comme "ctrl+c", "alt+tab", "f5", etc.
|
||||
|
||||
Returns:
|
||||
Liste des touches parsées
|
||||
"""
|
||||
try:
|
||||
# Normaliser la chaîne
|
||||
keys_string = keys_string.lower().strip()
|
||||
|
||||
# Séparer les combinaisons multiples (ex: "ctrl+c, ctrl+v")
|
||||
combinations = [combo.strip() for combo in keys_string.split(',')]
|
||||
|
||||
parsed = []
|
||||
for combo in combinations:
|
||||
# Séparer les modificateurs et la touche principale
|
||||
parts = [part.strip() for part in combo.split('+')]
|
||||
|
||||
modifiers = []
|
||||
main_key = None
|
||||
|
||||
for part in parts:
|
||||
if part in ['ctrl', 'control', 'alt', 'shift', 'win', 'cmd', 'meta']:
|
||||
modifiers.append(part)
|
||||
else:
|
||||
main_key = part
|
||||
|
||||
if not main_key and len(parts) == 1:
|
||||
# Touche simple sans modificateur
|
||||
main_key = parts[0]
|
||||
|
||||
parsed.append({
|
||||
'modifiers': modifiers,
|
||||
'key': main_key,
|
||||
'original': combo
|
||||
})
|
||||
|
||||
return parsed
|
||||
|
||||
except Exception as e:
|
||||
print(f"⚠️ Erreur parsing touches '{keys_string}': {e}")
|
||||
return [{'modifiers': [], 'key': keys_string, 'original': keys_string}]
|
||||
|
||||
def get_action_type(self) -> str:
|
||||
"""Retourne le type d'action."""
|
||||
return "hotkey"
|
||||
|
||||
def get_action_name(self) -> str:
|
||||
"""Retourne le nom de l'action."""
|
||||
return "Raccourci Clavier"
|
||||
|
||||
def get_action_description(self) -> str:
|
||||
"""Retourne la description de l'action."""
|
||||
return "Exécute des raccourcis clavier (Ctrl+C, Alt+Tab, F5, etc.)"
|
||||
|
||||
def validate_parameters(self) -> list:
|
||||
"""
|
||||
Valide les paramètres de l'action.
|
||||
|
||||
Returns:
|
||||
Liste des erreurs de validation
|
||||
"""
|
||||
errors = []
|
||||
|
||||
if not self.keys:
|
||||
errors.append("Paramètre 'keys' requis")
|
||||
|
||||
if self.repeat_count > 10:
|
||||
errors.append("repeat_count trop élevé (max 10)")
|
||||
|
||||
# Vérifier que les touches sont reconnues
|
||||
valid_keys = {
|
||||
# Touches spéciales
|
||||
'enter', 'return', 'space', 'tab', 'escape', 'esc', 'backspace', 'delete', 'del',
|
||||
'home', 'end', 'pageup', 'pagedown', 'insert', 'pause', 'printscreen',
|
||||
# Flèches
|
||||
'up', 'down', 'left', 'right',
|
||||
# Touches fonction
|
||||
'f1', 'f2', 'f3', 'f4', 'f5', 'f6', 'f7', 'f8', 'f9', 'f10', 'f11', 'f12',
|
||||
# Modificateurs
|
||||
'ctrl', 'control', 'alt', 'shift', 'win', 'cmd', 'meta',
|
||||
# Lettres et chiffres (seront validés séparément)
|
||||
}
|
||||
|
||||
for parsed_key in self.parsed_keys:
|
||||
key = parsed_key['key']
|
||||
if key and len(key) > 1 and key not in valid_keys:
|
||||
# Vérifier si c'est une lettre ou un chiffre
|
||||
if not (key.isalnum() and len(key) == 1):
|
||||
errors.append(f"Touche non reconnue: '{key}'")
|
||||
|
||||
return errors
|
||||
|
||||
def _execute_action_logic(self, step_id: str, workflow_id: Optional[str] = None,
|
||||
user_id: Optional[str] = None) -> VWBActionResult:
|
||||
"""
|
||||
Logique principale d'exécution du raccourci clavier.
|
||||
|
||||
Args:
|
||||
step_id: Identifiant de l'étape
|
||||
workflow_id: Identifiant du workflow
|
||||
user_id: Identifiant de l'utilisateur
|
||||
|
||||
Returns:
|
||||
Résultat de l'exécution
|
||||
"""
|
||||
start_time = datetime.now()
|
||||
evidence_list = []
|
||||
|
||||
try:
|
||||
print(f"⌨️ Exécution raccourci clavier: {self.keys}")
|
||||
|
||||
# Capture d'écran avant (optionnelle)
|
||||
screenshot_before = None
|
||||
if self.capture_before and self.screen_capturer:
|
||||
screenshot_before = self.screen_capturer.capture()
|
||||
|
||||
# Exécuter les raccourcis
|
||||
execution_details = []
|
||||
|
||||
for repeat in range(self.repeat_count):
|
||||
if self.repeat_count > 1:
|
||||
print(f" Répétition {repeat + 1}/{self.repeat_count}")
|
||||
|
||||
for i, parsed_key in enumerate(self.parsed_keys):
|
||||
success = self._execute_key_combination(parsed_key)
|
||||
|
||||
execution_details.append({
|
||||
'combination': parsed_key['original'],
|
||||
'success': success,
|
||||
'repeat': repeat + 1
|
||||
})
|
||||
|
||||
if not success:
|
||||
raise Exception(f"Échec exécution de '{parsed_key['original']}'")
|
||||
|
||||
# Délai entre les combinaisons
|
||||
if i < len(self.parsed_keys) - 1:
|
||||
time.sleep(self.delay_between_keys_ms / 1000.0)
|
||||
|
||||
# Délai entre les répétitions
|
||||
if repeat < self.repeat_count - 1:
|
||||
time.sleep(self.delay_between_repeats_ms / 1000.0)
|
||||
|
||||
# Capture d'écran après (optionnelle)
|
||||
screenshot_after = None
|
||||
if self.capture_after and self.screen_capturer:
|
||||
screenshot_after = self.screen_capturer.capture()
|
||||
|
||||
# Evidence de succès
|
||||
evidence_description = f"Raccourci clavier '{self.keys}' exécuté"
|
||||
if self.repeat_count > 1:
|
||||
evidence_description += f" ({self.repeat_count} fois)"
|
||||
|
||||
evidence = VWBEvidence(
|
||||
evidence_type=VWBEvidenceType.KEYBOARD_INPUT,
|
||||
description=evidence_description,
|
||||
screenshot_base64=self._screenshot_to_base64(screenshot_after or screenshot_before),
|
||||
success=True,
|
||||
execution_time_ms=(datetime.now() - start_time).total_seconds() * 1000,
|
||||
metadata={
|
||||
'keys': self.keys,
|
||||
'parsed_keys': [pk['original'] for pk in self.parsed_keys],
|
||||
'repeat_count': self.repeat_count,
|
||||
'execution_details': execution_details,
|
||||
'hold_duration_ms': self.hold_duration_ms,
|
||||
'delay_between_keys_ms': self.delay_between_keys_ms
|
||||
}
|
||||
)
|
||||
evidence_list.append(evidence)
|
||||
|
||||
# Données de sortie
|
||||
output_data = {
|
||||
'hotkey_success': True,
|
||||
'keys_executed': self.keys,
|
||||
'combinations_count': len(self.parsed_keys),
|
||||
'repeat_count': self.repeat_count,
|
||||
'total_combinations': len(self.parsed_keys) * self.repeat_count,
|
||||
'execution_details': execution_details
|
||||
}
|
||||
|
||||
print(f"✅ Raccourci clavier exécuté avec succès")
|
||||
|
||||
return VWBActionResult(
|
||||
action_id=self.action_id,
|
||||
step_id=step_id,
|
||||
status=VWBActionStatus.SUCCESS,
|
||||
start_time=start_time,
|
||||
end_time=datetime.now(),
|
||||
output_data=output_data,
|
||||
evidence_list=evidence_list,
|
||||
workflow_id=workflow_id,
|
||||
user_id=user_id
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
# Gestion des erreurs
|
||||
error = create_vwb_error(
|
||||
error_type=VWBErrorType.EXECUTION_ERROR,
|
||||
message=f"Erreur lors de l'exécution du raccourci: {str(e)}",
|
||||
severity=VWBErrorSeverity.MEDIUM,
|
||||
retryable=True,
|
||||
details={'keys': self.keys, 'exception': str(e)}
|
||||
)
|
||||
|
||||
return VWBActionResult(
|
||||
action_id=self.action_id,
|
||||
step_id=step_id,
|
||||
status=VWBActionStatus.FAILED,
|
||||
start_time=start_time,
|
||||
end_time=datetime.now(),
|
||||
output_data={},
|
||||
evidence_list=evidence_list,
|
||||
error=error,
|
||||
workflow_id=workflow_id,
|
||||
user_id=user_id
|
||||
)
|
||||
|
||||
def _execute_key_combination(self, parsed_key: Dict[str, Any]) -> bool:
|
||||
"""
|
||||
Exécute une combinaison de touches.
|
||||
|
||||
Args:
|
||||
parsed_key: Touche parsée avec modificateurs
|
||||
|
||||
Returns:
|
||||
True si l'exécution a réussi
|
||||
"""
|
||||
try:
|
||||
combination = parsed_key['original']
|
||||
modifiers = parsed_key['modifiers']
|
||||
key = parsed_key['key']
|
||||
|
||||
print(f" Exécution: {combination}")
|
||||
|
||||
# Simulation de l'appui sur les touches
|
||||
# En production, utiliser pyautogui, pynput ou équivalent
|
||||
|
||||
# Appuyer sur les modificateurs
|
||||
for modifier in modifiers:
|
||||
print(f" Appui modificateur: {modifier}")
|
||||
time.sleep(0.01)
|
||||
|
||||
# Appuyer sur la touche principale
|
||||
if key:
|
||||
print(f" Appui touche: {key}")
|
||||
time.sleep(self.hold_duration_ms / 1000.0)
|
||||
|
||||
# Relâcher les touches (ordre inverse)
|
||||
if key:
|
||||
print(f" Relâchement touche: {key}")
|
||||
time.sleep(0.01)
|
||||
|
||||
for modifier in reversed(modifiers):
|
||||
print(f" Relâchement modificateur: {modifier}")
|
||||
time.sleep(0.01)
|
||||
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Erreur exécution combinaison '{parsed_key['original']}': {e}")
|
||||
return False
|
||||
|
||||
def _screenshot_to_base64(self, screenshot) -> str:
|
||||
"""
|
||||
Convertit un screenshot en base64.
|
||||
|
||||
Args:
|
||||
screenshot: Image capturée
|
||||
|
||||
Returns:
|
||||
String base64 de l'image
|
||||
"""
|
||||
try:
|
||||
if not screenshot:
|
||||
return ""
|
||||
|
||||
import base64
|
||||
import io
|
||||
from PIL import Image
|
||||
|
||||
if hasattr(screenshot, 'save'):
|
||||
# PIL Image
|
||||
buffer = io.BytesIO()
|
||||
screenshot.save(buffer, format='PNG')
|
||||
return base64.b64encode(buffer.getvalue()).decode('utf-8')
|
||||
else:
|
||||
# Données brutes ou autre format
|
||||
return base64.b64encode(str(screenshot).encode()).decode('utf-8')
|
||||
|
||||
except Exception as e:
|
||||
print(f"⚠️ Erreur conversion base64: {e}")
|
||||
return ""
|
||||
@@ -0,0 +1,351 @@
|
||||
"""
|
||||
Action VWB - Capture d'Écran Evidence
|
||||
|
||||
Auteur : Dom, Alice, Kiro - 10 janvier 2026
|
||||
|
||||
Cette action capture l'écran pour créer une preuve visuelle (Evidence).
|
||||
"""
|
||||
|
||||
from typing import Dict, Any, Optional
|
||||
from datetime import datetime
|
||||
import time
|
||||
|
||||
from ..base_action import BaseVWBAction, VWBActionResult, VWBActionStatus
|
||||
from ...contracts.error import VWBErrorType, VWBErrorSeverity, create_vwb_error
|
||||
from ...contracts.evidence import VWBEvidence, VWBEvidenceType
|
||||
|
||||
|
||||
class VWBScreenshotEvidenceAction(BaseVWBAction):
|
||||
"""
|
||||
Action pour capturer l'écran et créer une Evidence.
|
||||
|
||||
Cette action capture l'état actuel de l'écran pour documenter
|
||||
une étape du workflow ou créer une preuve visuelle.
|
||||
"""
|
||||
|
||||
def __init__(self, action_id: str, parameters: Dict[str, Any], screen_capturer=None):
|
||||
"""
|
||||
Initialise l'action de capture d'écran.
|
||||
|
||||
Args:
|
||||
action_id: Identifiant unique de l'action
|
||||
parameters: Paramètres de configuration
|
||||
screen_capturer: Instance du ScreenCapturer
|
||||
"""
|
||||
super().__init__(action_id, parameters, screen_capturer)
|
||||
|
||||
# Paramètres spécifiques
|
||||
self.capture_region = parameters.get('capture_region') # None = écran complet
|
||||
self.evidence_title = parameters.get('evidence_title', 'Capture d\'écran')
|
||||
self.evidence_description = parameters.get('evidence_description', '')
|
||||
self.include_timestamp = parameters.get('include_timestamp', True)
|
||||
self.include_cursor = parameters.get('include_cursor', False)
|
||||
self.delay_before_capture_ms = parameters.get('delay_before_capture_ms', 0)
|
||||
self.quality = parameters.get('quality', 'high') # low, medium, high
|
||||
self.format = parameters.get('format', 'png') # png, jpg
|
||||
|
||||
# Validation des paramètres
|
||||
self._validate_screenshot_parameters()
|
||||
|
||||
def _validate_screenshot_parameters(self):
|
||||
"""Valide les paramètres spécifiques à la capture d'écran."""
|
||||
errors = []
|
||||
|
||||
if self.capture_region and not isinstance(self.capture_region, dict):
|
||||
errors.append("capture_region doit être un dictionnaire avec x, y, width, height")
|
||||
|
||||
if self.capture_region:
|
||||
required_keys = ['x', 'y', 'width', 'height']
|
||||
for key in required_keys:
|
||||
if key not in self.capture_region:
|
||||
errors.append(f"capture_region manque la clé '{key}'")
|
||||
elif not isinstance(self.capture_region[key], (int, float)):
|
||||
errors.append(f"capture_region['{key}'] doit être un nombre")
|
||||
|
||||
if not isinstance(self.delay_before_capture_ms, (int, float)) or self.delay_before_capture_ms < 0:
|
||||
errors.append("delay_before_capture_ms doit être un nombre positif")
|
||||
|
||||
if self.quality not in ['low', 'medium', 'high']:
|
||||
errors.append("quality doit être 'low', 'medium' ou 'high'")
|
||||
|
||||
if self.format not in ['png', 'jpg', 'jpeg']:
|
||||
errors.append("format doit être 'png', 'jpg' ou 'jpeg'")
|
||||
|
||||
if errors:
|
||||
raise ValueError(f"Paramètres invalides pour ScreenshotEvidence: {'; '.join(errors)}")
|
||||
|
||||
def get_action_type(self) -> str:
|
||||
"""Retourne le type d'action."""
|
||||
return "screenshot_evidence"
|
||||
|
||||
def get_action_name(self) -> str:
|
||||
"""Retourne le nom de l'action."""
|
||||
return "Capture d'Écran Evidence"
|
||||
|
||||
def get_action_description(self) -> str:
|
||||
"""Retourne la description de l'action."""
|
||||
return "Capture l'écran pour créer une preuve visuelle (Evidence)"
|
||||
|
||||
def validate_parameters(self) -> list:
|
||||
"""
|
||||
Valide les paramètres de l'action.
|
||||
|
||||
Returns:
|
||||
Liste des erreurs de validation
|
||||
"""
|
||||
errors = []
|
||||
|
||||
if self.capture_region:
|
||||
region = self.capture_region
|
||||
if region.get('width', 0) <= 0 or region.get('height', 0) <= 0:
|
||||
errors.append("Région de capture invalide (largeur/hauteur <= 0)")
|
||||
|
||||
return errors
|
||||
|
||||
def _execute_action_logic(self, step_id: str, workflow_id: Optional[str] = None,
|
||||
user_id: Optional[str] = None) -> VWBActionResult:
|
||||
"""
|
||||
Logique principale d'exécution de la capture d'écran.
|
||||
|
||||
Args:
|
||||
step_id: Identifiant de l'étape
|
||||
workflow_id: Identifiant du workflow
|
||||
user_id: Identifiant de l'utilisateur
|
||||
|
||||
Returns:
|
||||
Résultat de l'exécution
|
||||
"""
|
||||
start_time = datetime.now()
|
||||
evidence_list = []
|
||||
|
||||
try:
|
||||
print(f"📸 Capture d'écran Evidence: {self.evidence_title}")
|
||||
|
||||
# Vérifier la disponibilité du ScreenCapturer
|
||||
if not self.screen_capturer:
|
||||
raise Exception("ScreenCapturer non disponible")
|
||||
|
||||
# Délai avant capture si spécifié
|
||||
if self.delay_before_capture_ms > 0:
|
||||
print(f" Attente de {self.delay_before_capture_ms}ms avant capture")
|
||||
time.sleep(self.delay_before_capture_ms / 1000.0)
|
||||
|
||||
# Effectuer la capture
|
||||
capture_start = time.time()
|
||||
|
||||
if self.capture_region:
|
||||
# Capture d'une région spécifique
|
||||
region = self.capture_region
|
||||
print(f" Capture région: {region['x']},{region['y']} {region['width']}x{region['height']}")
|
||||
screenshot = self._capture_region(region)
|
||||
else:
|
||||
# Capture écran complet
|
||||
print(" Capture écran complet")
|
||||
screenshot = self.screen_capturer.capture()
|
||||
|
||||
capture_duration = (time.time() - capture_start) * 1000
|
||||
|
||||
if not screenshot:
|
||||
raise Exception("Échec de la capture d'écran")
|
||||
|
||||
# Obtenir les informations sur la capture
|
||||
capture_info = self._get_capture_info(screenshot)
|
||||
|
||||
# Créer la description de l'Evidence
|
||||
description = self.evidence_description or self.evidence_title
|
||||
if self.include_timestamp:
|
||||
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
description += f" (capturé le {timestamp})"
|
||||
|
||||
# Evidence de succès
|
||||
evidence = VWBEvidence(
|
||||
evidence_type=VWBEvidenceType.SCREENSHOT,
|
||||
description=description,
|
||||
screenshot_base64=self._screenshot_to_base64(screenshot),
|
||||
success=True,
|
||||
execution_time_ms=(datetime.now() - start_time).total_seconds() * 1000,
|
||||
metadata={
|
||||
'evidence_title': self.evidence_title,
|
||||
'capture_region': self.capture_region,
|
||||
'capture_duration_ms': capture_duration,
|
||||
'quality': self.quality,
|
||||
'format': self.format,
|
||||
'include_cursor': self.include_cursor,
|
||||
'capture_info': capture_info,
|
||||
'screen_resolution': capture_info.get('resolution', {}),
|
||||
'file_size_bytes': capture_info.get('size_bytes', 0)
|
||||
}
|
||||
)
|
||||
evidence_list.append(evidence)
|
||||
|
||||
# Données de sortie
|
||||
output_data = {
|
||||
'capture_success': True,
|
||||
'evidence_title': self.evidence_title,
|
||||
'capture_duration_ms': capture_duration,
|
||||
'capture_info': capture_info,
|
||||
'evidence_id': evidence.evidence_id
|
||||
}
|
||||
|
||||
print(f"✅ Capture d'écran réussie ({capture_info.get('resolution', {}).get('width', '?')}x{capture_info.get('resolution', {}).get('height', '?')})")
|
||||
|
||||
return VWBActionResult(
|
||||
action_id=self.action_id,
|
||||
step_id=step_id,
|
||||
status=VWBActionStatus.SUCCESS,
|
||||
start_time=start_time,
|
||||
end_time=datetime.now(),
|
||||
output_data=output_data,
|
||||
evidence_list=evidence_list,
|
||||
workflow_id=workflow_id,
|
||||
user_id=user_id
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
# Gestion des erreurs
|
||||
error = create_vwb_error(
|
||||
error_type=VWBErrorType.EXECUTION_ERROR,
|
||||
message=f"Erreur lors de la capture d'écran: {str(e)}",
|
||||
severity=VWBErrorSeverity.MEDIUM,
|
||||
retryable=True,
|
||||
details={'exception': str(e)}
|
||||
)
|
||||
|
||||
return VWBActionResult(
|
||||
action_id=self.action_id,
|
||||
step_id=step_id,
|
||||
status=VWBActionStatus.FAILED,
|
||||
start_time=start_time,
|
||||
end_time=datetime.now(),
|
||||
output_data={},
|
||||
evidence_list=evidence_list,
|
||||
error=error,
|
||||
workflow_id=workflow_id,
|
||||
user_id=user_id
|
||||
)
|
||||
|
||||
def _capture_region(self, region: Dict[str, int]):
|
||||
"""
|
||||
Capture une région spécifique de l'écran.
|
||||
|
||||
Args:
|
||||
region: Dictionnaire avec x, y, width, height
|
||||
|
||||
Returns:
|
||||
Image capturée ou None
|
||||
"""
|
||||
try:
|
||||
# En production, utiliser mss ou pyautogui pour capturer une région
|
||||
# Pour l'instant, simulation avec capture complète
|
||||
full_screenshot = self.screen_capturer.capture()
|
||||
|
||||
if not full_screenshot:
|
||||
return None
|
||||
|
||||
# Simuler le crop de la région
|
||||
# En production: full_screenshot.crop((x, y, x+width, y+height))
|
||||
print(f" Région simulée: {region}")
|
||||
return full_screenshot
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Erreur capture région: {e}")
|
||||
return None
|
||||
|
||||
def _get_capture_info(self, screenshot) -> Dict[str, Any]:
|
||||
"""
|
||||
Obtient les informations sur la capture.
|
||||
|
||||
Args:
|
||||
screenshot: Image capturée
|
||||
|
||||
Returns:
|
||||
Informations sur la capture
|
||||
"""
|
||||
try:
|
||||
info = {
|
||||
'timestamp': datetime.now().isoformat(),
|
||||
'quality': self.quality,
|
||||
'format': self.format
|
||||
}
|
||||
|
||||
# Tenter d'obtenir les dimensions
|
||||
if hasattr(screenshot, 'size'):
|
||||
# PIL Image
|
||||
info['resolution'] = {
|
||||
'width': screenshot.size[0],
|
||||
'height': screenshot.size[1]
|
||||
}
|
||||
elif hasattr(screenshot, 'width') and hasattr(screenshot, 'height'):
|
||||
info['resolution'] = {
|
||||
'width': screenshot.width,
|
||||
'height': screenshot.height
|
||||
}
|
||||
else:
|
||||
# Valeurs par défaut
|
||||
info['resolution'] = {
|
||||
'width': 1920,
|
||||
'height': 1080
|
||||
}
|
||||
|
||||
# Estimer la taille du fichier
|
||||
width = info['resolution']['width']
|
||||
height = info['resolution']['height']
|
||||
|
||||
if self.format.lower() in ['jpg', 'jpeg']:
|
||||
# JPEG approximation
|
||||
info['size_bytes'] = int(width * height * 0.1) # Compression ~10:1
|
||||
else:
|
||||
# PNG approximation
|
||||
info['size_bytes'] = int(width * height * 3) # RGB sans compression
|
||||
|
||||
return info
|
||||
|
||||
except Exception as e:
|
||||
print(f"⚠️ Erreur obtention info capture: {e}")
|
||||
return {
|
||||
'timestamp': datetime.now().isoformat(),
|
||||
'quality': self.quality,
|
||||
'format': self.format,
|
||||
'resolution': {'width': 1920, 'height': 1080},
|
||||
'size_bytes': 0
|
||||
}
|
||||
|
||||
def _screenshot_to_base64(self, screenshot) -> str:
|
||||
"""
|
||||
Convertit un screenshot en base64.
|
||||
|
||||
Args:
|
||||
screenshot: Image capturée
|
||||
|
||||
Returns:
|
||||
String base64 de l'image
|
||||
"""
|
||||
try:
|
||||
if not screenshot:
|
||||
return ""
|
||||
|
||||
import base64
|
||||
import io
|
||||
from PIL import Image
|
||||
|
||||
if hasattr(screenshot, 'save'):
|
||||
# PIL Image
|
||||
buffer = io.BytesIO()
|
||||
|
||||
# Ajuster la qualité selon le paramètre
|
||||
if self.format.lower() in ['jpg', 'jpeg']:
|
||||
quality_map = {'low': 60, 'medium': 80, 'high': 95}
|
||||
quality_value = quality_map.get(self.quality, 80)
|
||||
screenshot.save(buffer, format='JPEG', quality=quality_value)
|
||||
else:
|
||||
# PNG - pas de paramètre qualité
|
||||
screenshot.save(buffer, format='PNG')
|
||||
|
||||
return base64.b64encode(buffer.getvalue()).decode('utf-8')
|
||||
else:
|
||||
# Données brutes ou autre format
|
||||
return base64.b64encode(str(screenshot).encode()).decode('utf-8')
|
||||
|
||||
except Exception as e:
|
||||
print(f"⚠️ Erreur conversion base64: {e}")
|
||||
return ""
|
||||
@@ -0,0 +1,554 @@
|
||||
"""
|
||||
Action VWB Scroll To Anchor - Faire défiler jusqu'à un élément visuel
|
||||
|
||||
Auteur : Dom, Alice, Kiro - 10 janvier 2026
|
||||
|
||||
Cette action permet de faire défiler la page ou une zone jusqu'à ce qu'un élément
|
||||
identifié par une ancre visuelle soit visible à l'écran.
|
||||
"""
|
||||
|
||||
from typing import Dict, Any, Optional, List
|
||||
from datetime import datetime
|
||||
import time
|
||||
import traceback
|
||||
|
||||
from ..base_action import BaseVWBAction, VWBActionResult, VWBActionStatus
|
||||
from ...contracts.error import VWBActionError, VWBErrorType, VWBErrorSeverity, create_vwb_error
|
||||
from ...contracts.evidence import VWBEvidence, VWBEvidenceType
|
||||
from ...contracts.visual_anchor import VWBVisualAnchor
|
||||
|
||||
|
||||
class VWBScrollToAnchorAction(BaseVWBAction):
|
||||
"""
|
||||
Action VWB pour faire défiler jusqu'à un élément identifié par une ancre visuelle.
|
||||
|
||||
Cette action recherche un élément et fait défiler la page ou une zone
|
||||
jusqu'à ce que l'élément soit visible et accessible.
|
||||
"""
|
||||
|
||||
def __init__(self, action_id: str, parameters: Dict[str, Any], screen_capturer=None):
|
||||
"""
|
||||
Initialise l'action ScrollToAnchor.
|
||||
|
||||
Args:
|
||||
action_id: Identifiant unique de l'action
|
||||
parameters: Paramètres de configuration
|
||||
screen_capturer: Instance du ScreenCapturer (Option A thread-safe)
|
||||
"""
|
||||
super().__init__(
|
||||
action_id=action_id,
|
||||
name="Défiler vers un Élément",
|
||||
description="Fait défiler la page jusqu'à ce qu'un élément soit visible",
|
||||
parameters=parameters,
|
||||
screen_capturer=screen_capturer
|
||||
)
|
||||
|
||||
# Paramètres spécifiques à ScrollToAnchor
|
||||
self.visual_anchor = parameters.get('visual_anchor')
|
||||
self.scroll_direction = parameters.get('scroll_direction', 'vertical') # vertical, horizontal, both
|
||||
self.scroll_speed = parameters.get('scroll_speed', 'medium') # slow, medium, fast
|
||||
self.max_scroll_attempts = parameters.get('max_scroll_attempts', 10)
|
||||
self.scroll_step_pixels = parameters.get('scroll_step_pixels', 200)
|
||||
self.wait_after_scroll_ms = parameters.get('wait_after_scroll_ms', 500)
|
||||
self.target_position = parameters.get('target_position', 'center') # top, center, bottom
|
||||
self.confidence_threshold = parameters.get('confidence_threshold', 0.8)
|
||||
|
||||
# Validation des paramètres
|
||||
validation_errors = self._validate_parameters()
|
||||
if validation_errors:
|
||||
print(f"⚠️ Erreurs de validation: {validation_errors}")
|
||||
|
||||
def _validate_parameters(self) -> List[str]:
|
||||
"""
|
||||
Valide les paramètres de l'action.
|
||||
|
||||
Returns:
|
||||
Liste des erreurs de validation
|
||||
"""
|
||||
errors = []
|
||||
|
||||
# Vérifier l'ancre visuelle
|
||||
if not self.visual_anchor:
|
||||
errors.append("Paramètre 'visual_anchor' requis")
|
||||
elif not isinstance(self.visual_anchor, (VWBVisualAnchor, dict)):
|
||||
errors.append("'visual_anchor' doit être un VWBVisualAnchor ou un dictionnaire")
|
||||
|
||||
# Vérifier la direction de défilement
|
||||
valid_directions = ['vertical', 'horizontal', 'both']
|
||||
if self.scroll_direction not in valid_directions:
|
||||
errors.append(f"'scroll_direction' doit être l'un de : {valid_directions}")
|
||||
|
||||
# Vérifier la vitesse de défilement
|
||||
valid_speeds = ['slow', 'medium', 'fast']
|
||||
if self.scroll_speed not in valid_speeds:
|
||||
errors.append(f"'scroll_speed' doit être l'un de : {valid_speeds}")
|
||||
|
||||
# Vérifier la position cible
|
||||
valid_positions = ['top', 'center', 'bottom']
|
||||
if self.target_position not in valid_positions:
|
||||
errors.append(f"'target_position' doit être l'un de : {valid_positions}")
|
||||
|
||||
# Vérifier les valeurs numériques
|
||||
if self.max_scroll_attempts <= 0:
|
||||
errors.append("'max_scroll_attempts' doit être positif")
|
||||
|
||||
if self.scroll_step_pixels <= 0:
|
||||
errors.append("'scroll_step_pixels' doit être positif")
|
||||
|
||||
if self.wait_after_scroll_ms < 0:
|
||||
errors.append("'wait_after_scroll_ms' doit être positif ou nul")
|
||||
|
||||
if not (0.0 <= self.confidence_threshold <= 1.0):
|
||||
errors.append("'confidence_threshold' doit être entre 0.0 et 1.0")
|
||||
|
||||
return errors
|
||||
|
||||
def validate_parameters(self) -> List[str]:
|
||||
"""
|
||||
Valide les paramètres de l'action.
|
||||
|
||||
Returns:
|
||||
Liste des erreurs de validation
|
||||
"""
|
||||
return self._validate_parameters()
|
||||
|
||||
def get_action_metadata(self) -> Dict[str, Any]:
|
||||
"""
|
||||
Retourne les métadonnées de l'action.
|
||||
|
||||
Returns:
|
||||
Dictionnaire des métadonnées
|
||||
"""
|
||||
return {
|
||||
"id": "scroll_to_anchor",
|
||||
"name": "Défiler vers un Élément",
|
||||
"description": "Fait défiler la page jusqu'à ce qu'un élément soit visible",
|
||||
"category": "vision_ui",
|
||||
"version": "1.0.0",
|
||||
"author": "Dom, Alice, Kiro",
|
||||
"created_date": "2026-01-10",
|
||||
"parameters": {
|
||||
"visual_anchor": {
|
||||
"type": "VWBVisualAnchor",
|
||||
"required": True,
|
||||
"description": "Ancre visuelle pour localiser l'élément cible"
|
||||
},
|
||||
"scroll_direction": {
|
||||
"type": "string",
|
||||
"required": False,
|
||||
"default": "vertical",
|
||||
"options": ["vertical", "horizontal", "both"],
|
||||
"description": "Direction du défilement"
|
||||
},
|
||||
"scroll_speed": {
|
||||
"type": "string",
|
||||
"required": False,
|
||||
"default": "medium",
|
||||
"options": ["slow", "medium", "fast"],
|
||||
"description": "Vitesse du défilement"
|
||||
},
|
||||
"max_scroll_attempts": {
|
||||
"type": "number",
|
||||
"required": False,
|
||||
"default": 10,
|
||||
"min": 1,
|
||||
"description": "Nombre maximum de tentatives de défilement"
|
||||
},
|
||||
"scroll_step_pixels": {
|
||||
"type": "number",
|
||||
"required": False,
|
||||
"default": 200,
|
||||
"min": 50,
|
||||
"description": "Nombre de pixels par étape de défilement"
|
||||
},
|
||||
"wait_after_scroll_ms": {
|
||||
"type": "number",
|
||||
"required": False,
|
||||
"default": 500,
|
||||
"min": 0,
|
||||
"description": "Délai d'attente après chaque défilement"
|
||||
},
|
||||
"target_position": {
|
||||
"type": "string",
|
||||
"required": False,
|
||||
"default": "center",
|
||||
"options": ["top", "center", "bottom"],
|
||||
"description": "Position cible de l'élément dans la vue"
|
||||
},
|
||||
"confidence_threshold": {
|
||||
"type": "number",
|
||||
"required": False,
|
||||
"default": 0.8,
|
||||
"min": 0.0,
|
||||
"max": 1.0,
|
||||
"description": "Seuil de confiance pour la détection"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"element_found": {
|
||||
"type": "boolean",
|
||||
"description": "Indique si l'élément a été trouvé et rendu visible"
|
||||
},
|
||||
"scroll_distance": {
|
||||
"type": "object",
|
||||
"description": "Distance totale de défilement (x, y)"
|
||||
},
|
||||
"final_coordinates": {
|
||||
"type": "object",
|
||||
"description": "Coordonnées finales de l'élément"
|
||||
},
|
||||
"scroll_attempts": {
|
||||
"type": "number",
|
||||
"description": "Nombre de tentatives de défilement effectuées"
|
||||
}
|
||||
},
|
||||
"examples": [
|
||||
{
|
||||
"name": "Défiler vers un bouton",
|
||||
"description": "Fait défiler verticalement pour trouver un bouton",
|
||||
"parameters": {
|
||||
"scroll_direction": "vertical",
|
||||
"target_position": "center",
|
||||
"scroll_speed": "medium"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Défilement horizontal",
|
||||
"description": "Fait défiler horizontalement dans un carrousel",
|
||||
"parameters": {
|
||||
"scroll_direction": "horizontal",
|
||||
"scroll_step_pixels": 300,
|
||||
"max_scroll_attempts": 5
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
def execute_core(self, step_id: str) -> VWBActionResult:
|
||||
"""
|
||||
Exécute l'action de défilement vers l'ancre visuelle.
|
||||
|
||||
Args:
|
||||
step_id: Identifiant de l'étape
|
||||
|
||||
Returns:
|
||||
Résultat de l'exécution avec Evidence
|
||||
"""
|
||||
start_time = datetime.now()
|
||||
evidence_list = []
|
||||
|
||||
try:
|
||||
print(f"📜 Début ScrollToAnchor - Direction: {self.scroll_direction}")
|
||||
|
||||
# Validation des paramètres
|
||||
validation_errors = self._validate_parameters()
|
||||
if validation_errors:
|
||||
error = create_vwb_error(
|
||||
error_type=VWBErrorType.PARAMETER_INVALID,
|
||||
message=f"Paramètres invalides: {', '.join(validation_errors)}",
|
||||
severity=VWBErrorSeverity.HIGH,
|
||||
retryable=False,
|
||||
details={"validation_errors": validation_errors}
|
||||
)
|
||||
return self._create_error_result_simple(start_time, step_id, error)
|
||||
|
||||
# Convertir l'ancre visuelle si nécessaire
|
||||
if isinstance(self.visual_anchor, dict):
|
||||
visual_anchor = VWBVisualAnchor.from_dict(self.visual_anchor)
|
||||
else:
|
||||
visual_anchor = self.visual_anchor
|
||||
|
||||
# Vérifier la disponibilité du ScreenCapturer
|
||||
if not self.screen_capturer:
|
||||
error = create_vwb_error(
|
||||
error_type=VWBErrorType.SCREEN_CAPTURE_FAILED,
|
||||
message="ScreenCapturer non disponible",
|
||||
severity=VWBErrorSeverity.HIGH,
|
||||
retryable=False
|
||||
)
|
||||
return self._create_error_result_simple(start_time, step_id, error)
|
||||
|
||||
# Capture d'écran initiale
|
||||
initial_screenshot = self._capture_screen_safe()
|
||||
if not initial_screenshot:
|
||||
error = create_vwb_error(
|
||||
error_type=VWBErrorType.SCREEN_CAPTURE_FAILED,
|
||||
message="Impossible de capturer l'écran initial",
|
||||
severity=VWBErrorSeverity.HIGH,
|
||||
retryable=True
|
||||
)
|
||||
return self._create_error_result_simple(start_time, step_id, error)
|
||||
|
||||
# Vérifier si l'élément est déjà visible
|
||||
print(f"🔍 Vérification initiale de '{visual_anchor.label}'")
|
||||
element_found, element_coords, confidence = self._find_visual_element(
|
||||
initial_screenshot, visual_anchor, self.confidence_threshold
|
||||
)
|
||||
|
||||
total_scroll_x = 0
|
||||
total_scroll_y = 0
|
||||
scroll_attempts = 0
|
||||
|
||||
# Si l'élément n'est pas trouvé, commencer le défilement
|
||||
if not element_found:
|
||||
print(f"🔄 Élément non visible, début du défilement...")
|
||||
|
||||
# Calculer les paramètres de défilement
|
||||
scroll_delays = {'slow': 800, 'medium': 500, 'fast': 200}
|
||||
scroll_delay = scroll_delays.get(self.scroll_speed, 500)
|
||||
|
||||
# Boucle de défilement
|
||||
for attempt in range(self.max_scroll_attempts):
|
||||
scroll_attempts += 1
|
||||
print(f" Tentative {attempt + 1}/{self.max_scroll_attempts}")
|
||||
|
||||
# Effectuer le défilement
|
||||
scroll_x, scroll_y = self._perform_scroll_step()
|
||||
total_scroll_x += scroll_x
|
||||
total_scroll_y += scroll_y
|
||||
|
||||
# Attendre que le défilement soit effectif
|
||||
time.sleep(self.wait_after_scroll_ms / 1000.0)
|
||||
|
||||
# Nouvelle capture d'écran
|
||||
current_screenshot = self._capture_screen_safe()
|
||||
if not current_screenshot:
|
||||
continue
|
||||
|
||||
# Rechercher l'élément dans la nouvelle vue
|
||||
element_found, element_coords, confidence = self._find_visual_element(
|
||||
current_screenshot, visual_anchor, self.confidence_threshold
|
||||
)
|
||||
|
||||
if element_found:
|
||||
print(f"✅ Élément trouvé après {attempt + 1} tentatives!")
|
||||
break
|
||||
|
||||
# Attendre avant la prochaine tentative
|
||||
time.sleep(scroll_delay / 1000.0)
|
||||
|
||||
# Vérifier le résultat final
|
||||
if not element_found:
|
||||
error = create_vwb_error(
|
||||
error_type=VWBErrorType.ELEMENT_NOT_FOUND,
|
||||
message=f"Élément '{visual_anchor.label}' non trouvé après {scroll_attempts} tentatives de défilement",
|
||||
severity=VWBErrorSeverity.MEDIUM,
|
||||
retryable=True,
|
||||
details={
|
||||
"anchor_label": visual_anchor.label,
|
||||
"scroll_attempts": scroll_attempts,
|
||||
"total_scroll_distance": {"x": total_scroll_x, "y": total_scroll_y},
|
||||
"confidence_threshold": self.confidence_threshold
|
||||
}
|
||||
)
|
||||
|
||||
# Evidence d'échec
|
||||
final_screenshot = self._capture_screen_safe()
|
||||
evidence = VWBEvidence(
|
||||
evidence_type=VWBEvidenceType.SCREENSHOT,
|
||||
description=f"Élément '{visual_anchor.label}' non trouvé après défilement",
|
||||
screenshot_base64=self._encode_screenshot(final_screenshot) if final_screenshot else "",
|
||||
confidence_score=confidence,
|
||||
interaction_details={
|
||||
"scroll_attempts": scroll_attempts,
|
||||
"total_scroll_distance": {"x": total_scroll_x, "y": total_scroll_y}
|
||||
},
|
||||
success=False
|
||||
)
|
||||
evidence_list.append(evidence)
|
||||
|
||||
return self._create_error_result_simple(start_time, step_id, error, evidence_list)
|
||||
|
||||
# Ajustement de position si nécessaire
|
||||
if self.target_position != 'current':
|
||||
adjustment_success = self._adjust_element_position(element_coords)
|
||||
if adjustment_success:
|
||||
print(f"📍 Élément ajusté à la position '{self.target_position}'")
|
||||
|
||||
# Capture d'écran finale
|
||||
final_screenshot = self._capture_screen_safe()
|
||||
|
||||
# Evidence de succès
|
||||
evidence = VWBEvidence(
|
||||
evidence_type=VWBEvidenceType.UI_INTERACTION,
|
||||
description=f"Défilement réussi vers '{visual_anchor.label}'",
|
||||
screenshot_base64=self._encode_screenshot(final_screenshot) if final_screenshot else "",
|
||||
element_coordinates=element_coords,
|
||||
confidence_score=confidence,
|
||||
interaction_details={
|
||||
"scroll_direction": self.scroll_direction,
|
||||
"scroll_attempts": scroll_attempts,
|
||||
"total_scroll_distance": {"x": total_scroll_x, "y": total_scroll_y},
|
||||
"target_position": self.target_position,
|
||||
"scroll_speed": self.scroll_speed
|
||||
},
|
||||
success=True
|
||||
)
|
||||
evidence_list.append(evidence)
|
||||
|
||||
# Données de sortie
|
||||
output_data = {
|
||||
"element_found": True,
|
||||
"scroll_distance": {"x": total_scroll_x, "y": total_scroll_y},
|
||||
"final_coordinates": element_coords,
|
||||
"scroll_attempts": scroll_attempts,
|
||||
"confidence_score": confidence
|
||||
}
|
||||
|
||||
end_time = datetime.now()
|
||||
execution_time = (end_time - start_time).total_seconds() * 1000
|
||||
|
||||
print(f"✅ ScrollToAnchor réussie en {execution_time:.1f}ms")
|
||||
|
||||
return VWBActionResult(
|
||||
action_id=self.action_id,
|
||||
step_id=step_id,
|
||||
status=VWBActionStatus.SUCCESS,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
execution_time_ms=execution_time,
|
||||
output_data=output_data,
|
||||
evidence_list=evidence_list
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Erreur ScrollToAnchor: {e}")
|
||||
error = create_vwb_error(
|
||||
error_type=VWBErrorType.SYSTEM_ERROR,
|
||||
message=f"Erreur inattendue lors du défilement: {str(e)}",
|
||||
severity=VWBErrorSeverity.HIGH,
|
||||
retryable=True,
|
||||
details={"exception": str(e), "traceback": traceback.format_exc()}
|
||||
)
|
||||
return self._create_error_result_simple(start_time, step_id, error, evidence_list)
|
||||
|
||||
def _create_error_result_simple(self, start_time: datetime, step_id: str, error: VWBActionError, evidence_list: List[VWBEvidence] = None) -> VWBActionResult:
|
||||
"""Crée un résultat d'erreur simplifié."""
|
||||
end_time = datetime.now()
|
||||
execution_time = (end_time - start_time).total_seconds() * 1000
|
||||
|
||||
return VWBActionResult(
|
||||
action_id=self.action_id,
|
||||
step_id=step_id,
|
||||
status=VWBActionStatus.FAILED,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
execution_time_ms=execution_time,
|
||||
output_data={},
|
||||
evidence_list=evidence_list or [],
|
||||
error=error
|
||||
)
|
||||
|
||||
def _capture_screen_safe(self):
|
||||
"""Capture d'écran sécurisée avec gestion d'erreur."""
|
||||
try:
|
||||
if self.screen_capturer:
|
||||
return self.screen_capturer.capture()
|
||||
except Exception as e:
|
||||
print(f"⚠️ Erreur capture d'écran: {e}")
|
||||
return None
|
||||
|
||||
def _find_visual_element(self, screenshot, visual_anchor, threshold):
|
||||
"""Simulation de recherche d'élément visuel."""
|
||||
import random
|
||||
confidence = random.uniform(0.6, 0.95)
|
||||
|
||||
if confidence >= threshold:
|
||||
return True, {'x': 400, 'y': 300, 'width': 200, 'height': 50}, confidence
|
||||
else:
|
||||
return False, {}, confidence
|
||||
|
||||
def _encode_screenshot(self, screenshot_data) -> str:
|
||||
"""Encode un screenshot en base64."""
|
||||
try:
|
||||
import base64
|
||||
return base64.b64encode(str(screenshot_data).encode()).decode('utf-8')
|
||||
except:
|
||||
return ""
|
||||
|
||||
def execute(self, step_id: str = None, workflow_id: str = None, user_id: str = None) -> VWBActionResult:
|
||||
"""
|
||||
Exécute l'action de défilement vers l'ancre visuelle (méthode héritée).
|
||||
|
||||
Args:
|
||||
step_id: Identifiant de l'étape
|
||||
workflow_id: Identifiant du workflow
|
||||
user_id: Identifiant de l'utilisateur
|
||||
|
||||
Returns:
|
||||
Résultat de l'exécution avec Evidence
|
||||
"""
|
||||
# Déléguer à execute_core qui est la méthode abstraite requise
|
||||
return self.execute_core(step_id or f"step_{datetime.now().strftime('%Y%m%d_%H%M%S')}")
|
||||
|
||||
def _perform_scroll_step(self) -> tuple[int, int]:
|
||||
"""
|
||||
Effectue une étape de défilement.
|
||||
|
||||
Returns:
|
||||
Tuple (pixels_x, pixels_y) défilés
|
||||
"""
|
||||
scroll_x = 0
|
||||
scroll_y = 0
|
||||
|
||||
try:
|
||||
if self.scroll_direction in ['vertical', 'both']:
|
||||
# Défilement vertical vers le bas
|
||||
scroll_y = self.scroll_step_pixels
|
||||
print(f" ⬇️ Défilement vertical: {scroll_y}px")
|
||||
# En réalité: pyautogui.scroll(-scroll_y)
|
||||
|
||||
if self.scroll_direction in ['horizontal', 'both']:
|
||||
# Défilement horizontal vers la droite
|
||||
scroll_x = self.scroll_step_pixels
|
||||
print(f" ➡️ Défilement horizontal: {scroll_x}px")
|
||||
# En réalité: pyautogui.hscroll(scroll_x)
|
||||
|
||||
# Simuler le délai de défilement
|
||||
time.sleep(0.1)
|
||||
|
||||
except Exception as e:
|
||||
print(f"⚠️ Erreur lors du défilement: {e}")
|
||||
|
||||
return scroll_x, scroll_y
|
||||
|
||||
def _adjust_element_position(self, element_coords: Dict[str, int]) -> bool:
|
||||
"""
|
||||
Ajuste la position de l'élément selon la position cible.
|
||||
|
||||
Args:
|
||||
element_coords: Coordonnées actuelles de l'élément
|
||||
|
||||
Returns:
|
||||
True si l'ajustement a réussi
|
||||
"""
|
||||
try:
|
||||
# Obtenir les dimensions de l'écran
|
||||
# En réalité, on utiliserait pyautogui.size()
|
||||
screen_width, screen_height = 1920, 1080 # Valeurs par défaut
|
||||
|
||||
element_center_y = element_coords['y'] + element_coords['height'] // 2
|
||||
|
||||
# Calculer la position cible
|
||||
if self.target_position == 'top':
|
||||
target_y = screen_height * 0.2 # 20% du haut
|
||||
elif self.target_position == 'center':
|
||||
target_y = screen_height * 0.5 # Centre
|
||||
elif self.target_position == 'bottom':
|
||||
target_y = screen_height * 0.8 # 80% du haut
|
||||
else:
|
||||
return True # Pas d'ajustement nécessaire
|
||||
|
||||
# Calculer le défilement nécessaire
|
||||
adjustment_pixels = int(element_center_y - target_y)
|
||||
|
||||
if abs(adjustment_pixels) > 50: # Seuil minimum d'ajustement
|
||||
print(f"🎯 Ajustement de position: {adjustment_pixels}px")
|
||||
# En réalité: pyautogui.scroll(-adjustment_pixels // 10)
|
||||
time.sleep(0.2)
|
||||
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
print(f"⚠️ Erreur ajustement position: {e}")
|
||||
return False
|
||||
428
visual_workflow_builder/backend/actions/vision_ui/type_secret.py
Normal file
428
visual_workflow_builder/backend/actions/vision_ui/type_secret.py
Normal file
@@ -0,0 +1,428 @@
|
||||
"""
|
||||
Action VWB - Saisie de Secret
|
||||
|
||||
Auteur : Dom, Alice, Kiro - 10 janvier 2026
|
||||
|
||||
Cette action saisit un mot de passe ou secret dans un champ identifié par une ancre visuelle.
|
||||
Inclut des mesures de sécurité pour éviter la journalisation des secrets.
|
||||
"""
|
||||
|
||||
from typing import Dict, Any, Optional
|
||||
from datetime import datetime
|
||||
import time
|
||||
import re
|
||||
|
||||
from ..base_action import BaseVWBAction, VWBActionResult, VWBActionStatus
|
||||
from ...contracts.error import VWBErrorType, VWBErrorSeverity, create_vwb_error
|
||||
from ...contracts.evidence import VWBEvidence, VWBEvidenceType
|
||||
from ...contracts.visual_anchor import VWBVisualAnchor
|
||||
|
||||
|
||||
class VWBTypeSecretAction(BaseVWBAction):
|
||||
"""
|
||||
Action pour saisir un secret (mot de passe) dans un champ UI.
|
||||
|
||||
Cette action localise un champ de saisie et y tape un secret de manière sécurisée,
|
||||
en évitant la journalisation du contenu sensible.
|
||||
"""
|
||||
|
||||
def __init__(self, action_id: str, parameters: Dict[str, Any], screen_capturer=None):
|
||||
"""
|
||||
Initialise l'action de saisie de secret.
|
||||
|
||||
Args:
|
||||
action_id: Identifiant unique de l'action
|
||||
parameters: Paramètres de configuration
|
||||
screen_capturer: Instance du ScreenCapturer
|
||||
"""
|
||||
super().__init__(
|
||||
action_id=action_id,
|
||||
name="Saisie de Secret",
|
||||
description="Saisit un mot de passe ou secret dans un champ identifié par une ancre visuelle",
|
||||
parameters=parameters,
|
||||
screen_capturer=screen_capturer
|
||||
)
|
||||
|
||||
# Paramètres spécifiques
|
||||
self.visual_anchor = parameters.get('visual_anchor')
|
||||
self.secret_value = parameters.get('secret_value', '')
|
||||
self.secret_ref = parameters.get('secret_ref') # Référence sécurisée
|
||||
self.clear_field_first = parameters.get('clear_field_first', True)
|
||||
self.click_before_typing = parameters.get('click_before_typing', True)
|
||||
self.press_enter_after = parameters.get('press_enter_after', False)
|
||||
self.typing_speed_ms = parameters.get('typing_speed_ms', 30) # Plus rapide pour secrets
|
||||
self.confidence_threshold = parameters.get('confidence_threshold', 0.8)
|
||||
self.mask_in_evidence = parameters.get('mask_in_evidence', True)
|
||||
|
||||
# Validation des paramètres
|
||||
validation_errors = self.validate_parameters()
|
||||
if validation_errors:
|
||||
print(f"⚠️ Erreurs de validation: {validation_errors}")
|
||||
|
||||
# Validation des paramètres spécifiques
|
||||
secret_errors = self._validate_secret_parameters()
|
||||
if secret_errors:
|
||||
print(f"⚠️ Erreurs de validation secret: {secret_errors}")
|
||||
|
||||
def _validate_secret_parameters(self):
|
||||
"""Valide les paramètres spécifiques à la saisie de secret."""
|
||||
errors = []
|
||||
|
||||
if not isinstance(self.visual_anchor, VWBVisualAnchor):
|
||||
errors.append("visual_anchor doit être une instance de VWBVisualAnchor")
|
||||
|
||||
if not self.secret_value and not self.secret_ref:
|
||||
errors.append("secret_value ou secret_ref doit être fourni")
|
||||
|
||||
if self.secret_value and len(self.secret_value.strip()) == 0:
|
||||
errors.append("secret_value ne peut pas être vide")
|
||||
|
||||
if not isinstance(self.typing_speed_ms, (int, float)) or self.typing_speed_ms < 0:
|
||||
errors.append("typing_speed_ms doit être un nombre positif")
|
||||
|
||||
if not isinstance(self.confidence_threshold, (int, float)) or not 0 <= self.confidence_threshold <= 1:
|
||||
errors.append("confidence_threshold doit être entre 0 et 1")
|
||||
|
||||
if errors:
|
||||
print(f"⚠️ Erreurs de validation secret: {errors}")
|
||||
|
||||
return errors
|
||||
|
||||
def get_action_type(self) -> str:
|
||||
"""Retourne le type d'action."""
|
||||
return "type_secret"
|
||||
|
||||
def get_action_name(self) -> str:
|
||||
"""Retourne le nom de l'action."""
|
||||
return "Saisie de Secret"
|
||||
|
||||
def get_action_description(self) -> str:
|
||||
"""Retourne la description de l'action."""
|
||||
return "Saisit un mot de passe ou secret dans un champ identifié par une ancre visuelle"
|
||||
|
||||
def validate_parameters(self) -> list:
|
||||
"""
|
||||
Valide les paramètres de l'action.
|
||||
|
||||
Returns:
|
||||
Liste des erreurs de validation
|
||||
"""
|
||||
errors = []
|
||||
|
||||
if not self.visual_anchor:
|
||||
errors.append("Paramètre 'visual_anchor' requis")
|
||||
|
||||
if not self.secret_value and not self.secret_ref:
|
||||
errors.append("Paramètre 'secret_value' ou 'secret_ref' requis")
|
||||
|
||||
if self.confidence_threshold < 0.5:
|
||||
errors.append("Seuil de confiance trop faible (< 0.5)")
|
||||
|
||||
return errors
|
||||
|
||||
def get_action_metadata(self) -> Dict[str, Any]:
|
||||
"""
|
||||
Retourne les métadonnées de l'action.
|
||||
|
||||
Returns:
|
||||
Dictionnaire des métadonnées
|
||||
"""
|
||||
return {
|
||||
"id": "type_secret",
|
||||
"name": "Saisie de Secret",
|
||||
"description": "Saisit un mot de passe ou secret dans un champ identifié par une ancre visuelle",
|
||||
"category": "vision_ui",
|
||||
"version": "1.0.0",
|
||||
"author": "Dom, Alice, Kiro",
|
||||
"created_date": "2026-01-10",
|
||||
"parameters": {
|
||||
"visual_anchor": {
|
||||
"type": "VWBVisualAnchor",
|
||||
"required": True,
|
||||
"description": "Ancre visuelle pour localiser le champ de saisie"
|
||||
},
|
||||
"secret_value": {
|
||||
"type": "string",
|
||||
"required": False,
|
||||
"description": "Valeur du secret à saisir (sensible)"
|
||||
},
|
||||
"secret_ref": {
|
||||
"type": "string",
|
||||
"required": False,
|
||||
"description": "Référence sécurisée vers le secret"
|
||||
},
|
||||
"clear_field_first": {
|
||||
"type": "boolean",
|
||||
"required": False,
|
||||
"default": True,
|
||||
"description": "Vider le champ avant la saisie"
|
||||
},
|
||||
"press_enter_after": {
|
||||
"type": "boolean",
|
||||
"required": False,
|
||||
"default": False,
|
||||
"description": "Appuyer sur Entrée après la saisie"
|
||||
},
|
||||
"confidence_threshold": {
|
||||
"type": "number",
|
||||
"required": False,
|
||||
"default": 0.8,
|
||||
"min": 0.0,
|
||||
"max": 1.0,
|
||||
"description": "Seuil de confiance pour la détection"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"typing_success": {
|
||||
"type": "boolean",
|
||||
"description": "Indique si la saisie a réussi"
|
||||
},
|
||||
"secret_length": {
|
||||
"type": "number",
|
||||
"description": "Longueur du secret saisi"
|
||||
},
|
||||
"masked_secret": {
|
||||
"type": "string",
|
||||
"description": "Version masquée du secret pour les logs"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def _get_secret_to_type(self) -> str:
|
||||
"""
|
||||
Récupère le secret à saisir de manière sécurisée.
|
||||
|
||||
Returns:
|
||||
Le secret à saisir
|
||||
"""
|
||||
if self.secret_ref:
|
||||
# Récupération depuis une référence sécurisée
|
||||
# TODO: Implémenter la récupération depuis un coffre-fort
|
||||
print("🔐 Récupération du secret depuis la référence sécurisée")
|
||||
return self.secret_ref # Placeholder
|
||||
else:
|
||||
return self.secret_value
|
||||
|
||||
def _mask_secret(self, secret: str) -> str:
|
||||
"""
|
||||
Masque un secret pour la journalisation.
|
||||
|
||||
Args:
|
||||
secret: Secret à masquer
|
||||
|
||||
Returns:
|
||||
Secret masqué
|
||||
"""
|
||||
if not secret:
|
||||
return ""
|
||||
|
||||
if len(secret) <= 2:
|
||||
return "*" * len(secret)
|
||||
elif len(secret) <= 6:
|
||||
return secret[0] + "*" * (len(secret) - 2) + secret[-1]
|
||||
else:
|
||||
return secret[:2] + "*" * (len(secret) - 4) + secret[-2:]
|
||||
|
||||
def execute_core(self, step_id: str) -> VWBActionResult:
|
||||
"""
|
||||
Logique principale d'exécution de la saisie de secret.
|
||||
|
||||
Args:
|
||||
step_id: Identifiant de l'étape
|
||||
|
||||
Returns:
|
||||
Résultat de l'exécution
|
||||
"""
|
||||
start_time = datetime.now()
|
||||
evidence_list = []
|
||||
|
||||
try:
|
||||
# Récupérer le secret de manière sécurisée
|
||||
secret_to_type = self._get_secret_to_type()
|
||||
masked_secret = self._mask_secret(secret_to_type)
|
||||
|
||||
print(f"🔐 Saisie de secret dans '{self.visual_anchor.label}' (longueur: {len(secret_to_type)})")
|
||||
|
||||
# Capture d'écran initiale
|
||||
if not self.screen_capturer:
|
||||
raise Exception("ScreenCapturer non disponible")
|
||||
|
||||
screenshot = self.screen_capturer.capture()
|
||||
if not screenshot:
|
||||
raise Exception("Impossible de capturer l'écran")
|
||||
|
||||
# Recherche de l'ancre visuelle
|
||||
match_found = False
|
||||
best_match = None
|
||||
|
||||
# Simulation de recherche d'ancre (à remplacer par vraie implémentation)
|
||||
import random
|
||||
confidence = random.uniform(0.7, 0.95)
|
||||
|
||||
if confidence >= self.confidence_threshold:
|
||||
match_found = True
|
||||
best_match = {
|
||||
'confidence': confidence,
|
||||
'bbox': {'x': 300, 'y': 250, 'width': 200, 'height': 30},
|
||||
'center': {'x': 400, 'y': 265}
|
||||
}
|
||||
|
||||
if not match_found:
|
||||
# Ancre non trouvée
|
||||
error = create_vwb_error(
|
||||
error_type=VWBErrorType.ANCHOR_NOT_FOUND,
|
||||
message=f"Champ secret '{self.visual_anchor.label}' non trouvé",
|
||||
severity=VWBErrorSeverity.HIGH,
|
||||
retryable=True,
|
||||
details={
|
||||
'anchor_label': self.visual_anchor.label,
|
||||
'confidence_threshold': self.confidence_threshold
|
||||
}
|
||||
)
|
||||
|
||||
# Evidence d'échec (sans révéler le secret)
|
||||
evidence = VWBEvidence(
|
||||
evidence_type=VWBEvidenceType.SCREENSHOT,
|
||||
description=f"Échec saisie secret - champ non trouvé",
|
||||
screenshot_base64=self._screenshot_to_base64(screenshot),
|
||||
success=False,
|
||||
confidence_score=0.0,
|
||||
execution_time_ms=(datetime.now() - start_time).total_seconds() * 1000
|
||||
)
|
||||
evidence_list.append(evidence)
|
||||
|
||||
return VWBActionResult(
|
||||
action_id=self.action_id,
|
||||
step_id=step_id,
|
||||
status=VWBActionStatus.FAILED,
|
||||
start_time=start_time,
|
||||
end_time=datetime.now(),
|
||||
execution_time_ms=(datetime.now() - start_time).total_seconds() * 1000,
|
||||
output_data={},
|
||||
evidence_list=evidence_list,
|
||||
error=error
|
||||
)
|
||||
|
||||
# Cliquer sur le champ si demandé
|
||||
if self.click_before_typing:
|
||||
print(f" Clic sur le champ à ({best_match['center']['x']}, {best_match['center']['y']})")
|
||||
time.sleep(0.1)
|
||||
|
||||
# Vider le champ si demandé
|
||||
if self.clear_field_first:
|
||||
print(" Vidage du champ existant")
|
||||
time.sleep(0.1)
|
||||
|
||||
# Saisir le secret
|
||||
print(f" Saisie du secret ({masked_secret}) à {self.typing_speed_ms}ms/char")
|
||||
typing_start = time.time()
|
||||
|
||||
# Simulation de la saisie caractère par caractère
|
||||
for i, char in enumerate(secret_to_type):
|
||||
time.sleep(self.typing_speed_ms / 1000.0)
|
||||
# En production, utiliser pyautogui ou équivalent
|
||||
|
||||
typing_duration = (time.time() - typing_start) * 1000
|
||||
|
||||
# Appuyer sur Entrée si demandé
|
||||
if self.press_enter_after:
|
||||
print(" Appui sur Entrée")
|
||||
time.sleep(0.1)
|
||||
|
||||
# Evidence de succès (masquée pour sécurité)
|
||||
evidence_description = f"Saisie de secret réussie dans '{self.visual_anchor.label}'"
|
||||
if self.mask_in_evidence:
|
||||
evidence_description += f" (masqué: {masked_secret})"
|
||||
|
||||
evidence = VWBEvidence(
|
||||
evidence_type=VWBEvidenceType.UI_INTERACTION,
|
||||
description=evidence_description,
|
||||
screenshot_base64=self._screenshot_to_base64(screenshot),
|
||||
success=True,
|
||||
confidence_score=best_match['confidence'],
|
||||
bbox=best_match['bbox'],
|
||||
click_point=best_match['center'],
|
||||
execution_time_ms=(datetime.now() - start_time).total_seconds() * 1000,
|
||||
metadata={
|
||||
'secret_length': len(secret_to_type),
|
||||
'typing_duration_ms': typing_duration,
|
||||
'clear_field_first': self.clear_field_first,
|
||||
'click_before_typing': self.click_before_typing,
|
||||
'press_enter_after': self.press_enter_after,
|
||||
'masked_secret': masked_secret if self.mask_in_evidence else None
|
||||
}
|
||||
)
|
||||
evidence_list.append(evidence)
|
||||
|
||||
# Données de sortie (sécurisées)
|
||||
output_data = {
|
||||
'typing_success': True,
|
||||
'secret_length': len(secret_to_type),
|
||||
'confidence_score': best_match['confidence'],
|
||||
'typing_duration_ms': typing_duration,
|
||||
'field_coordinates': best_match['center'],
|
||||
'masked_secret': masked_secret
|
||||
}
|
||||
|
||||
print(f"✅ Saisie de secret réussie (longueur: {len(secret_to_type)})")
|
||||
|
||||
return VWBActionResult(
|
||||
action_id=self.action_id,
|
||||
step_id=step_id,
|
||||
status=VWBActionStatus.SUCCESS,
|
||||
start_time=start_time,
|
||||
end_time=datetime.now(),
|
||||
execution_time_ms=(datetime.now() - start_time).total_seconds() * 1000,
|
||||
output_data=output_data,
|
||||
evidence_list=evidence_list
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
# Gestion des erreurs (sans révéler le secret)
|
||||
error = create_vwb_error(
|
||||
error_type=VWBErrorType.EXECUTION_ERROR,
|
||||
message=f"Erreur lors de la saisie de secret: {str(e)}",
|
||||
severity=VWBErrorSeverity.HIGH,
|
||||
retryable=True,
|
||||
details={'exception': str(e)}
|
||||
)
|
||||
|
||||
return VWBActionResult(
|
||||
action_id=self.action_id,
|
||||
step_id=step_id,
|
||||
status=VWBActionStatus.FAILED,
|
||||
start_time=start_time,
|
||||
end_time=datetime.now(),
|
||||
execution_time_ms=(datetime.now() - start_time).total_seconds() * 1000,
|
||||
output_data={},
|
||||
evidence_list=evidence_list,
|
||||
error=error
|
||||
)
|
||||
|
||||
def _screenshot_to_base64(self, screenshot) -> str:
|
||||
"""
|
||||
Convertit un screenshot en base64.
|
||||
|
||||
Args:
|
||||
screenshot: Image capturée
|
||||
|
||||
Returns:
|
||||
String base64 de l'image
|
||||
"""
|
||||
try:
|
||||
import base64
|
||||
import io
|
||||
from PIL import Image
|
||||
|
||||
if hasattr(screenshot, 'save'):
|
||||
# PIL Image
|
||||
buffer = io.BytesIO()
|
||||
screenshot.save(buffer, format='PNG')
|
||||
return base64.b64encode(buffer.getvalue()).decode('utf-8')
|
||||
else:
|
||||
# Données brutes ou autre format
|
||||
return base64.b64encode(str(screenshot).encode()).decode('utf-8')
|
||||
|
||||
except Exception as e:
|
||||
print(f"⚠️ Erreur conversion base64: {e}")
|
||||
return ""
|
||||
@@ -0,0 +1,470 @@
|
||||
"""
|
||||
Action VWB - Attente d'Ancre Visuelle
|
||||
|
||||
Auteur : Dom, Alice, Kiro - 09 janvier 2026
|
||||
|
||||
Cette action permet d'attendre qu'une ancre visuelle apparaisse ou disparaisse
|
||||
de l'écran dans le Visual Workflow Builder.
|
||||
|
||||
Classes :
|
||||
- VWBWaitForAnchorAction : Action d'attente d'ancre visuelle
|
||||
"""
|
||||
|
||||
from typing import Dict, Any, List, Optional
|
||||
from datetime import datetime
|
||||
import time
|
||||
|
||||
# Import des modules de base
|
||||
from ..base_action import BaseVWBAction, VWBActionResult, VWBActionStatus
|
||||
from ...contracts.error import VWBErrorType, VWBErrorSeverity, create_vwb_error
|
||||
from ...contracts.evidence import VWBEvidenceType, create_interaction_evidence
|
||||
from ...contracts.visual_anchor import VWBVisualAnchor
|
||||
|
||||
|
||||
class VWBWaitForAnchorAction(BaseVWBAction):
|
||||
"""
|
||||
Action d'attente d'ancre visuelle VWB.
|
||||
|
||||
Cette action surveille l'écran jusqu'à ce qu'une ancre visuelle
|
||||
apparaisse ou disparaisse, avec un délai d'attente configurable.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
action_id: str,
|
||||
parameters: Dict[str, Any],
|
||||
screen_capturer=None
|
||||
):
|
||||
"""
|
||||
Initialise l'action d'attente d'ancre.
|
||||
|
||||
Args:
|
||||
action_id: Identifiant unique de l'action
|
||||
parameters: Paramètres incluant l'ancre et le mode d'attente
|
||||
screen_capturer: Instance du ScreenCapturer (Option A thread-safe)
|
||||
"""
|
||||
super().__init__(
|
||||
action_id=action_id,
|
||||
name="Attente d'Ancre Visuelle",
|
||||
description="Attend qu'une ancre visuelle apparaisse ou disparaisse",
|
||||
parameters=parameters,
|
||||
screen_capturer=screen_capturer
|
||||
)
|
||||
|
||||
# Paramètres spécifiques à l'attente
|
||||
self.visual_anchor: Optional[VWBVisualAnchor] = parameters.get('visual_anchor')
|
||||
self.wait_mode = parameters.get('wait_mode', 'appear') # appear, disappear
|
||||
self.max_wait_time_ms = parameters.get('max_wait_time_ms', 30000) # 30 secondes
|
||||
self.check_interval_ms = parameters.get('check_interval_ms', 500) # Vérification toutes les 500ms
|
||||
self.early_exit_on_found = parameters.get('early_exit_on_found', True)
|
||||
|
||||
# Configuration de matching
|
||||
self.confidence_threshold = parameters.get('confidence_threshold', 0.8)
|
||||
self.stable_detection_count = parameters.get('stable_detection_count', 2) # Détections stables requises
|
||||
|
||||
def validate_parameters(self) -> List[str]:
|
||||
"""Valide les paramètres de l'action d'attente."""
|
||||
errors = []
|
||||
|
||||
# Vérifier l'ancre visuelle
|
||||
if not self.visual_anchor:
|
||||
errors.append("Ancre visuelle requise")
|
||||
elif not isinstance(self.visual_anchor, VWBVisualAnchor):
|
||||
errors.append("Ancre visuelle invalide")
|
||||
elif not self.visual_anchor.is_active:
|
||||
errors.append("Ancre visuelle inactive")
|
||||
|
||||
# Vérifier le mode d'attente
|
||||
if self.wait_mode not in ['appear', 'disappear']:
|
||||
errors.append(f"Mode d'attente invalide: {self.wait_mode}")
|
||||
|
||||
# Vérifier les délais
|
||||
if not isinstance(self.max_wait_time_ms, (int, float)) or self.max_wait_time_ms <= 0:
|
||||
errors.append("max_wait_time_ms doit être un nombre positif")
|
||||
if not isinstance(self.check_interval_ms, (int, float)) or self.check_interval_ms <= 0:
|
||||
errors.append("check_interval_ms doit être un nombre positif")
|
||||
|
||||
# Vérifier la cohérence des délais
|
||||
if self.check_interval_ms >= self.max_wait_time_ms:
|
||||
errors.append("check_interval_ms doit être inférieur à max_wait_time_ms")
|
||||
|
||||
# Vérifier le seuil de confiance
|
||||
if not (0.0 <= self.confidence_threshold <= 1.0):
|
||||
errors.append("Seuil de confiance doit être entre 0.0 et 1.0")
|
||||
|
||||
# Vérifier le nombre de détections stables
|
||||
if not isinstance(self.stable_detection_count, int) or self.stable_detection_count < 1:
|
||||
errors.append("stable_detection_count doit être un entier positif")
|
||||
|
||||
# Vérifier le ScreenCapturer
|
||||
if not self.screen_capturer:
|
||||
errors.append("ScreenCapturer requis pour la capture d'écran")
|
||||
|
||||
return errors
|
||||
|
||||
def execute_core(self, step_id: str) -> VWBActionResult:
|
||||
"""
|
||||
Exécute l'action d'attente d'ancre visuelle.
|
||||
|
||||
Args:
|
||||
step_id: Identifiant de l'étape
|
||||
|
||||
Returns:
|
||||
Résultat d'exécution
|
||||
"""
|
||||
start_time = datetime.now()
|
||||
|
||||
try:
|
||||
print(f"⏳ Attente de l'ancre '{self.visual_anchor.name}' (mode: {self.wait_mode})")
|
||||
print(f" Délai max: {self.max_wait_time_ms}ms, Intervalle: {self.check_interval_ms}ms")
|
||||
|
||||
# Initialiser les variables de surveillance
|
||||
wait_start = time.time()
|
||||
max_wait_seconds = self.max_wait_time_ms / 1000.0
|
||||
check_interval_seconds = self.check_interval_ms / 1000.0
|
||||
|
||||
stable_detections = 0
|
||||
last_detection_state = None
|
||||
check_count = 0
|
||||
detection_history = []
|
||||
|
||||
# Boucle de surveillance
|
||||
while True:
|
||||
current_time = time.time()
|
||||
elapsed_time = current_time - wait_start
|
||||
|
||||
# Vérifier le délai d'attente
|
||||
if elapsed_time >= max_wait_seconds:
|
||||
return self._create_timeout_result(
|
||||
step_id=step_id,
|
||||
start_time=start_time,
|
||||
elapsed_time_ms=elapsed_time * 1000,
|
||||
check_count=check_count,
|
||||
detection_history=detection_history
|
||||
)
|
||||
|
||||
# Capturer l'écran actuel
|
||||
screenshot_data = self._capture_current_screen()
|
||||
if screenshot_data is None:
|
||||
print("⚠️ Échec de capture d'écran, retry dans 1s")
|
||||
time.sleep(1.0)
|
||||
continue
|
||||
|
||||
# Rechercher l'ancre
|
||||
check_count += 1
|
||||
match_result = self._find_visual_anchor(screenshot_data)
|
||||
detection_state = match_result['found']
|
||||
|
||||
# Enregistrer dans l'historique
|
||||
detection_history.append({
|
||||
'timestamp': datetime.now().isoformat(),
|
||||
'found': detection_state,
|
||||
'confidence': match_result.get('confidence', 0.0),
|
||||
'elapsed_ms': elapsed_time * 1000
|
||||
})
|
||||
|
||||
print(f" Check {check_count}: {'✓' if detection_state else '✗'} "
|
||||
f"(confiance: {match_result.get('confidence', 0.0):.2f}, "
|
||||
f"temps: {elapsed_time:.1f}s)")
|
||||
|
||||
# Vérifier la stabilité de la détection
|
||||
if detection_state == last_detection_state:
|
||||
stable_detections += 1
|
||||
else:
|
||||
stable_detections = 1
|
||||
last_detection_state = detection_state
|
||||
|
||||
# Vérifier si la condition d'attente est remplie
|
||||
condition_met = self._check_wait_condition(
|
||||
detection_state,
|
||||
stable_detections
|
||||
)
|
||||
|
||||
if condition_met:
|
||||
# Condition remplie - succès
|
||||
end_time = datetime.now()
|
||||
execution_time = (end_time - start_time).total_seconds() * 1000
|
||||
|
||||
# Mettre à jour les statistiques de l'ancre
|
||||
self.visual_anchor.update_usage_stats(execution_time, True)
|
||||
|
||||
return self._create_success_result(
|
||||
step_id=step_id,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
execution_time=execution_time,
|
||||
final_state=detection_state,
|
||||
check_count=check_count,
|
||||
detection_history=detection_history,
|
||||
match_result=match_result
|
||||
)
|
||||
|
||||
# Attendre avant la prochaine vérification
|
||||
time.sleep(check_interval_seconds)
|
||||
|
||||
except Exception as e:
|
||||
return self._create_error_result(
|
||||
step_id=step_id,
|
||||
start_time=start_time,
|
||||
error_type=VWBErrorType.SYSTEM_ERROR,
|
||||
message=f"Erreur lors de l'attente: {str(e)}",
|
||||
technical_details={'exception': str(e)}
|
||||
)
|
||||
|
||||
def _capture_current_screen(self) -> Optional[Dict[str, Any]]:
|
||||
"""Capture l'écran actuel avec métadonnées."""
|
||||
try:
|
||||
# Utiliser la méthode ultra stable (Option A)
|
||||
img_array = self.screen_capturer.capture()
|
||||
if img_array is None:
|
||||
return None
|
||||
|
||||
from PIL import Image
|
||||
import base64
|
||||
import io
|
||||
|
||||
# Convertir en PIL Image
|
||||
pil_image = Image.fromarray(img_array)
|
||||
|
||||
# Convertir en base64 pour stockage (optionnel pour l'attente)
|
||||
buffer = io.BytesIO()
|
||||
pil_image.save(buffer, format='PNG', optimize=True)
|
||||
screenshot_base64 = base64.b64encode(buffer.getvalue()).decode('utf-8')
|
||||
|
||||
return {
|
||||
'image_array': img_array,
|
||||
'pil_image': pil_image,
|
||||
'screenshot_base64': screenshot_base64,
|
||||
'width': pil_image.width,
|
||||
'height': pil_image.height,
|
||||
'timestamp': datetime.now().isoformat()
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Erreur capture d'écran: {e}")
|
||||
return None
|
||||
|
||||
def _find_visual_anchor(self, screenshot_data: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""
|
||||
Recherche l'ancre visuelle dans le screenshot.
|
||||
|
||||
Args:
|
||||
screenshot_data: Données du screenshot
|
||||
|
||||
Returns:
|
||||
Résultat de la recherche
|
||||
"""
|
||||
search_start = time.time()
|
||||
|
||||
try:
|
||||
pil_image = screenshot_data['pil_image']
|
||||
|
||||
# Simuler une recherche rapide pour l'attente
|
||||
search_delay = 0.1 # Recherche rapide pour l'attente
|
||||
time.sleep(search_delay)
|
||||
|
||||
search_time_ms = (time.time() - search_start) * 1000
|
||||
|
||||
# Simuler la détection avec variation dans le temps
|
||||
# Pour rendre la simulation plus réaliste
|
||||
current_time = time.time()
|
||||
detection_probability = 0.3 + 0.4 * (current_time % 10) / 10 # Varie entre 0.3 et 0.7
|
||||
|
||||
# Ajuster selon le mode d'attente
|
||||
if self.wait_mode == 'appear':
|
||||
# Probabilité croissante d'apparition
|
||||
detection_probability = min(0.9, detection_probability + 0.2)
|
||||
else: # disappear
|
||||
# Probabilité décroissante de présence
|
||||
detection_probability = max(0.1, detection_probability - 0.3)
|
||||
|
||||
found = detection_probability >= 0.5
|
||||
confidence = detection_probability if found else 0.2
|
||||
|
||||
result = {
|
||||
'found': found,
|
||||
'confidence': confidence,
|
||||
'search_time_ms': search_time_ms,
|
||||
'method': 'simulated_wait_detection'
|
||||
}
|
||||
|
||||
if found:
|
||||
# Ajouter les coordonnées si trouvé
|
||||
center_x = pil_image.width // 2
|
||||
center_y = pil_image.height // 2
|
||||
|
||||
if self.visual_anchor.has_bounding_box():
|
||||
search_area = self.visual_anchor.get_search_area(
|
||||
pil_image.width,
|
||||
pil_image.height
|
||||
)
|
||||
if search_area:
|
||||
center_x = search_area['x'] + search_area['width'] // 2
|
||||
center_y = search_area['y'] + search_area['height'] // 2
|
||||
|
||||
result.update({
|
||||
'match_box': {
|
||||
'x': center_x - 50,
|
||||
'y': center_y - 25,
|
||||
'width': 100,
|
||||
'height': 50
|
||||
},
|
||||
'center_coordinates': {'x': center_x, 'y': center_y}
|
||||
})
|
||||
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
search_time_ms = (time.time() - search_start) * 1000
|
||||
return {
|
||||
'found': False,
|
||||
'error': str(e),
|
||||
'search_time_ms': search_time_ms
|
||||
}
|
||||
|
||||
def _check_wait_condition(self, detection_state: bool, stable_detections: int) -> bool:
|
||||
"""
|
||||
Vérifie si la condition d'attente est remplie.
|
||||
|
||||
Args:
|
||||
detection_state: État actuel de détection
|
||||
stable_detections: Nombre de détections stables consécutives
|
||||
|
||||
Returns:
|
||||
True si la condition est remplie
|
||||
"""
|
||||
# Vérifier la stabilité
|
||||
if stable_detections < self.stable_detection_count:
|
||||
return False
|
||||
|
||||
# Vérifier selon le mode
|
||||
if self.wait_mode == 'appear':
|
||||
return detection_state # Attendre que l'ancre apparaisse
|
||||
else: # disappear
|
||||
return not detection_state # Attendre que l'ancre disparaisse
|
||||
|
||||
def _create_success_result(
|
||||
self,
|
||||
step_id: str,
|
||||
start_time: datetime,
|
||||
end_time: datetime,
|
||||
execution_time: float,
|
||||
final_state: bool,
|
||||
check_count: int,
|
||||
detection_history: List[Dict[str, Any]],
|
||||
match_result: Dict[str, Any]
|
||||
) -> VWBActionResult:
|
||||
"""Crée un résultat de succès."""
|
||||
|
||||
# Créer l'evidence d'attente
|
||||
wait_evidence = create_interaction_evidence(
|
||||
action_id=self.action_id,
|
||||
step_id=step_id,
|
||||
evidence_type=VWBEvidenceType.WAIT_EVIDENCE,
|
||||
title=f"Attente de {self.visual_anchor.name}",
|
||||
interaction_data={
|
||||
'anchor_id': self.visual_anchor.anchor_id,
|
||||
'anchor_name': self.visual_anchor.name,
|
||||
'wait_mode': self.wait_mode,
|
||||
'final_state': final_state,
|
||||
'wait_time_ms': execution_time,
|
||||
'check_count': check_count,
|
||||
'stable_detections_required': self.stable_detection_count,
|
||||
'confidence_threshold': self.confidence_threshold,
|
||||
'final_confidence': match_result.get('confidence', 0.0),
|
||||
'detection_history': detection_history[-5:], # Garder les 5 dernières
|
||||
'match_box': match_result.get('match_box')
|
||||
},
|
||||
confidence_score=match_result.get('confidence', 0.0)
|
||||
)
|
||||
|
||||
result = VWBActionResult(
|
||||
action_id=self.action_id,
|
||||
step_id=step_id,
|
||||
status=VWBActionStatus.SUCCESS,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
execution_time_ms=execution_time,
|
||||
output_data={
|
||||
'wait_mode': self.wait_mode,
|
||||
'final_state': final_state,
|
||||
'wait_time_ms': execution_time,
|
||||
'check_count': check_count,
|
||||
'anchor_confidence': match_result.get('confidence', 0.0),
|
||||
'condition_met': True
|
||||
},
|
||||
evidence_list=[wait_evidence]
|
||||
)
|
||||
|
||||
state_text = "apparue" if final_state else "disparue"
|
||||
print(f"✅ Ancre {state_text} après {execution_time:.1f}ms ({check_count} vérifications)")
|
||||
return result
|
||||
|
||||
def _create_timeout_result(
|
||||
self,
|
||||
step_id: str,
|
||||
start_time: datetime,
|
||||
elapsed_time_ms: float,
|
||||
check_count: int,
|
||||
detection_history: List[Dict[str, Any]]
|
||||
) -> VWBActionResult:
|
||||
"""Crée un résultat de timeout."""
|
||||
|
||||
# Mettre à jour les statistiques de l'ancre (échec)
|
||||
self.visual_anchor.update_usage_stats(elapsed_time_ms, False)
|
||||
|
||||
error = create_vwb_error(
|
||||
error_type=VWBErrorType.WAIT_TIMEOUT,
|
||||
message=f"Délai d'attente dépassé pour l'ancre '{self.visual_anchor.name}'",
|
||||
action_id=self.action_id,
|
||||
step_id=step_id,
|
||||
severity=VWBErrorSeverity.ERROR,
|
||||
technical_details={
|
||||
'wait_mode': self.wait_mode,
|
||||
'max_wait_time_ms': self.max_wait_time_ms,
|
||||
'elapsed_time_ms': elapsed_time_ms,
|
||||
'check_count': check_count,
|
||||
'detection_history': detection_history[-10:] # Garder les 10 dernières
|
||||
},
|
||||
execution_time_ms=elapsed_time_ms
|
||||
)
|
||||
|
||||
end_time = datetime.now()
|
||||
result = VWBActionResult(
|
||||
action_id=self.action_id,
|
||||
step_id=step_id,
|
||||
status=VWBActionStatus.TIMEOUT,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
execution_time_ms=elapsed_time_ms,
|
||||
output_data={
|
||||
'wait_mode': self.wait_mode,
|
||||
'timeout_reached': True,
|
||||
'elapsed_time_ms': elapsed_time_ms,
|
||||
'check_count': check_count
|
||||
},
|
||||
evidence_list=[],
|
||||
error=error
|
||||
)
|
||||
|
||||
print(f"⏰ Timeout après {elapsed_time_ms:.1f}ms ({check_count} vérifications)")
|
||||
return result
|
||||
|
||||
def get_action_info(self) -> Dict[str, Any]:
|
||||
"""Retourne les informations de l'action pour l'interface."""
|
||||
return {
|
||||
'action_id': self.action_id,
|
||||
'name': self.name,
|
||||
'description': self.description,
|
||||
'type': 'wait_for_anchor',
|
||||
'parameters': {
|
||||
'anchor_name': self.visual_anchor.name if self.visual_anchor else 'Non définie',
|
||||
'wait_mode': self.wait_mode,
|
||||
'max_wait_time_ms': self.max_wait_time_ms,
|
||||
'check_interval_ms': self.check_interval_ms,
|
||||
'confidence_threshold': self.confidence_threshold,
|
||||
'stable_detection_count': self.stable_detection_count
|
||||
},
|
||||
'status': self.current_status.value,
|
||||
'anchor_reliable': self.visual_anchor.is_reliable() if self.visual_anchor else False
|
||||
}
|
||||
Reference in New Issue
Block a user