Feat: Action keyboard_shortcut avec support humanisation
- Touches simples et combinaisons (Ctrl+S, Alt+F4, etc.) - Mapping complet des touches (F1-F12, flèches, navigation) - Support répétition avec délai configurable - Focus optionnel sur ancre visuelle avant raccourci - Intégration humanizer pour délais naturels Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Dom
13
visual_workflow_builder/backend/actions/control/__init__.py
Normal file
13
visual_workflow_builder/backend/actions/control/__init__.py
Normal file
@@ -0,0 +1,13 @@
|
||||
"""
|
||||
Actions de Contrôle VWB
|
||||
Auteur : Dom, Claude - 14 janvier 2026
|
||||
|
||||
Ce module contient les actions de contrôle de flux :
|
||||
- keyboard_shortcut : Raccourcis clavier
|
||||
"""
|
||||
|
||||
from .keyboard_shortcut import VWBKeyboardShortcutAction
|
||||
|
||||
__all__ = [
|
||||
'VWBKeyboardShortcutAction',
|
||||
]
|
||||
@@ -0,0 +1,423 @@
|
||||
"""
|
||||
Action Raccourci Clavier - Appuyer sur des touches ou combinaisons
|
||||
Auteur : Dom, Claude - 14 janvier 2026
|
||||
|
||||
Cette action permet d'exécuter des raccourcis clavier simples ou complexes,
|
||||
avec support de l'humanisation pour éviter la détection.
|
||||
"""
|
||||
|
||||
from typing import Dict, Any, List, Optional
|
||||
from datetime import datetime
|
||||
import time
|
||||
|
||||
from ..base_action import BaseVWBAction, VWBActionResult, VWBActionStatus
|
||||
from ...contracts.error import VWBErrorType, create_vwb_error
|
||||
from ...contracts.visual_anchor import VWBVisualAnchor
|
||||
|
||||
|
||||
class VWBKeyboardShortcutAction(BaseVWBAction):
|
||||
"""
|
||||
Action pour exécuter des raccourcis clavier.
|
||||
|
||||
Supporte :
|
||||
- Touches simples (Enter, Tab, Escape, F1-F12, etc.)
|
||||
- Combinaisons avec modificateurs (Ctrl+S, Alt+F4, etc.)
|
||||
- Combinaisons personnalisées
|
||||
- Répétition avec délai
|
||||
- Humanisation des délais
|
||||
"""
|
||||
|
||||
# Mapping des noms de touches vers pyautogui
|
||||
KEY_MAPPING = {
|
||||
# Touches spéciales
|
||||
'Enter': 'enter',
|
||||
'Tab': 'tab',
|
||||
'Escape': 'escape',
|
||||
'Space': 'space',
|
||||
'Backspace': 'backspace',
|
||||
'Delete': 'delete',
|
||||
|
||||
# Flèches
|
||||
'ArrowUp': 'up',
|
||||
'ArrowDown': 'down',
|
||||
'ArrowLeft': 'left',
|
||||
'ArrowRight': 'right',
|
||||
|
||||
# Navigation
|
||||
'Home': 'home',
|
||||
'End': 'end',
|
||||
'PageUp': 'pageup',
|
||||
'PageDown': 'pagedown',
|
||||
|
||||
# Fonction
|
||||
'F1': 'f1', 'F2': 'f2', 'F3': 'f3', 'F4': 'f4',
|
||||
'F5': 'f5', 'F6': 'f6', 'F7': 'f7', 'F8': 'f8',
|
||||
'F9': 'f9', 'F10': 'f10', 'F11': 'f11', 'F12': 'f12',
|
||||
|
||||
# Modificateurs
|
||||
'Ctrl': 'ctrl',
|
||||
'Alt': 'alt',
|
||||
'Shift': 'shift',
|
||||
'Meta': 'win', # Windows key
|
||||
'Win': 'win',
|
||||
'Command': 'command', # Mac
|
||||
|
||||
# Autres
|
||||
'Insert': 'insert',
|
||||
'PrintScreen': 'printscreen',
|
||||
'Pause': 'pause',
|
||||
'CapsLock': 'capslock',
|
||||
'NumLock': 'numlock',
|
||||
'ScrollLock': 'scrolllock',
|
||||
}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
action_id: str,
|
||||
parameters: Dict[str, Any],
|
||||
screen_capturer=None
|
||||
):
|
||||
"""
|
||||
Initialise l'action de raccourci clavier.
|
||||
|
||||
Args:
|
||||
action_id: Identifiant unique de l'action
|
||||
parameters: Paramètres du raccourci
|
||||
screen_capturer: Instance du ScreenCapturer (optionnel)
|
||||
"""
|
||||
super().__init__(
|
||||
action_id=action_id,
|
||||
name="Raccourci Clavier",
|
||||
description="Appuie sur une touche ou combinaison de touches",
|
||||
parameters=parameters,
|
||||
screen_capturer=screen_capturer
|
||||
)
|
||||
|
||||
# Paramètres du raccourci
|
||||
self.visual_anchor: Optional[VWBVisualAnchor] = parameters.get('visual_anchor')
|
||||
self.key = parameters.get('key', 'Enter')
|
||||
self.modifiers = parameters.get('modifiers', '')
|
||||
self.custom_combination = parameters.get('custom_combination', '')
|
||||
self.repeat_count = parameters.get('repeat_count', 1)
|
||||
self.delay_between_ms = parameters.get('delay_between_ms', 100)
|
||||
|
||||
# Humanisation
|
||||
self.humanize = parameters.get('humanize', True)
|
||||
self.humanize_profile = parameters.get('humanize_profile', 'normal')
|
||||
|
||||
def validate_parameters(self) -> List[str]:
|
||||
"""Valide les paramètres de l'action."""
|
||||
errors = []
|
||||
|
||||
# Vérifier qu'on a soit une touche, soit une combinaison personnalisée
|
||||
if not self.key and not self.custom_combination:
|
||||
errors.append("Touche ou combinaison personnalisée requise")
|
||||
|
||||
# Vérifier le repeat_count
|
||||
if self.repeat_count < 1 or self.repeat_count > 100:
|
||||
errors.append("repeat_count doit être entre 1 et 100")
|
||||
|
||||
# Vérifier le délai
|
||||
if self.delay_between_ms < 10 or self.delay_between_ms > 5000:
|
||||
errors.append("delay_between_ms doit être entre 10 et 5000")
|
||||
|
||||
return errors
|
||||
|
||||
def execute_core(self, step_id: str) -> VWBActionResult:
|
||||
"""
|
||||
Exécute le raccourci clavier.
|
||||
|
||||
Args:
|
||||
step_id: Identifiant de l'étape
|
||||
|
||||
Returns:
|
||||
Résultat d'exécution
|
||||
"""
|
||||
start_time = datetime.now()
|
||||
|
||||
try:
|
||||
# Focus sur l'élément si ancre fournie
|
||||
if self.visual_anchor:
|
||||
if not self._focus_anchor():
|
||||
return self._create_error_result(
|
||||
step_id=step_id,
|
||||
start_time=start_time,
|
||||
error_type=VWBErrorType.ELEMENT_NOT_FOUND,
|
||||
message="Impossible de focus sur l'ancre visuelle"
|
||||
)
|
||||
|
||||
# Déterminer les touches à appuyer
|
||||
keys = self._parse_keys()
|
||||
if not keys:
|
||||
return self._create_error_result(
|
||||
step_id=step_id,
|
||||
start_time=start_time,
|
||||
error_type=VWBErrorType.PARAMETER_INVALID,
|
||||
message="Aucune touche valide à appuyer"
|
||||
)
|
||||
|
||||
# Exécuter le raccourci
|
||||
success = self._execute_shortcut(keys)
|
||||
|
||||
end_time = datetime.now()
|
||||
execution_time = (end_time - start_time).total_seconds() * 1000
|
||||
|
||||
if success:
|
||||
key_desc = self._get_key_description()
|
||||
print(f"⌨️ Raccourci exécuté: {key_desc} (x{self.repeat_count})")
|
||||
|
||||
return VWBActionResult(
|
||||
action_id=self.action_id,
|
||||
step_id=step_id,
|
||||
status=VWBActionStatus.SUCCESS,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
execution_time_ms=execution_time,
|
||||
output_data={
|
||||
'keys': keys,
|
||||
'repeat_count': self.repeat_count,
|
||||
'description': key_desc
|
||||
},
|
||||
evidence_list=self.evidence_list.copy()
|
||||
)
|
||||
else:
|
||||
return self._create_error_result(
|
||||
step_id=step_id,
|
||||
start_time=start_time,
|
||||
error_type=VWBErrorType.SYSTEM_ERROR,
|
||||
message="Échec de l'exécution du raccourci"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
return self._create_error_result(
|
||||
step_id=step_id,
|
||||
start_time=start_time,
|
||||
error_type=VWBErrorType.SYSTEM_ERROR,
|
||||
message=f"Erreur: {str(e)}",
|
||||
technical_details={'exception': str(e)}
|
||||
)
|
||||
|
||||
def _parse_keys(self) -> List[str]:
|
||||
"""
|
||||
Parse et retourne la liste des touches à appuyer.
|
||||
|
||||
Returns:
|
||||
Liste des touches au format pyautogui
|
||||
"""
|
||||
keys = []
|
||||
|
||||
# Utiliser la combinaison personnalisée si fournie
|
||||
if self.custom_combination:
|
||||
parts = self.custom_combination.replace('+', ' ').split()
|
||||
for part in parts:
|
||||
key = self._map_key(part.strip())
|
||||
if key:
|
||||
keys.append(key)
|
||||
return keys
|
||||
|
||||
# Sinon, construire à partir de key et modifiers
|
||||
if self.modifiers:
|
||||
modifier_parts = self.modifiers.replace('+', ' ').split()
|
||||
for mod in modifier_parts:
|
||||
key = self._map_key(mod.strip())
|
||||
if key:
|
||||
keys.append(key)
|
||||
|
||||
if self.key:
|
||||
key = self._map_key(self.key)
|
||||
if key:
|
||||
keys.append(key)
|
||||
|
||||
return keys
|
||||
|
||||
def _map_key(self, key_name: str) -> Optional[str]:
|
||||
"""
|
||||
Mappe un nom de touche vers le format pyautogui.
|
||||
|
||||
Args:
|
||||
key_name: Nom de la touche
|
||||
|
||||
Returns:
|
||||
Nom de la touche au format pyautogui ou None
|
||||
"""
|
||||
# Vérifier le mapping
|
||||
if key_name in self.KEY_MAPPING:
|
||||
return self.KEY_MAPPING[key_name]
|
||||
|
||||
# Vérifier le mapping insensible à la casse
|
||||
for k, v in self.KEY_MAPPING.items():
|
||||
if k.lower() == key_name.lower():
|
||||
return v
|
||||
|
||||
# Si c'est une lettre ou un chiffre simple
|
||||
if len(key_name) == 1 and key_name.isalnum():
|
||||
return key_name.lower()
|
||||
|
||||
# Retourner tel quel si c'est déjà au format pyautogui
|
||||
return key_name.lower()
|
||||
|
||||
def _execute_shortcut(self, keys: List[str]) -> bool:
|
||||
"""
|
||||
Exécute le raccourci clavier.
|
||||
|
||||
Args:
|
||||
keys: Liste des touches à appuyer
|
||||
|
||||
Returns:
|
||||
True si succès
|
||||
"""
|
||||
try:
|
||||
import pyautogui
|
||||
|
||||
for i in range(self.repeat_count):
|
||||
# Délai humanisé avant l'action
|
||||
if self.humanize and i > 0:
|
||||
self._humanized_delay()
|
||||
elif i > 0:
|
||||
time.sleep(self.delay_between_ms / 1000)
|
||||
|
||||
# Exécuter le raccourci
|
||||
if len(keys) == 1:
|
||||
# Touche simple
|
||||
pyautogui.press(keys[0])
|
||||
else:
|
||||
# Combinaison (hotkey)
|
||||
pyautogui.hotkey(*keys)
|
||||
|
||||
return True
|
||||
|
||||
except ImportError:
|
||||
print("⚠️ pyautogui non disponible - simulation du raccourci")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Erreur exécution raccourci: {e}")
|
||||
return False
|
||||
|
||||
def _humanized_delay(self):
|
||||
"""Applique un délai humanisé entre les répétitions."""
|
||||
try:
|
||||
from ...utils.humanizer import Humanizer, HumanProfile
|
||||
|
||||
profile_map = {
|
||||
'fast': HumanProfile.FAST,
|
||||
'normal': HumanProfile.NORMAL,
|
||||
'slow': HumanProfile.SLOW,
|
||||
'stealth': HumanProfile.STEALTH,
|
||||
}
|
||||
profile = profile_map.get(self.humanize_profile, HumanProfile.NORMAL)
|
||||
|
||||
humanizer = Humanizer(profile=profile)
|
||||
humanizer.wait(self.delay_between_ms, int(self.delay_between_ms * 1.5))
|
||||
|
||||
except ImportError:
|
||||
time.sleep(self.delay_between_ms / 1000)
|
||||
|
||||
def _focus_anchor(self) -> bool:
|
||||
"""
|
||||
Met le focus sur l'ancre visuelle avant d'appuyer sur les touches.
|
||||
|
||||
Returns:
|
||||
True si le focus a réussi
|
||||
"""
|
||||
if not self.visual_anchor:
|
||||
return True
|
||||
|
||||
try:
|
||||
# Utiliser la recherche visuelle pour trouver l'élément
|
||||
from ...catalog_routes import find_visual_anchor_on_screen
|
||||
|
||||
# Obtenir l'image de l'ancre
|
||||
anchor_image = None
|
||||
bounding_box = None
|
||||
|
||||
if isinstance(self.visual_anchor, dict):
|
||||
anchor_image = self.visual_anchor.get('screenshot') or self.visual_anchor.get('image_base64')
|
||||
bounding_box = self.visual_anchor.get('bounding_box')
|
||||
elif isinstance(self.visual_anchor, VWBVisualAnchor):
|
||||
anchor_image = self.visual_anchor.screenshot_base64
|
||||
if self.visual_anchor.has_bounding_box():
|
||||
bounding_box = self.visual_anchor.bounding_box
|
||||
|
||||
if not anchor_image:
|
||||
print("⚠️ Pas d'image d'ancre, utilisation des coordonnées statiques")
|
||||
if bounding_box:
|
||||
x = bounding_box.get('x', 0) + bounding_box.get('width', 0) // 2
|
||||
y = bounding_box.get('y', 0) + bounding_box.get('height', 0) // 2
|
||||
return self._click_at(x, y)
|
||||
return False
|
||||
|
||||
# Rechercher l'élément
|
||||
result = find_visual_anchor_on_screen(
|
||||
anchor_image_base64=anchor_image,
|
||||
confidence_threshold=0.7,
|
||||
bounding_box=bounding_box
|
||||
)
|
||||
|
||||
if result and result.get('found'):
|
||||
x = result.get('center_x', 0)
|
||||
y = result.get('center_y', 0)
|
||||
return self._click_at(x, y)
|
||||
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
print(f"⚠️ Erreur focus: {e}")
|
||||
return False
|
||||
|
||||
def _click_at(self, x: int, y: int) -> bool:
|
||||
"""Clique à une position pour mettre le focus."""
|
||||
try:
|
||||
if self.humanize:
|
||||
from ...utils.humanizer import Humanizer, HumanProfile
|
||||
|
||||
profile_map = {
|
||||
'fast': HumanProfile.FAST,
|
||||
'normal': HumanProfile.NORMAL,
|
||||
'slow': HumanProfile.SLOW,
|
||||
'stealth': HumanProfile.STEALTH,
|
||||
}
|
||||
profile = profile_map.get(self.humanize_profile, HumanProfile.NORMAL)
|
||||
|
||||
humanizer = Humanizer(profile=profile)
|
||||
humanizer.click(x, y)
|
||||
else:
|
||||
import pyautogui
|
||||
pyautogui.click(x, y)
|
||||
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Erreur clic: {e}")
|
||||
return False
|
||||
|
||||
def _get_key_description(self) -> str:
|
||||
"""Retourne une description lisible du raccourci."""
|
||||
if self.custom_combination:
|
||||
return self.custom_combination
|
||||
|
||||
parts = []
|
||||
if self.modifiers:
|
||||
parts.append(self.modifiers)
|
||||
if self.key:
|
||||
parts.append(self.key)
|
||||
|
||||
return '+'.join(parts) if parts else 'Unknown'
|
||||
|
||||
def get_action_info(self) -> Dict[str, Any]:
|
||||
"""Retourne les informations de l'action pour l'interface."""
|
||||
return {
|
||||
'action_id': self.action_id,
|
||||
'name': self.name,
|
||||
'description': self.description,
|
||||
'type': 'keyboard_shortcut',
|
||||
'parameters': {
|
||||
'key': self.key,
|
||||
'modifiers': self.modifiers,
|
||||
'custom_combination': self.custom_combination,
|
||||
'repeat_count': self.repeat_count,
|
||||
'delay_between_ms': self.delay_between_ms,
|
||||
'humanize': self.humanize
|
||||
},
|
||||
'status': self.current_status.value
|
||||
}
|
||||
Reference in New Issue
Block a user