809 lines
29 KiB
Python
809 lines
29 KiB
Python
#!/usr/bin/env python3
|
||
"""
|
||
RPA Vision V3 - Catalogue de Primitives Gestuelles
|
||
|
||
Bibliothèque de gestes universels Windows (raccourcis clavier) que le système
|
||
connaît nativement, sans apprentissage visuel.
|
||
|
||
Trois usages :
|
||
1. Chat : l'utilisateur demande "ferme la fenêtre" → match direct → exécution
|
||
2. Replay : une action enregistrée correspond à un geste connu → substitution
|
||
automatique par le raccourci clavier (plus fiable que le clic visuel)
|
||
3. Workflows : enrichissement automatique des workflows avec les primitives
|
||
|
||
Auteur: Dom — Mars 2026
|
||
"""
|
||
|
||
import logging
|
||
import re
|
||
import unicodedata
|
||
import uuid
|
||
from dataclasses import dataclass, field
|
||
from difflib import SequenceMatcher
|
||
from typing import Dict, List, Optional, Tuple
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
|
||
SAVE_COMMAND_LABELS = {"enregistrer", "save", "sauvegarder"}
|
||
SAVE_AS_LABELS = {"enregistrer sous", "save as", "sauvegarder sous"}
|
||
FILE_MENU_LABELS = {"fichier", "file", "menu fichier", "file menu"}
|
||
|
||
|
||
@dataclass
|
||
class Gesture:
|
||
"""Un geste primitif universel."""
|
||
id: str
|
||
name: str
|
||
description: str
|
||
keys: List[str] # Ex: ["alt", "f4"], ["ctrl", "t"]
|
||
aliases: List[str] = field(default_factory=list) # Termes alternatifs
|
||
tags: List[str] = field(default_factory=list)
|
||
context: str = "windows" # "windows", "chrome", "explorer", etc.
|
||
category: str = "window" # "window", "navigation", "editing", "system"
|
||
|
||
def to_replay_action(self) -> Dict:
|
||
"""Convertir en action de replay pour l'Agent V1."""
|
||
return {
|
||
"action_id": f"gesture_{self.id}_{uuid.uuid4().hex[:6]}",
|
||
"type": "key_combo",
|
||
"keys": self.keys,
|
||
"gesture_id": self.id,
|
||
"gesture_name": self.name,
|
||
}
|
||
|
||
|
||
# =============================================================================
|
||
# Catalogue des primitives
|
||
# =============================================================================
|
||
|
||
GESTURES: List[Gesture] = [
|
||
# --- Gestion de fenêtres ---
|
||
Gesture(
|
||
id="win_close", name="Fermer la fenêtre",
|
||
description="Fermer la fenêtre active",
|
||
keys=["alt", "f4"],
|
||
aliases=["fermer", "close", "quitter la fenêtre", "fermer l'application",
|
||
"fermer le programme", "close window"],
|
||
tags=["fenêtre", "fermer", "close"],
|
||
category="window",
|
||
),
|
||
Gesture(
|
||
id="win_maximize", name="Agrandir la fenêtre",
|
||
description="Agrandir la fenêtre au maximum",
|
||
keys=["super", "up"],
|
||
aliases=["agrandir", "maximize", "plein écran", "maximiser",
|
||
"fullscreen", "agrandir la fenêtre"],
|
||
tags=["fenêtre", "agrandir", "maximize"],
|
||
category="window",
|
||
),
|
||
Gesture(
|
||
id="win_minimize", name="Réduire la fenêtre",
|
||
description="Réduire la fenêtre dans la barre des tâches",
|
||
keys=["super", "down"],
|
||
aliases=["réduire", "minimize", "minimiser", "réduire la fenêtre",
|
||
"mettre en bas"],
|
||
tags=["fenêtre", "réduire", "minimize"],
|
||
category="window",
|
||
),
|
||
Gesture(
|
||
id="win_minimize_all", name="Afficher le bureau",
|
||
description="Réduire toutes les fenêtres (afficher le bureau)",
|
||
keys=["super", "d"],
|
||
aliases=["bureau", "desktop", "afficher le bureau", "tout réduire",
|
||
"montrer le bureau", "show desktop"],
|
||
tags=["bureau", "desktop", "minimize all"],
|
||
category="window",
|
||
),
|
||
Gesture(
|
||
id="win_switch", name="Basculer entre fenêtres",
|
||
description="Basculer vers la fenêtre suivante",
|
||
keys=["alt", "tab"],
|
||
aliases=["basculer", "switch", "changer de fenêtre",
|
||
"fenêtre suivante", "alt tab"],
|
||
tags=["fenêtre", "basculer", "switch"],
|
||
category="window",
|
||
),
|
||
Gesture(
|
||
id="win_snap_left", name="Fenêtre à gauche",
|
||
description="Ancrer la fenêtre à gauche de l'écran",
|
||
keys=["super", "left"],
|
||
aliases=["fenêtre à gauche", "snap left", "ancrer à gauche",
|
||
"moitié gauche"],
|
||
tags=["fenêtre", "snap", "gauche"],
|
||
category="window",
|
||
),
|
||
Gesture(
|
||
id="win_snap_right", name="Fenêtre à droite",
|
||
description="Ancrer la fenêtre à droite de l'écran",
|
||
keys=["super", "right"],
|
||
aliases=["fenêtre à droite", "snap right", "ancrer à droite",
|
||
"moitié droite"],
|
||
tags=["fenêtre", "snap", "droite"],
|
||
category="window",
|
||
),
|
||
Gesture(
|
||
id="win_restore", name="Restaurer la fenêtre",
|
||
description="Restaurer la taille normale de la fenêtre",
|
||
keys=["super", "down"],
|
||
aliases=["restaurer", "restore", "taille normale",
|
||
"fenêtre normale"],
|
||
tags=["fenêtre", "restaurer", "restore"],
|
||
category="window",
|
||
),
|
||
|
||
# --- Navigation Chrome / navigateur ---
|
||
Gesture(
|
||
id="chrome_new_tab", name="Nouvel onglet",
|
||
description="Ouvrir un nouvel onglet dans le navigateur",
|
||
keys=["ctrl", "t"],
|
||
aliases=["nouvel onglet", "new tab", "ouvrir un onglet",
|
||
"ajouter un onglet", "nouveau tab"],
|
||
tags=["chrome", "onglet", "tab", "nouveau"],
|
||
context="chrome",
|
||
category="navigation",
|
||
),
|
||
Gesture(
|
||
id="chrome_close_tab", name="Fermer l'onglet",
|
||
description="Fermer l'onglet actif du navigateur",
|
||
keys=["ctrl", "w"],
|
||
aliases=["fermer l'onglet", "close tab", "fermer le tab",
|
||
"fermer cet onglet"],
|
||
tags=["chrome", "onglet", "fermer"],
|
||
context="chrome",
|
||
category="navigation",
|
||
),
|
||
Gesture(
|
||
id="chrome_next_tab", name="Onglet suivant",
|
||
description="Passer à l'onglet suivant",
|
||
keys=["ctrl", "tab"],
|
||
aliases=["onglet suivant", "next tab", "tab suivant",
|
||
"prochain onglet"],
|
||
tags=["chrome", "onglet", "suivant"],
|
||
context="chrome",
|
||
category="navigation",
|
||
),
|
||
Gesture(
|
||
id="chrome_prev_tab", name="Onglet précédent",
|
||
description="Passer à l'onglet précédent",
|
||
keys=["ctrl", "shift", "tab"],
|
||
aliases=["onglet précédent", "previous tab", "tab précédent",
|
||
"onglet d'avant"],
|
||
tags=["chrome", "onglet", "précédent"],
|
||
context="chrome",
|
||
category="navigation",
|
||
),
|
||
Gesture(
|
||
id="chrome_reopen_tab", name="Rouvrir le dernier onglet",
|
||
description="Rouvrir le dernier onglet fermé",
|
||
keys=["ctrl", "shift", "t"],
|
||
aliases=["rouvrir l'onglet", "reopen tab", "onglet fermé",
|
||
"restaurer l'onglet"],
|
||
tags=["chrome", "onglet", "rouvrir"],
|
||
context="chrome",
|
||
category="navigation",
|
||
),
|
||
Gesture(
|
||
id="chrome_address_bar", name="Barre d'adresse",
|
||
description="Sélectionner la barre d'adresse du navigateur",
|
||
keys=["ctrl", "l"],
|
||
aliases=["barre d'adresse", "address bar", "url bar",
|
||
"aller à l'adresse", "sélectionner l'url"],
|
||
tags=["chrome", "url", "adresse"],
|
||
context="chrome",
|
||
category="navigation",
|
||
),
|
||
Gesture(
|
||
id="chrome_refresh", name="Rafraîchir la page",
|
||
description="Recharger la page web actuelle",
|
||
keys=["f5"],
|
||
aliases=["rafraîchir", "refresh", "recharger", "actualiser",
|
||
"reload"],
|
||
tags=["chrome", "rafraîchir", "reload"],
|
||
context="chrome",
|
||
category="navigation",
|
||
),
|
||
Gesture(
|
||
id="chrome_back", name="Page précédente",
|
||
description="Retourner à la page précédente",
|
||
keys=["alt", "left"],
|
||
aliases=["retour", "back", "page précédente", "revenir en arrière",
|
||
"page d'avant"],
|
||
tags=["chrome", "retour", "back"],
|
||
context="chrome",
|
||
category="navigation",
|
||
),
|
||
Gesture(
|
||
id="chrome_forward", name="Page suivante",
|
||
description="Aller à la page suivante",
|
||
keys=["alt", "right"],
|
||
aliases=["avancer", "forward", "page suivante"],
|
||
tags=["chrome", "avancer", "forward"],
|
||
context="chrome",
|
||
category="navigation",
|
||
),
|
||
Gesture(
|
||
id="chrome_find", name="Rechercher dans la page",
|
||
description="Ouvrir la barre de recherche dans la page",
|
||
keys=["ctrl", "f"],
|
||
aliases=["rechercher", "find", "chercher dans la page", "ctrl f",
|
||
"trouver"],
|
||
tags=["chrome", "rechercher", "find"],
|
||
context="chrome",
|
||
category="navigation",
|
||
),
|
||
Gesture(
|
||
id="chrome_new_window", name="Nouvelle fenêtre",
|
||
description="Ouvrir une nouvelle fenêtre de navigateur",
|
||
keys=["ctrl", "n"],
|
||
aliases=["nouvelle fenêtre", "new window", "ouvrir une fenêtre"],
|
||
tags=["chrome", "fenêtre", "nouveau"],
|
||
context="chrome",
|
||
category="navigation",
|
||
),
|
||
|
||
# --- Édition / presse-papier ---
|
||
Gesture(
|
||
id="edit_copy", name="Copier",
|
||
description="Copier la sélection dans le presse-papier",
|
||
keys=["ctrl", "c"],
|
||
aliases=["copier", "copy", "ctrl c"],
|
||
tags=["édition", "copier", "presse-papier"],
|
||
category="editing",
|
||
),
|
||
Gesture(
|
||
id="edit_paste", name="Coller",
|
||
description="Coller le contenu du presse-papier",
|
||
keys=["ctrl", "v"],
|
||
aliases=["coller", "paste", "ctrl v"],
|
||
tags=["édition", "coller", "presse-papier"],
|
||
category="editing",
|
||
),
|
||
Gesture(
|
||
id="edit_cut", name="Couper",
|
||
description="Couper la sélection",
|
||
keys=["ctrl", "x"],
|
||
aliases=["couper", "cut", "ctrl x"],
|
||
tags=["édition", "couper"],
|
||
category="editing",
|
||
),
|
||
Gesture(
|
||
id="edit_undo", name="Annuler",
|
||
description="Annuler la dernière action",
|
||
keys=["ctrl", "z"],
|
||
aliases=["annuler", "undo", "défaire", "ctrl z"],
|
||
tags=["édition", "annuler", "undo"],
|
||
category="editing",
|
||
),
|
||
Gesture(
|
||
id="edit_redo", name="Rétablir",
|
||
description="Rétablir l'action annulée",
|
||
keys=["ctrl", "y"],
|
||
aliases=["rétablir", "redo", "refaire", "ctrl y"],
|
||
tags=["édition", "rétablir", "redo"],
|
||
category="editing",
|
||
),
|
||
Gesture(
|
||
id="edit_select_all", name="Tout sélectionner",
|
||
description="Sélectionner tout le contenu",
|
||
keys=["ctrl", "a"],
|
||
aliases=["tout sélectionner", "select all", "sélectionner tout",
|
||
"ctrl a"],
|
||
tags=["édition", "sélection", "tout"],
|
||
category="editing",
|
||
),
|
||
Gesture(
|
||
id="edit_save", name="Enregistrer",
|
||
description="Enregistrer le document/fichier actuel",
|
||
keys=["ctrl", "s"],
|
||
aliases=["enregistrer", "save", "sauvegarder", "ctrl s"],
|
||
tags=["édition", "enregistrer", "save"],
|
||
category="editing",
|
||
),
|
||
|
||
# --- Système ---
|
||
Gesture(
|
||
id="sys_start_menu", name="Menu Démarrer",
|
||
description="Ouvrir le menu Démarrer Windows",
|
||
keys=["super"],
|
||
aliases=["menu démarrer", "start menu", "démarrer", "windows",
|
||
"touche windows"],
|
||
tags=["système", "démarrer", "menu"],
|
||
category="system",
|
||
),
|
||
Gesture(
|
||
id="sys_task_manager", name="Gestionnaire des tâches",
|
||
description="Ouvrir le gestionnaire des tâches",
|
||
keys=["ctrl", "shift", "escape"],
|
||
aliases=["gestionnaire des tâches", "task manager",
|
||
"gestionnaire tâches", "processes"],
|
||
tags=["système", "tâches", "processus"],
|
||
category="system",
|
||
),
|
||
Gesture(
|
||
id="sys_lock", name="Verrouiller le PC",
|
||
description="Verrouiller la session Windows",
|
||
keys=["super", "l"],
|
||
aliases=["verrouiller", "lock", "verrouiller le pc",
|
||
"verrouiller la session"],
|
||
tags=["système", "verrouiller", "lock"],
|
||
category="system",
|
||
),
|
||
Gesture(
|
||
id="sys_screenshot", name="Capture d'écran",
|
||
description="Prendre une capture d'écran",
|
||
keys=["super", "shift", "s"],
|
||
aliases=["capture d'écran", "screenshot", "capture écran",
|
||
"impr écran"],
|
||
tags=["système", "capture", "screenshot"],
|
||
category="system",
|
||
),
|
||
Gesture(
|
||
id="sys_explorer", name="Ouvrir l'explorateur",
|
||
description="Ouvrir l'explorateur de fichiers Windows",
|
||
keys=["super", "e"],
|
||
aliases=["explorateur", "explorer", "ouvrir l'explorateur",
|
||
"mes fichiers", "file explorer", "explorateur de fichiers"],
|
||
tags=["système", "explorateur"],
|
||
category="system",
|
||
),
|
||
Gesture(
|
||
id="sys_run", name="Exécuter (Run)",
|
||
description="Ouvrir la boîte de dialogue Exécuter",
|
||
keys=["super", "r"],
|
||
aliases=["exécuter", "run", "boîte exécuter"],
|
||
tags=["système", "exécuter", "run"],
|
||
category="system",
|
||
),
|
||
Gesture(
|
||
id="sys_settings", name="Paramètres Windows",
|
||
description="Ouvrir les paramètres Windows",
|
||
keys=["super", "i"],
|
||
aliases=["paramètres", "settings", "réglages",
|
||
"paramètres windows"],
|
||
tags=["système", "paramètres", "settings"],
|
||
category="system",
|
||
),
|
||
|
||
# --- Navigation texte ---
|
||
Gesture(
|
||
id="nav_home", name="Début de ligne",
|
||
description="Aller au début de la ligne",
|
||
keys=["home"],
|
||
aliases=["début de ligne", "home", "début"],
|
||
tags=["navigation", "texte", "début"],
|
||
category="editing",
|
||
),
|
||
Gesture(
|
||
id="nav_end", name="Fin de ligne",
|
||
description="Aller à la fin de la ligne",
|
||
keys=["end"],
|
||
aliases=["fin de ligne", "end", "fin"],
|
||
tags=["navigation", "texte", "fin"],
|
||
category="editing",
|
||
),
|
||
Gesture(
|
||
id="nav_enter", name="Valider / Entrée",
|
||
description="Appuyer sur Entrée",
|
||
keys=["enter"],
|
||
aliases=["entrée", "enter", "valider", "confirmer", "ok"],
|
||
tags=["navigation", "entrée", "valider"],
|
||
category="editing",
|
||
),
|
||
Gesture(
|
||
id="nav_escape", name="Échap / Annuler",
|
||
description="Appuyer sur Échap (fermer popup, annuler)",
|
||
keys=["escape"],
|
||
aliases=["échap", "escape", "esc", "annuler", "fermer le popup",
|
||
"fermer la popup", "fermer le dialogue"],
|
||
tags=["navigation", "échap", "annuler", "popup"],
|
||
category="editing",
|
||
),
|
||
Gesture(
|
||
id="nav_tab", name="Champ suivant",
|
||
description="Passer au champ suivant (Tab)",
|
||
keys=["tab"],
|
||
aliases=["tab", "champ suivant", "suivant", "prochain champ",
|
||
"tabulation"],
|
||
tags=["navigation", "tab", "champ"],
|
||
category="editing",
|
||
),
|
||
]
|
||
|
||
|
||
class GestureCatalog:
|
||
"""
|
||
Catalogue de gestes primitifs avec matching sémantique.
|
||
|
||
Utilisé par :
|
||
- Le chat (match direct quand l'utilisateur demande un geste)
|
||
- Le replay (substitution automatique d'actions enregistrées)
|
||
"""
|
||
|
||
def __init__(self, gestures: List[Gesture] = None):
|
||
self.gestures = gestures or GESTURES
|
||
# Index pour recherche rapide
|
||
self._by_id: Dict[str, Gesture] = {g.id: g for g in self.gestures}
|
||
# Pré-calculer les termes de recherche normalisés
|
||
self._search_index: List[Tuple[Gesture, List[str]]] = []
|
||
for g in self.gestures:
|
||
terms = [g.name.lower(), g.description.lower()]
|
||
terms.extend(a.lower() for a in g.aliases)
|
||
terms.extend(t.lower() for t in g.tags)
|
||
self._search_index.append((g, terms))
|
||
|
||
logger.info(f"GestureCatalog: {len(self.gestures)} primitives chargées")
|
||
|
||
def match(self, query: str, min_score: float = 0.45) -> Optional[Tuple[Gesture, float]]:
|
||
"""
|
||
Trouver le geste le plus proche d'une requête textuelle.
|
||
|
||
Returns:
|
||
(Gesture, score) si match trouvé, None sinon.
|
||
"""
|
||
query_lower = query.lower().strip()
|
||
if not query_lower:
|
||
return None
|
||
|
||
best_gesture = None
|
||
best_score = 0.0
|
||
|
||
for gesture, terms in self._search_index:
|
||
score = self._compute_score(query_lower, terms, gesture)
|
||
if score > best_score:
|
||
best_score = score
|
||
best_gesture = gesture
|
||
|
||
if best_gesture and best_score >= min_score:
|
||
logger.debug(f"Gesture match: '{query}' → {best_gesture.id} (score={best_score:.2f})")
|
||
return (best_gesture, best_score)
|
||
|
||
return None
|
||
|
||
def match_action(self, action: Dict) -> Optional[Gesture]:
|
||
"""
|
||
Détecter si une action de workflow correspond à un geste primitif.
|
||
|
||
Utilisé pendant le replay pour auto-substituer les actions visuelles
|
||
par des raccourcis clavier plus fiables.
|
||
|
||
Patterns détectés :
|
||
- Clic sur boutons de contrôle fenêtre (X, □, ─)
|
||
- key_combo qui matche déjà un geste
|
||
- Actions avec target_text contenant des mots-clés de geste
|
||
"""
|
||
action_type = action.get("type", "")
|
||
|
||
# key_combo → vérifier si c'est déjà un geste connu
|
||
if action_type == "key_combo":
|
||
keys = action.get("keys", [])
|
||
return self._match_by_keys(keys)
|
||
|
||
# Clic sur un bouton de contrôle de fenêtre
|
||
if action_type == "click":
|
||
return self._match_click_as_gesture(action)
|
||
|
||
return None
|
||
|
||
def get_by_id(self, gesture_id: str) -> Optional[Gesture]:
|
||
return self._by_id.get(gesture_id)
|
||
|
||
def get_by_category(self, category: str) -> List[Gesture]:
|
||
return [g for g in self.gestures if g.category == category]
|
||
|
||
def get_by_context(self, context: str) -> List[Gesture]:
|
||
"""Gestes applicables à un contexte (inclut toujours 'windows')."""
|
||
return [
|
||
g for g in self.gestures
|
||
if g.context == context or g.context == "windows"
|
||
]
|
||
|
||
def list_all(self) -> List[Dict]:
|
||
"""Lister tous les gestes pour l'affichage."""
|
||
return [
|
||
{
|
||
"id": g.id,
|
||
"name": g.name,
|
||
"description": g.description,
|
||
"keys": "+".join(g.keys),
|
||
"category": g.category,
|
||
"context": g.context,
|
||
}
|
||
for g in self.gestures
|
||
]
|
||
|
||
# =========================================================================
|
||
# Scoring interne
|
||
# =========================================================================
|
||
|
||
def _compute_score(self, query: str, terms: List[str], gesture: Gesture) -> float:
|
||
"""Calculer le score de correspondance entre une requête et un geste."""
|
||
best = 0.0
|
||
query_words = set(query.split())
|
||
|
||
for term in terms:
|
||
# Match exact
|
||
if query == term:
|
||
return 1.0
|
||
|
||
# Contenu dans l'un ou l'autre sens
|
||
if query in term:
|
||
score = len(query) / len(term) * 0.95
|
||
best = max(best, score)
|
||
continue
|
||
if term in query:
|
||
# Si le terme est un alias exact (mot unique) présent dans la requête
|
||
# c'est un signal très fort : "copier le texte" contient "copier"
|
||
if term in query_words:
|
||
best = max(best, 0.85)
|
||
else:
|
||
score = len(term) / len(query) * 0.9
|
||
best = max(best, score)
|
||
continue
|
||
|
||
# Similarité de séquence
|
||
ratio = SequenceMatcher(None, query, term).ratio()
|
||
best = max(best, ratio)
|
||
|
||
# Bonus si tous les mots de la requête sont présents dans les termes
|
||
all_terms_text = " ".join(terms)
|
||
matched_words = sum(1 for w in query_words if w in all_terms_text)
|
||
if query_words:
|
||
word_ratio = matched_words / len(query_words)
|
||
if word_ratio >= 0.8:
|
||
best = max(best, 0.5 + word_ratio * 0.4)
|
||
|
||
return best
|
||
|
||
def _match_by_keys(self, keys: List[str]) -> Optional[Gesture]:
|
||
"""Trouver un geste par sa combinaison de touches exacte."""
|
||
keys_normalized = [k.lower() for k in keys]
|
||
for gesture in self.gestures:
|
||
if gesture.keys == keys_normalized:
|
||
return gesture
|
||
return None
|
||
|
||
def _match_click_as_gesture(self, action: Dict) -> Optional[Gesture]:
|
||
"""
|
||
Détecter si un clic correspond à un geste primitif.
|
||
|
||
Patterns :
|
||
- Clic en haut à droite de la fenêtre (x > 95%, y < 5%) → fermer
|
||
- target_text contenant ✕, ×, X, □, ─, etc.
|
||
- Commande applicative "Enregistrer" sûre → Ctrl+S
|
||
"""
|
||
# Vérifier le target_text
|
||
target_text = (
|
||
action.get("target_text", "") or
|
||
action.get("target_spec", {}).get("by_text", "")
|
||
).strip()
|
||
|
||
if target_text:
|
||
target_lower = target_text.lower()
|
||
# Bouton fermer
|
||
if target_lower in ("✕", "×", "x", "close", "fermer"):
|
||
return self._by_id.get("win_close")
|
||
# Bouton maximiser
|
||
if target_lower in ("□", "☐", "maximize", "agrandir"):
|
||
return self._by_id.get("win_maximize")
|
||
# Bouton minimiser
|
||
if target_lower in ("─", "—", "_", "minimize", "réduire"):
|
||
return self._by_id.get("win_minimize")
|
||
|
||
if self._is_save_command_action(action):
|
||
return self._by_id.get("edit_save")
|
||
|
||
# Vérifier la position relative (coin haut-droite = fermer)
|
||
x_pct = action.get("x_pct", 0)
|
||
y_pct = action.get("y_pct", 0)
|
||
|
||
if x_pct > 0.96 and y_pct < 0.04:
|
||
return self._by_id.get("win_close")
|
||
if 0.92 < x_pct < 0.96 and y_pct < 0.04:
|
||
return self._by_id.get("win_maximize")
|
||
if 0.88 < x_pct < 0.92 and y_pct < 0.04:
|
||
return self._by_id.get("win_minimize")
|
||
|
||
return None
|
||
|
||
def _normalize_ui_text(self, value: str) -> str:
|
||
"""Normaliser un libellé UI pour comparer accents, casse et raccourcis."""
|
||
text = str(value or "").strip().lower()
|
||
text = unicodedata.normalize("NFKD", text)
|
||
text = "".join(ch for ch in text if not unicodedata.combining(ch))
|
||
text = text.replace("’", "'")
|
||
text = re.sub(r"\s+", " ", text)
|
||
text = re.sub(r"\s*\([^)]*ctrl\s*\+?\s*s[^)]*\)\s*$", "", text)
|
||
text = re.sub(r"\s+ctrl\s*\+?\s*s\s*$", "", text)
|
||
return text.strip()
|
||
|
||
def _action_text_candidates(self, action: Dict) -> List[str]:
|
||
"""Retourner les libellés utiles d'une action et de son target_spec."""
|
||
target_spec = action.get("target_spec") or {}
|
||
candidates = [
|
||
action.get("target_text", ""),
|
||
action.get("target_description", ""),
|
||
action.get("description", ""),
|
||
target_spec.get("by_text", ""),
|
||
target_spec.get("target_text", ""),
|
||
target_spec.get("vlm_description", ""),
|
||
]
|
||
return [str(c) for c in candidates if c]
|
||
|
||
def _action_role_text(self, action: Dict) -> str:
|
||
target_spec = action.get("target_spec") or {}
|
||
uia = action.get("uia_snapshot") or {}
|
||
role_parts = [
|
||
action.get("role", ""),
|
||
action.get("control_type", ""),
|
||
target_spec.get("by_role", ""),
|
||
target_spec.get("role", ""),
|
||
target_spec.get("control_type", ""),
|
||
uia.get("control_type", ""),
|
||
uia.get("class_name", ""),
|
||
]
|
||
return " ".join(self._normalize_ui_text(part) for part in role_parts if part)
|
||
|
||
def _action_context_text(self, action: Dict) -> str:
|
||
target_spec = action.get("target_spec") or {}
|
||
hints = target_spec.get("context_hints") or {}
|
||
context_parts = [
|
||
action.get("window_title", ""),
|
||
target_spec.get("window_title", ""),
|
||
target_spec.get("vlm_description", ""),
|
||
hints.get("window_title", ""),
|
||
hints.get("interaction", ""),
|
||
hints.get("source", ""),
|
||
hints.get("menu_path", ""),
|
||
]
|
||
return " ".join(self._normalize_ui_text(part) for part in context_parts if part)
|
||
|
||
def _is_file_menu_action(self, action: Dict) -> bool:
|
||
labels = {self._normalize_ui_text(text) for text in self._action_text_candidates(action)}
|
||
return bool(labels & FILE_MENU_LABELS)
|
||
|
||
def _is_save_command_label(self, action: Dict) -> bool:
|
||
for text in self._action_text_candidates(action):
|
||
label = self._normalize_ui_text(text)
|
||
if not label:
|
||
continue
|
||
if any(save_as in label for save_as in SAVE_AS_LABELS):
|
||
return False
|
||
if label in SAVE_COMMAND_LABELS:
|
||
return True
|
||
return False
|
||
|
||
def _is_save_dialog_action(self, action: Dict) -> bool:
|
||
context = self._action_context_text(action)
|
||
if any(save_as in context for save_as in SAVE_AS_LABELS):
|
||
return True
|
||
dialog_markers = (
|
||
"save dialog",
|
||
"save_dialog",
|
||
"dialog",
|
||
"boite de dialogue",
|
||
"fenetre enregistrer sous",
|
||
"confirmer l'enregistrement",
|
||
"save changes",
|
||
)
|
||
return any(marker in context for marker in dialog_markers)
|
||
|
||
def _is_save_command_action(self, action: Dict) -> bool:
|
||
if not self._is_save_command_label(action):
|
||
return False
|
||
if self._is_save_dialog_action(action):
|
||
return False
|
||
|
||
role = self._action_role_text(action)
|
||
context = self._action_context_text(action)
|
||
command_markers = (
|
||
"menu",
|
||
"menuitem",
|
||
"item de menu",
|
||
"toolbar",
|
||
"barre d'outils",
|
||
"tool bar",
|
||
"ruban",
|
||
"ribbon",
|
||
"commande",
|
||
"command",
|
||
)
|
||
return any(marker in role or marker in context for marker in command_markers)
|
||
|
||
def _substitute_action(
|
||
self,
|
||
action: Dict,
|
||
gesture: Gesture,
|
||
*,
|
||
original_type: str,
|
||
source_action_ids: Optional[List[str]] = None,
|
||
reason: str = "",
|
||
) -> Dict:
|
||
new_action = gesture.to_replay_action()
|
||
new_action["action_id"] = action.get("action_id", new_action["action_id"])
|
||
new_action["original_type"] = original_type
|
||
if source_action_ids:
|
||
new_action["substitution_source_action_ids"] = source_action_ids
|
||
if reason:
|
||
new_action["substitution_reason"] = reason
|
||
return new_action
|
||
|
||
def optimize_replay_actions(self, actions: List[Dict]) -> List[Dict]:
|
||
"""
|
||
Optimiser une liste d'actions de replay en substituant les gestes connus.
|
||
|
||
Pour chaque action, si elle correspond à un geste primitif,
|
||
on la remplace par le raccourci clavier équivalent.
|
||
|
||
Retourne la liste d'actions optimisée (les originales non-matchées
|
||
sont conservées telles quelles).
|
||
"""
|
||
optimized = []
|
||
substitutions = 0
|
||
|
||
for action in actions:
|
||
if (
|
||
action.get("type") == "click"
|
||
and optimized
|
||
and optimized[-1].get("type") == "click"
|
||
and self._is_file_menu_action(optimized[-1])
|
||
and self._is_save_command_label(action)
|
||
and not self._is_save_dialog_action(action)
|
||
):
|
||
gesture = self._by_id.get("edit_save")
|
||
previous = optimized.pop()
|
||
source_ids = [
|
||
source_id for source_id in (
|
||
previous.get("action_id"),
|
||
action.get("action_id"),
|
||
)
|
||
if source_id
|
||
]
|
||
optimized.append(
|
||
self._substitute_action(
|
||
action,
|
||
gesture,
|
||
original_type="click_sequence",
|
||
source_action_ids=source_ids,
|
||
reason="file_menu_save_to_ctrl_s",
|
||
)
|
||
)
|
||
substitutions += 1
|
||
logger.debug("Séquence Fichier > Enregistrer substituée par Ctrl+S")
|
||
continue
|
||
|
||
gesture = self.match_action(action)
|
||
if gesture and action.get("type") != "key_combo":
|
||
# Substituer par le raccourci clavier
|
||
new_action = self._substitute_action(
|
||
action,
|
||
gesture,
|
||
original_type=action.get("type", ""),
|
||
reason=f"{gesture.id}_gesture_substitution",
|
||
)
|
||
optimized.append(new_action)
|
||
substitutions += 1
|
||
logger.debug(
|
||
f"Geste substitué: {action.get('type')} → {gesture.id} ({gesture.name})"
|
||
)
|
||
else:
|
||
optimized.append(action)
|
||
|
||
if substitutions:
|
||
logger.info(
|
||
f"Replay optimisé: {substitutions} action(s) substituée(s) par des primitives"
|
||
)
|
||
|
||
return optimized
|
||
|
||
|
||
# Singleton
|
||
_catalog: Optional[GestureCatalog] = None
|
||
|
||
|
||
def get_gesture_catalog() -> GestureCatalog:
|
||
global _catalog
|
||
if _catalog is None:
|
||
_catalog = GestureCatalog()
|
||
return _catalog
|