From 99f89317cbb8f11587d166f16ffb91aeea542cc1 Mon Sep 17 00:00:00 2001 From: Dom Date: Fri, 29 May 2026 13:45:44 +0200 Subject: [PATCH] feat(lea): substitute save menu gesture --- agent_chat/gesture_catalog.py | 172 ++++++++++++++++++++++++++++- tests/unit/test_gesture_catalog.py | 95 ++++++++++++++++ 2 files changed, 263 insertions(+), 4 deletions(-) diff --git a/agent_chat/gesture_catalog.py b/agent_chat/gesture_catalog.py index 379170bec..9c1923e81 100644 --- a/agent_chat/gesture_catalog.py +++ b/agent_chat/gesture_catalog.py @@ -16,6 +16,7 @@ Auteur: Dom — Mars 2026 import logging import re +import unicodedata import uuid from dataclasses import dataclass, field from difflib import SequenceMatcher @@ -24,6 +25,11 @@ from typing import Dict, List, Optional, Tuple logger = logging.getLogger(__name__) +SAVE_COMMAND_LABELS = {"enregistrer", "save", "sauvegarder"} +SAVE_AS_LABELS = {"enregistrer sous", "save as", "sauvegarder sous"} +FILE_MENU_LABELS = {"fichier", "file", "menu fichier", "file menu"} + + @dataclass class Gesture: """Un geste primitif universel.""" @@ -564,6 +570,7 @@ class GestureCatalog: Patterns : - Clic en haut à droite de la fenêtre (x > 95%, y < 5%) → fermer - target_text contenant ✕, ×, X, □, ─, etc. + - Commande applicative "Enregistrer" sûre → Ctrl+S """ # Vérifier le target_text target_text = ( @@ -583,6 +590,9 @@ class GestureCatalog: if target_lower in ("─", "—", "_", "minimize", "réduire"): return self._by_id.get("win_minimize") + if self._is_save_command_action(action): + return self._by_id.get("edit_save") + # Vérifier la position relative (coin haut-droite = fermer) x_pct = action.get("x_pct", 0) y_pct = action.get("y_pct", 0) @@ -596,6 +606,128 @@ class GestureCatalog: return None + def _normalize_ui_text(self, value: str) -> str: + """Normaliser un libellé UI pour comparer accents, casse et raccourcis.""" + text = str(value or "").strip().lower() + text = unicodedata.normalize("NFKD", text) + text = "".join(ch for ch in text if not unicodedata.combining(ch)) + text = text.replace("’", "'") + text = re.sub(r"\s+", " ", text) + text = re.sub(r"\s*\([^)]*ctrl\s*\+?\s*s[^)]*\)\s*$", "", text) + text = re.sub(r"\s+ctrl\s*\+?\s*s\s*$", "", text) + return text.strip() + + def _action_text_candidates(self, action: Dict) -> List[str]: + """Retourner les libellés utiles d'une action et de son target_spec.""" + target_spec = action.get("target_spec") or {} + candidates = [ + action.get("target_text", ""), + action.get("target_description", ""), + action.get("description", ""), + target_spec.get("by_text", ""), + target_spec.get("target_text", ""), + target_spec.get("vlm_description", ""), + ] + return [str(c) for c in candidates if c] + + def _action_role_text(self, action: Dict) -> str: + target_spec = action.get("target_spec") or {} + uia = action.get("uia_snapshot") or {} + role_parts = [ + action.get("role", ""), + action.get("control_type", ""), + target_spec.get("by_role", ""), + target_spec.get("role", ""), + target_spec.get("control_type", ""), + uia.get("control_type", ""), + uia.get("class_name", ""), + ] + return " ".join(self._normalize_ui_text(part) for part in role_parts if part) + + def _action_context_text(self, action: Dict) -> str: + target_spec = action.get("target_spec") or {} + hints = target_spec.get("context_hints") or {} + context_parts = [ + action.get("window_title", ""), + target_spec.get("window_title", ""), + target_spec.get("vlm_description", ""), + hints.get("window_title", ""), + hints.get("interaction", ""), + hints.get("source", ""), + hints.get("menu_path", ""), + ] + return " ".join(self._normalize_ui_text(part) for part in context_parts if part) + + def _is_file_menu_action(self, action: Dict) -> bool: + labels = {self._normalize_ui_text(text) for text in self._action_text_candidates(action)} + return bool(labels & FILE_MENU_LABELS) + + def _is_save_command_label(self, action: Dict) -> bool: + for text in self._action_text_candidates(action): + label = self._normalize_ui_text(text) + if not label: + continue + if any(save_as in label for save_as in SAVE_AS_LABELS): + return False + if label in SAVE_COMMAND_LABELS: + return True + return False + + def _is_save_dialog_action(self, action: Dict) -> bool: + context = self._action_context_text(action) + if any(save_as in context for save_as in SAVE_AS_LABELS): + return True + dialog_markers = ( + "save dialog", + "save_dialog", + "dialog", + "boite de dialogue", + "fenetre enregistrer sous", + "confirmer l'enregistrement", + "save changes", + ) + return any(marker in context for marker in dialog_markers) + + def _is_save_command_action(self, action: Dict) -> bool: + if not self._is_save_command_label(action): + return False + if self._is_save_dialog_action(action): + return False + + role = self._action_role_text(action) + context = self._action_context_text(action) + command_markers = ( + "menu", + "menuitem", + "item de menu", + "toolbar", + "barre d'outils", + "tool bar", + "ruban", + "ribbon", + "commande", + "command", + ) + return any(marker in role or marker in context for marker in command_markers) + + def _substitute_action( + self, + action: Dict, + gesture: Gesture, + *, + original_type: str, + source_action_ids: Optional[List[str]] = None, + reason: str = "", + ) -> Dict: + new_action = gesture.to_replay_action() + new_action["action_id"] = action.get("action_id", new_action["action_id"]) + new_action["original_type"] = original_type + if source_action_ids: + new_action["substitution_source_action_ids"] = source_action_ids + if reason: + new_action["substitution_reason"] = reason + return new_action + def optimize_replay_actions(self, actions: List[Dict]) -> List[Dict]: """ Optimiser une liste d'actions de replay en substituant les gestes connus. @@ -610,13 +742,45 @@ class GestureCatalog: substitutions = 0 for action in actions: + if ( + action.get("type") == "click" + and optimized + and optimized[-1].get("type") == "click" + and self._is_file_menu_action(optimized[-1]) + and self._is_save_command_label(action) + and not self._is_save_dialog_action(action) + ): + gesture = self._by_id.get("edit_save") + previous = optimized.pop() + source_ids = [ + source_id for source_id in ( + previous.get("action_id"), + action.get("action_id"), + ) + if source_id + ] + optimized.append( + self._substitute_action( + action, + gesture, + original_type="click_sequence", + source_action_ids=source_ids, + reason="file_menu_save_to_ctrl_s", + ) + ) + substitutions += 1 + logger.debug("Séquence Fichier > Enregistrer substituée par Ctrl+S") + continue + gesture = self.match_action(action) if gesture and action.get("type") != "key_combo": # Substituer par le raccourci clavier - new_action = gesture.to_replay_action() - # Conserver l'action_id original pour le tracking - new_action["action_id"] = action.get("action_id", new_action["action_id"]) - new_action["original_type"] = action.get("type") + new_action = self._substitute_action( + action, + gesture, + original_type=action.get("type", ""), + reason=f"{gesture.id}_gesture_substitution", + ) optimized.append(new_action) substitutions += 1 logger.debug( diff --git a/tests/unit/test_gesture_catalog.py b/tests/unit/test_gesture_catalog.py index c51025b8a..8c468e073 100644 --- a/tests/unit/test_gesture_catalog.py +++ b/tests/unit/test_gesture_catalog.py @@ -413,6 +413,101 @@ class TestReplayOptimization: assert optimized[0]["keys"] == ["alt", "f4"] assert optimized[0]["action_id"] == "t1" + def test_optimize_file_save_sequence_to_ctrl_s(self, catalog): + """Fichier > Enregistrer est remplace par Ctrl+S.""" + actions = [ + { + "type": "click", + "target_text": "Fichier", + "target_spec": {"by_text": "Fichier", "by_role": "menuitem"}, + "action_id": "file_menu", + }, + { + "type": "click", + "target_text": "Enregistrer", + "target_spec": {"by_text": "Enregistrer", "by_role": "menuitem"}, + "action_id": "save_menu", + }, + ] + optimized = catalog.optimize_replay_actions(actions) + assert len(optimized) == 1 + assert optimized[0]["type"] == "key_combo" + assert optimized[0]["keys"] == ["ctrl", "s"] + assert optimized[0]["gesture_id"] == "edit_save" + assert optimized[0]["action_id"] == "save_menu" + assert optimized[0]["original_type"] == "click_sequence" + assert optimized[0]["substitution_source_action_ids"] == ["file_menu", "save_menu"] + assert optimized[0]["substitution_reason"] == "file_menu_save_to_ctrl_s" + + def test_optimize_menu_save_action_to_ctrl_s(self, catalog): + """Une commande de menu Enregistrer isolee est remplacee par Ctrl+S.""" + actions = [ + { + "type": "click", + "target_spec": {"by_text": "Enregistrer", "by_role": "menuitem"}, + "action_id": "save_menu", + }, + ] + optimized = catalog.optimize_replay_actions(actions) + assert len(optimized) == 1 + assert optimized[0]["type"] == "key_combo" + assert optimized[0]["keys"] == ["ctrl", "s"] + assert optimized[0]["gesture_id"] == "edit_save" + + def test_optimize_toolbar_save_action_to_ctrl_s(self, catalog): + """Un bouton Enregistrer de barre d'outils est remplace par Ctrl+S.""" + actions = [ + { + "type": "click", + "target_spec": { + "by_text": "Enregistrer", + "by_role": "button", + "vlm_description": "Bouton Enregistrer dans la barre d'outils", + }, + "action_id": "save_toolbar", + }, + ] + optimized = catalog.optimize_replay_actions(actions) + assert len(optimized) == 1 + assert optimized[0]["type"] == "key_combo" + assert optimized[0]["keys"] == ["ctrl", "s"] + assert optimized[0]["gesture_id"] == "edit_save" + + def test_save_dialog_button_not_substituted(self, catalog): + """Le bouton Enregistrer d'un dialogue Enregistrer sous reste un clic.""" + actions = [ + { + "type": "click", + "target_spec": { + "by_text": "Enregistrer", + "by_role": "button", + "window_title": "Enregistrer sous", + "context_hints": { + "interaction": "save_dialog_primary_button", + }, + }, + "action_id": "save_dialog", + }, + ] + optimized = catalog.optimize_replay_actions(actions) + assert len(optimized) == 1 + assert optimized[0] is actions[0] + assert optimized[0]["type"] == "click" + + def test_save_as_menu_not_substituted(self, catalog): + """Enregistrer sous n'est pas equivalent a Ctrl+S.""" + actions = [ + { + "type": "click", + "target_spec": {"by_text": "Enregistrer sous", "by_role": "menuitem"}, + "action_id": "save_as_menu", + }, + ] + optimized = catalog.optimize_replay_actions(actions) + assert len(optimized) == 1 + assert optimized[0] is actions[0] + assert optimized[0]["type"] == "click" + def test_optimize_action_without_id(self, catalog): """Action substituee sans action_id recoit un id genere.""" actions = [{"type": "click", "x_pct": 0.97, "y_pct": 0.02}]