Files
rpa_vision_v3/tests/unit/test_gesture_catalog.py
Dom cf495dd82f feat: chat unifié, GestureCatalog, Copilot, Léa UI, extraction données, vérification replay
Refonte majeure du système Agent Chat et ajout de nombreux modules :

- Chat unifié : suppression du dual Workflows/Agent Libre, tout passe par /api/chat
  avec résolution en 3 niveaux (workflow → geste → "montre-moi")
- GestureCatalog : 38 raccourcis clavier universels Windows avec matching sémantique,
  substitution automatique dans les replays, et endpoint /api/gestures
- Mode Copilot : exécution pas-à-pas des workflows avec validation humaine via WebSocket
  (approve/skip/abort) avant chaque action
- Léa UI (agent_v0/lea_ui/) : interface PyQt5 pour Windows avec overlay transparent
  pour feedback visuel pendant le replay
- Data Extraction (core/extraction/) : moteur d'extraction visuelle de données
  (OCR + VLM → SQLite), avec schémas YAML et export CSV/Excel
- ReplayVerifier (agent_v0/server_v1/) : vérification post-action par comparaison
  de screenshots, avec logique de retry (max 3)
- IntentParser durci : meilleur fallback regex, type GREETING, patterns améliorés
- Dashboard : nouvelles pages gestures, streaming, extractions
- Tests : 63 tests GestureCatalog, 47 tests extraction, corrections tests existants
- Dépréciation : /api/agent/plan et /api/agent/execute retournent HTTP 410,
  suppression du code hardcodé _plan_to_replay_actions

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-15 10:02:09 +01:00

578 lines
24 KiB
Python

"""
Tests unitaires pour le GestureCatalog - Catalogue de primitives gestuelles.
Couvre :
- Matching textuel (exact, partiel, seuil, absence de faux positifs)
- Matching d'actions (position de clic, key_combo, target_text)
- Optimisation de replay (substitution, préservation, listes mixtes)
- Utilitaires (get_by_id, get_by_category, get_by_context, list_all, to_replay_action)
Auteur: Dom - Mars 2026
"""
import pytest
from agent_chat.gesture_catalog import Gesture, GestureCatalog, GESTURES
@pytest.fixture
def catalog():
"""Instance fraiche du catalogue avec les gestes par defaut."""
return GestureCatalog()
# =============================================================================
# 1. Tests de matching textuel
# =============================================================================
class TestGestureMatching:
"""Match de requetes textuelles vers des gestes primitifs."""
def test_exact_match_name_copier(self, catalog):
"""Match exact sur le nom 'Copier'."""
result = catalog.match("copier")
assert result is not None
gesture, score = result
assert gesture.id == "edit_copy"
assert score == 1.0
def test_exact_match_alias_nouvel_onglet(self, catalog):
"""Match exact sur l'alias 'nouvel onglet'."""
result = catalog.match("nouvel onglet")
assert result is not None
gesture, score = result
assert gesture.id == "chrome_new_tab"
assert score == 1.0
def test_exact_match_alias_fermer(self, catalog):
"""Match exact sur l'alias 'fermer'."""
result = catalog.match("fermer")
assert result is not None
gesture, score = result
assert gesture.id == "win_close"
assert score == 1.0
def test_exact_match_alias_coller(self, catalog):
"""Match exact sur l'alias 'coller'."""
result = catalog.match("coller")
assert result is not None
gesture, score = result
assert gesture.id == "edit_paste"
assert score == 1.0
def test_exact_match_alias_annuler(self, catalog):
"""Match exact sur l'alias 'annuler'."""
result = catalog.match("annuler")
assert result is not None
gesture, score = result
# 'annuler' est alias de edit_undo ET nav_escape ; les deux sont valides
assert gesture.id in ("edit_undo", "nav_escape")
assert score == 1.0
def test_partial_match_ferme_la_fenetre(self, catalog):
"""'ferme la fenetre' doit matcher win_close."""
result = catalog.match("ferme la fenêtre")
assert result is not None
gesture, score = result
assert gesture.id == "win_close"
assert score >= 0.5
def test_partial_match_ouvre_un_nouvel_onglet(self, catalog):
"""'ouvre un nouvel onglet' doit matcher chrome_new_tab."""
result = catalog.match("ouvre un nouvel onglet")
assert result is not None
gesture, score = result
assert gesture.id == "chrome_new_tab"
assert score >= 0.5
def test_partial_match_copier_le_texte(self, catalog):
"""'copier le texte' contient l'alias 'copier' => edit_copy."""
result = catalog.match("copier le texte")
assert result is not None
gesture, score = result
assert gesture.id == "edit_copy"
assert score >= 0.7
def test_partial_match_agrandir_la_fenetre(self, catalog):
"""'agrandir la fenetre' doit matcher win_maximize."""
result = catalog.match("agrandir la fenêtre")
assert result is not None
gesture, score = result
assert gesture.id == "win_maximize"
assert score >= 0.7
def test_partial_match_close_window(self, catalog):
"""'close window' (anglais) doit matcher win_close."""
result = catalog.match("close window")
assert result is not None
gesture, score = result
assert gesture.id == "win_close"
assert score == 1.0 # alias exact
def test_no_false_positive_recherche_google(self, catalog):
"""'recherche google' ne doit pas matcher un geste a min_score=0.75."""
result = catalog.match("recherche google", min_score=0.75)
assert result is None
def test_no_false_positive_blah_blah(self, catalog):
"""Requete sans rapport ne matche pas."""
result = catalog.match("blah blah test", min_score=0.5)
assert result is None
def test_no_false_positive_facturer_client(self, catalog):
"""'facturer le client Acme' ne doit pas matcher a min_score=0.65."""
result = catalog.match("facturer le client Acme", min_score=0.65)
assert result is None
def test_no_false_positive_dossier_patient(self, catalog):
"""'ouvrir le dossier patient' ne doit pas matcher a min_score=0.7."""
result = catalog.match("ouvrir le dossier patient", min_score=0.7)
assert result is None
def test_min_score_threshold_rejects_weak(self, catalog):
"""Un seuil eleve rejette les matchs faibles."""
# Avec min_score=1.0 seul un match exact passe
result_strict = catalog.match("ferme la fenêtre", min_score=1.0)
assert result_strict is None
# Avec min_score plus bas ca passe
result_relaxed = catalog.match("ferme la fenêtre", min_score=0.4)
assert result_relaxed is not None
def test_min_score_threshold_allows_exact(self, catalog):
"""Un match exact passe meme avec un seuil eleve."""
result = catalog.match("copier", min_score=0.99)
assert result is not None
assert result[1] == 1.0
def test_empty_query_returns_none(self, catalog):
"""Requete vide retourne None."""
assert catalog.match("") is None
assert catalog.match(" ") is None
def test_all_gestures_self_match(self, catalog):
"""Chaque geste doit matcher sur son propre nom avec score >= 0.9."""
for gesture in catalog.gestures:
result = catalog.match(gesture.name)
assert result is not None, f"Le geste '{gesture.id}' ne matche pas sur son propre nom '{gesture.name}'"
matched_gesture, score = result
assert score >= 0.9, (
f"Le geste '{gesture.id}' matche sur son nom avec score={score:.2f}, "
f"attendu >= 0.9"
)
def test_all_gestures_alias_match(self, catalog):
"""Chaque alias de geste doit matcher avec score >= 0.8."""
for gesture in catalog.gestures:
for alias in gesture.aliases:
result = catalog.match(alias)
assert result is not None, (
f"L'alias '{alias}' du geste '{gesture.id}' ne matche pas"
)
_, score = result
assert score >= 0.8, (
f"L'alias '{alias}' du geste '{gesture.id}' matche avec score={score:.2f}, "
f"attendu >= 0.8"
)
def test_case_insensitive_match(self, catalog):
"""Le matching est insensible a la casse."""
result = catalog.match("COPIER")
assert result is not None
assert result[0].id == "edit_copy"
assert result[1] == 1.0
# =============================================================================
# 2. Tests de matching d'actions
# =============================================================================
class TestActionMatching:
"""Match d'actions de workflow vers des gestes primitifs."""
def test_click_close_button_position(self, catalog):
"""Clic en haut a droite (x > 96%, y < 4%) => fermer fenetre."""
action = {"type": "click", "x_pct": 0.97, "y_pct": 0.02}
gesture = catalog.match_action(action)
assert gesture is not None
assert gesture.id == "win_close"
def test_click_maximize_button_position(self, catalog):
"""Clic sur la zone maximize (92% < x < 96%, y < 4%)."""
action = {"type": "click", "x_pct": 0.94, "y_pct": 0.02}
gesture = catalog.match_action(action)
assert gesture is not None
assert gesture.id == "win_maximize"
def test_click_minimize_button_position(self, catalog):
"""Clic sur la zone minimize (88% < x < 92%, y < 4%)."""
action = {"type": "click", "x_pct": 0.90, "y_pct": 0.02}
gesture = catalog.match_action(action)
assert gesture is not None
assert gesture.id == "win_minimize"
def test_click_center_no_match(self, catalog):
"""Clic au centre de l'ecran ne matche pas un geste."""
action = {"type": "click", "x_pct": 0.5, "y_pct": 0.5}
gesture = catalog.match_action(action)
assert gesture is None
def test_click_top_left_no_match(self, catalog):
"""Clic en haut a gauche ne matche pas un bouton de fenetre."""
action = {"type": "click", "x_pct": 0.05, "y_pct": 0.02}
gesture = catalog.match_action(action)
assert gesture is None
def test_key_combo_ctrl_t(self, catalog):
"""key_combo ctrl+t => chrome_new_tab."""
action = {"type": "key_combo", "keys": ["ctrl", "t"]}
gesture = catalog.match_action(action)
assert gesture is not None
assert gesture.id == "chrome_new_tab"
def test_key_combo_alt_f4(self, catalog):
"""key_combo alt+f4 => win_close."""
action = {"type": "key_combo", "keys": ["alt", "f4"]}
gesture = catalog.match_action(action)
assert gesture is not None
assert gesture.id == "win_close"
def test_key_combo_ctrl_c(self, catalog):
"""key_combo ctrl+c => edit_copy."""
action = {"type": "key_combo", "keys": ["ctrl", "c"]}
gesture = catalog.match_action(action)
assert gesture is not None
assert gesture.id == "edit_copy"
def test_key_combo_unknown(self, catalog):
"""key_combo inconnu ne matche pas."""
action = {"type": "key_combo", "keys": ["ctrl", "shift", "alt", "p"]}
gesture = catalog.match_action(action)
assert gesture is None
def test_target_text_close_symbol(self, catalog):
"""Clic sur target_text unicode de fermeture => win_close."""
action = {"type": "click", "x_pct": 0.5, "y_pct": 0.5, "target_text": "\u2715"}
gesture = catalog.match_action(action)
assert gesture is not None
assert gesture.id == "win_close"
def test_target_text_close_x(self, catalog):
"""Clic sur target_text 'X' => win_close."""
action = {"type": "click", "x_pct": 0.5, "y_pct": 0.5, "target_text": "X"}
gesture = catalog.match_action(action)
assert gesture is not None
assert gesture.id == "win_close"
def test_target_text_close_word(self, catalog):
"""Clic sur target_text 'Fermer' => win_close."""
action = {"type": "click", "x_pct": 0.5, "y_pct": 0.5, "target_text": "Fermer"}
gesture = catalog.match_action(action)
assert gesture is not None
assert gesture.id == "win_close"
def test_target_text_maximize_symbol(self, catalog):
"""Clic sur target_text '' => win_maximize."""
action = {"type": "click", "x_pct": 0.5, "y_pct": 0.5, "target_text": "\u25a1"}
gesture = catalog.match_action(action)
assert gesture is not None
assert gesture.id == "win_maximize"
def test_target_text_minimize_symbol(self, catalog):
"""Clic sur target_text '' => win_minimize."""
action = {"type": "click", "x_pct": 0.5, "y_pct": 0.5, "target_text": "\u2500"}
gesture = catalog.match_action(action)
assert gesture is not None
assert gesture.id == "win_minimize"
def test_target_text_via_target_spec(self, catalog):
"""target_text dans target_spec.by_text est aussi pris en compte."""
action = {
"type": "click",
"x_pct": 0.5,
"y_pct": 0.5,
"target_spec": {"by_text": "close"},
}
gesture = catalog.match_action(action)
assert gesture is not None
assert gesture.id == "win_close"
def test_unknown_action_type(self, catalog):
"""Type d'action inconnu ne matche pas."""
action = {"type": "scroll", "x_pct": 0.5, "y_pct": 0.5}
gesture = catalog.match_action(action)
assert gesture is None
def test_target_text_priority_over_position(self, catalog):
"""target_text prime sur la position du clic."""
# Clic en position close mais target_text dit minimize
action = {"type": "click", "x_pct": 0.97, "y_pct": 0.02, "target_text": "\u2500"}
gesture = catalog.match_action(action)
assert gesture is not None
assert gesture.id == "win_minimize"
def test_close_position_boundary_not_matched(self, catalog):
"""Position juste en dessous du seuil close (x=0.96, y=0.04) => pas de match."""
action = {"type": "click", "x_pct": 0.96, "y_pct": 0.04}
gesture = catalog.match_action(action)
# 0.96 n'est pas > 0.96, et 0.04 n'est pas < 0.04 => pas de match position
assert gesture is None
# =============================================================================
# 3. Tests d'optimisation de replay
# =============================================================================
class TestReplayOptimization:
"""Optimisation d'actions de replay par substitution de gestes."""
def test_optimize_close_click(self, catalog):
"""Un clic sur X (haut-droite) est remplace par Alt+F4."""
actions = [{"type": "click", "x_pct": 0.97, "y_pct": 0.02, "action_id": "a1"}]
optimized = catalog.optimize_replay_actions(actions)
assert len(optimized) == 1
assert optimized[0]["type"] == "key_combo"
assert optimized[0]["keys"] == ["alt", "f4"]
assert optimized[0]["action_id"] == "a1"
assert optimized[0]["gesture_id"] == "win_close"
def test_optimize_preserves_action_id(self, catalog):
"""L'action_id original est preserve apres substitution."""
actions = [{"type": "click", "x_pct": 0.97, "y_pct": 0.02, "action_id": "original_42"}]
optimized = catalog.optimize_replay_actions(actions)
assert optimized[0]["action_id"] == "original_42"
def test_optimize_preserves_normal_clicks(self, catalog):
"""Les clics normaux (centre) ne sont pas modifies."""
actions = [{"type": "click", "x_pct": 0.5, "y_pct": 0.5, "action_id": "a2"}]
optimized = catalog.optimize_replay_actions(actions)
assert len(optimized) == 1
assert optimized[0]["type"] == "click"
assert optimized[0]["action_id"] == "a2"
def test_optimize_mixed_actions(self, catalog):
"""Mix d'actions optimisables et normales."""
actions = [
{"type": "click", "x_pct": 0.5, "y_pct": 0.5, "action_id": "a1"},
{"type": "click", "x_pct": 0.97, "y_pct": 0.02, "action_id": "a2"},
{"type": "click", "x_pct": 0.3, "y_pct": 0.7, "action_id": "a3"},
{"type": "click", "x_pct": 0.94, "y_pct": 0.02, "action_id": "a4"},
]
optimized = catalog.optimize_replay_actions(actions)
assert len(optimized) == 4
# Premier : normal
assert optimized[0]["type"] == "click"
assert optimized[0]["action_id"] == "a1"
# Deuxieme : substitue (close)
assert optimized[1]["type"] == "key_combo"
assert optimized[1]["gesture_id"] == "win_close"
assert optimized[1]["action_id"] == "a2"
# Troisieme : normal
assert optimized[2]["type"] == "click"
assert optimized[2]["action_id"] == "a3"
# Quatrieme : substitue (maximize)
assert optimized[3]["type"] == "key_combo"
assert optimized[3]["gesture_id"] == "win_maximize"
assert optimized[3]["action_id"] == "a4"
def test_optimize_empty_list(self, catalog):
"""Liste vide => liste vide."""
optimized = catalog.optimize_replay_actions([])
assert optimized == []
def test_key_combo_not_double_substituted(self, catalog):
"""Un key_combo existant n'est pas substitue inutilement."""
actions = [
{"type": "key_combo", "keys": ["ctrl", "t"], "action_id": "k1"},
]
optimized = catalog.optimize_replay_actions(actions)
assert len(optimized) == 1
# L'action est conservee telle quelle (pas de substitution)
assert optimized[0]["type"] == "key_combo"
assert optimized[0]["keys"] == ["ctrl", "t"]
assert optimized[0]["action_id"] == "k1"
# Pas de champ gesture_id ajoute (action inchangee)
assert optimized[0] is actions[0]
def test_optimize_sets_original_type(self, catalog):
"""L'action substituee conserve le type original dans original_type."""
actions = [{"type": "click", "x_pct": 0.97, "y_pct": 0.02, "action_id": "a1"}]
optimized = catalog.optimize_replay_actions(actions)
assert optimized[0]["original_type"] == "click"
def test_optimize_target_text_substitution(self, catalog):
"""Un clic sur target_text 'Fermer' est substitue."""
actions = [
{"type": "click", "x_pct": 0.5, "y_pct": 0.5,
"target_text": "Fermer", "action_id": "t1"},
]
optimized = catalog.optimize_replay_actions(actions)
assert optimized[0]["type"] == "key_combo"
assert optimized[0]["keys"] == ["alt", "f4"]
assert optimized[0]["action_id"] == "t1"
def test_optimize_action_without_id(self, catalog):
"""Action substituee sans action_id recoit un id genere."""
actions = [{"type": "click", "x_pct": 0.97, "y_pct": 0.02}]
optimized = catalog.optimize_replay_actions(actions)
assert "action_id" in optimized[0]
# Le to_replay_action genere un id qui commence par "gesture_"
assert optimized[0]["action_id"].startswith("gesture_")
# =============================================================================
# 4. Tests utilitaires
# =============================================================================
class TestCatalogUtilities:
"""Tests des methodes utilitaires du catalogue."""
def test_get_by_id_existing(self, catalog):
"""get_by_id retourne le bon geste."""
gesture = catalog.get_by_id("win_close")
assert gesture is not None
assert gesture.id == "win_close"
assert gesture.name == "Fermer la fen\u00eatre"
assert gesture.keys == ["alt", "f4"]
def test_get_by_id_nonexistent(self, catalog):
"""get_by_id retourne None pour un id inconnu."""
gesture = catalog.get_by_id("geste_inexistant")
assert gesture is None
def test_get_by_category_window(self, catalog):
"""get_by_category('window') retourne les gestes de fenetre."""
window_gestures = catalog.get_by_category("window")
assert len(window_gestures) > 0
for g in window_gestures:
assert g.category == "window"
# Verifier qu'on retrouve bien win_close, win_maximize, win_minimize
ids = {g.id for g in window_gestures}
assert "win_close" in ids
assert "win_maximize" in ids
assert "win_minimize" in ids
def test_get_by_category_navigation(self, catalog):
"""get_by_category('navigation') retourne les gestes chrome."""
nav_gestures = catalog.get_by_category("navigation")
assert len(nav_gestures) > 0
for g in nav_gestures:
assert g.category == "navigation"
ids = {g.id for g in nav_gestures}
assert "chrome_new_tab" in ids
def test_get_by_category_editing(self, catalog):
"""get_by_category('editing') retourne les gestes d'edition."""
edit_gestures = catalog.get_by_category("editing")
assert len(edit_gestures) > 0
for g in edit_gestures:
assert g.category == "editing"
ids = {g.id for g in edit_gestures}
assert "edit_copy" in ids
assert "edit_paste" in ids
def test_get_by_category_system(self, catalog):
"""get_by_category('system') retourne les gestes systeme."""
sys_gestures = catalog.get_by_category("system")
assert len(sys_gestures) > 0
for g in sys_gestures:
assert g.category == "system"
ids = {g.id for g in sys_gestures}
assert "sys_start_menu" in ids
def test_get_by_category_empty(self, catalog):
"""get_by_category pour une categorie inconnue retourne une liste vide."""
gestures = catalog.get_by_category("categorie_inexistante")
assert gestures == []
def test_get_by_context_chrome(self, catalog):
"""get_by_context('chrome') inclut les gestes chrome ET windows."""
chrome_gestures = catalog.get_by_context("chrome")
contexts = {g.context for g in chrome_gestures}
# Doit inclure les gestes chrome et les gestes universels (windows)
assert "chrome" in contexts
assert "windows" in contexts
def test_get_by_context_windows_only(self, catalog):
"""get_by_context('windows') retourne uniquement les gestes universels."""
win_gestures = catalog.get_by_context("windows")
for g in win_gestures:
assert g.context == "windows"
def test_list_all_returns_all(self, catalog):
"""list_all retourne autant d'elements que de gestes."""
all_gestures = catalog.list_all()
assert len(all_gestures) == len(GESTURES)
assert len(all_gestures) == len(catalog.gestures)
def test_list_all_format(self, catalog):
"""list_all retourne des dicts avec les bonnes cles."""
all_gestures = catalog.list_all()
expected_keys = {"id", "name", "description", "keys", "category", "context"}
for entry in all_gestures:
assert set(entry.keys()) == expected_keys
def test_list_all_keys_format(self, catalog):
"""Les keys dans list_all sont jointes par '+'."""
all_gestures = catalog.list_all()
for entry in all_gestures:
assert isinstance(entry["keys"], str)
# Au moins un element => pas vide
assert len(entry["keys"]) > 0
def test_to_replay_action_format(self):
"""Verifier le format de l'action de replay genere par un geste."""
gesture = Gesture(
id="test_gesture",
name="Test Gesture",
description="Un geste de test",
keys=["ctrl", "shift", "x"],
)
action = gesture.to_replay_action()
assert action["type"] == "key_combo"
assert action["keys"] == ["ctrl", "shift", "x"]
assert action["gesture_id"] == "test_gesture"
assert action["gesture_name"] == "Test Gesture"
assert action["action_id"].startswith("gesture_test_gesture_")
# L'action_id a un suffixe hex de 6 chars
suffix = action["action_id"].split("_")[-1]
assert len(suffix) == 6
def test_to_replay_action_unique_ids(self):
"""Chaque appel a to_replay_action genere un action_id unique."""
gesture = Gesture(
id="test_unique",
name="Test Unique",
description="Verifier unicite des IDs",
keys=["f1"],
)
ids = {gesture.to_replay_action()["action_id"] for _ in range(100)}
assert len(ids) == 100
def test_gesture_dataclass_defaults(self):
"""Verifier les valeurs par defaut de la dataclass Gesture."""
gesture = Gesture(
id="minimal",
name="Minimal",
description="Minimal gesture",
keys=["a"],
)
assert gesture.aliases == []
assert gesture.tags == []
assert gesture.context == "windows"
assert gesture.category == "window"
def test_custom_catalog(self):
"""Un catalogue peut etre instancie avec des gestes personnalises."""
custom_gestures = [
Gesture(id="custom1", name="Custom One", description="Custom 1", keys=["f12"]),
Gesture(id="custom2", name="Custom Two", description="Custom 2", keys=["f11"]),
]
catalog = GestureCatalog(gestures=custom_gestures)
assert len(catalog.gestures) == 2
assert catalog.get_by_id("custom1") is not None
assert catalog.get_by_id("win_close") is None