Files
rpa_vision_v3/tests/unit/test_env_setup.py
Dom 7df51d2c79 snapshot: WIP 5j replay reliability (B1 watchdog + dialog handlers + grounding drift)
Snapshot avant correction du blocage relance Léa (3 incidents 24h: SSH refusé,
polls morts ×2). Point de rollback stable.

Contenu:
- agent_v1/core/executor.py: 5 patchs dialog handling (saveas drift, close_tab
  hotkey fallback, confirm_save Unicode apostrophe, foreground dialog
  recontextualization, runtime_dialog in-loop) + helpers normalize_window_hint,
  requires_post_verify_window_transition
- agent_v1/core/grounding.py: garde drift template fix (fallback_x/y plumbed)
- server_v1/replay_watchdog.py (NEW): orphan watchdog B1, scan 10s timeout 30s
- server_v1/api_stream.py: dispatched_action plumbing, watchdog lifespan,
  metrics endpoint
- server_v1/replay_engine.py: _schedule_retry préserve original_action +
  dispatched_action
- stream_processor.py: gardes _infer_tab_switch_target (no false switch_tab
  on save_as dialog open) + _attach_expected_window_before
- tests/integration: test_replay_watchdog.py (8 cas), test_stream_processor.py
- tests/unit: test_executor_verify_window_guard.py (start_button, close_tab,
  runtime_dialog, post_verify, transition fallbacks)

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-24 16:48:37 +02:00

1179 lines
51 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
# tests/unit/test_env_setup.py
"""
Tests unitaires pour la phase de setup environnement (pré-replay).
Vérifie que les fonctions d'extraction d'apps et de génération
d'actions de setup 100% visuelles fonctionnent correctement.
"""
import pytest
import sys
from pathlib import Path
# Ajouter le répertoire racine au path pour l'import
ROOT = Path(__file__).parent.parent.parent
sys.path.insert(0, str(ROOT))
from agent_v0.server_v1.api_stream import (
_extract_required_apps_from_events,
_extract_required_apps_from_workflow,
_trim_redundant_setup_events,
_resolve_launch_command,
_infer_app_from_window_titles,
_generate_setup_actions,
_get_visual_search_info,
_APP_LAUNCH_COMMANDS,
_APP_VISUAL_SEARCH,
_SETUP_IGNORE_APPS,
)
# =========================================================================
# Tests pour _resolve_launch_command
# =========================================================================
class TestResolveLaunchCommand:
"""Tests pour la résolution des commandes de lancement."""
def test_known_app(self):
"""Les apps connues retournent la bonne commande."""
assert _resolve_launch_command("Notepad.exe") == "notepad"
assert _resolve_launch_command("notepad.exe") == "notepad"
def test_known_app_case_insensitive(self):
"""Le mapping est insensible à la casse."""
assert _resolve_launch_command("NOTEPAD.EXE") == "notepad"
assert _resolve_launch_command("Chrome.exe") == "chrome"
def test_unknown_app_strips_exe(self):
"""Les apps inconnues utilisent le nom sans .exe."""
assert _resolve_launch_command("MonApp.exe") == "MonApp"
assert _resolve_launch_command("customtool.exe") == "customtool"
def test_no_exe_extension(self):
"""Les noms sans .exe sont retournés tels quels."""
assert _resolve_launch_command("notepad") == "notepad"
def test_all_mapped_apps(self):
"""Toutes les apps du mapping sont résolvables."""
for app_name, expected_cmd in _APP_LAUNCH_COMMANDS.items():
assert _resolve_launch_command(app_name) == expected_cmd
# =========================================================================
# Tests pour _get_visual_search_info
# =========================================================================
class TestGetVisualSearchInfo:
"""Tests pour la résolution des infos de recherche visuelle."""
def test_known_app_notepad(self):
"""Notepad retourne les infos visuelles françaises."""
info = _get_visual_search_info("Notepad.exe")
assert info["search_text"] == "Bloc-notes"
assert info["display_name"] == "Bloc-notes"
assert "Bloc-notes" in info["vlm_description"]
def test_known_app_calc(self):
"""Calculatrice retourne les infos visuelles françaises."""
info = _get_visual_search_info("calc.exe")
assert info["search_text"] == "Calculatrice"
def test_known_app_word(self):
"""Word retourne les infos visuelles."""
info = _get_visual_search_info("winword.exe")
assert info["search_text"] == "Word"
assert info["display_name"] == "Microsoft Word"
def test_case_insensitive(self):
"""Le mapping est insensible à la casse."""
info = _get_visual_search_info("NOTEPAD.EXE")
assert info["search_text"] == "Bloc-notes"
def test_unknown_app_fallback(self):
"""Une app inconnue utilise le nom sans .exe comme fallback."""
info = _get_visual_search_info("MonApp.exe")
assert info["search_text"] == "MonApp"
assert info["display_name"] == "MonApp"
assert "MonApp" in info["vlm_description"]
def test_unknown_app_no_exe(self):
"""Une app sans .exe utilise le nom tel quel."""
info = _get_visual_search_info("myapp")
assert info["search_text"] == "myapp"
def test_all_visual_apps_have_required_keys(self):
"""Toutes les apps du mapping visuel ont les clés requises."""
for app_name, info in _APP_VISUAL_SEARCH.items():
assert "search_text" in info, f"{app_name} manque search_text"
assert "display_name" in info, f"{app_name} manque display_name"
assert "vlm_description" in info, f"{app_name} manque vlm_description"
# =========================================================================
# Tests pour _infer_app_from_window_titles
# =========================================================================
class TestInferAppFromWindowTitles:
"""Tests pour l'inférence d'app depuis les titres de fenêtres."""
def test_notepad_french(self):
"""Détecte Notepad depuis un titre français."""
app, cmd, title = _infer_app_from_window_titles(["Sans titre Bloc-notes"])
assert app == "Notepad.exe"
assert cmd == "notepad"
assert title == "Sans titre Bloc-notes"
def test_notepad_english(self):
"""Détecte Notepad depuis un titre anglais."""
app, cmd, title = _infer_app_from_window_titles(["Untitled - Notepad"])
assert app == "Notepad.exe"
assert cmd == "notepad"
def test_word(self):
"""Détecte Word."""
app, cmd, _ = _infer_app_from_window_titles(["Document1 - Word"])
assert app == "winword.exe"
assert cmd == "winword"
def test_excel(self):
"""Détecte Excel."""
app, cmd, _ = _infer_app_from_window_titles(["Classeur1 - Excel"])
assert app == "excel.exe"
assert cmd == "excel"
def test_chrome(self):
"""Détecte Chrome."""
app, cmd, _ = _infer_app_from_window_titles(["Google - Chrome"])
assert app == "chrome.exe"
assert cmd == "chrome"
def test_explorer_ignored(self):
"""Explorer est ignoré (app système)."""
app, cmd, _ = _infer_app_from_window_titles(["Explorateur de fichiers"])
assert app == ""
assert cmd == ""
def test_unknown_title(self):
"""Un titre inconnu retourne des chaînes vides."""
app, cmd, _ = _infer_app_from_window_titles(["Ma Super App Custom"])
assert app == ""
assert cmd == ""
def test_empty_list(self):
"""Une liste vide retourne des chaînes vides."""
app, cmd, _ = _infer_app_from_window_titles([])
assert app == ""
assert cmd == ""
def test_first_match_wins(self):
"""Le premier titre reconnu est utilisé."""
app, cmd, title = _infer_app_from_window_titles([
"Rechercher", # Pas reconnu
"*test Bloc-notes", # Notepad
"Document1 - Excel", # Excel (pas utilisé car Notepad est trouvé avant)
])
assert app == "Notepad.exe"
assert title == "*test Bloc-notes"
def test_returns_matched_title(self):
"""Le titre matché est celui de l'app, pas le premier de la liste."""
app, cmd, title = _infer_app_from_window_titles([
"Rechercher", # Pas reconnu
"Sans titre Bloc-notes", # Notepad → ce titre
])
assert title == "Sans titre Bloc-notes"
# =========================================================================
# Tests pour _extract_required_apps_from_events
# =========================================================================
class TestExtractRequiredAppsFromEvents:
"""Tests pour l'extraction d'apps depuis les événements bruts."""
def _make_events(self, focus_changes):
"""Helper : créer des événements bruts à partir de changements de focus."""
events = []
for from_info, to_info in focus_changes:
events.append({
"session_id": "test_sess",
"event": {
"type": "window_focus_change",
"from": from_info,
"to": to_info,
},
})
return events
def test_notepad_session(self):
"""Détecte Notepad comme app principale."""
events = self._make_events([
(None, {"app_name": "explorer.exe", "title": "Explorateur"}),
({"app_name": "explorer.exe"}, {"app_name": "SearchHost.exe", "title": "Rechercher"}),
({"app_name": "SearchHost.exe"}, {"app_name": "Notepad.exe", "title": "Bloc-notes"}),
({"app_name": "Notepad.exe"}, {"app_name": "Notepad.exe", "title": "Sans titre Bloc-notes"}),
({"app_name": "Notepad.exe"}, {"app_name": "Notepad.exe", "title": "*test Bloc-notes"}),
])
result = _extract_required_apps_from_events(events)
assert result["primary_app"] == "Notepad.exe"
assert result["primary_launch_cmd"] == "notepad"
# Le premier app hors ignorées est Notepad
assert result["first_window_title"] == "Bloc-notes"
def test_extracts_searchhost_launch_result_target(self):
"""Récupère le vrai clic SearchHost qui lance l'app."""
events = [
{"event": {"type": "window_focus_change", "from": None, "to": {
"app_name": "explorer.exe", "title": "Explorateur"}}},
{"event": {"type": "window_focus_change", "from": {
"app_name": "explorer.exe", "title": "Explorateur"}, "to": {
"app_name": "SearchHost.exe", "title": "Rechercher"}}},
{"event": {"type": "text_input", "text": "bloc", "window": {
"app_name": "SearchHost.exe", "title": "Rechercher"}}},
{"event": {"type": "mouse_click", "button": "left", "pos": [1449, 641],
"timestamp": 10.0,
"screen_metadata": {"screen_resolution": [2560, 1600]},
"window": {"app_name": "SearchHost.exe", "title": "Rechercher"},
"window_capture": {
"click_relative": [681, 448],
"window_size": [1287, 1407],
}}},
{"event": {"type": "window_focus_change", "from": {
"app_name": "SearchHost.exe", "title": "Rechercher"}, "to": {
"app_name": "explorer.exe", "title": "unknown_window"},
"timestamp": 10.4}},
{"event": {"type": "window_focus_change", "from": {
"app_name": "explorer.exe", "title": "unknown_window"}, "to": {
"app_name": "Notepad.exe", "title": "Sans titre Bloc-notes"},
"timestamp": 11.1}},
]
result = _extract_required_apps_from_events(events)
target = result["launch_result_target"]
assert result["primary_app"] == "Notepad.exe"
assert target["window_title"] == "Rechercher"
assert target["expected_window_before"] == "Rechercher"
assert target["x_pct"] == pytest.approx(1449 / 2560, rel=0, abs=1e-6)
assert target["y_pct"] == pytest.approx(641 / 1600, rel=0, abs=1e-6)
assert target["original_position"]["x_relative"] == "au centre"
assert target["original_position"]["y_relative"] == "au milieu"
assert target["window_capture"]["click_relative"] == [681, 448]
def test_extracts_start_menu_target(self):
"""Récupère le vrai clic Démarrer qui ouvre SearchHost."""
events = [
{"event": {"type": "window_focus_change", "from": None, "to": {
"app_name": "explorer.exe", "title": "Explorateur"}}},
{"event": {"type": "mouse_click", "button": "left", "pos": [993, 1559],
"timestamp": 1.0,
"screen_metadata": {"screen_resolution": [2560, 1600]},
"window": {"app_name": "explorer.exe", "title": "Explorateur"}}},
{"event": {"type": "window_focus_change", "from": {
"app_name": "explorer.exe", "title": "Explorateur"}, "to": {
"app_name": "SearchHost.exe", "title": "Rechercher"},
"timestamp": 1.2}},
{"event": {"type": "mouse_click", "button": "left", "pos": [1449, 641],
"timestamp": 4.0,
"screen_metadata": {"screen_resolution": [2560, 1600]},
"window": {"app_name": "SearchHost.exe", "title": "Rechercher"}}},
{"event": {"type": "window_focus_change", "from": {
"app_name": "SearchHost.exe", "title": "Rechercher"}, "to": {
"app_name": "Notepad.exe", "title": "Sans titre Bloc-notes"},
"timestamp": 4.4}},
]
result = _extract_required_apps_from_events(events)
target = result["start_menu_target"]
assert target["x_pct"] == pytest.approx(993 / 2560, rel=0, abs=1e-6)
assert target["y_pct"] == pytest.approx(1559 / 1600, rel=0, abs=1e-6)
assert target["original_position"]["x_relative"] == "au centre"
assert target["original_position"]["y_relative"] == "en bas"
assert "en bas" in target["position_desc"]
def test_extracts_start_menu_target_anchor_from_session_shot(self, tmp_path):
"""Le clic Démarrer récupère aussi une ancre image depuis le shot source."""
from PIL import Image
session_dir = tmp_path / "sess"
shots_dir = session_dir / "shots"
shots_dir.mkdir(parents=True)
Image.new("RGB", (2560, 1600), color="white").save(
shots_dir / "shot_start_full.png"
)
events = [
{"event": {"type": "window_focus_change", "from": None, "to": {
"app_name": "explorer.exe", "title": "Explorateur"}}},
{"event": {"type": "mouse_click", "button": "left", "pos": [993, 1559],
"timestamp": 1.0,
"screenshot_id": "shot_start",
"screen_metadata": {"screen_resolution": [2560, 1600]},
"window": {"app_name": "explorer.exe", "title": "Explorateur"}}},
{"event": {"type": "window_focus_change", "from": {
"app_name": "explorer.exe", "title": "Explorateur"}, "to": {
"app_name": "SearchHost.exe", "title": "Rechercher"},
"timestamp": 1.2}},
{"event": {"type": "window_focus_change", "from": {
"app_name": "SearchHost.exe", "title": "Rechercher"}, "to": {
"app_name": "Notepad.exe", "title": "Sans titre Bloc-notes"},
"timestamp": 2.0}},
]
result = _extract_required_apps_from_events(
events,
session_dir=str(session_dir),
)
target = result["start_menu_target"]
assert target["anchor_image_base64"]
def test_extracts_direct_typing_search_interaction(self):
"""Détecte qu'aucun clic SearchHost n'est requis avant la saisie."""
events = [
{"event": {"type": "window_focus_change", "from": None, "to": {
"app_name": "explorer.exe", "title": "Explorateur"}}},
{"event": {"type": "mouse_click", "button": "left", "pos": [993, 1559],
"timestamp": 1.0,
"screen_metadata": {"screen_resolution": [2560, 1600]},
"window": {"app_name": "explorer.exe", "title": "Explorateur"}}},
{"event": {"type": "window_focus_change", "from": {
"app_name": "explorer.exe", "title": "Explorateur"}, "to": {
"app_name": "SearchHost.exe", "title": "Rechercher"},
"timestamp": 1.2}},
{"event": {"type": "text_input", "text": "bloc",
"window": {"app_name": "SearchHost.exe", "title": "Rechercher"},
"timestamp": 2.0}},
{"event": {"type": "window_focus_change", "from": {
"app_name": "SearchHost.exe", "title": "Rechercher"}, "to": {
"app_name": "Notepad.exe", "title": "Sans titre Bloc-notes"},
"timestamp": 2.4}},
]
result = _extract_required_apps_from_events(events)
assert result["search_box_interaction"]["mode"] == "direct_typing"
assert result["search_box_interaction"]["window_title"] == "Rechercher"
def test_empty_events(self):
"""Pas d'événements → dict vide."""
assert _extract_required_apps_from_events([]) == {}
def test_only_system_apps(self):
"""Que des apps système → dict vide."""
events = self._make_events([
(None, {"app_name": "explorer.exe", "title": "Bureau"}),
(None, {"app_name": "SearchHost.exe", "title": "Rechercher"}),
])
assert _extract_required_apps_from_events(events) == {}
def test_multiple_apps_picks_most_frequent(self):
"""L'app la plus fréquente (hors système) est choisie."""
events = self._make_events([
(None, {"app_name": "Notepad.exe", "title": "Bloc-notes"}),
(None, {"app_name": "calc.exe", "title": "Calculatrice"}),
(None, {"app_name": "calc.exe", "title": "Calculatrice"}),
(None, {"app_name": "calc.exe", "title": "Calculatrice"}),
])
result = _extract_required_apps_from_events(events)
assert result["primary_app"] == "calc.exe"
assert result["primary_launch_cmd"] == "calc"
class TestTrimRedundantSetupEvents:
"""Tests pour la coupe du préambule déjà couvert par le setup."""
def test_trims_until_first_primary_app_focus(self):
raw_events = [
{"event": {"type": "window_focus_change", "to": {
"app_name": "explorer.exe", "title": "Explorateur"}}},
{"event": {"type": "mouse_click", "pos": [993, 1559], "window": {
"app_name": "explorer.exe", "title": "Explorateur"}}},
{"event": {"type": "window_focus_change", "to": {
"app_name": "SearchHost.exe", "title": "Rechercher"}}},
{"event": {"type": "text_input", "text": "bloc", "window": {
"app_name": "SearchHost.exe", "title": "Rechercher"}}},
{"event": {"type": "mouse_click", "pos": [1449, 641], "window": {
"app_name": "SearchHost.exe", "title": "Rechercher"}}},
{"event": {"type": "window_focus_change", "to": {
"app_name": "Notepad.exe",
"title": "http192.168.1.408765dossier.htmlid=.txt Bloc-notes",
}}},
{"event": {"type": "mouse_click", "pos": [1514, 562], "window": {
"app_name": "Notepad.exe", "title": "*test Bloc-notes"}}},
{"event": {"type": "text_input", "text": "test", "window": {
"app_name": "Notepad.exe", "title": "*test Bloc-notes"}}},
]
app_info = {
"primary_app": "Notepad.exe",
"first_window_title": "Bloc-notes",
}
trimmed = _trim_redundant_setup_events(raw_events, app_info)
assert len(trimmed) == 2
assert trimmed[0]["event"]["type"] == "mouse_click"
assert trimmed[0]["event"]["pos"] == [1514, 562]
assert trimmed[1]["event"]["type"] == "text_input"
def test_keeps_events_when_no_matching_focus_found(self):
raw_events = [
{"event": {"type": "mouse_click", "pos": [10, 10], "window": {
"app_name": "explorer.exe", "title": "Explorateur"}}},
{"event": {"type": "text_input", "text": "abc", "window": {
"app_name": "explorer.exe", "title": "Explorateur"}}},
]
app_info = {
"primary_app": "Notepad.exe",
"first_window_title": "Bloc-notes",
}
trimmed = _trim_redundant_setup_events(raw_events, app_info)
assert trimmed == raw_events
def test_prefers_neutral_title_focus_after_non_neutral_first_focus(self):
"""Cas observé sess_20260520T102916_066851 : premier focus Notepad
a un titre non-neutre (http...txt), suivi d'un clic intra-Notepad
et d'un focus vers 'Sans titre' (= état initial neutre que le setup
auto produit). Le trim doit couper jusqu'au focus neutre pour
éliminer le clic intra-Notepad redondant.
"""
raw_events = [
{"event": {"type": "window_focus_change", "to": {
"app_name": "SearchHost.exe", "title": "Rechercher"}}},
{"event": {"type": "mouse_click", "pos": [681, 448], "window": {
"app_name": "SearchHost.exe", "title": "Rechercher"}}},
{"event": {"type": "window_focus_change", "to": {
"app_name": "Notepad.exe",
"title": "http192.168.1.408765dossier.htmlid=.txt Bloc-notes",
}}},
{"event": {"type": "mouse_click", "pos": [1191, 40], "window": {
"app_name": "Notepad.exe",
"title": "http192.168.1.408765dossier.htmlid=.txt Bloc-notes",
}}},
{"event": {"type": "window_focus_change", "to": {
"app_name": "Notepad.exe", "title": "Sans titre Bloc-notes"}}},
{"event": {"type": "text_input", "text": "test", "window": {
"app_name": "Notepad.exe", "title": "*test Bloc-notes"}}},
]
app_info = {
"primary_app": "Notepad.exe",
"first_window_title": (
"http192.168.1.408765dossier.htmlid=.txt Bloc-notes"
),
}
trimmed = _trim_redundant_setup_events(raw_events, app_info)
# Le clic intra-Notepad (event idx 3) doit être supprimé : il
# bascule vers 'Sans titre' qui est déjà l'état setup, donc
# rejoué il n'a aucun effet visuel et déclenche retry_threshold.
assert len(trimmed) == 1
assert trimmed[0]["event"]["type"] == "text_input"
assert trimmed[0]["event"]["text"] == "test"
def test_neutral_focus_outside_lookahead_window_is_ignored(self):
"""Filet de sécurité : un focus 'Sans titre' qui arrive trop loin
après le premier focus primary_app n'est pas considéré comme
l'état de bootstrap. Évite de couper un workflow qui re-visite
'Sans titre' bien après le démarrage."""
# 30 events séparent le premier focus du focus neutre
raw_events = [
{"event": {"type": "window_focus_change", "to": {
"app_name": "Notepad.exe",
"title": "rapport_final.txt Bloc-notes"}}},
]
# Bourrer avec des events utiles intra-Notepad
for i in range(30):
raw_events.append({"event": {
"type": "mouse_click", "pos": [100 + i, 200],
"window": {"app_name": "Notepad.exe",
"title": "rapport_final.txt Bloc-notes"},
}})
raw_events.append({"event": {"type": "window_focus_change", "to": {
"app_name": "Notepad.exe", "title": "Sans titre Bloc-notes"}}})
raw_events.append({"event": {"type": "text_input", "text": "x",
"window": {"app_name": "Notepad.exe",
"title": "Sans titre Bloc-notes"}}})
app_info = {
"primary_app": "Notepad.exe",
"first_window_title": "rapport_final.txt Bloc-notes",
}
trimmed = _trim_redundant_setup_events(raw_events, app_info)
# Doit garder les 30 clicks + focus tardif + text_input = 32 events
# (cut uniquement au premier focus primary_app, comportement legacy)
assert len(trimmed) == 32
assert trimmed[0]["event"]["type"] == "mouse_click"
assert trimmed[0]["event"]["pos"] == [100, 200]
def test_keeps_legacy_behavior_when_first_focus_already_neutral(self):
"""Non-régression : si le premier focus primary_app est déjà sur
un titre neutre (cas normal), on coupe au premier focus comme
avant — pas de chasse au neutral_idx inutile."""
raw_events = [
{"event": {"type": "window_focus_change", "to": {
"app_name": "SearchHost.exe", "title": "Rechercher"}}},
{"event": {"type": "window_focus_change", "to": {
"app_name": "Notepad.exe", "title": "Sans titre Bloc-notes"}}},
{"event": {"type": "text_input", "text": "hello",
"window": {"app_name": "Notepad.exe",
"title": "Sans titre Bloc-notes"}}},
]
app_info = {
"primary_app": "Notepad.exe",
"first_window_title": "Sans titre Bloc-notes",
}
trimmed = _trim_redundant_setup_events(raw_events, app_info)
assert len(trimmed) == 1
assert trimmed[0]["event"]["type"] == "text_input"
def test_neutral_detection_recognizes_office_default_titles(self):
"""Word, Excel, PowerPoint utilisent leurs propres titres
par défaut (Document1, Classeur1, etc.) que le setup auto
amène également."""
raw_events = [
{"event": {"type": "window_focus_change", "to": {
"app_name": "winword.exe",
"title": "rapport.docx - Word"}}},
{"event": {"type": "mouse_click", "pos": [100, 40],
"window": {"app_name": "winword.exe",
"title": "rapport.docx - Word"}}},
{"event": {"type": "window_focus_change", "to": {
"app_name": "winword.exe", "title": "Document1 - Word"}}},
{"event": {"type": "text_input", "text": "abc",
"window": {"app_name": "winword.exe",
"title": "Document1 - Word"}}},
]
app_info = {
"primary_app": "winword.exe",
"first_window_title": "rapport.docx - Word",
}
trimmed = _trim_redundant_setup_events(raw_events, app_info)
assert len(trimmed) == 1
assert trimmed[0]["event"]["type"] == "text_input"
# =========================================================================
# Tests pour _extract_required_apps_from_workflow
# =========================================================================
class TestExtractRequiredAppsFromWorkflow:
"""Tests pour l'extraction d'apps depuis un workflow structuré."""
def test_workflow_dict_with_notepad(self):
"""Détecte Notepad depuis un workflow dict."""
workflow = {
"nodes": [
{
"node_id": "node_000",
"template": {
"window": {
"title_pattern": "Sans titre Bloc-notes",
"title_contains": "Bloc-notes",
},
},
},
],
"edges": [],
"metadata": {
"source_session_id": "sess_test",
"machine_id": "DESKTOP-TEST",
},
}
result = _extract_required_apps_from_workflow(workflow)
assert result["primary_app"] == "Notepad.exe"
assert result["primary_launch_cmd"] == "notepad"
def test_workflow_no_recognizable_titles(self):
"""Workflow sans titres reconnaissables → dict vide."""
workflow = {
"nodes": [
{
"node_id": "node_000",
"template": {
"window": {"title_pattern": "Rechercher"},
},
},
],
"edges": [],
"metadata": {},
}
result = _extract_required_apps_from_workflow(workflow)
assert result == {}
def test_empty_workflow(self):
"""Workflow vide → dict vide."""
assert _extract_required_apps_from_workflow({"nodes": [], "edges": []}) == {}
assert _extract_required_apps_from_workflow({}) == {}
# =========================================================================
# Tests pour _generate_setup_actions — 100% visuel
# =========================================================================
class TestGenerateSetupActions:
"""Tests pour la génération des actions de setup."""
def test_notepad_setup_uses_run_dialog(self):
"""Bloc-notes utilise désormais le setup sémantique Win+R."""
app_info = {
"primary_app": "Notepad.exe",
"primary_launch_cmd": "notepad",
"first_window_title": "Sans titre Bloc-notes",
}
actions = _generate_setup_actions(app_info)
assert len(actions) == 7
assert actions[0]["type"] == "key_combo"
assert actions[0]["keys"] == ["win", "r"]
assert actions[0]["_setup_step"] == "open_run_dialog"
assert actions[1]["type"] == "wait"
assert actions[1]["duration_ms"] == 500
assert actions[2]["type"] == "type"
assert actions[2]["text"] == "notepad"
assert actions[3]["type"] == "wait"
assert actions[3]["duration_ms"] == 300
assert actions[4]["type"] == "key_combo"
assert actions[4]["keys"] == ["enter"]
assert actions[5]["type"] == "wait"
assert actions[5]["duration_ms"] == 2000
assert actions[6]["type"] == "verify_screen"
assert actions[6]["expected_window_title_contains"] == ["Bloc-notes", "notepad"]
# Toutes les actions sont marquées comme phase setup
for action in actions:
assert action.get("_setup_phase") is True
assert action.get("_setup_strategy") == "run_dialog"
def test_visual_setup_keeps_no_key_combo_for_word(self):
"""Le setup visuel classique ne doit pas introduire de key_combo."""
app_info = {
"primary_app": "winword.exe",
"primary_launch_cmd": "winword",
"first_window_title": "Document1 - Word",
}
actions = _generate_setup_actions(app_info)
key_combos = [a for a in actions if a["type"] == "key_combo"]
assert key_combos == []
def test_all_clicks_are_visual_for_visual_setup(self):
"""Tous les clics du setup visuel doivent avoir visual_mode=True."""
app_info = {
"primary_app": "winword.exe",
"primary_launch_cmd": "winword",
"first_window_title": "Document1 - Word",
}
actions = _generate_setup_actions(app_info)
clicks = [a for a in actions if a["type"] == "click"]
assert len(clicks) >= 3 # Démarrer, recherche, résultat
for click in clicks:
assert click.get("visual_mode") is True, (
f"Clic {click.get('action_id')} n'a pas visual_mode=True"
)
assert click.get("target_spec"), (
f"Clic {click.get('action_id')} n'a pas de target_spec"
)
# Chaque target_spec doit avoir by_text et by_role
spec = click["target_spec"]
assert "by_text" in spec, f"target_spec sans by_text : {spec}"
assert "by_role" in spec, f"target_spec sans by_role : {spec}"
assert "vlm_description" in spec, f"target_spec sans vlm_description : {spec}"
def test_clicks_have_fallback_coordinates(self):
"""Tous les clics visuels ont des coordonnées de fallback."""
app_info = {
"primary_app": "winword.exe",
"primary_launch_cmd": "winword",
"first_window_title": "Document1 - Word",
}
actions = _generate_setup_actions(app_info)
clicks = [a for a in actions if a["type"] == "click"]
for click in clicks:
assert "x_pct" in click, f"Clic {click['action_id']} sans x_pct"
assert "y_pct" in click, f"Clic {click['action_id']} sans y_pct"
assert isinstance(click["x_pct"], (int, float))
assert isinstance(click["y_pct"], (int, float))
def test_heavy_app_longer_wait(self):
"""Les apps lourdes ont un temps d'attente plus long."""
app_info = {
"primary_app": "winword.exe",
"primary_launch_cmd": "winword",
"first_window_title": "Document1 - Word",
}
actions = _generate_setup_actions(app_info)
wait_action = [a for a in actions if a.get("_setup_step") == "wait_app_launch"][0]
assert wait_action["duration_ms"] == 3000
def test_light_app_shorter_wait(self):
"""Les apps légères ont un temps d'attente standard."""
app_info = {
"primary_app": "Notepad.exe",
"primary_launch_cmd": "notepad",
"first_window_title": "Bloc-notes",
}
actions = _generate_setup_actions(app_info)
wait_action = [a for a in actions if a.get("_setup_step") == "wait_app_launch"][0]
assert wait_action["duration_ms"] == 2000
def test_word_uses_visual_search_text(self):
"""Word utilise 'Word' comme texte de recherche visuelle, pas 'winword'."""
app_info = {
"primary_app": "winword.exe",
"primary_launch_cmd": "winword",
"first_window_title": "Document1 - Word",
}
actions = _generate_setup_actions(app_info)
# Le type doit utiliser le texte visuel, pas la commande shell
type_action = [a for a in actions if a["type"] == "type"][0]
assert type_action["text"] == "Word"
# Le clic sur le résultat doit utiliser le display_name
click_result = [a for a in actions if a.get("_setup_step") == "click_app_result"][0]
assert click_result["target_spec"]["by_text"] == "Microsoft Word"
def test_prefers_recorded_searchhost_click_target(self):
"""Le setup réutilise la vraie cible SearchHost quand elle existe."""
app_info = {
"primary_app": "winword.exe",
"primary_launch_cmd": "winword",
"first_window_title": "Document1 - Word",
"launch_result_target": {
"x_pct": 0.566016,
"y_pct": 0.400625,
"window_title": "Rechercher",
"expected_window_before": "Rechercher",
"original_position": {
"x_relative": "au centre",
"y_relative": "au milieu",
},
"window_capture": {
"click_relative": [681, 448],
"window_size": [1287, 1407],
},
"position_desc": "au milieu au centre",
},
}
actions = _generate_setup_actions(app_info)
click_result = [a for a in actions if a.get("_setup_step") == "click_app_result"][0]
assert click_result["x_pct"] == pytest.approx(0.566016)
assert click_result["y_pct"] == pytest.approx(0.400625)
assert click_result["expected_window_before"] == "Rechercher"
assert click_result["target_spec"]["by_text"] == "Microsoft Word"
assert click_result["target_spec"]["by_role"] == "search_result"
assert click_result["target_spec"]["allow_position_fallback"] is True
assert click_result["target_spec"]["window_title"] == "Rechercher"
assert click_result["target_spec"]["original_position"]["x_relative"] == "au centre"
assert click_result["target_spec"]["window_capture"]["window_size"] == [1287, 1407]
assert "résultat de recherche" in click_result["target_spec"]["vlm_description"]
def test_prefers_recorded_start_button_target(self):
"""Le setup visuel réutilise le vrai clic Démarrer quand il existe."""
app_info = {
"primary_app": "winword.exe",
"primary_launch_cmd": "winword",
"first_window_title": "Document1 - Word",
"start_menu_target": {
"x_pct": 0.387891,
"y_pct": 0.974375,
"anchor_image_base64": "abc123",
"original_position": {
"x_relative": "au centre",
"y_relative": "en bas",
},
"position_desc": "en bas au centre",
},
}
actions = _generate_setup_actions(app_info)
click_start = [a for a in actions if a.get("_setup_step") == "click_start_menu"][0]
assert click_start["x_pct"] == pytest.approx(0.387891)
assert click_start["y_pct"] == pytest.approx(0.974375)
assert click_start["target_spec"]["by_text"] == ""
assert click_start["target_spec"]["by_role"] == "start_button"
assert click_start["target_spec"]["screen_scope"] == "full_screen"
assert click_start["target_spec"]["allow_position_fallback"] is True
assert click_start["target_spec"]["anchor_image_base64"] == "abc123"
assert click_start["target_spec"]["original_position"]["y_relative"] == "en bas"
assert "icône Windows" in click_start["target_spec"]["vlm_description"]
def test_skips_search_click_for_direct_typing(self):
"""Quand la session tape directement dans SearchHost, on saute
click_search et son wait/verify dédiés. La garde
verify_start_menu_open reste obligatoire et précède le type."""
app_info = {
"primary_app": "winword.exe",
"primary_launch_cmd": "winword",
"first_window_title": "Document1 - Word",
"search_box_interaction": {
"mode": "direct_typing",
"window_title": "Rechercher",
},
}
actions = _generate_setup_actions(app_info)
setup_steps = [a.get("_setup_step") for a in actions]
assert "click_search_box" not in setup_steps
assert "wait_search_ready" not in setup_steps
assert "verify_search_box_active" not in setup_steps
# Garde générique conservée — c'est elle qui sécurise la frappe.
assert "verify_start_menu_open" in setup_steps
idx_type = setup_steps.index("type_app_name")
assert actions[idx_type]["type"] == "type"
assert actions[idx_type]["text"] == "Word"
def test_verify_screen_final_present_with_title(self):
"""Le setup run_dialog termine par une vérification souple sur le titre app."""
app_info = {
"primary_app": "Notepad.exe",
"primary_launch_cmd": "notepad",
"first_window_title": "Sans titre Bloc-notes",
}
actions = _generate_setup_actions(app_info)
final_verifies = [
a for a in actions
if a.get("type") == "verify_screen"
and a.get("_setup_step") == "verify_app_ready"
]
assert len(final_verifies) == 1
assert "Bloc-notes" in final_verifies[0]["expected_window_title_contains"]
def test_run_dialog_keeps_final_verify_even_without_exact_title(self):
"""Le setup run_dialog garde une vérification finale générique."""
app_info = {
"primary_app": "Notepad.exe",
"primary_launch_cmd": "notepad",
"first_window_title": "",
}
actions = _generate_setup_actions(app_info)
# Aucun verify_screen ne doit porter _expected_title.
final_verifies = [
a for a in actions
if a.get("type") == "verify_screen"
and a.get("_setup_step") == "verify_app_ready"
]
assert len(final_verifies) == 1
assert "notepad" in [p.lower() for p in final_verifies[0]["expected_window_title_contains"]]
def test_empty_app_info(self):
"""Dict vide → pas d'actions."""
assert _generate_setup_actions({}) == []
def test_system_app_ignored(self):
"""Les apps système sont ignorées."""
app_info = {
"primary_app": "explorer.exe",
"primary_launch_cmd": "explorer",
"first_window_title": "Explorateur",
}
assert _generate_setup_actions(app_info) == []
def test_no_launch_cmd(self):
"""Pas de commande de lancement → pas d'actions."""
app_info = {
"primary_app": "Notepad.exe",
"primary_launch_cmd": "",
"first_window_title": "Bloc-notes",
}
assert _generate_setup_actions(app_info) == []
def test_action_ids_unique(self):
"""Tous les action_id sont uniques."""
app_info = {
"primary_app": "Notepad.exe",
"primary_launch_cmd": "notepad",
"first_window_title": "Bloc-notes",
}
actions = _generate_setup_actions(app_info)
ids = [a["action_id"] for a in actions]
assert len(ids) == len(set(ids))
def test_custom_prefix(self):
"""Le préfixe personnalisé est utilisé dans les action_id."""
app_info = {
"primary_app": "Notepad.exe",
"primary_launch_cmd": "notepad",
"first_window_title": "Bloc-notes",
}
actions = _generate_setup_actions(app_info, setup_id_prefix="mysetup")
for action in actions:
assert "mysetup" in action["action_id"]
def test_unknown_app_uses_fallback_visual_info(self):
"""Une app inconnue utilise le nom de l'exécutable comme texte de recherche."""
app_info = {
"primary_app": "MonAppMedical.exe",
"primary_launch_cmd": "MonAppMedical",
"first_window_title": "Mon App",
}
actions = _generate_setup_actions(app_info)
# Le type doit utiliser le nom sans .exe
type_action = [a for a in actions if a["type"] == "type"][0]
assert type_action["text"] == "MonAppMedical"
# =========================================================================
# Tests des gardes visuelles du setup (verify_screen titre fenêtre)
# =========================================================================
class TestSetupVisualGuards:
"""Couvre les gardes visuelles insérées entre les étapes du setup
auto Windows (post-blocage `position_fallback` live du 22 mai 2026).
Sans ces gardes, un clic Démarrer qui touche en fait le systray
overflow popup laissait le setup taper « bloc » dans la mauvaise
fenêtre, et seul le `click_result` final remontait l'erreur — trop
tard. Les `verify_screen` titre-fenêtre stoppent net après chaque
étape critique.
"""
def test_verify_start_menu_open_inserted_after_wait_start(self):
"""Une garde verify_screen est insérée juste après wait_start_menu."""
app_info = {
"primary_app": "winword.exe",
"primary_launch_cmd": "winword",
"first_window_title": "Document1 - Word",
}
actions = _generate_setup_actions(app_info)
steps = [a.get("_setup_step") for a in actions]
# Ordre : click_start_menu → wait_start_menu → verify_start_menu_open
assert "verify_start_menu_open" in steps
idx_wait = steps.index("wait_start_menu")
idx_verify = steps.index("verify_start_menu_open")
assert idx_verify == idx_wait + 1
verify = actions[idx_verify]
assert verify["type"] == "verify_screen"
assert verify.get("_setup_phase") is True
patterns = verify.get("expected_window_title_contains") or []
assert isinstance(patterns, list) and patterns
lowered = [p.lower() for p in patterns]
# Doit couvrir au minimum FR + EN + l'app SearchHost / StartMenu
assert any("recherch" in p for p in lowered), patterns
assert any("search" in p for p in lowered), patterns
def test_verify_search_box_active_inserted_when_click_then_type(self):
"""Quand le setup clique sur la barre Rechercher puis attend,
une garde verify_screen suit l'attente pour bloquer la frappe
si le focus n'est pas réellement dans la barre."""
app_info = {
"primary_app": "winword.exe",
"primary_launch_cmd": "winword",
"first_window_title": "Document1 - Word",
"search_box_interaction": {
"mode": "click_then_type",
"window_title": "Rechercher",
"x_pct": 0.10, "y_pct": 0.95,
},
}
actions = _generate_setup_actions(app_info)
steps = [a.get("_setup_step") for a in actions]
assert "verify_search_box_active" in steps
idx_wait_ready = steps.index("wait_search_ready")
idx_verify = steps.index("verify_search_box_active")
idx_type = steps.index("type_app_name")
# Ordre : wait_search_ready → verify_search_box_active → type_app_name
assert idx_verify == idx_wait_ready + 1
assert idx_type == idx_verify + 1
verify = actions[idx_verify]
assert verify["type"] == "verify_screen"
patterns = verify.get("expected_window_title_contains") or []
assert "Rechercher" in patterns or any(
p.lower() == "rechercher" for p in patterns
)
def test_no_verify_search_box_when_direct_typing(self):
"""En mode direct_typing on n'a pas de click sur la barre — donc
pas de verify_search_box_active dédié (la garde verify_start_menu_open
suffit, on tape directement après)."""
app_info = {
"primary_app": "winword.exe",
"primary_launch_cmd": "winword",
"first_window_title": "Document1 - Word",
"search_box_interaction": {
"mode": "direct_typing",
"window_title": "Rechercher",
},
}
actions = _generate_setup_actions(app_info)
steps = [a.get("_setup_step") for a in actions]
assert "verify_search_box_active" not in steps
# La garde verify_start_menu_open reste présente (couvre la frappe).
assert "verify_start_menu_open" in steps
idx_verify = steps.index("verify_start_menu_open")
idx_type = steps.index("type_app_name")
assert idx_type > idx_verify, (
"type_app_name doit suivre verify_start_menu_open en direct_typing"
)
def test_verify_search_results_visible_inserted_before_click_result(self):
"""Dernier filet : la barre Rechercher (et ses résultats) doit
être encore active juste avant `click_app_result`. Sans cette
garde finale, un focus perdu pendant `wait_search_results`
peut faire cliquer le `click_app_result` dans la mauvaise
surface (constat live 2026-05-22 — fenêtre observée
``Fenêtre de dépassement de capacité de la barre d'état
système.``)."""
app_info = {
"primary_app": "winword.exe",
"primary_launch_cmd": "winword",
"first_window_title": "Document1 - Word",
}
actions = _generate_setup_actions(app_info)
steps = [a.get("_setup_step") for a in actions]
assert "verify_search_results_visible" in steps
idx_wait_results = steps.index("wait_search_results")
idx_verify = steps.index("verify_search_results_visible")
idx_click_result = steps.index("click_app_result")
# Ordre : wait_search_results → verify_search_results_visible → click_app_result
assert idx_verify == idx_wait_results + 1
assert idx_click_result == idx_verify + 1
verify = actions[idx_verify]
assert verify["type"] == "verify_screen"
patterns = verify.get("expected_window_title_contains") or []
assert isinstance(patterns, list) and patterns
lowered = [p.lower() for p in patterns]
assert any("recherch" in p for p in lowered), patterns
assert any("search" in p for p in lowered), patterns
def test_verify_search_results_visible_present_in_direct_typing(self):
"""La garde finale avant click_app_result reste obligatoire
quelle que soit la modalité de la barre Rechercher."""
app_info = {
"primary_app": "winword.exe",
"primary_launch_cmd": "winword",
"first_window_title": "Document1 - Word",
"search_box_interaction": {
"mode": "direct_typing",
"window_title": "Rechercher",
},
}
actions = _generate_setup_actions(app_info)
steps = [a.get("_setup_step") for a in actions]
assert "verify_search_results_visible" in steps
def test_setup_guards_have_short_timeout(self):
"""Les gardes verify_screen ont un timeout court (≤ 2 s) — c'est
un check titre, pas un wait long."""
app_info = {
"primary_app": "winword.exe",
"primary_launch_cmd": "winword",
"first_window_title": "Document1 - Word",
"search_box_interaction": {
"mode": "click_then_type",
"window_title": "Rechercher",
},
}
actions = _generate_setup_actions(app_info)
guards = [
a for a in actions
if a.get("_setup_step") in (
"verify_start_menu_open",
"verify_search_box_active",
"verify_search_results_visible",
)
]
assert guards, "il doit exister au moins une garde verify_screen"
for g in guards:
assert g.get("timeout_ms", 5000) <= 2000
# =========================================================================
# Tests d'intégration : pipeline complet events → setup visuel
# =========================================================================
class TestSetupPipeline:
"""Tests du pipeline complet : extraction + génération du setup."""
def test_full_pipeline_from_events(self):
"""Pipeline complet depuis des événements bruts de type Notepad."""
events = [
{"event": {"type": "window_focus_change", "from": None,
"to": {"app_name": "explorer.exe", "title": "Bureau"}}},
{"event": {"type": "window_focus_change",
"from": {"app_name": "explorer.exe"},
"to": {"app_name": "Notepad.exe", "title": "Sans titre Bloc-notes"}}},
{"event": {"type": "window_focus_change",
"from": {"app_name": "Notepad.exe"},
"to": {"app_name": "Notepad.exe", "title": "*test Bloc-notes"}}},
]
app_info = _extract_required_apps_from_events(events)
assert app_info["primary_app"] == "Notepad.exe"
actions = _generate_setup_actions(app_info)
assert len(actions) == 7
types = [a["type"] for a in actions]
steps = [a.get("_setup_step") for a in actions]
expected_step_order = [
"open_run_dialog",
"wait_run_dialog",
"type_launch_command",
"wait_launch_command",
"submit_run_dialog",
"wait_app_launch",
"verify_app_ready",
]
assert steps == expected_step_order, steps
assert types.count("key_combo") == 2
idx_type = steps.index("type_launch_command")
assert actions[idx_type]["text"] == "notepad"
def test_full_pipeline_from_workflow(self):
"""Pipeline complet depuis un workflow structuré."""
workflow = {
"nodes": [
{"node_id": "n0", "template": {
"window": {"title_pattern": "Sans titre Bloc-notes"},
}},
{"node_id": "n1", "template": {
"window": {"title_pattern": "*test Bloc-notes"},
}},
],
"edges": [],
"metadata": {},
}
app_info = _extract_required_apps_from_workflow(workflow)
assert app_info["primary_app"] == "Notepad.exe"
actions = _generate_setup_actions(app_info)
assert len(actions) == 7
# Le texte tapé doit être la commande shell pour le setup Win+R.
type_action = [a for a in actions if a["type"] == "type"][0]
assert type_action["text"] == "notepad"
# Le setup Notepad s'appuie maintenant sur deux key_combo.
key_combos = [a for a in actions if a["type"] == "key_combo"]
assert len(key_combos) == 2