Dom/rpa_vision_v3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

snapshot: WIP 5j replay reliability (B1 watchdog + dialog handlers + grounding drift)

Browse Source 
Snapshot avant correction du blocage relance Léa (3 incidents 24h: SSH refusé,
polls morts ×2). Point de rollback stable.

Contenu:
- agent_v1/core/executor.py: 5 patchs dialog handling (saveas drift, close_tab
  hotkey fallback, confirm_save Unicode apostrophe, foreground dialog
  recontextualization, runtime_dialog in-loop) + helpers normalize_window_hint,
  requires_post_verify_window_transition
- agent_v1/core/grounding.py: garde drift template fix (fallback_x/y plumbed)
- server_v1/replay_watchdog.py (NEW): orphan watchdog B1, scan 10s timeout 30s
- server_v1/api_stream.py: dispatched_action plumbing, watchdog lifespan,
  metrics endpoint
- server_v1/replay_engine.py: _schedule_retry préserve original_action +
  dispatched_action
- stream_processor.py: gardes _infer_tab_switch_target (no false switch_tab
  on save_as dialog open) + _attach_expected_window_before
- tests/integration: test_replay_watchdog.py (8 cas), test_stream_processor.py
- tests/unit: test_executor_verify_window_guard.py (start_button, close_tab,
  runtime_dialog, post_verify, transition fallbacks)

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Dom
2026-05-24 16:48:37 +02:00
parent 5ea4960e65
commit 7df51d2c79
47 changed files with 9811 additions and 451 deletions
Download Patch File Download Diff File Expand all files Collapse all files

734
tests/unit/test_env_setup.py
View File

@@ -16,6 +16,7 @@ sys.path.insert(0, str(ROOT))
from agent_v0.server_v1.api_stream import (
_extract_required_apps_from_events,
_extract_required_apps_from_workflow,
_trim_redundant_setup_events,
_resolve_launch_command,
_infer_app_from_window_titles,
_generate_setup_actions,
@@ -220,6 +221,139 @@ class TestExtractRequiredAppsFromEvents:
# Le premier app hors ignorées est Notepad
assert result["first_window_title"] == "Bloc-notes"
def test_extracts_searchhost_launch_result_target(self):
"""Récupère le vrai clic SearchHost qui lance l'app."""
events = [
{"event": {"type": "window_focus_change", "from": None, "to": {
"app_name": "explorer.exe", "title": "Explorateur"}}},
{"event": {"type": "window_focus_change", "from": {
"app_name": "explorer.exe", "title": "Explorateur"}, "to": {
"app_name": "SearchHost.exe", "title": "Rechercher"}}},
{"event": {"type": "text_input", "text": "bloc", "window": {
"app_name": "SearchHost.exe", "title": "Rechercher"}}},
{"event": {"type": "mouse_click", "button": "left", "pos": [1449, 641],
"timestamp": 10.0,
"screen_metadata": {"screen_resolution": [2560, 1600]},
"window": {"app_name": "SearchHost.exe", "title": "Rechercher"},
"window_capture": {
"click_relative": [681, 448],
"window_size": [1287, 1407],
}}},
{"event": {"type": "window_focus_change", "from": {
"app_name": "SearchHost.exe", "title": "Rechercher"}, "to": {
"app_name": "explorer.exe", "title": "unknown_window"},
"timestamp": 10.4}},
{"event": {"type": "window_focus_change", "from": {
"app_name": "explorer.exe", "title": "unknown_window"}, "to": {
"app_name": "Notepad.exe", "title": "Sans titre – Bloc-notes"},
"timestamp": 11.1}},
]
result = _extract_required_apps_from_events(events)
target = result["launch_result_target"]
assert result["primary_app"] == "Notepad.exe"
assert target["window_title"] == "Rechercher"
assert target["expected_window_before"] == "Rechercher"
assert target["x_pct"] == pytest.approx(1449 / 2560, rel=0, abs=1e-6)
assert target["y_pct"] == pytest.approx(641 / 1600, rel=0, abs=1e-6)
assert target["original_position"]["x_relative"] == "au centre"
assert target["original_position"]["y_relative"] == "au milieu"
assert target["window_capture"]["click_relative"] == [681, 448]
def test_extracts_start_menu_target(self):
"""Récupère le vrai clic Démarrer qui ouvre SearchHost."""
events = [
{"event": {"type": "window_focus_change", "from": None, "to": {
"app_name": "explorer.exe", "title": "Explorateur"}}},
{"event": {"type": "mouse_click", "button": "left", "pos": [993, 1559],
"timestamp": 1.0,
"screen_metadata": {"screen_resolution": [2560, 1600]},
"window": {"app_name": "explorer.exe", "title": "Explorateur"}}},
{"event": {"type": "window_focus_change", "from": {
"app_name": "explorer.exe", "title": "Explorateur"}, "to": {
"app_name": "SearchHost.exe", "title": "Rechercher"},
"timestamp": 1.2}},
{"event": {"type": "mouse_click", "button": "left", "pos": [1449, 641],
"timestamp": 4.0,
"screen_metadata": {"screen_resolution": [2560, 1600]},
"window": {"app_name": "SearchHost.exe", "title": "Rechercher"}}},
{"event": {"type": "window_focus_change", "from": {
"app_name": "SearchHost.exe", "title": "Rechercher"}, "to": {
"app_name": "Notepad.exe", "title": "Sans titre – Bloc-notes"},
"timestamp": 4.4}},
]
result = _extract_required_apps_from_events(events)
target = result["start_menu_target"]
assert target["x_pct"] == pytest.approx(993 / 2560, rel=0, abs=1e-6)
assert target["y_pct"] == pytest.approx(1559 / 1600, rel=0, abs=1e-6)
assert target["original_position"]["x_relative"] == "au centre"
assert target["original_position"]["y_relative"] == "en bas"
assert "en bas" in target["position_desc"]
def test_extracts_start_menu_target_anchor_from_session_shot(self, tmp_path):
"""Le clic Démarrer récupère aussi une ancre image depuis le shot source."""
from PIL import Image
session_dir = tmp_path / "sess"
shots_dir = session_dir / "shots"
shots_dir.mkdir(parents=True)
Image.new("RGB", (2560, 1600), color="white").save(
shots_dir / "shot_start_full.png"
)
events = [
{"event": {"type": "window_focus_change", "from": None, "to": {
"app_name": "explorer.exe", "title": "Explorateur"}}},
{"event": {"type": "mouse_click", "button": "left", "pos": [993, 1559],
"timestamp": 1.0,
"screenshot_id": "shot_start",
"screen_metadata": {"screen_resolution": [2560, 1600]},
"window": {"app_name": "explorer.exe", "title": "Explorateur"}}},
{"event": {"type": "window_focus_change", "from": {
"app_name": "explorer.exe", "title": "Explorateur"}, "to": {
"app_name": "SearchHost.exe", "title": "Rechercher"},
"timestamp": 1.2}},
{"event": {"type": "window_focus_change", "from": {
"app_name": "SearchHost.exe", "title": "Rechercher"}, "to": {
"app_name": "Notepad.exe", "title": "Sans titre – Bloc-notes"},
"timestamp": 2.0}},
]
result = _extract_required_apps_from_events(
events,
session_dir=str(session_dir),
)
target = result["start_menu_target"]
assert target["anchor_image_base64"]
def test_extracts_direct_typing_search_interaction(self):
"""Détecte qu'aucun clic SearchHost n'est requis avant la saisie."""
events = [
{"event": {"type": "window_focus_change", "from": None, "to": {
"app_name": "explorer.exe", "title": "Explorateur"}}},
{"event": {"type": "mouse_click", "button": "left", "pos": [993, 1559],
"timestamp": 1.0,
"screen_metadata": {"screen_resolution": [2560, 1600]},
"window": {"app_name": "explorer.exe", "title": "Explorateur"}}},
{"event": {"type": "window_focus_change", "from": {
"app_name": "explorer.exe", "title": "Explorateur"}, "to": {
"app_name": "SearchHost.exe", "title": "Rechercher"},
"timestamp": 1.2}},
{"event": {"type": "text_input", "text": "bloc",
"window": {"app_name": "SearchHost.exe", "title": "Rechercher"},
"timestamp": 2.0}},
{"event": {"type": "window_focus_change", "from": {
"app_name": "SearchHost.exe", "title": "Rechercher"}, "to": {
"app_name": "Notepad.exe", "title": "Sans titre – Bloc-notes"},
"timestamp": 2.4}},
]
result = _extract_required_apps_from_events(events)
assert result["search_box_interaction"]["mode"] == "direct_typing"
assert result["search_box_interaction"]["window_title"] == "Rechercher"
def test_empty_events(self):
"""Pas d'événements → dict vide."""
assert _extract_required_apps_from_events([]) == {}
@@ -245,6 +379,187 @@ class TestExtractRequiredAppsFromEvents:
assert result["primary_launch_cmd"] == "calc"
class TestTrimRedundantSetupEvents:
"""Tests pour la coupe du préambule déjà couvert par le setup."""
def test_trims_until_first_primary_app_focus(self):
raw_events = [
{"event": {"type": "window_focus_change", "to": {
"app_name": "explorer.exe", "title": "Explorateur"}}},
{"event": {"type": "mouse_click", "pos": [993, 1559], "window": {
"app_name": "explorer.exe", "title": "Explorateur"}}},
{"event": {"type": "window_focus_change", "to": {
"app_name": "SearchHost.exe", "title": "Rechercher"}}},
{"event": {"type": "text_input", "text": "bloc", "window": {
"app_name": "SearchHost.exe", "title": "Rechercher"}}},
{"event": {"type": "mouse_click", "pos": [1449, 641], "window": {
"app_name": "SearchHost.exe", "title": "Rechercher"}}},
{"event": {"type": "window_focus_change", "to": {
"app_name": "Notepad.exe",
"title": "http192.168.1.408765dossier.htmlid=.txt – Bloc-notes",
}}},
{"event": {"type": "mouse_click", "pos": [1514, 562], "window": {
"app_name": "Notepad.exe", "title": "*test – Bloc-notes"}}},
{"event": {"type": "text_input", "text": "test", "window": {
"app_name": "Notepad.exe", "title": "*test – Bloc-notes"}}},
]
app_info = {
"primary_app": "Notepad.exe",
"first_window_title": "Bloc-notes",
}
trimmed = _trim_redundant_setup_events(raw_events, app_info)
assert len(trimmed) == 2
assert trimmed[0]["event"]["type"] == "mouse_click"
assert trimmed[0]["event"]["pos"] == [1514, 562]
assert trimmed[1]["event"]["type"] == "text_input"
def test_keeps_events_when_no_matching_focus_found(self):
raw_events = [
{"event": {"type": "mouse_click", "pos": [10, 10], "window": {
"app_name": "explorer.exe", "title": "Explorateur"}}},
{"event": {"type": "text_input", "text": "abc", "window": {
"app_name": "explorer.exe", "title": "Explorateur"}}},
]
app_info = {
"primary_app": "Notepad.exe",
"first_window_title": "Bloc-notes",
}
trimmed = _trim_redundant_setup_events(raw_events, app_info)
assert trimmed == raw_events
def test_prefers_neutral_title_focus_after_non_neutral_first_focus(self):
"""Cas observé sess_20260520T102916_066851 : premier focus Notepad
a un titre non-neutre (http...txt), suivi d'un clic intra-Notepad
et d'un focus vers 'Sans titre' (= état initial neutre que le setup
auto produit). Le trim doit couper jusqu'au focus neutre pour
éliminer le clic intra-Notepad redondant.
"""
raw_events = [
{"event": {"type": "window_focus_change", "to": {
"app_name": "SearchHost.exe", "title": "Rechercher"}}},
{"event": {"type": "mouse_click", "pos": [681, 448], "window": {
"app_name": "SearchHost.exe", "title": "Rechercher"}}},
{"event": {"type": "window_focus_change", "to": {
"app_name": "Notepad.exe",
"title": "http192.168.1.408765dossier.htmlid=.txt – Bloc-notes",
}}},
{"event": {"type": "mouse_click", "pos": [1191, 40], "window": {
"app_name": "Notepad.exe",
"title": "http192.168.1.408765dossier.htmlid=.txt – Bloc-notes",
}}},
{"event": {"type": "window_focus_change", "to": {
"app_name": "Notepad.exe", "title": "Sans titre – Bloc-notes"}}},
{"event": {"type": "text_input", "text": "test", "window": {
"app_name": "Notepad.exe", "title": "*test – Bloc-notes"}}},
]
app_info = {
"primary_app": "Notepad.exe",
"first_window_title": (
"http192.168.1.408765dossier.htmlid=.txt – Bloc-notes"
),
}
trimmed = _trim_redundant_setup_events(raw_events, app_info)
# Le clic intra-Notepad (event idx 3) doit être supprimé : il
# bascule vers 'Sans titre' qui est déjà l'état setup, donc
# rejoué il n'a aucun effet visuel et déclenche retry_threshold.
assert len(trimmed) == 1
assert trimmed[0]["event"]["type"] == "text_input"
assert trimmed[0]["event"]["text"] == "test"
def test_neutral_focus_outside_lookahead_window_is_ignored(self):
"""Filet de sécurité : un focus 'Sans titre' qui arrive trop loin
après le premier focus primary_app n'est pas considéré comme
l'état de bootstrap. Évite de couper un workflow qui re-visite
'Sans titre' bien après le démarrage."""
# 30 events séparent le premier focus du focus neutre
raw_events = [
{"event": {"type": "window_focus_change", "to": {
"app_name": "Notepad.exe",
"title": "rapport_final.txt – Bloc-notes"}}},
]
# Bourrer avec des events utiles intra-Notepad
for i in range(30):
raw_events.append({"event": {
"type": "mouse_click", "pos": [100 + i, 200],
"window": {"app_name": "Notepad.exe",
"title": "rapport_final.txt – Bloc-notes"},
}})
raw_events.append({"event": {"type": "window_focus_change", "to": {
"app_name": "Notepad.exe", "title": "Sans titre – Bloc-notes"}}})
raw_events.append({"event": {"type": "text_input", "text": "x",
"window": {"app_name": "Notepad.exe",
"title": "Sans titre – Bloc-notes"}}})
app_info = {
"primary_app": "Notepad.exe",
"first_window_title": "rapport_final.txt – Bloc-notes",
}
trimmed = _trim_redundant_setup_events(raw_events, app_info)
# Doit garder les 30 clicks + focus tardif + text_input = 32 events
# (cut uniquement au premier focus primary_app, comportement legacy)
assert len(trimmed) == 32
assert trimmed[0]["event"]["type"] == "mouse_click"
assert trimmed[0]["event"]["pos"] == [100, 200]
def test_keeps_legacy_behavior_when_first_focus_already_neutral(self):
"""Non-régression : si le premier focus primary_app est déjà sur
un titre neutre (cas normal), on coupe au premier focus comme
avant — pas de chasse au neutral_idx inutile."""
raw_events = [
{"event": {"type": "window_focus_change", "to": {
"app_name": "SearchHost.exe", "title": "Rechercher"}}},
{"event": {"type": "window_focus_change", "to": {
"app_name": "Notepad.exe", "title": "Sans titre – Bloc-notes"}}},
{"event": {"type": "text_input", "text": "hello",
"window": {"app_name": "Notepad.exe",
"title": "Sans titre – Bloc-notes"}}},
]
app_info = {
"primary_app": "Notepad.exe",
"first_window_title": "Sans titre – Bloc-notes",
}
trimmed = _trim_redundant_setup_events(raw_events, app_info)
assert len(trimmed) == 1
assert trimmed[0]["event"]["type"] == "text_input"
def test_neutral_detection_recognizes_office_default_titles(self):
"""Word, Excel, PowerPoint utilisent leurs propres titres
par défaut (Document1, Classeur1, etc.) que le setup auto
amène également."""
raw_events = [
{"event": {"type": "window_focus_change", "to": {
"app_name": "winword.exe",
"title": "rapport.docx - Word"}}},
{"event": {"type": "mouse_click", "pos": [100, 40],
"window": {"app_name": "winword.exe",
"title": "rapport.docx - Word"}}},
{"event": {"type": "window_focus_change", "to": {
"app_name": "winword.exe", "title": "Document1 - Word"}}},
{"event": {"type": "text_input", "text": "abc",
"window": {"app_name": "winword.exe",
"title": "Document1 - Word"}}},
]
app_info = {
"primary_app": "winword.exe",
"first_window_title": "rapport.docx - Word",
}
trimmed = _trim_redundant_setup_events(raw_events, app_info)
assert len(trimmed) == 1
assert trimmed[0]["event"]["type"] == "text_input"
# =========================================================================
# Tests pour _extract_required_apps_from_workflow
# =========================================================================
@@ -304,10 +619,10 @@ class TestExtractRequiredAppsFromWorkflow:
# =========================================================================
class TestGenerateSetupActions:
"""Tests pour la génération des actions de setup 100% visuelles."""
"""Tests pour la génération des actions de setup."""
def test_notepad_setup_visual(self):
"""Génère les bonnes actions visuelles pour lancer Notepad."""
def test_notepad_setup_uses_run_dialog(self):
"""Bloc-notes utilise désormais le setup sémantique Win+R."""
app_info = {
"primary_app": "Notepad.exe",
"primary_launch_cmd": "notepad",
@@ -315,74 +630,52 @@ class TestGenerateSetupActions:
}
actions = _generate_setup_actions(app_info)
# 9 actions : click_start, wait, click_search, wait, type, wait, click_result, wait, verify
assert len(actions) == 9
assert len(actions) == 7
# Étape 1 : clic visuel sur le bouton Démarrer
assert actions[0]["type"] == "click"
assert actions[0]["visual_mode"] is True
assert actions[0]["target_spec"]["by_role"] == "start_button"
assert actions[0]["target_spec"]["by_text"] == "Démarrer"
assert actions[0]["type"] == "key_combo"
assert actions[0]["keys"] == ["win", "r"]
assert actions[0]["_setup_step"] == "open_run_dialog"
# Étape 2 : attente menu Démarrer
assert actions[1]["type"] == "wait"
assert actions[1]["duration_ms"] == 1000
assert actions[1]["duration_ms"] == 500
# Étape 3 : clic visuel sur la barre de recherche
assert actions[2]["type"] == "click"
assert actions[2]["visual_mode"] is True
assert actions[2]["target_spec"]["by_role"] == "search_box"
assert actions[2]["type"] == "type"
assert actions[2]["text"] == "notepad"
# Étape 4 : attente barre de recherche active
assert actions[3]["type"] == "wait"
assert actions[3]["duration_ms"] == 500
assert actions[3]["duration_ms"] == 300
# Étape 5 : taper le nom visuel français
assert actions[4]["type"] == "type"
assert actions[4]["text"] == "Bloc-notes"
assert actions[4]["type"] == "key_combo"
assert actions[4]["keys"] == ["enter"]
# Étape 6 : attente résultats
assert actions[5]["type"] == "wait"
assert actions[5]["duration_ms"] == 1200
assert actions[5]["duration_ms"] == 2000
# Étape 7 : clic visuel sur le résultat
assert actions[6]["type"] == "click"
assert actions[6]["visual_mode"] is True
assert actions[6]["target_spec"]["by_text"] == "Bloc-notes"
assert actions[6]["target_spec"]["by_role"] == "app_icon"
# Étape 8 : attente lancement (app légère = 2000ms)
assert actions[7]["type"] == "wait"
assert actions[7]["duration_ms"] == 2000
# Étape 9 : vérification visuelle
assert actions[8]["type"] == "verify_screen"
assert actions[8]["_expected_title"] == "Sans titre – Bloc-notes"
assert actions[6]["type"] == "verify_screen"
assert actions[6]["expected_window_title_contains"] == ["Bloc-notes", "notepad"]
# Toutes les actions sont marquées comme phase setup
for action in actions:
assert action.get("_setup_phase") is True
assert action.get("_setup_strategy") == "run_dialog"
def test_no_key_combo_in_setup(self):
"""AUCUNE action key_combo ne doit être générée dans le setup."""
def test_visual_setup_keeps_no_key_combo_for_word(self):
"""Le setup visuel classique ne doit pas introduire de key_combo."""
app_info = {
"primary_app": "Notepad.exe",
"primary_launch_cmd": "notepad",
"first_window_title": "Bloc-notes",
"primary_app": "winword.exe",
"primary_launch_cmd": "winword",
"first_window_title": "Document1 - Word",
}
actions = _generate_setup_actions(app_info)
key_combos = [a for a in actions if a["type"] == "key_combo"]
assert key_combos == [], (
"Le setup 100% visuel ne doit JAMAIS contenir de key_combo. "
f"Trouvé : {key_combos}"
)
assert key_combos == []
def test_all_clicks_are_visual(self):
"""Tous les clics du setup doivent avoir visual_mode=True et un target_spec."""
def test_all_clicks_are_visual_for_visual_setup(self):
"""Tous les clics du setup visuel doivent avoir visual_mode=True."""
app_info = {
"primary_app": "Notepad.exe",
"primary_launch_cmd": "notepad",
"first_window_title": "Bloc-notes",
"primary_app": "winword.exe",
"primary_launch_cmd": "winword",
"first_window_title": "Document1 - Word",
}
actions = _generate_setup_actions(app_info)
clicks = [a for a in actions if a["type"] == "click"]
@@ -402,11 +695,11 @@ class TestGenerateSetupActions:
assert "vlm_description" in spec, f"target_spec sans vlm_description : {spec}"
def test_clicks_have_fallback_coordinates(self):
"""Tous les clics visuels ont des coordonnées de fallback (x_pct, y_pct)."""
"""Tous les clics visuels ont des coordonnées de fallback."""
app_info = {
"primary_app": "Notepad.exe",
"primary_launch_cmd": "notepad",
"first_window_title": "Bloc-notes",
"primary_app": "winword.exe",
"primary_launch_cmd": "winword",
"first_window_title": "Document1 - Word",
}
actions = _generate_setup_actions(app_info)
clicks = [a for a in actions if a["type"] == "click"]
@@ -456,28 +749,130 @@ class TestGenerateSetupActions:
click_result = [a for a in actions if a.get("_setup_step") == "click_app_result"][0]
assert click_result["target_spec"]["by_text"] == "Microsoft Word"
def test_verify_screen_present_with_title(self):
"""Un verify_screen est ajouté quand un titre de fenêtre est connu."""
def test_prefers_recorded_searchhost_click_target(self):
"""Le setup réutilise la vraie cible SearchHost quand elle existe."""
app_info = {
"primary_app": "winword.exe",
"primary_launch_cmd": "winword",
"first_window_title": "Document1 - Word",
"launch_result_target": {
"x_pct": 0.566016,
"y_pct": 0.400625,
"window_title": "Rechercher",
"expected_window_before": "Rechercher",
"original_position": {
"x_relative": "au centre",
"y_relative": "au milieu",
},
"window_capture": {
"click_relative": [681, 448],
"window_size": [1287, 1407],
},
"position_desc": "au milieu au centre",
},
}
actions = _generate_setup_actions(app_info)
click_result = [a for a in actions if a.get("_setup_step") == "click_app_result"][0]
assert click_result["x_pct"] == pytest.approx(0.566016)
assert click_result["y_pct"] == pytest.approx(0.400625)
assert click_result["expected_window_before"] == "Rechercher"
assert click_result["target_spec"]["by_text"] == "Microsoft Word"
assert click_result["target_spec"]["by_role"] == "search_result"
assert click_result["target_spec"]["allow_position_fallback"] is True
assert click_result["target_spec"]["window_title"] == "Rechercher"
assert click_result["target_spec"]["original_position"]["x_relative"] == "au centre"
assert click_result["target_spec"]["window_capture"]["window_size"] == [1287, 1407]
assert "résultat de recherche" in click_result["target_spec"]["vlm_description"]
def test_prefers_recorded_start_button_target(self):
"""Le setup visuel réutilise le vrai clic Démarrer quand il existe."""
app_info = {
"primary_app": "winword.exe",
"primary_launch_cmd": "winword",
"first_window_title": "Document1 - Word",
"start_menu_target": {
"x_pct": 0.387891,
"y_pct": 0.974375,
"anchor_image_base64": "abc123",
"original_position": {
"x_relative": "au centre",
"y_relative": "en bas",
},
"position_desc": "en bas au centre",
},
}
actions = _generate_setup_actions(app_info)
click_start = [a for a in actions if a.get("_setup_step") == "click_start_menu"][0]
assert click_start["x_pct"] == pytest.approx(0.387891)
assert click_start["y_pct"] == pytest.approx(0.974375)
assert click_start["target_spec"]["by_text"] == ""
assert click_start["target_spec"]["by_role"] == "start_button"
assert click_start["target_spec"]["screen_scope"] == "full_screen"
assert click_start["target_spec"]["allow_position_fallback"] is True
assert click_start["target_spec"]["anchor_image_base64"] == "abc123"
assert click_start["target_spec"]["original_position"]["y_relative"] == "en bas"
assert "icône Windows" in click_start["target_spec"]["vlm_description"]
def test_skips_search_click_for_direct_typing(self):
"""Quand la session tape directement dans SearchHost, on saute
click_search et son wait/verify dédiés. La garde
verify_start_menu_open reste obligatoire et précède le type."""
app_info = {
"primary_app": "winword.exe",
"primary_launch_cmd": "winword",
"first_window_title": "Document1 - Word",
"search_box_interaction": {
"mode": "direct_typing",
"window_title": "Rechercher",
},
}
actions = _generate_setup_actions(app_info)
setup_steps = [a.get("_setup_step") for a in actions]
assert "click_search_box" not in setup_steps
assert "wait_search_ready" not in setup_steps
assert "verify_search_box_active" not in setup_steps
# Garde générique conservée — c'est elle qui sécurise la frappe.
assert "verify_start_menu_open" in setup_steps
idx_type = setup_steps.index("type_app_name")
assert actions[idx_type]["type"] == "type"
assert actions[idx_type]["text"] == "Word"
def test_verify_screen_final_present_with_title(self):
"""Le setup run_dialog termine par une vérification souple sur le titre app."""
app_info = {
"primary_app": "Notepad.exe",
"primary_launch_cmd": "notepad",
"first_window_title": "Sans titre – Bloc-notes",
}
actions = _generate_setup_actions(app_info)
verify = [a for a in actions if a.get("type") == "verify_screen"]
assert len(verify) == 1
assert verify[0]["_expected_title"] == "Sans titre – Bloc-notes"
final_verifies = [
a for a in actions
if a.get("type") == "verify_screen"
and a.get("_setup_step") == "verify_app_ready"
]
assert len(final_verifies) == 1
assert "Bloc-notes" in final_verifies[0]["expected_window_title_contains"]
def test_no_verify_without_title(self):
"""Pas de verify_screen si aucun titre de fenêtre n'est connu."""
def test_run_dialog_keeps_final_verify_even_without_exact_title(self):
"""Le setup run_dialog garde une vérification finale générique."""
app_info = {
"primary_app": "Notepad.exe",
"primary_launch_cmd": "notepad",
"first_window_title": "",
}
actions = _generate_setup_actions(app_info)
verify = [a for a in actions if a.get("type") == "verify_screen"]
assert len(verify) == 0
# Aucun verify_screen ne doit porter _expected_title.
final_verifies = [
a for a in actions
if a.get("type") == "verify_screen"
and a.get("_setup_step") == "verify_app_ready"
]
assert len(final_verifies) == 1
assert "notepad" in [p.lower() for p in final_verifies[0]["expected_window_title_contains"]]
def test_empty_app_info(self):
"""Dict vide → pas d'actions."""
@@ -537,12 +932,184 @@ class TestGenerateSetupActions:
assert type_action["text"] == "MonAppMedical"
# =========================================================================
# Tests des gardes visuelles du setup (verify_screen titre fenêtre)
# =========================================================================
class TestSetupVisualGuards:
"""Couvre les gardes visuelles insérées entre les étapes du setup
auto Windows (post-blocage `position_fallback` live du 22 mai 2026).
Sans ces gardes, un clic Démarrer qui touche en fait le systray
overflow popup laissait le setup taper « bloc » dans la mauvaise
fenêtre, et seul le `click_result` final remontait l'erreur — trop
tard. Les `verify_screen` titre-fenêtre stoppent net après chaque
étape critique.
"""
def test_verify_start_menu_open_inserted_after_wait_start(self):
"""Une garde verify_screen est insérée juste après wait_start_menu."""
app_info = {
"primary_app": "winword.exe",
"primary_launch_cmd": "winword",
"first_window_title": "Document1 - Word",
}
actions = _generate_setup_actions(app_info)
steps = [a.get("_setup_step") for a in actions]
# Ordre : click_start_menu → wait_start_menu → verify_start_menu_open
assert "verify_start_menu_open" in steps
idx_wait = steps.index("wait_start_menu")
idx_verify = steps.index("verify_start_menu_open")
assert idx_verify == idx_wait + 1
verify = actions[idx_verify]
assert verify["type"] == "verify_screen"
assert verify.get("_setup_phase") is True
patterns = verify.get("expected_window_title_contains") or []
assert isinstance(patterns, list) and patterns
lowered = [p.lower() for p in patterns]
# Doit couvrir au minimum FR + EN + l'app SearchHost / StartMenu
assert any("recherch" in p for p in lowered), patterns
assert any("search" in p for p in lowered), patterns
def test_verify_search_box_active_inserted_when_click_then_type(self):
"""Quand le setup clique sur la barre Rechercher puis attend,
une garde verify_screen suit l'attente pour bloquer la frappe
si le focus n'est pas réellement dans la barre."""
app_info = {
"primary_app": "winword.exe",
"primary_launch_cmd": "winword",
"first_window_title": "Document1 - Word",
"search_box_interaction": {
"mode": "click_then_type",
"window_title": "Rechercher",
"x_pct": 0.10, "y_pct": 0.95,
},
}
actions = _generate_setup_actions(app_info)
steps = [a.get("_setup_step") for a in actions]
assert "verify_search_box_active" in steps
idx_wait_ready = steps.index("wait_search_ready")
idx_verify = steps.index("verify_search_box_active")
idx_type = steps.index("type_app_name")
# Ordre : wait_search_ready → verify_search_box_active → type_app_name
assert idx_verify == idx_wait_ready + 1
assert idx_type == idx_verify + 1
verify = actions[idx_verify]
assert verify["type"] == "verify_screen"
patterns = verify.get("expected_window_title_contains") or []
assert "Rechercher" in patterns or any(
p.lower() == "rechercher" for p in patterns
)
def test_no_verify_search_box_when_direct_typing(self):
"""En mode direct_typing on n'a pas de click sur la barre — donc
pas de verify_search_box_active dédié (la garde verify_start_menu_open
suffit, on tape directement après)."""
app_info = {
"primary_app": "winword.exe",
"primary_launch_cmd": "winword",
"first_window_title": "Document1 - Word",
"search_box_interaction": {
"mode": "direct_typing",
"window_title": "Rechercher",
},
}
actions = _generate_setup_actions(app_info)
steps = [a.get("_setup_step") for a in actions]
assert "verify_search_box_active" not in steps
# La garde verify_start_menu_open reste présente (couvre la frappe).
assert "verify_start_menu_open" in steps
idx_verify = steps.index("verify_start_menu_open")
idx_type = steps.index("type_app_name")
assert idx_type > idx_verify, (
"type_app_name doit suivre verify_start_menu_open en direct_typing"
)
def test_verify_search_results_visible_inserted_before_click_result(self):
"""Dernier filet : la barre Rechercher (et ses résultats) doit
être encore active juste avant `click_app_result`. Sans cette
garde finale, un focus perdu pendant `wait_search_results`
peut faire cliquer le `click_app_result` dans la mauvaise
surface (constat live 2026-05-22 — fenêtre observée
``Fenêtre de dépassement de capacité de la barre d'état
système.``)."""
app_info = {
"primary_app": "winword.exe",
"primary_launch_cmd": "winword",
"first_window_title": "Document1 - Word",
}
actions = _generate_setup_actions(app_info)
steps = [a.get("_setup_step") for a in actions]
assert "verify_search_results_visible" in steps
idx_wait_results = steps.index("wait_search_results")
idx_verify = steps.index("verify_search_results_visible")
idx_click_result = steps.index("click_app_result")
# Ordre : wait_search_results → verify_search_results_visible → click_app_result
assert idx_verify == idx_wait_results + 1
assert idx_click_result == idx_verify + 1
verify = actions[idx_verify]
assert verify["type"] == "verify_screen"
patterns = verify.get("expected_window_title_contains") or []
assert isinstance(patterns, list) and patterns
lowered = [p.lower() for p in patterns]
assert any("recherch" in p for p in lowered), patterns
assert any("search" in p for p in lowered), patterns
def test_verify_search_results_visible_present_in_direct_typing(self):
"""La garde finale avant click_app_result reste obligatoire
quelle que soit la modalité de la barre Rechercher."""
app_info = {
"primary_app": "winword.exe",
"primary_launch_cmd": "winword",
"first_window_title": "Document1 - Word",
"search_box_interaction": {
"mode": "direct_typing",
"window_title": "Rechercher",
},
}
actions = _generate_setup_actions(app_info)
steps = [a.get("_setup_step") for a in actions]
assert "verify_search_results_visible" in steps
def test_setup_guards_have_short_timeout(self):
"""Les gardes verify_screen ont un timeout court (≤ 2 s) — c'est
un check titre, pas un wait long."""
app_info = {
"primary_app": "winword.exe",
"primary_launch_cmd": "winword",
"first_window_title": "Document1 - Word",
"search_box_interaction": {
"mode": "click_then_type",
"window_title": "Rechercher",
},
}
actions = _generate_setup_actions(app_info)
guards = [
a for a in actions
if a.get("_setup_step") in (
"verify_start_menu_open",
"verify_search_box_active",
"verify_search_results_visible",
)
]
assert guards, "il doit exister au moins une garde verify_screen"
for g in guards:
assert g.get("timeout_ms", 5000) <= 2000
# =========================================================================
# Tests d'intégration : pipeline complet events → setup visuel
# =========================================================================
class TestSetupPipeline:
"""Tests du pipeline complet : extraction + génération visuelle."""
"""Tests du pipeline complet : extraction + génération du setup."""
def test_full_pipeline_from_events(self):
"""Pipeline complet depuis des événements bruts de type Notepad."""
@@ -561,24 +1128,25 @@ class TestSetupPipeline:
assert app_info["primary_app"] == "Notepad.exe"
actions = _generate_setup_actions(app_info)
assert len(actions) >= 8 # Au minimum 8 actions visuelles (sans verify si pas de titre)
assert len(actions) == 7
# Vérifier l'ordre logique 100% visuel
types = [a["type"] for a in actions]
assert types[0] == "click" # Clic Démarrer
assert types[1] == "wait" # Attente menu
assert types[2] == "click" # Clic barre de recherche
assert types[3] == "wait" # Attente barre active
assert types[4] == "type" # Taper le nom
assert types[5] == "wait" # Attente résultats
assert types[6] == "click" # Clic sur le résultat
assert types[7] == "wait" # Attente lancement
steps = [a.get("_setup_step") for a in actions]
expected_step_order = [
"open_run_dialog",
"wait_run_dialog",
"type_launch_command",
"wait_launch_command",
"submit_run_dialog",
"wait_app_launch",
"verify_app_ready",
]
assert steps == expected_step_order, steps
# AUCUN key_combo dans le pipeline
assert "key_combo" not in types, "Le pipeline ne doit contenir aucun key_combo"
assert types.count("key_combo") == 2
# Le texte tapé est le nom visuel français
assert actions[4]["text"] == "Bloc-notes"
idx_type = steps.index("type_launch_command")
assert actions[idx_type]["text"] == "notepad"
def test_full_pipeline_from_workflow(self):
"""Pipeline complet depuis un workflow structuré."""
@@ -599,12 +1167,12 @@ class TestSetupPipeline:
assert app_info["primary_app"] == "Notepad.exe"
actions = _generate_setup_actions(app_info)
assert len(actions) >= 8
assert len(actions) == 7
# Le texte tapé doit être le nom visuel, pas la commande shell
# Le texte tapé doit être la commande shell pour le setup Win+R.
type_action = [a for a in actions if a["type"] == "type"][0]
assert type_action["text"] == "Bloc-notes"
assert type_action["text"] == "notepad"
# Aucun key_combo
# Le setup Notepad s'appuie maintenant sur deux key_combo.
key_combos = [a for a in actions if a["type"] == "key_combo"]
assert key_combos == []
assert len(key_combos) == 2