581 lines
19 KiB
Python
581 lines
19 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
|
|
import pytest
|
|
|
|
import tools.extract_competences_from_session as extractor
|
|
from tools.extract_competences_from_session import build_report, render_markdown_report
|
|
|
|
|
|
def _write_raw_jsonl(path, events):
|
|
lines = [
|
|
json.dumps(
|
|
{
|
|
"session_id": "sess_extract_test",
|
|
"timestamp": float(index),
|
|
"event": event,
|
|
"machine_id": "windows_vm",
|
|
}
|
|
)
|
|
for index, event in enumerate(events)
|
|
]
|
|
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
|
|
|
|
|
def test_dry_run_extracts_click_wait_state_candidate(tmp_path):
|
|
session_path = tmp_path / "live_events.jsonl"
|
|
output_dir = tmp_path / "observed"
|
|
_write_raw_jsonl(
|
|
session_path,
|
|
[
|
|
{"type": "heartbeat", "active_window_title": "Bureau"},
|
|
{
|
|
"type": "mouse_click",
|
|
"button": "left",
|
|
"window": {"title": "Bureau", "app_name": "explorer.exe"},
|
|
"uia_snapshot": {
|
|
"name": "Rechercher",
|
|
"control_type": "bouton",
|
|
"automation_id": "SearchButton",
|
|
"parent_path": [{"name": "Barre des taches", "control_type": "volet"}],
|
|
},
|
|
},
|
|
{
|
|
"type": "window_focus_change",
|
|
"to": {"title": "Rechercher", "app_name": "SearchHost.exe"},
|
|
"window": {"title": "Rechercher", "app_name": "SearchHost.exe"},
|
|
},
|
|
],
|
|
)
|
|
|
|
report = build_report(
|
|
session_path=session_path,
|
|
machine_id="windows_vm",
|
|
output_dir=output_dir,
|
|
)
|
|
|
|
assert report["mode"] == "dry_run"
|
|
assert report["summary"]["would_write"] == 0
|
|
assert report["summary"]["candidates_generated"] == 1
|
|
candidate = report["candidates"][0]
|
|
assert candidate["validator_status"] == "would_pass"
|
|
assert candidate["apply_eligible"] is True
|
|
assert candidate["primitive_refs"] == ["click_anchor", "wait_for_state"]
|
|
assert candidate["segment"] == {"keep": [0, 1, 2], "method": [1, 2], "success": [2]}
|
|
assert candidate["t2_gaps_detected"] == [
|
|
"click_target_semantics_not_observed_offline",
|
|
"no_ocr_offline",
|
|
]
|
|
assert not (output_dir / f"{candidate['competence_id']}.yaml").exists()
|
|
|
|
|
|
def test_dry_run_rejects_click_without_uia_anchor(tmp_path):
|
|
session_path = tmp_path / "live_events.jsonl"
|
|
_write_raw_jsonl(
|
|
session_path,
|
|
[
|
|
{
|
|
"type": "mouse_click",
|
|
"button": "left",
|
|
"window": {"title": "Bureau", "app_name": "explorer.exe"},
|
|
},
|
|
{
|
|
"type": "window_focus_change",
|
|
"to": {"title": "Rechercher", "app_name": "SearchHost.exe"},
|
|
},
|
|
],
|
|
)
|
|
|
|
report = build_report(session_path=session_path, machine_id="windows_vm")
|
|
|
|
assert report["summary"]["candidates_generated"] == 0
|
|
assert report["summary"]["candidates_rejected"] == 1
|
|
assert report["rejected"][0]["reason"] == "click without uia_snapshot anchor"
|
|
assert report["rejected"][0]["validator_codes"] == ["anchor_ref_uia_missing"]
|
|
|
|
|
|
def test_dry_run_rejects_weak_uia_click_anchor(tmp_path):
|
|
session_path = tmp_path / "live_events.jsonl"
|
|
_write_raw_jsonl(
|
|
session_path,
|
|
[
|
|
{
|
|
"type": "mouse_click",
|
|
"button": "left",
|
|
"window": {"title": "Rechercher", "app_name": "SearchHost.exe"},
|
|
"uia_snapshot": {
|
|
"name": "Aujourd'hui",
|
|
"control_type": "Groupe",
|
|
"automation_id": "0",
|
|
},
|
|
},
|
|
{
|
|
"type": "window_focus_change",
|
|
"to": {"title": "unknown_window", "app_name": "explorer.exe"},
|
|
},
|
|
],
|
|
)
|
|
|
|
report = build_report(session_path=session_path, machine_id="windows_vm")
|
|
|
|
assert report["summary"]["candidates_generated"] == 0
|
|
assert report["rejected"][0]["reason"] == "click with too generic anchor"
|
|
assert report["rejected"][0]["validator_codes"] == ["anchor_ref_too_generic"]
|
|
|
|
|
|
def test_dry_run_rejects_systemtrayicon_anchor(tmp_path):
|
|
session_path = tmp_path / "live_events.jsonl"
|
|
_write_raw_jsonl(
|
|
session_path,
|
|
[
|
|
{
|
|
"type": "mouse_click",
|
|
"button": "left",
|
|
"window": {"title": "Shell_TrayWnd", "app_name": "explorer.exe"},
|
|
"uia_snapshot": {
|
|
"name": "SystemTrayIcon",
|
|
"control_type": "bouton",
|
|
"automation_id": "SystemTrayIcon",
|
|
},
|
|
},
|
|
{
|
|
"type": "window_focus_change",
|
|
"to": {"title": "unknown_window", "app_name": "explorer.exe"},
|
|
},
|
|
],
|
|
)
|
|
|
|
report = build_report(session_path=session_path, machine_id="windows_vm")
|
|
|
|
assert report["summary"]["candidates_generated"] == 0
|
|
assert report["rejected"][0]["reason"] == "click on fragile system tray anchor"
|
|
assert report["rejected"][0]["validator_codes"] == ["anchor_ref_systray_fragile"]
|
|
|
|
|
|
def test_dry_run_rejects_dom_autogenerated_anchor(tmp_path):
|
|
session_path = tmp_path / "live_events.jsonl"
|
|
_write_raw_jsonl(
|
|
session_path,
|
|
[
|
|
{
|
|
"type": "mouse_click",
|
|
"button": "left",
|
|
"window": {"title": "Chrome", "app_name": "chrome.exe"},
|
|
"uia_snapshot": {
|
|
"name": "Continuer",
|
|
"control_type": "bouton",
|
|
"automation_id": "so_iazxhgsedkduppcyhoay_73",
|
|
},
|
|
},
|
|
{
|
|
"type": "window_focus_change",
|
|
"to": {"title": "Chrome", "app_name": "chrome.exe"},
|
|
},
|
|
],
|
|
)
|
|
|
|
report = build_report(session_path=session_path, machine_id="windows_vm")
|
|
|
|
assert report["summary"]["candidates_generated"] == 0
|
|
assert report["rejected"][0]["reason"] == "click on autogenerated DOM anchor"
|
|
assert report["rejected"][0]["validator_codes"] == ["anchor_ref_dom_autogenerated"]
|
|
|
|
|
|
def test_dry_run_rejects_unknown_window_title(tmp_path):
|
|
session_path = tmp_path / "live_events.jsonl"
|
|
_write_raw_jsonl(
|
|
session_path,
|
|
[
|
|
{
|
|
"type": "mouse_click",
|
|
"button": "left",
|
|
"window": {"title": "unknown_window", "app_name": "explorer.exe"},
|
|
"uia_snapshot": {
|
|
"name": "Ouvrir",
|
|
"control_type": "bouton",
|
|
"automation_id": "OpenButton",
|
|
},
|
|
},
|
|
{
|
|
"type": "window_focus_change",
|
|
"to": {"title": "Explorateur", "app_name": "explorer.exe"},
|
|
},
|
|
],
|
|
)
|
|
|
|
report = build_report(session_path=session_path, machine_id="windows_vm")
|
|
|
|
assert report["summary"]["candidates_generated"] == 0
|
|
assert report["rejected"][0]["reason"] == "click in unknown or overflow window"
|
|
assert report["rejected"][0]["validator_codes"] == ["anchor_ref_unknown_window"]
|
|
|
|
|
|
def test_dry_run_rejects_browser_contextual_anchor(tmp_path):
|
|
session_path = tmp_path / "live_events.jsonl"
|
|
_write_raw_jsonl(
|
|
session_path,
|
|
[
|
|
{
|
|
"type": "mouse_click",
|
|
"button": "left",
|
|
"window": {"title": "Dashboard - Google Chrome", "app_name": "chrome.exe"},
|
|
"uia_snapshot": {
|
|
"name": "Nouvel onglet",
|
|
"control_type": "Bouton",
|
|
"class_name": "TabStripControlButton",
|
|
"automation_id": "",
|
|
"parent_path": [{"name": "", "control_type": "tabulation"}],
|
|
},
|
|
},
|
|
{
|
|
"type": "window_focus_change",
|
|
"to": {"title": "Nouvel onglet - Google Chrome", "app_name": "chrome.exe"},
|
|
},
|
|
],
|
|
)
|
|
|
|
report = build_report(session_path=session_path, machine_id="windows_vm")
|
|
|
|
assert report["summary"]["candidates_generated"] == 0
|
|
assert report["rejected"][0]["reason"] == "click on contextual browser chrome anchor"
|
|
assert report["rejected"][0]["validator_codes"] == ["anchor_ref_browser_contextual"]
|
|
|
|
|
|
def test_dry_run_rejects_contextual_add_tab_button_anchor(tmp_path):
|
|
session_path = tmp_path / "live_events.jsonl"
|
|
_write_raw_jsonl(
|
|
session_path,
|
|
[
|
|
{
|
|
"type": "mouse_click",
|
|
"button": "left",
|
|
"window": {"title": "agent_debug.log - Bloc-notes", "app_name": "Notepad.exe"},
|
|
"uia_snapshot": {
|
|
"name": "Ajouter un nouvel onglet",
|
|
"control_type": "bouton",
|
|
"class_name": "Button",
|
|
"automation_id": "AddButton",
|
|
"parent_path": [
|
|
{"name": "Bureau 1", "control_type": "volet"},
|
|
{"name": "agent_debug.log - Bloc-notes", "control_type": "fenetre"},
|
|
{"name": "", "control_type": "volet"},
|
|
{"name": "", "control_type": "onglet"},
|
|
],
|
|
},
|
|
},
|
|
{
|
|
"type": "window_focus_change",
|
|
"to": {"title": "agent_debug.log - Bloc-notes", "app_name": "Notepad.exe"},
|
|
},
|
|
],
|
|
)
|
|
|
|
report = build_report(session_path=session_path, machine_id="windows_vm")
|
|
|
|
assert report["summary"]["candidates_generated"] == 0
|
|
assert report["rejected"][0]["reason"] == "click on contextual UI chrome button"
|
|
assert report["rejected"][0]["validator_codes"] == ["anchor_ref_contextual_button"]
|
|
|
|
|
|
def test_dry_run_rejects_too_generic_anchor(tmp_path):
|
|
session_path = tmp_path / "live_events.jsonl"
|
|
_write_raw_jsonl(
|
|
session_path,
|
|
[
|
|
{
|
|
"type": "mouse_click",
|
|
"button": "left",
|
|
"window": {"title": "Application", "app_name": "app.exe"},
|
|
"uia_snapshot": {
|
|
"name": "button_12",
|
|
"control_type": "bouton",
|
|
"automation_id": "",
|
|
},
|
|
},
|
|
{
|
|
"type": "window_focus_change",
|
|
"to": {"title": "Application", "app_name": "app.exe"},
|
|
},
|
|
],
|
|
)
|
|
|
|
report = build_report(session_path=session_path, machine_id="windows_vm")
|
|
|
|
assert report["summary"]["candidates_generated"] == 0
|
|
assert report["rejected"][0]["reason"] == "click with too generic anchor"
|
|
assert report["rejected"][0]["validator_codes"] == ["anchor_ref_too_generic"]
|
|
|
|
|
|
def test_dry_run_rejects_empty_region_anchor(tmp_path):
|
|
session_path = tmp_path / "live_events.jsonl"
|
|
_write_raw_jsonl(
|
|
session_path,
|
|
[
|
|
{
|
|
"type": "mouse_click",
|
|
"button": "left",
|
|
"window": {"title": "Application", "app_name": "app.exe"},
|
|
"uia_snapshot": {
|
|
"name": "",
|
|
"control_type": "région",
|
|
"automation_id": "",
|
|
},
|
|
},
|
|
{
|
|
"type": "window_focus_change",
|
|
"to": {"title": "Application", "app_name": "app.exe"},
|
|
},
|
|
],
|
|
)
|
|
|
|
report = build_report(session_path=session_path, machine_id="windows_vm")
|
|
|
|
assert report["summary"]["candidates_generated"] == 0
|
|
assert report["rejected"][0]["reason"] == "click with too generic anchor"
|
|
assert report["rejected"][0]["validator_codes"] == ["anchor_ref_too_generic"]
|
|
|
|
|
|
def test_dry_run_hard_caps_candidates(tmp_path):
|
|
session_path = tmp_path / "live_events.jsonl"
|
|
_write_raw_jsonl(session_path, [])
|
|
|
|
with pytest.raises(ValueError, match="hard-cap"):
|
|
build_report(session_path=session_path, machine_id="windows_vm", max_candidates=11)
|
|
|
|
|
|
def test_apply_requires_allow_list(tmp_path):
|
|
session_path = tmp_path / "live_events.jsonl"
|
|
_write_raw_jsonl(session_path, [])
|
|
|
|
with pytest.raises(ValueError, match="--allow-list is required"):
|
|
build_report(session_path=session_path, machine_id="windows_vm", mode="apply")
|
|
|
|
|
|
def test_apply_rejects_unknown_id_in_allow_list(tmp_path):
|
|
session_path = tmp_path / "live_events.jsonl"
|
|
output_dir = tmp_path / "observed"
|
|
_write_raw_jsonl(
|
|
session_path,
|
|
[
|
|
{
|
|
"type": "key_combo",
|
|
"keys": ["win", "e"],
|
|
"window": {"title": "Bureau", "app_name": "explorer.exe"},
|
|
},
|
|
{
|
|
"type": "window_focus_change",
|
|
"to": {"title": "Executer", "app_name": "explorer.exe"},
|
|
},
|
|
],
|
|
)
|
|
|
|
with pytest.raises(ValueError, match="--allow-list-id-not-found: missing_id"):
|
|
build_report(
|
|
session_path=session_path,
|
|
machine_id="windows_vm",
|
|
output_dir=output_dir,
|
|
mode="apply",
|
|
allow_list=["missing_id"],
|
|
)
|
|
|
|
assert not list(output_dir.glob("*.yaml"))
|
|
|
|
|
|
def test_apply_atomic_rollback_on_validation_failure(tmp_path, monkeypatch):
|
|
session_path = tmp_path / "live_events.jsonl"
|
|
output_dir = tmp_path / "observed"
|
|
_write_raw_jsonl(
|
|
session_path,
|
|
[
|
|
{
|
|
"type": "key_combo",
|
|
"keys": ["win", "e"],
|
|
"window": {"title": "Bureau", "app_name": "explorer.exe"},
|
|
},
|
|
{
|
|
"type": "window_focus_change",
|
|
"to": {"title": "Executer", "app_name": "explorer.exe"},
|
|
},
|
|
],
|
|
)
|
|
|
|
def fail_validation(paths, *, repo_root):
|
|
raise ValueError("apply-validation-failed: forced")
|
|
|
|
monkeypatch.setattr(extractor, "_validate_apply_yaml_files", fail_validation)
|
|
|
|
with pytest.raises(ValueError, match="apply-validation-failed: forced"):
|
|
build_report(
|
|
session_path=session_path,
|
|
machine_id="windows_vm",
|
|
output_dir=output_dir,
|
|
mode="apply",
|
|
allow_list=["key_win_e_wait_explorer_exe"],
|
|
)
|
|
|
|
assert not list(output_dir.glob("*.yaml"))
|
|
|
|
|
|
def test_apply_writes_only_allowed_ids(tmp_path):
|
|
session_path = tmp_path / "live_events.jsonl"
|
|
output_dir = tmp_path / "observed"
|
|
_write_raw_jsonl(
|
|
session_path,
|
|
[
|
|
{
|
|
"type": "key_combo",
|
|
"keys": ["win", "e"],
|
|
"window": {"title": "Bureau", "app_name": "explorer.exe"},
|
|
},
|
|
{
|
|
"type": "window_focus_change",
|
|
"to": {"title": "Executer", "app_name": "explorer.exe"},
|
|
},
|
|
{
|
|
"type": "key_combo",
|
|
"keys": ["ctrl", "p"],
|
|
"window": {"title": "Bloc-notes", "app_name": "Notepad.exe"},
|
|
},
|
|
{
|
|
"type": "window_focus_change",
|
|
"to": {"title": "Enregistrer sous", "app_name": "Notepad.exe"},
|
|
},
|
|
],
|
|
)
|
|
|
|
report = build_report(
|
|
session_path=session_path,
|
|
machine_id="windows_vm",
|
|
output_dir=output_dir,
|
|
mode="apply",
|
|
allow_list=["key_win_e_wait_explorer_exe"],
|
|
)
|
|
|
|
assert report["mode"] == "apply"
|
|
assert report["allow_list"] == ["key_win_e_wait_explorer_exe"]
|
|
assert report["summary"]["would_write"] == 1
|
|
assert report["summary"]["written"] == 1
|
|
assert report["applied"] == [
|
|
{
|
|
"competence_id": "key_win_e_wait_explorer_exe",
|
|
"path": str(output_dir / "key_win_e_wait_explorer_exe.yaml"),
|
|
}
|
|
]
|
|
assert (output_dir / "key_win_e_wait_explorer_exe.yaml").is_file()
|
|
assert not (output_dir / "key_ctrl_p_wait_notepad_exe.yaml").exists()
|
|
|
|
|
|
def test_apply_respects_max_candidates_cap(tmp_path):
|
|
session_path = tmp_path / "live_events.jsonl"
|
|
_write_raw_jsonl(session_path, [])
|
|
|
|
with pytest.raises(ValueError, match="hard-cap"):
|
|
build_report(
|
|
session_path=session_path,
|
|
machine_id="windows_vm",
|
|
mode="apply",
|
|
allow_list=["key_win_r_wait_explorer_exe"],
|
|
max_candidates=11,
|
|
)
|
|
|
|
|
|
def test_markdown_report_includes_candidate_summary(tmp_path):
|
|
session_path = tmp_path / "live_events.jsonl"
|
|
_write_raw_jsonl(
|
|
session_path,
|
|
[
|
|
{
|
|
"type": "key_combo",
|
|
"keys": ["ctrl", "s"],
|
|
"window": {"title": "Bloc-notes", "app_name": "Notepad.exe"},
|
|
},
|
|
{
|
|
"type": "window_focus_change",
|
|
"to": {"title": "Enregistrer sous", "app_name": "Notepad.exe"},
|
|
},
|
|
],
|
|
)
|
|
report = build_report(session_path=session_path, machine_id="windows_vm")
|
|
|
|
markdown = render_markdown_report(report)
|
|
|
|
assert "# Extraction report" in markdown
|
|
assert "key_ctrl_s_wait_notepad_exe" in markdown
|
|
assert "wait_for_state" in markdown
|
|
|
|
|
|
def test_azerty_ctrl_s_trace_is_normalized_for_candidate(tmp_path):
|
|
session_path = tmp_path / "live_events.jsonl"
|
|
_write_raw_jsonl(
|
|
session_path,
|
|
[
|
|
{
|
|
"type": "key_combo",
|
|
"keys": ["shift", "ctrl", "@"],
|
|
"window": {"title": "WordPad", "app_name": "WordPad.exe"},
|
|
},
|
|
{
|
|
"type": "window_focus_change",
|
|
"to": {"title": "Enregistrer sous", "app_name": "WordPad.exe"},
|
|
},
|
|
],
|
|
)
|
|
|
|
report = build_report(session_path=session_path, machine_id="windows_vm")
|
|
|
|
candidate = report["candidates"][0]
|
|
assert candidate["competence_id"] == "key_ctrl_s_wait_wordpad_exe"
|
|
assert candidate["validator_status"] == "would_pass"
|
|
assert candidate["apply_eligible"] is True
|
|
|
|
|
|
def test_ctrl_s_control_character_trace_is_normalized_for_candidate(tmp_path):
|
|
session_path = tmp_path / "live_events.jsonl"
|
|
_write_raw_jsonl(
|
|
session_path,
|
|
[
|
|
{
|
|
"type": "key_combo",
|
|
"keys": ["shift", "ctrl", "\x13"],
|
|
"window": {"title": "Bloc-notes", "app_name": "Notepad.exe"},
|
|
},
|
|
{
|
|
"type": "window_focus_change",
|
|
"to": {"title": "Enregistrer sous", "app_name": "Notepad.exe"},
|
|
},
|
|
],
|
|
)
|
|
|
|
report = build_report(session_path=session_path, machine_id="windows_vm")
|
|
|
|
candidate = report["candidates"][0]
|
|
assert candidate["competence_id"] == "key_ctrl_s_wait_notepad_exe"
|
|
assert candidate["validator_status"] == "would_pass"
|
|
|
|
|
|
def test_text_input_candidate_is_below_apply_threshold(tmp_path):
|
|
session_path = tmp_path / "live_events.jsonl"
|
|
_write_raw_jsonl(
|
|
session_path,
|
|
[
|
|
{
|
|
"type": "text_input",
|
|
"text": "hello",
|
|
"window": {"title": "Bloc-notes", "app_name": "Notepad.exe"},
|
|
},
|
|
{
|
|
"type": "heartbeat",
|
|
"window": {"title": "Bloc-notes", "app_name": "Notepad.exe"},
|
|
},
|
|
],
|
|
)
|
|
|
|
report = build_report(session_path=session_path, machine_id="windows_vm")
|
|
|
|
candidate = report["candidates"][0]
|
|
assert candidate["primitive_refs"] == ["text_input_focused"]
|
|
assert candidate["confidence"] < report["summary"]["apply_min_confidence"]
|
|
assert candidate["apply_eligible"] is False
|
|
assert "below_apply_confidence_threshold" in candidate["quality_flags"]
|