feat(navigation): brique login visuel OCR-ancre + action navigate au replay
- core/navigation/ : visual_verifier (presence=OCR, role=VLM ancre sur tokens), grounding (OCR-anchor first, VLM fallback, cache coords valide par la vue), visual_login (verify_before/after, DETTE-023), action_resolver (pont runtime) - api_stream/replay_engine : dispatch action navigate server-side, never-fail -> needs_review, import depuis core.navigation (boot 5005 garanti) - 131 tests verts (wiring boot, e2e handler, unit modules) Chantier Qwen 01-02/07/2026, revue croisee Claude (plan deploy v2). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
205
tests/unit/test_action_resolver.py
Normal file
205
tests/unit/test_action_resolver.py
Normal file
@@ -0,0 +1,205 @@
|
||||
"""Tests for core/navigation/action_resolver.py — coordinate conversion + OCR adapters."""
|
||||
|
||||
import json
|
||||
import pytest
|
||||
from core.navigation.action_resolver import (
|
||||
NavigateCoords,
|
||||
NavigateResult,
|
||||
grounded_to_coords,
|
||||
make_ocr_simple_from_detailed,
|
||||
navigate_login,
|
||||
)
|
||||
from core.navigation.grounding import (
|
||||
CoordsCache,
|
||||
GroundedElement,
|
||||
OcrTokenInfo,
|
||||
OcrDetailedClient,
|
||||
)
|
||||
from core.navigation.visual_verifier import VlmClient
|
||||
|
||||
|
||||
# ── Mock factories ─────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def mock_ocr_detailed_client_factory(tokens: list):
|
||||
def client(image_path: str) -> list:
|
||||
return tokens
|
||||
return client
|
||||
|
||||
|
||||
def mock_vlm_client_factory(response_json: dict):
|
||||
def client(image_path: str, prompt: str) -> str:
|
||||
return json.dumps(response_json)
|
||||
return client
|
||||
|
||||
|
||||
# ── grounded_to_coords tests ───────────────────────────────────────────
|
||||
|
||||
|
||||
class TestGroundedToCoords:
|
||||
def test_basic_conversion(self):
|
||||
el = GroundedElement(
|
||||
role="bouton", text="Connexion",
|
||||
bbox=(200, 50, 400, 100), center=(300, 75),
|
||||
confidence=0.9, method="ocr_anchor",
|
||||
)
|
||||
coords = grounded_to_coords(el, 1920, 1080)
|
||||
assert coords.x_pct == pytest.approx(300 / 1920, abs=0.01)
|
||||
assert coords.y_pct == pytest.approx(75 / 1080, abs=0.01)
|
||||
assert coords.method == "ocr_anchor"
|
||||
assert coords.bbox_pct is not None
|
||||
|
||||
def test_to_dict(self):
|
||||
coords = NavigateCoords(x_pct=0.15, y_pct=0.07, method="ocr_anchor")
|
||||
d = coords.to_dict()
|
||||
assert d["x_pct"] == 0.15
|
||||
assert d["y_pct"] == 0.07
|
||||
assert d["method"] == "ocr_anchor"
|
||||
|
||||
def test_to_dict_with_bbox(self):
|
||||
coords = NavigateCoords(
|
||||
x_pct=0.15, y_pct=0.07,
|
||||
bbox_pct=(0.10, 0.05, 0.20, 0.09),
|
||||
method="vlm_grounder",
|
||||
)
|
||||
d = coords.to_dict()
|
||||
assert "bbox_pct" in d
|
||||
assert len(d["bbox_pct"]) == 4
|
||||
|
||||
|
||||
# ── make_ocr_simple_from_detailed tests ────────────────────────────────
|
||||
|
||||
|
||||
class TestMakeOcrSimpleFromDetailed:
|
||||
def test_conversion(self):
|
||||
tokens = [
|
||||
OcrTokenInfo(text="Login", bbox=(100, 50, 250, 90)),
|
||||
OcrTokenInfo(text="Password", bbox=(100, 100, 250, 140)),
|
||||
]
|
||||
detailed = mock_ocr_detailed_client_factory(tokens)
|
||||
simple = make_ocr_simple_from_detailed(detailed)
|
||||
result = simple("/tmp/test.png")
|
||||
assert result == ["Login", "Password"]
|
||||
|
||||
def test_empty_tokens(self):
|
||||
detailed = mock_ocr_detailed_client_factory([])
|
||||
simple = make_ocr_simple_from_detailed(detailed)
|
||||
result = simple("/tmp/test.png")
|
||||
assert result == []
|
||||
|
||||
|
||||
# ── navigate_login tests ───────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestNavigateLogin:
|
||||
def test_full_success(self):
|
||||
"""All fields grounded → NavigateResult with coords."""
|
||||
ocr = mock_ocr_detailed_client_factory([
|
||||
OcrTokenInfo(text="Login", bbox=(100, 50, 250, 90), confidence=0.95),
|
||||
OcrTokenInfo(text="Mot de passe", bbox=(100, 100, 250, 140), confidence=0.95),
|
||||
OcrTokenInfo(text="Connexion", bbox=(100, 150, 250, 190), confidence=0.95),
|
||||
])
|
||||
vlm = mock_vlm_client_factory({
|
||||
"confirmed": [
|
||||
{"index": 1, "role_confirmed": True, "actual_role": "champ", "confidence": 0.9},
|
||||
{"index": 2, "role_confirmed": True, "actual_role": "champ", "confidence": 0.9},
|
||||
{"index": 3, "role_confirmed": True, "actual_role": "bouton", "confidence": 0.9},
|
||||
],
|
||||
"overall_confidence": 0.9,
|
||||
})
|
||||
result = navigate_login(
|
||||
"/tmp/login.png",
|
||||
ocr_client=ocr, vlm_client=vlm,
|
||||
skip_pre_verify=True,
|
||||
)
|
||||
assert result.all_resolved == True
|
||||
assert result.login_coords is not None
|
||||
assert result.password_coords is not None
|
||||
assert result.submit_coords is not None
|
||||
assert result.submit_coords.x_pct > 0
|
||||
assert result.submit_coords.y_pct > 0
|
||||
|
||||
def test_no_clients_error(self):
|
||||
"""Missing OCR/VLM clients → error."""
|
||||
result = navigate_login("/tmp/login.png", ocr_client=None, vlm_client=None)
|
||||
assert result.all_resolved == False
|
||||
assert "required" in result.error
|
||||
|
||||
def test_pre_verify_fail(self):
|
||||
"""Pre-verify fails → early abort."""
|
||||
ocr = mock_ocr_detailed_client_factory([
|
||||
OcrTokenInfo(text="Accueil", bbox=(0, 0, 100, 40)),
|
||||
])
|
||||
vlm = mock_vlm_client_factory({})
|
||||
result = navigate_login(
|
||||
"/tmp/page.png",
|
||||
ocr_client=ocr, vlm_client=vlm,
|
||||
skip_pre_verify=False,
|
||||
)
|
||||
assert result.all_resolved == False
|
||||
assert result.pre_verify is not None
|
||||
assert result.pre_verify.match == False
|
||||
|
||||
def test_skip_pre_verify(self):
|
||||
"""Skip pre-verify → proceed to grounding even if form incomplete."""
|
||||
ocr = mock_ocr_detailed_client_factory([
|
||||
OcrTokenInfo(text="Login", bbox=(100, 50, 250, 90)),
|
||||
OcrTokenInfo(text="Mot de passe", bbox=(100, 100, 250, 140)),
|
||||
OcrTokenInfo(text="Connexion", bbox=(100, 150, 250, 190)),
|
||||
])
|
||||
vlm = mock_vlm_client_factory({})
|
||||
result = navigate_login(
|
||||
"/tmp/login.png",
|
||||
ocr_client=ocr, vlm_client=vlm,
|
||||
skip_pre_verify=True,
|
||||
)
|
||||
assert result.pre_verify is None # skipped
|
||||
assert result.all_resolved == True
|
||||
|
||||
|
||||
# ── NavigateResult dataclass tests ─────────────────────────────────────
|
||||
|
||||
|
||||
class TestNavigateResult:
|
||||
def test_default(self):
|
||||
result = NavigateResult()
|
||||
assert result.all_resolved == False
|
||||
assert result.login_coords is None
|
||||
assert result.error == ""
|
||||
|
||||
def test_with_coords(self):
|
||||
result = NavigateResult(
|
||||
login_coords=NavigateCoords(x_pct=0.15, y_pct=0.07, method="ocr_anchor"),
|
||||
all_resolved=True,
|
||||
)
|
||||
assert result.login_coords.x_pct == 0.15
|
||||
|
||||
|
||||
# ── Import validation ──────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestImportValidation:
|
||||
def test_action_resolver_imports(self):
|
||||
"""Verify action_resolver module imports cleanly."""
|
||||
from core.navigation.action_resolver import (
|
||||
NavigateCoords,
|
||||
NavigateResult,
|
||||
grounded_to_coords,
|
||||
make_ocr_detailed_from_grid,
|
||||
make_ocr_simple_from_detailed,
|
||||
navigate_login,
|
||||
)
|
||||
assert NavigateCoords is not None
|
||||
assert NavigateResult is not None
|
||||
|
||||
def test_navigation_package_handler(self):
|
||||
"""Verify _handle_navigate_action is importable from package."""
|
||||
from core.navigation import _handle_navigate_action
|
||||
assert callable(_handle_navigate_action)
|
||||
|
||||
def test_navigation_package_exports(self):
|
||||
"""Verify package __all__ includes navigate exports."""
|
||||
import core.navigation as nav
|
||||
assert "navigate_login" in nav.__all__
|
||||
assert "NavigateResult" in nav.__all__
|
||||
assert "_handle_navigate_action" in nav.__all__
|
||||
406
tests/unit/test_grounding.py
Normal file
406
tests/unit/test_grounding.py
Normal file
@@ -0,0 +1,406 @@
|
||||
"""Tests for core/navigation/grounding.py — OCR-anchored grounding + VLM fallback + coords cache."""
|
||||
|
||||
import json
|
||||
import pytest
|
||||
from core.navigation.grounding import (
|
||||
OcrTokenInfo,
|
||||
GroundedElement,
|
||||
CoordsCacheEntry,
|
||||
CoordsCache,
|
||||
bbox_center,
|
||||
make_element_key,
|
||||
ocr_anchor_ground,
|
||||
build_grounder_prompt,
|
||||
parse_grounder_response,
|
||||
ground_element,
|
||||
)
|
||||
from core.navigation.visual_verifier import normalize_text
|
||||
|
||||
|
||||
# ── Mock factories ─────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def mock_ocr_detailed_client_factory(tokens: list):
|
||||
"""Factory for mock OcrDetailedClient returning List[OcrTokenInfo]."""
|
||||
def client(image_path: str) -> list:
|
||||
return tokens
|
||||
return client
|
||||
|
||||
|
||||
def mock_vlm_client_factory(response_json: dict):
|
||||
"""Factory for mock VlmClient returning given JSON."""
|
||||
def client(image_path: str, prompt: str) -> str:
|
||||
return json.dumps(response_json)
|
||||
return client
|
||||
|
||||
|
||||
# ── bbox_center tests ──────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestBboxCenter:
|
||||
def test_basic(self):
|
||||
assert bbox_center((100, 200, 300, 400)) == (200, 300)
|
||||
|
||||
def test_zero_origin(self):
|
||||
assert bbox_center((0, 0, 100, 100)) == (50, 50)
|
||||
|
||||
def test_symmetric(self):
|
||||
assert bbox_center((10, 10, 20, 20)) == (15, 15)
|
||||
|
||||
|
||||
# ── make_element_key tests ─────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestMakeElementKey:
|
||||
def test_basic(self):
|
||||
key = make_element_key("bouton", "Rechercher")
|
||||
assert key == "bouton:rechercher"
|
||||
|
||||
def test_normalized(self):
|
||||
key = make_element_key("champ", "Nom Prénom")
|
||||
assert "nom" in key and "prenom" in key
|
||||
|
||||
def test_consistent(self):
|
||||
# Same element always produces same key
|
||||
assert make_element_key("bouton", "Connexion") == make_element_key("bouton", "CONNEXION")
|
||||
|
||||
|
||||
# ── ocr_anchor_ground tests ────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestOcrAnchorGround:
|
||||
def test_exact_match(self):
|
||||
tokens = [OcrTokenInfo(text="Rechercher", bbox=(100, 50, 250, 90), confidence=0.95)]
|
||||
result = ocr_anchor_ground(tokens, {"role": "bouton", "text": "Rechercher"})
|
||||
assert result is not None
|
||||
assert result.method == "ocr_anchor"
|
||||
assert result.bbox == (100, 50, 250, 90)
|
||||
assert result.center == (175, 70)
|
||||
assert result.confidence == 0.95
|
||||
|
||||
def test_fuzzy_match(self):
|
||||
tokens = [OcrTokenInfo(text="Rechércher", bbox=(100, 50, 250, 90))]
|
||||
result = ocr_anchor_ground(tokens, {"role": "bouton", "text": "Rechercher"})
|
||||
assert result is not None
|
||||
assert result.source_ocr_text == "Rechércher"
|
||||
|
||||
def test_no_match(self):
|
||||
tokens = [OcrTokenInfo(text="Accueil", bbox=(100, 50, 250, 90))]
|
||||
result = ocr_anchor_ground(tokens, {"role": "bouton", "text": "Rechercher"})
|
||||
assert result is None
|
||||
|
||||
def test_token_without_bbox(self):
|
||||
tokens = [OcrTokenInfo(text="Rechercher", bbox=None)]
|
||||
result = ocr_anchor_ground(tokens, {"role": "bouton", "text": "Rechercher"})
|
||||
assert result is None # found text but no bbox → can't ground
|
||||
|
||||
def test_no_text_target(self):
|
||||
tokens = [OcrTokenInfo(text="Dashboard", bbox=(0, 0, 1920, 1080))]
|
||||
result = ocr_anchor_ground(tokens, {"role": "page"}) # no text key
|
||||
assert result is None # no text to match
|
||||
|
||||
def test_multiple_tokens_first_match(self):
|
||||
tokens = [
|
||||
OcrTokenInfo(text="Accueil", bbox=(0, 0, 100, 40)),
|
||||
OcrTokenInfo(text="Connexion", bbox=(200, 50, 350, 90)),
|
||||
]
|
||||
result = ocr_anchor_ground(tokens, {"role": "bouton", "text": "Connexion"})
|
||||
assert result is not None
|
||||
assert result.bbox == (200, 50, 350, 90)
|
||||
|
||||
|
||||
# ── build_grounder_prompt tests ────────────────────────────────────────
|
||||
|
||||
|
||||
class TestBuildGrounderPrompt:
|
||||
def test_basic_prompt(self):
|
||||
prompt = build_grounder_prompt({"role": "bouton", "text": "Connexion"})
|
||||
assert "bouton" in prompt
|
||||
assert "Connexion" in prompt
|
||||
assert "bbox" in prompt
|
||||
|
||||
def test_with_context(self):
|
||||
prompt = build_grounder_prompt(
|
||||
{"role": "champ", "text": "Login"},
|
||||
context="page login DPI",
|
||||
)
|
||||
assert "page login DPI" in prompt
|
||||
|
||||
def test_with_extra(self):
|
||||
prompt = build_grounder_prompt(
|
||||
{"role": "champ", "text": "IPP", "extra": "colonne gauche"},
|
||||
)
|
||||
assert "colonne gauche" in prompt
|
||||
|
||||
|
||||
# ── parse_grounder_response tests ──────────────────────────────────────
|
||||
|
||||
|
||||
class TestParseGrounderResponse:
|
||||
def test_valid_response(self):
|
||||
vlm_text = json.dumps({
|
||||
"found": True,
|
||||
"bbox": [0.1, 0.2, 0.3, 0.4],
|
||||
"confidence": 0.92,
|
||||
"description": "login button",
|
||||
})
|
||||
result = parse_grounder_response(vlm_text, 1920, 1080, {"role": "bouton", "text": "Connexion"})
|
||||
assert result is not None
|
||||
assert result.method == "vlm_grounder"
|
||||
assert result.bbox == (192, 216, 576, 432) # 0.1*1920, 0.2*1080, 0.3*1920, 0.4*1080
|
||||
assert result.confidence == 0.92
|
||||
|
||||
def test_not_found(self):
|
||||
vlm_text = json.dumps({"found": False, "bbox": [], "confidence": 0.0})
|
||||
result = parse_grounder_response(vlm_text, 1920, 1080, {"role": "bouton", "text": "Connexion"})
|
||||
assert result is None
|
||||
|
||||
def test_json_in_markdown(self):
|
||||
vlm_text = "```json\n{\"found\": true, \"bbox\": [0.5, 0.5, 0.6, 0.6], \"confidence\": 0.8}\n```"
|
||||
result = parse_grounder_response(vlm_text, 1920, 1080, {"role": "bouton", "text": "Connexion"})
|
||||
assert result is not None
|
||||
|
||||
def test_garbled_response(self):
|
||||
result = parse_grounder_response("I cannot find the element", 1920, 1080, {"role": "bouton", "text": "Connexion"})
|
||||
assert result is None
|
||||
|
||||
def test_invalid_bbox_format(self):
|
||||
vlm_text = json.dumps({"found": True, "bbox": [0.1, 0.2], "confidence": 0.8})
|
||||
result = parse_grounder_response(vlm_text, 1920, 1080, {"role": "bouton", "text": "Connexion"})
|
||||
assert result is None # bbox must have 4 values
|
||||
|
||||
def test_confidence_as_string(self):
|
||||
vlm_text = json.dumps({"found": True, "bbox": [0.1, 0.2, 0.3, 0.4], "confidence": "0.85"})
|
||||
result = parse_grounder_response(vlm_text, 1920, 1080, {"role": "bouton", "text": "Connexion"})
|
||||
assert result is not None
|
||||
assert result.confidence == 0.85
|
||||
|
||||
def test_bbox_clamped_to_screen(self):
|
||||
vlm_text = json.dumps({"found": True, "bbox": [-0.1, -0.1, 1.5, 1.5], "confidence": 0.7})
|
||||
result = parse_grounder_response(vlm_text, 1920, 1080, {"role": "bouton", "text": "Connexion"})
|
||||
assert result is not None
|
||||
assert result.bbox[0] >= 0
|
||||
assert result.bbox[1] >= 0
|
||||
assert result.bbox[2] <= 1920
|
||||
assert result.bbox[3] <= 1080
|
||||
|
||||
|
||||
# ── ground_element (composition) tests ─────────────────────────────────
|
||||
|
||||
|
||||
class TestGroundElement:
|
||||
def test_ocr_anchor_success(self):
|
||||
"""OCR finds text with bbox → grounded via OCR (deterministic)."""
|
||||
ocr = mock_ocr_detailed_client_factory([
|
||||
OcrTokenInfo(text="Connexion", bbox=(200, 50, 350, 90), confidence=0.95),
|
||||
])
|
||||
vlm = mock_vlm_client_factory({})
|
||||
result = ground_element(
|
||||
"/tmp/login.png",
|
||||
{"role": "bouton", "text": "Connexion"},
|
||||
ocr_client=ocr,
|
||||
vlm_client=vlm,
|
||||
)
|
||||
assert result is not None
|
||||
assert result.method == "ocr_anchor"
|
||||
assert result.bbox == (200, 50, 350, 90)
|
||||
|
||||
def test_vlm_fallback(self):
|
||||
"""OCR doesn't find text → VLM grounder succeeds."""
|
||||
ocr = mock_ocr_detailed_client_factory([
|
||||
OcrTokenInfo(text="Accueil", bbox=(0, 0, 100, 40)),
|
||||
])
|
||||
vlm = mock_vlm_client_factory({
|
||||
"found": True,
|
||||
"bbox": [0.2, 0.3, 0.4, 0.5],
|
||||
"confidence": 0.85,
|
||||
})
|
||||
result = ground_element(
|
||||
"/tmp/login.png",
|
||||
{"role": "bouton", "text": "Connexion"},
|
||||
ocr_client=ocr,
|
||||
vlm_client=vlm,
|
||||
)
|
||||
assert result is not None
|
||||
assert result.method == "vlm_grounder"
|
||||
|
||||
def test_not_found_any_method(self):
|
||||
"""Both OCR and VLM fail → None."""
|
||||
ocr = mock_ocr_detailed_client_factory([OcrTokenInfo(text="Accueil", bbox=(0, 0, 100, 40))])
|
||||
vlm = mock_vlm_client_factory({"found": False, "bbox": [], "confidence": 0.0})
|
||||
result = ground_element(
|
||||
"/tmp/login.png",
|
||||
{"role": "bouton", "text": "Connexion"},
|
||||
ocr_client=ocr,
|
||||
vlm_client=vlm,
|
||||
)
|
||||
assert result is None
|
||||
|
||||
def test_ocr_error_vlm_fallback(self):
|
||||
"""OCR engine fails → VLM fallback."""
|
||||
def failing_ocr(image_path):
|
||||
raise RuntimeError("OCR engine down")
|
||||
vlm = mock_vlm_client_factory({
|
||||
"found": True,
|
||||
"bbox": [0.2, 0.3, 0.4, 0.5],
|
||||
"confidence": 0.8,
|
||||
})
|
||||
result = ground_element(
|
||||
"/tmp/login.png",
|
||||
{"role": "bouton", "text": "Connexion"},
|
||||
ocr_client=failing_ocr,
|
||||
vlm_client=vlm,
|
||||
)
|
||||
assert result is not None
|
||||
assert result.method == "vlm_grounder"
|
||||
|
||||
def test_vlm_error_ocr_success(self):
|
||||
"""VLM fails but OCR succeeds → OCR anchor used."""
|
||||
ocr = mock_ocr_detailed_client_factory([
|
||||
OcrTokenInfo(text="Connexion", bbox=(200, 50, 350, 90)),
|
||||
])
|
||||
def failing_vlm(image_path, prompt):
|
||||
raise RuntimeError("VLM down")
|
||||
result = ground_element(
|
||||
"/tmp/login.png",
|
||||
{"role": "bouton", "text": "Connexion"},
|
||||
ocr_client=ocr,
|
||||
vlm_client=failing_vlm,
|
||||
)
|
||||
assert result is not None
|
||||
assert result.method == "ocr_anchor"
|
||||
|
||||
def test_both_fail(self):
|
||||
"""OCR + VLM both fail → None."""
|
||||
def failing_ocr(image_path):
|
||||
raise RuntimeError("OCR down")
|
||||
def failing_vlm(image_path, prompt):
|
||||
raise RuntimeError("VLM down")
|
||||
result = ground_element(
|
||||
"/tmp/login.png",
|
||||
{"role": "bouton", "text": "Connexion"},
|
||||
ocr_client=failing_ocr,
|
||||
vlm_client=failing_vlm,
|
||||
)
|
||||
assert result is None
|
||||
|
||||
def test_no_text_target(self):
|
||||
"""Target without text → VLM grounder skipped, None."""
|
||||
ocr = mock_ocr_detailed_client_factory([])
|
||||
vlm = mock_vlm_client_factory({})
|
||||
result = ground_element(
|
||||
"/tmp/page.png",
|
||||
{"role": "page"},
|
||||
ocr_client=ocr,
|
||||
vlm_client=vlm,
|
||||
)
|
||||
assert result is None
|
||||
|
||||
def test_cache_hit(self):
|
||||
"""Cached coords exist → returned directly."""
|
||||
cache = CoordsCache()
|
||||
cache.put("bouton:connexion", (200, 50, 350, 90), (275, 70), "ocr_anchor")
|
||||
|
||||
ocr = mock_ocr_detailed_client_factory([])
|
||||
vlm = mock_vlm_client_factory({})
|
||||
result = ground_element(
|
||||
"/tmp/login.png",
|
||||
{"role": "bouton", "text": "Connexion"},
|
||||
ocr_client=ocr,
|
||||
vlm_client=vlm,
|
||||
coords_cache=cache,
|
||||
)
|
||||
assert result is not None
|
||||
assert result.method == "cache"
|
||||
assert result.bbox == (200, 50, 350, 90)
|
||||
|
||||
def test_cache_stored_on_ocr_anchor(self):
|
||||
"""OCR anchor result → stored in cache."""
|
||||
cache = CoordsCache()
|
||||
ocr = mock_ocr_detailed_client_factory([
|
||||
OcrTokenInfo(text="Connexion", bbox=(200, 50, 350, 90)),
|
||||
])
|
||||
vlm = mock_vlm_client_factory({})
|
||||
ground_element(
|
||||
"/tmp/login.png",
|
||||
{"role": "bouton", "text": "Connexion"},
|
||||
ocr_client=ocr,
|
||||
vlm_client=vlm,
|
||||
coords_cache=cache,
|
||||
)
|
||||
cached = cache.get("bouton:connexion")
|
||||
assert cached is not None
|
||||
assert cached.bbox == (200, 50, 350, 90)
|
||||
assert cached.method == "ocr_anchor"
|
||||
|
||||
def test_cache_stored_on_vlm_grounder(self):
|
||||
"""VLM grounder result → stored in cache."""
|
||||
cache = CoordsCache()
|
||||
ocr = mock_ocr_detailed_client_factory([])
|
||||
vlm = mock_vlm_client_factory({
|
||||
"found": True,
|
||||
"bbox": [0.2, 0.3, 0.4, 0.5],
|
||||
"confidence": 0.85,
|
||||
})
|
||||
ground_element(
|
||||
"/tmp/login.png",
|
||||
{"role": "bouton", "text": "Connexion"},
|
||||
ocr_client=ocr,
|
||||
vlm_client=vlm,
|
||||
coords_cache=cache,
|
||||
)
|
||||
cached = cache.get("bouton:connexion")
|
||||
assert cached is not None
|
||||
assert cached.method == "vlm_grounder"
|
||||
|
||||
|
||||
# ── CoordsCache tests ──────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestCoordsCache:
|
||||
def test_put_and_get(self):
|
||||
cache = CoordsCache()
|
||||
cache.put("bouton:connexion", (200, 50, 350, 90), (275, 70), "ocr_anchor")
|
||||
entry = cache.get("bouton:connexion")
|
||||
assert entry is not None
|
||||
assert entry.bbox == (200, 50, 350, 90)
|
||||
|
||||
def test_get_missing(self):
|
||||
cache = CoordsCache()
|
||||
assert cache.get("bouton:connexion") is None
|
||||
|
||||
def test_invalidate(self):
|
||||
cache = CoordsCache()
|
||||
cache.put("bouton:connexion", (200, 50, 350, 90), (275, 70), "ocr_anchor")
|
||||
cache.invalidate("bouton:connexion")
|
||||
assert cache.get("bouton:connexion") is None
|
||||
|
||||
def test_clear(self):
|
||||
cache = CoordsCache()
|
||||
cache.put("a", (0, 0, 10, 10), (5, 5), "ocr_anchor")
|
||||
cache.put("b", (0, 0, 20, 20), (10, 10), "vlm_grounder")
|
||||
cache.clear()
|
||||
assert cache.get("a") is None
|
||||
assert cache.get("b") is None
|
||||
|
||||
def test_keys(self):
|
||||
cache = CoordsCache()
|
||||
cache.put("a", (0, 0, 10, 10), (5, 5), "ocr_anchor")
|
||||
cache.put("b", (0, 0, 20, 20), (10, 10), "vlm_grounder")
|
||||
assert sorted(cache.keys()) == ["a", "b"]
|
||||
|
||||
def test_update_existing(self):
|
||||
cache = CoordsCache()
|
||||
cache.put("bouton:connexion", (200, 50, 350, 90), (275, 70), "ocr_anchor")
|
||||
cache.put("bouton:connexion", (300, 60, 400, 100), (350, 80), "vlm_grounder")
|
||||
entry = cache.get("bouton:connexion")
|
||||
assert entry is not None
|
||||
assert entry.bbox == (300, 60, 400, 100) # updated
|
||||
assert entry.validation_count == 2
|
||||
|
||||
def test_validation_count_increments(self):
|
||||
cache = CoordsCache()
|
||||
cache.put("a", (0, 0, 10, 10), (5, 5), "ocr_anchor")
|
||||
assert cache.get("a").validation_count == 1
|
||||
cache.put("a", (0, 0, 10, 10), (5, 5), "ocr_anchor")
|
||||
assert cache.get("a").validation_count == 2
|
||||
151
tests/unit/test_navigate_handler_e2e.py
Normal file
151
tests/unit/test_navigate_handler_e2e.py
Normal file
@@ -0,0 +1,151 @@
|
||||
"""End-to-end mocked test for navigate action handler — 3 edge-case scenarios.
|
||||
|
||||
Tests the _handle_navigate_action handler with mocked OCR/VLM, verifying:
|
||||
- Nominal: all resolved, coords populated in variables
|
||||
- OCR miss + VLM fail: no phantom coords, all_resolved=False
|
||||
- No screenshot: error="no_screenshot", False return
|
||||
|
||||
NOTE: The handler uses lazy imports inside its body. Mock targets must be
|
||||
at the source module (core.navigation.action_resolver.navigate_login) rather
|
||||
than the package-level re-export (core.navigation.navigate_login).
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from unittest.mock import patch, MagicMock
|
||||
from core.navigation.action_resolver import NavigateCoords, NavigateResult
|
||||
from core.navigation import _handle_navigate_action
|
||||
|
||||
|
||||
def _patch_all_deps(navigate_login_result=None, navigate_login_side_effect=None):
|
||||
"""Return stacked patches for handler's lazy imports + navigate_login."""
|
||||
nl_mock = MagicMock(return_value=navigate_login_result) if navigate_login_result else None
|
||||
if navigate_login_side_effect:
|
||||
nl_mock = MagicMock(side_effect=navigate_login_side_effect)
|
||||
|
||||
return (
|
||||
patch("core.llm.extract_grid_from_image", return_value=[]),
|
||||
patch("core.extraction.vlm_client.make_vllm_client", return_value=MagicMock()),
|
||||
patch("core.navigation.action_resolver.make_ocr_detailed_from_grid",
|
||||
return_value=MagicMock(return_value=[])),
|
||||
patch("core.navigation.action_resolver.navigate_login", nl_mock),
|
||||
)
|
||||
|
||||
|
||||
class TestNominalCase:
|
||||
"""All fields grounded → coords populated, all_resolved=True."""
|
||||
|
||||
def test_nominal_coords_populated(self):
|
||||
mock_result = NavigateResult(
|
||||
login_coords=NavigateCoords(x_pct=0.15, y_pct=0.07, method="ocr_anchor"),
|
||||
password_coords=NavigateCoords(x_pct=0.15, y_pct=0.25, method="ocr_anchor"),
|
||||
submit_coords=NavigateCoords(x_pct=0.50, y_pct=0.35, method="ocr_anchor"),
|
||||
all_resolved=True,
|
||||
)
|
||||
|
||||
action = {"parameters": {"action": "login"}}
|
||||
replay_state = {
|
||||
"last_screenshot_path": "/tmp/login_screen.png",
|
||||
"screen_width": 1920,
|
||||
"screen_height": 1080,
|
||||
}
|
||||
|
||||
p1, p2, p3, p4 = _patch_all_deps(navigate_login_result=mock_result)
|
||||
with p1, p2, p3, p4:
|
||||
result = _handle_navigate_action(action, replay_state, "test-session")
|
||||
|
||||
assert result is True
|
||||
vars_ = replay_state["variables"]
|
||||
assert "navigate_login_coords" in vars_
|
||||
assert vars_["navigate_login_coords"]["x_pct"] == 0.15
|
||||
assert "navigate_password_coords" in vars_
|
||||
assert "navigate_submit_coords" in vars_
|
||||
assert vars_["navigate_result"]["all_resolved"] is True
|
||||
|
||||
|
||||
class TestOcrMissVlmFail:
|
||||
"""OCR misses target + VLM grounder also fails → no phantom coords."""
|
||||
|
||||
def test_no_phantom_coords_on_failure(self):
|
||||
mock_result = NavigateResult(
|
||||
login_coords=None,
|
||||
password_coords=None,
|
||||
submit_coords=None,
|
||||
all_resolved=False,
|
||||
error="grounding failed — no login form elements found",
|
||||
)
|
||||
|
||||
action = {"parameters": {"action": "login"}}
|
||||
replay_state = {
|
||||
"last_screenshot_path": "/tmp/no_login_form.png",
|
||||
"screen_width": 1920,
|
||||
"screen_height": 1080,
|
||||
}
|
||||
|
||||
p1, p2, p3, p4 = _patch_all_deps(navigate_login_result=mock_result)
|
||||
with p1, p2, p3, p4:
|
||||
result = _handle_navigate_action(action, replay_state, "test-session")
|
||||
|
||||
assert result is False
|
||||
vars_ = replay_state["variables"]
|
||||
# No coords keys should be present (coords are None → not stored)
|
||||
assert "navigate_login_coords" not in vars_
|
||||
assert "navigate_password_coords" not in vars_
|
||||
assert "navigate_submit_coords" not in vars_
|
||||
# Error must be non-empty
|
||||
assert vars_["navigate_result"]["all_resolved"] is False
|
||||
assert "grounding failed" in vars_["navigate_result"]["error"]
|
||||
|
||||
|
||||
class TestNoScreenshot:
|
||||
"""No screenshot in replay_state → error="no_screenshot", False."""
|
||||
|
||||
def test_no_screenshot_error(self):
|
||||
action = {"parameters": {"action": "login"}}
|
||||
replay_state = {} # No screenshot at all
|
||||
|
||||
result = _handle_navigate_action(action, replay_state, "test-session")
|
||||
|
||||
assert result is False
|
||||
vars_ = replay_state["variables"]
|
||||
assert vars_["navigate_login_coords"]["error"] == "no_screenshot"
|
||||
|
||||
def test_empty_screenshot_path(self):
|
||||
action = {"parameters": {"action": "login"}}
|
||||
replay_state = {"last_screenshot_path": ""}
|
||||
|
||||
result = _handle_navigate_action(action, replay_state, "test-session")
|
||||
|
||||
assert result is False
|
||||
vars_ = replay_state["variables"]
|
||||
assert vars_["navigate_login_coords"]["error"] == "no_screenshot"
|
||||
|
||||
|
||||
class TestNeverFailReplay:
|
||||
"""Handler must never raise — even on malformed input, returns False."""
|
||||
|
||||
def test_missing_parameters(self):
|
||||
action = {} # No "parameters" key
|
||||
replay_state = {"last_screenshot_path": "/tmp/x.png"}
|
||||
|
||||
mock_result = NavigateResult(all_resolved=False, error="no params")
|
||||
p1, p2, p3, p4 = _patch_all_deps(navigate_login_result=mock_result)
|
||||
with p1, p2, p3, p4:
|
||||
result = _handle_navigate_action(action, replay_state, "test-session")
|
||||
assert result is False
|
||||
|
||||
def test_exception_in_inner_call(self):
|
||||
action = {"parameters": {"action": "login"}}
|
||||
replay_state = {
|
||||
"last_screenshot_path": "/tmp/login.png",
|
||||
"screen_width": 1920,
|
||||
"screen_height": 1080,
|
||||
}
|
||||
|
||||
p1, p2, p3, p4 = _patch_all_deps(navigate_login_side_effect=RuntimeError("boom"))
|
||||
with p1, p2, p3, p4:
|
||||
result = _handle_navigate_action(action, replay_state, "test-session")
|
||||
|
||||
assert result is False
|
||||
vars_ = replay_state["variables"]
|
||||
assert vars_["navigate_result"]["all_resolved"] is False
|
||||
assert "boom" in vars_["navigate_result"]["error"]
|
||||
62
tests/unit/test_navigate_wiring.py
Normal file
62
tests/unit/test_navigate_wiring.py
Normal file
@@ -0,0 +1,62 @@
|
||||
"""Boot non-regression test for navigate wiring — catches import/regression bugs.
|
||||
|
||||
This test would have caught the ImportError where _handle_navigate_action
|
||||
was incorrectly imported from replay_engine instead of core/navigation.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
class TestApiStreamImports:
|
||||
"""(1) api_stream must import without error."""
|
||||
|
||||
def test_import_api_stream(self):
|
||||
from agent_v0.server_v1 import api_stream
|
||||
assert api_stream is not None
|
||||
|
||||
|
||||
class TestAllowedActionTypes:
|
||||
"""(2) 'navigate' must be in both _ALLOWED and _SERVER_SIDE."""
|
||||
|
||||
def test_navigate_in_allowed(self):
|
||||
from agent_v0.server_v1.replay_engine import _ALLOWED_ACTION_TYPES
|
||||
assert "navigate" in _ALLOWED_ACTION_TYPES
|
||||
|
||||
def test_navigate_in_server_side(self):
|
||||
from agent_v0.server_v1.replay_engine import _SERVER_SIDE_ACTION_TYPES
|
||||
assert "navigate" in _SERVER_SIDE_ACTION_TYPES
|
||||
|
||||
|
||||
class TestNavigateHandlerCallable:
|
||||
"""(3) _handle_navigate_action must be callable with correct signature."""
|
||||
|
||||
def test_handler_imported_from_core_navigation(self):
|
||||
from core.navigation import _handle_navigate_action
|
||||
assert callable(_handle_navigate_action)
|
||||
|
||||
def test_handler_imported_in_api_stream(self):
|
||||
from agent_v0.server_v1 import api_stream
|
||||
handler = api_stream._handle_navigate_action
|
||||
assert callable(handler)
|
||||
|
||||
def test_handler_signature(self):
|
||||
"""Signature: (action: dict, replay_state: dict, session_id: str) -> bool."""
|
||||
from core.navigation import _handle_navigate_action
|
||||
import inspect
|
||||
sig = inspect.signature(_handle_navigate_action)
|
||||
params = list(sig.parameters.keys())
|
||||
assert params == ["action", "replay_state", "session_id"]
|
||||
assert sig.return_annotation == bool
|
||||
|
||||
|
||||
class TestDispatchBlockExists:
|
||||
"""Verify the navigate dispatch block is wired in api_stream."""
|
||||
|
||||
def test_navigate_dispatch_reference(self):
|
||||
"""Source must contain the navigate dispatch elif block."""
|
||||
import agent_v0.server_v1.api_stream as mod
|
||||
source = inspect.getsource(mod)
|
||||
assert "type_ == \"navigate\"" in source
|
||||
|
||||
|
||||
import inspect
|
||||
336
tests/unit/test_visual_login.py
Normal file
336
tests/unit/test_visual_login.py
Normal file
@@ -0,0 +1,336 @@
|
||||
"""Tests for core/navigation/visual_login.py — login form resolution + verification."""
|
||||
|
||||
import json
|
||||
import pytest
|
||||
from core.navigation.visual_login import (
|
||||
LoginFormConfig,
|
||||
LoginResolution,
|
||||
dpi_urgences_login_config,
|
||||
verify_login_visible,
|
||||
verify_login_success,
|
||||
resolve_login_form,
|
||||
_ocr_detailed_to_simple,
|
||||
)
|
||||
from core.navigation.grounding import (
|
||||
CoordsCache,
|
||||
GroundedElement,
|
||||
OcrTokenInfo,
|
||||
OcrDetailedClient,
|
||||
)
|
||||
from core.navigation.visual_verifier import (
|
||||
ScreenMatchResult,
|
||||
VlmClient,
|
||||
OcrClient,
|
||||
)
|
||||
|
||||
|
||||
# ── Mock factories ─────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def mock_ocr_detailed_client_factory(tokens: list):
|
||||
"""Factory for mock OcrDetailedClient."""
|
||||
def client(image_path: str) -> list:
|
||||
return tokens
|
||||
return client
|
||||
|
||||
|
||||
def mock_ocr_simple_client_factory(tokens: list):
|
||||
"""Factory for mock OcrClient (text-only)."""
|
||||
def client(image_path: str) -> list:
|
||||
return tokens
|
||||
return client
|
||||
|
||||
|
||||
def mock_vlm_client_factory(response_json: dict):
|
||||
"""Factory for mock VlmClient."""
|
||||
def client(image_path: str, prompt: str) -> str:
|
||||
return json.dumps(response_json)
|
||||
return client
|
||||
|
||||
|
||||
# ── Default config tests ───────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestDpiUrgencesLoginConfig:
|
||||
def test_default_config(self):
|
||||
config = dpi_urgences_login_config()
|
||||
assert config.login_field["role"] == "champ"
|
||||
assert config.login_field["text"] == "Login"
|
||||
assert config.password_field["text"] == "Mot de passe"
|
||||
assert config.submit_button["text"] == "Connexion"
|
||||
assert len(config.success_elements) >= 1
|
||||
assert config.context != ""
|
||||
|
||||
def test_config_fields_are_dicts(self):
|
||||
config = dpi_urgences_login_config()
|
||||
assert isinstance(config.login_field, dict)
|
||||
assert isinstance(config.password_field, dict)
|
||||
assert isinstance(config.submit_button, dict)
|
||||
|
||||
|
||||
# ── _ocr_detailed_to_simple tests ────────────────────────────────────
|
||||
|
||||
|
||||
class TestOcrDetailedToSimple:
|
||||
def test_conversion(self):
|
||||
tokens = [
|
||||
OcrTokenInfo(text="Login", bbox=(100, 50, 200, 90)),
|
||||
OcrTokenInfo(text="Password", bbox=(100, 100, 200, 140)),
|
||||
]
|
||||
detailed = mock_ocr_detailed_client_factory(tokens)
|
||||
simple = _ocr_detailed_to_simple(detailed)
|
||||
result = simple("/tmp/test.png")
|
||||
assert result == ["Login", "Password"]
|
||||
|
||||
def test_empty_tokens(self):
|
||||
detailed = mock_ocr_detailed_client_factory([])
|
||||
simple = _ocr_detailed_to_simple(detailed)
|
||||
result = simple("/tmp/test.png")
|
||||
assert result == []
|
||||
|
||||
|
||||
# ── verify_login_visible tests ────────────────────────────────────────
|
||||
|
||||
|
||||
class TestVerifyLoginVisible:
|
||||
def test_form_visible(self):
|
||||
"""All 3 fields found by OCR + roles confirmed → match."""
|
||||
config = LoginFormConfig(
|
||||
login_field={"role": "champ", "text": "Login"},
|
||||
password_field={"role": "champ", "text": "Mot de passe"},
|
||||
submit_button={"role": "bouton", "text": "Connexion"},
|
||||
context="DPI login",
|
||||
)
|
||||
ocr = mock_ocr_simple_client_factory(["Login", "Mot de passe", "Connexion"])
|
||||
vlm = mock_vlm_client_factory({
|
||||
"confirmed": [
|
||||
{"index": 1, "role_confirmed": True, "actual_role": "champ", "confidence": 0.9},
|
||||
{"index": 2, "role_confirmed": True, "actual_role": "champ", "confidence": 0.9},
|
||||
{"index": 3, "role_confirmed": True, "actual_role": "bouton", "confidence": 0.9},
|
||||
],
|
||||
"overall_confidence": 0.9,
|
||||
})
|
||||
result = verify_login_visible("/tmp/login.png", config, ocr, vlm)
|
||||
assert result.match == True
|
||||
|
||||
def test_form_missing_button(self):
|
||||
"""Connexion button not found by OCR → mismatch."""
|
||||
config = LoginFormConfig(
|
||||
login_field={"role": "champ", "text": "Login"},
|
||||
password_field={"role": "champ", "text": "Mot de passe"},
|
||||
submit_button={"role": "bouton", "text": "Connexion"},
|
||||
)
|
||||
ocr = mock_ocr_simple_client_factory(["Login", "Mot de passe"]) # missing Connexion
|
||||
vlm = mock_vlm_client_factory({})
|
||||
result = verify_login_visible("/tmp/login.png", config, ocr, vlm)
|
||||
assert result.match == False
|
||||
|
||||
def test_form_wrong_role(self):
|
||||
"""OCR finds text but VLM says button is a label → mismatch."""
|
||||
config = LoginFormConfig(
|
||||
login_field={"role": "champ", "text": "Login"},
|
||||
password_field={"role": "champ", "text": "Mot de passe"},
|
||||
submit_button={"role": "bouton", "text": "Connexion"},
|
||||
)
|
||||
ocr = mock_ocr_simple_client_factory(["Login", "Mot de passe", "Connexion"])
|
||||
vlm = mock_vlm_client_factory({
|
||||
"confirmed": [
|
||||
{"index": 1, "role_confirmed": True, "actual_role": "champ", "confidence": 0.9},
|
||||
{"index": 2, "role_confirmed": True, "actual_role": "champ", "confidence": 0.9},
|
||||
{"index": 3, "role_confirmed": False, "actual_role": "label", "confidence": 0.5},
|
||||
],
|
||||
"overall_confidence": 0.5,
|
||||
})
|
||||
result = verify_login_visible("/tmp/login.png", config, ocr, vlm)
|
||||
assert result.match == False
|
||||
|
||||
|
||||
# ── verify_login_success tests ────────────────────────────────────────
|
||||
|
||||
|
||||
class TestVerifyLoginSuccess:
|
||||
def test_dashboard_visible(self):
|
||||
"""Dashboard found by OCR + role confirmed → success."""
|
||||
config = LoginFormConfig(
|
||||
login_field={"role": "champ", "text": "Login"},
|
||||
password_field={"role": "champ", "text": "Mot de passe"},
|
||||
submit_button={"role": "bouton", "text": "Connexion"},
|
||||
success_elements=[{"role": "page", "text": "Dashboard"}],
|
||||
)
|
||||
ocr = mock_ocr_simple_client_factory(["Dashboard", "Accueil"])
|
||||
vlm = mock_vlm_client_factory({
|
||||
"confirmed": [
|
||||
{"index": 1, "role_confirmed": True, "actual_role": "page", "confidence": 0.92},
|
||||
],
|
||||
"overall_confidence": 0.92,
|
||||
})
|
||||
result = verify_login_success("/tmp/dashboard.png", config, ocr, vlm)
|
||||
assert result.match == True
|
||||
|
||||
def test_no_success_elements(self):
|
||||
"""Config has no success_elements → can't verify."""
|
||||
config = LoginFormConfig(
|
||||
login_field={"role": "champ", "text": "Login"},
|
||||
password_field={"role": "champ", "text": "Mot de passe"},
|
||||
submit_button={"role": "bouton", "text": "Connexion"},
|
||||
success_elements=[], # empty!
|
||||
)
|
||||
ocr = mock_ocr_simple_client_factory(["Dashboard"])
|
||||
vlm = mock_vlm_client_factory({})
|
||||
result = verify_login_success("/tmp/page.png", config, ocr, vlm)
|
||||
assert result.match == False
|
||||
assert "no success_elements" in result.reason
|
||||
|
||||
def test_still_on_login_page(self):
|
||||
"""After login, still seeing login form → mismatch."""
|
||||
config = LoginFormConfig(
|
||||
login_field={"role": "champ", "text": "Login"},
|
||||
password_field={"role": "champ", "text": "Mot de passe"},
|
||||
submit_button={"role": "bouton", "text": "Connexion"},
|
||||
success_elements=[{"role": "page", "text": "Dashboard"}],
|
||||
)
|
||||
# OCR sees login form texts, not Dashboard
|
||||
ocr = mock_ocr_simple_client_factory(["Login", "Mot de passe", "Connexion"])
|
||||
vlm = mock_vlm_client_factory({})
|
||||
result = verify_login_success("/tmp/still_login.png", config, ocr, vlm)
|
||||
assert result.match == False
|
||||
|
||||
|
||||
# ── resolve_login_form tests ──────────────────────────────────────────
|
||||
|
||||
|
||||
class TestResolveLoginForm:
|
||||
def test_all_fields_ocr_anchor(self):
|
||||
"""All 3 fields found by OCR with bbox → full resolution."""
|
||||
config = LoginFormConfig(
|
||||
login_field={"role": "champ", "text": "Login"},
|
||||
password_field={"role": "champ", "text": "Mot de passe"},
|
||||
submit_button={"role": "bouton", "text": "Connexion"},
|
||||
)
|
||||
ocr = mock_ocr_detailed_client_factory([
|
||||
OcrTokenInfo(text="Login", bbox=(100, 50, 250, 90)),
|
||||
OcrTokenInfo(text="Mot de passe", bbox=(100, 100, 250, 140)),
|
||||
OcrTokenInfo(text="Connexion", bbox=(100, 150, 250, 190)),
|
||||
])
|
||||
vlm = mock_vlm_client_factory({})
|
||||
result = resolve_login_form("/tmp/login.png", config, ocr, vlm)
|
||||
assert result.all_resolved == True
|
||||
assert result.login_field is not None
|
||||
assert result.login_field.method == "ocr_anchor"
|
||||
assert result.password_field is not None
|
||||
assert result.submit_button is not None
|
||||
assert result.method == "ocr_anchor"
|
||||
|
||||
def test_partial_ocr_vlm_fallback(self):
|
||||
"""Login + password by OCR, button by VLM → mixed method."""
|
||||
config = LoginFormConfig(
|
||||
login_field={"role": "champ", "text": "Login"},
|
||||
password_field={"role": "champ", "text": "Password"},
|
||||
submit_button={"role": "bouton", "text": "Connexion"},
|
||||
)
|
||||
ocr = mock_ocr_detailed_client_factory([
|
||||
OcrTokenInfo(text="Login", bbox=(100, 50, 250, 90)),
|
||||
OcrTokenInfo(text="Password", bbox=(100, 100, 250, 140)),
|
||||
# Connexion not in OCR → VLM fallback
|
||||
])
|
||||
vlm = mock_vlm_client_factory({
|
||||
"found": True,
|
||||
"bbox": [0.2, 0.4, 0.4, 0.5],
|
||||
"confidence": 0.85,
|
||||
})
|
||||
result = resolve_login_form("/tmp/login.png", config, ocr, vlm)
|
||||
assert result.all_resolved == True
|
||||
assert result.login_field.method == "ocr_anchor"
|
||||
assert result.submit_button.method == "vlm_grounder"
|
||||
assert result.method == "mixed"
|
||||
|
||||
def test_incomplete_resolution(self):
|
||||
"""Button not found by OCR or VLM → incomplete."""
|
||||
config = LoginFormConfig(
|
||||
login_field={"role": "champ", "text": "Login"},
|
||||
password_field={"role": "champ", "text": "Password"},
|
||||
submit_button={"role": "bouton", "text": "Connexion"},
|
||||
)
|
||||
ocr = mock_ocr_detailed_client_factory([
|
||||
OcrTokenInfo(text="Login", bbox=(100, 50, 250, 90)),
|
||||
OcrTokenInfo(text="Password", bbox=(100, 100, 250, 140)),
|
||||
])
|
||||
vlm = mock_vlm_client_factory({"found": False, "bbox": [], "confidence": 0.0})
|
||||
result = resolve_login_form("/tmp/login.png", config, ocr, vlm)
|
||||
assert result.all_resolved == False
|
||||
assert result.submit_button is None
|
||||
|
||||
def test_cache_hit(self):
|
||||
"""All fields cached → returned directly."""
|
||||
cache = CoordsCache()
|
||||
cache.put("champ:login", (100, 50, 250, 90), (175, 70), "ocr_anchor")
|
||||
cache.put("champ:mot de passe", (100, 100, 250, 140), (175, 120), "ocr_anchor")
|
||||
cache.put("bouton:connexion", (100, 150, 250, 190), (175, 170), "ocr_anchor")
|
||||
|
||||
config = LoginFormConfig(
|
||||
login_field={"role": "champ", "text": "Login"},
|
||||
password_field={"role": "champ", "text": "Mot de passe"},
|
||||
submit_button={"role": "bouton", "text": "Connexion"},
|
||||
)
|
||||
ocr = mock_ocr_detailed_client_factory([])
|
||||
vlm = mock_vlm_client_factory({})
|
||||
result = resolve_login_form(
|
||||
"/tmp/login.png", config, ocr, vlm, coords_cache=cache,
|
||||
)
|
||||
assert result.all_resolved == True
|
||||
assert result.method == "cache"
|
||||
assert result.login_field.center == (175, 70)
|
||||
|
||||
def test_with_dpi_default_config(self):
|
||||
"""Full flow with dpi_urgences_login_config."""
|
||||
config = dpi_urgences_login_config()
|
||||
ocr = mock_ocr_detailed_client_factory([
|
||||
OcrTokenInfo(text="Login", bbox=(100, 50, 250, 90)),
|
||||
OcrTokenInfo(text="Mot de passe", bbox=(100, 100, 250, 140)),
|
||||
OcrTokenInfo(text="Connexion", bbox=(100, 150, 250, 190)),
|
||||
])
|
||||
vlm = mock_vlm_client_factory({})
|
||||
result = resolve_login_form("/tmp/login.png", config, ocr, vlm)
|
||||
assert result.all_resolved == True
|
||||
|
||||
|
||||
# ── LoginResolution describe tests ────────────────────────────────────
|
||||
|
||||
|
||||
class TestLoginResolutionDescribe:
|
||||
def test_all_resolved(self):
|
||||
resolution = LoginResolution(
|
||||
login_field=GroundedElement(
|
||||
role="champ", text="Login",
|
||||
bbox=(100, 50, 250, 90), center=(175, 70),
|
||||
confidence=0.9, method="ocr_anchor",
|
||||
),
|
||||
password_field=GroundedElement(
|
||||
role="champ", text="Mot de passe",
|
||||
bbox=(100, 100, 250, 140), center=(175, 120),
|
||||
confidence=0.9, method="ocr_anchor",
|
||||
),
|
||||
submit_button=GroundedElement(
|
||||
role="bouton", text="Connexion",
|
||||
bbox=(100, 150, 250, 190), center=(175, 170),
|
||||
confidence=0.9, method="ocr_anchor",
|
||||
),
|
||||
all_resolved=True,
|
||||
method="ocr_anchor",
|
||||
)
|
||||
desc = resolution.describe()
|
||||
assert "OK" in desc
|
||||
assert "login@" in desc
|
||||
assert "button@" in desc
|
||||
|
||||
def test_incomplete(self):
|
||||
resolution = LoginResolution(
|
||||
login_field=None,
|
||||
password_field=None,
|
||||
submit_button=None,
|
||||
all_resolved=False,
|
||||
method="",
|
||||
)
|
||||
desc = resolution.describe()
|
||||
assert "INCOMPLETE" in desc
|
||||
assert "NOT FOUND" in desc
|
||||
490
tests/unit/test_visual_verifier.py
Normal file
490
tests/unit/test_visual_verifier.py
Normal file
@@ -0,0 +1,490 @@
|
||||
"""Tests for core/navigation/visual_verifier.py — OCR-anchored architecture.
|
||||
|
||||
Tests pure functions (normalize_text, fuzzy_match, ocr_presence_check,
|
||||
build_role_confirm_prompt, parse_role_confirm_response) offline,
|
||||
then verifies verify_screen_match with mock OcrClient + VlmClient.
|
||||
"""
|
||||
|
||||
import json
|
||||
import pytest
|
||||
from core.navigation.visual_verifier import (
|
||||
normalize_text,
|
||||
fuzzy_match,
|
||||
ocr_presence_check,
|
||||
build_role_confirm_prompt,
|
||||
parse_role_confirm_response,
|
||||
verify_screen_match,
|
||||
verify_before,
|
||||
verify_after,
|
||||
ScreenMatchResult,
|
||||
OcrPresenceResult,
|
||||
)
|
||||
|
||||
|
||||
# ── Mock factories ─────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def mock_ocr_client_factory(tokens: list):
|
||||
"""Factory that creates a mock OcrClient returning the given tokens."""
|
||||
def client(image_path: str) -> list:
|
||||
return tokens
|
||||
return client
|
||||
|
||||
|
||||
def mock_vlm_client_factory(response_json: dict):
|
||||
"""Factory that creates a mock VlmClient returning the given JSON."""
|
||||
def client(image_path: str, prompt: str) -> str:
|
||||
return json.dumps(response_json)
|
||||
return client
|
||||
|
||||
|
||||
# ── normalize_text tests ──────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestNormalizeText:
|
||||
def test_lowercase(self):
|
||||
assert normalize_text("RECHERCHER") == "rechercher"
|
||||
|
||||
def test_strip_accents(self):
|
||||
assert normalize_text("Recherché") == "recherche"
|
||||
|
||||
def test_collapse_whitespace(self):
|
||||
assert normalize_text(" hello world ") == "hello world"
|
||||
|
||||
def test_combined(self):
|
||||
assert normalize_text(" Nom Prénom ") == "nom prenom"
|
||||
|
||||
def test_empty(self):
|
||||
assert normalize_text("") == ""
|
||||
|
||||
def test_numbers_preserved(self):
|
||||
assert normalize_text("IPP 12345") == "ipp 12345"
|
||||
|
||||
|
||||
# ── fuzzy_match tests ─────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestFuzzyMatch:
|
||||
def test_exact_match(self):
|
||||
assert fuzzy_match("Rechercher", "Rechercher") == True
|
||||
|
||||
def test_case_insensitive(self):
|
||||
assert fuzzy_match("rechercher", "RECHERCHER") == True
|
||||
|
||||
def test_accent_match(self):
|
||||
assert fuzzy_match("Recherché", "Recherche") == True
|
||||
|
||||
def test_substring_containment(self):
|
||||
# Short text contained in longer OCR token
|
||||
assert fuzzy_match("Rechercher", "Bouton Rechercher") == True
|
||||
|
||||
def test_reverse_containment(self):
|
||||
# OCR token contained in expected text
|
||||
assert fuzzy_match("Nom Prénom Patient", "Nom") == True
|
||||
|
||||
def test_fuzzy_ratio(self):
|
||||
# Similar but not exact/substring — ratio ~0.90
|
||||
assert fuzzy_match("Connexion", "Connection", threshold=0.8) == True
|
||||
|
||||
def test_no_match(self):
|
||||
assert fuzzy_match("Dashboard", "Login", threshold=0.8) == False
|
||||
|
||||
def test_custom_threshold(self):
|
||||
# "Connection" vs "Connexion" ratio ~0.90, passes at 0.8 but fails at 0.95
|
||||
assert fuzzy_match("Connexion", "Connection", threshold=0.95) == False
|
||||
|
||||
|
||||
# ── ocr_presence_check tests ──────────────────────────────────────────
|
||||
|
||||
|
||||
class TestOcrPresenceCheck:
|
||||
def test_all_found(self):
|
||||
tokens = ["Rechercher", "Connexion", "Nom Patient"]
|
||||
elements = [
|
||||
{"role": "bouton", "text": "Rechercher"},
|
||||
{"role": "bouton", "text": "Connexion"},
|
||||
]
|
||||
result = ocr_presence_check(tokens, elements)
|
||||
assert result.all_found == True
|
||||
assert result.presence_ratio == 1.0
|
||||
assert len(result.missing) == 0
|
||||
assert result.found_texts["Rechercher"] == "Rechercher"
|
||||
|
||||
def test_partial_found(self):
|
||||
tokens = ["Rechercher"]
|
||||
elements = [
|
||||
{"role": "bouton", "text": "Rechercher"},
|
||||
{"role": "bouton", "text": "Connexion"},
|
||||
]
|
||||
result = ocr_presence_check(tokens, elements)
|
||||
assert result.all_found == False
|
||||
assert result.presence_ratio == 0.5
|
||||
assert "bouton: Connexion" in result.missing
|
||||
|
||||
def test_none_found(self):
|
||||
tokens = ["Accueil", "Paramètres"]
|
||||
elements = [
|
||||
{"role": "bouton", "text": "Rechercher"},
|
||||
]
|
||||
result = ocr_presence_check(tokens, elements)
|
||||
assert result.all_found == False
|
||||
assert result.presence_ratio == 0.0
|
||||
assert "bouton: Rechercher" in result.missing
|
||||
|
||||
def test_fuzzy_match_in_presence(self):
|
||||
tokens = ["Rechércher"] # OCR with accent variation
|
||||
elements = [{"role": "bouton", "text": "Rechercher"}]
|
||||
result = ocr_presence_check(tokens, elements)
|
||||
assert result.all_found == True
|
||||
|
||||
def test_empty_tokens(self):
|
||||
result = ocr_presence_check([], [{"role": "bouton", "text": "Login"}])
|
||||
assert result.all_found == False
|
||||
assert result.presence_ratio == 0.0
|
||||
|
||||
def test_empty_elements(self):
|
||||
result = ocr_presence_check(["Login", "Password"], [])
|
||||
assert result.all_found == True
|
||||
assert result.presence_ratio == 1.0
|
||||
|
||||
def test_no_text_key(self):
|
||||
elements = [{"role": "page"}] # no text key
|
||||
result = ocr_presence_check(["Dashboard"], elements)
|
||||
assert result.all_found == True # no text to check → trivially found
|
||||
|
||||
def test_multiple_elements_same_text(self):
|
||||
tokens = ["Connexion"]
|
||||
elements = [
|
||||
{"role": "bouton", "text": "Connexion"},
|
||||
{"role": "label", "text": "Connexion"},
|
||||
]
|
||||
result = ocr_presence_check(tokens, elements)
|
||||
assert result.all_found == True
|
||||
|
||||
|
||||
# ── build_role_confirm_prompt tests ───────────────────────────────────
|
||||
|
||||
|
||||
class TestBuildRoleConfirmPrompt:
|
||||
def test_basic_prompt(self):
|
||||
found = [
|
||||
{"text": "Rechercher", "expected_role": "bouton", "matched_ocr": "Rechercher"},
|
||||
]
|
||||
expected = [{"role": "bouton", "text": "Rechercher"}]
|
||||
prompt = build_role_confirm_prompt(found, expected)
|
||||
assert "Text \"Rechercher\"" in prompt
|
||||
assert "expected role: bouton" in prompt
|
||||
assert "role_confirmed" in prompt
|
||||
|
||||
def test_with_context(self):
|
||||
found = [
|
||||
{"text": "Connexion", "expected_role": "bouton", "matched_ocr": "Connexion"},
|
||||
]
|
||||
expected = [{"role": "bouton", "text": "Connexion"}]
|
||||
prompt = build_role_confirm_prompt(found, expected, context="page login DPI")
|
||||
assert "Context: page login DPI" in prompt
|
||||
|
||||
def test_multiple_elements(self):
|
||||
found = [
|
||||
{"text": "Login", "expected_role": "champ", "matched_ocr": "Login"},
|
||||
{"text": "Password", "expected_role": "champ", "matched_ocr": "Password"},
|
||||
{"text": "Connexion", "expected_role": "bouton", "matched_ocr": "Connexion"},
|
||||
]
|
||||
expected = [
|
||||
{"role": "champ", "text": "Login"},
|
||||
{"role": "champ", "text": "Password"},
|
||||
{"role": "bouton", "text": "Connexion"},
|
||||
]
|
||||
prompt = build_role_confirm_prompt(found, expected)
|
||||
assert "1." in prompt
|
||||
assert "2." in prompt
|
||||
assert "3." in prompt
|
||||
|
||||
def test_no_self_declaration(self):
|
||||
"""Prompt must NOT ask VLM to declare presence — only role."""
|
||||
found = [
|
||||
{"text": "Login", "expected_role": "champ", "matched_ocr": "Login"},
|
||||
]
|
||||
expected = [{"role": "champ", "text": "Login"}]
|
||||
prompt = build_role_confirm_prompt(found, expected)
|
||||
assert "present" not in prompt.lower() or "confirmed" in prompt.lower()
|
||||
|
||||
|
||||
# ── parse_role_confirm_response tests ─────────────────────────────────
|
||||
|
||||
|
||||
class TestParseRoleConfirmResponse:
|
||||
def test_valid_json(self):
|
||||
data = json.dumps({
|
||||
"confirmed": [
|
||||
{"index": 1, "role_confirmed": True, "actual_role": "bouton", "confidence": 0.92},
|
||||
],
|
||||
"overall_confidence": 0.92,
|
||||
})
|
||||
result = parse_role_confirm_response(data)
|
||||
assert len(result["confirmed"]) == 1
|
||||
assert result["overall_confidence"] == 0.92
|
||||
|
||||
def test_json_in_markdown(self):
|
||||
vlm_text = "```json\n{\"confirmed\": [], \"overall_confidence\": 0.0}\n```"
|
||||
result = parse_role_confirm_response(vlm_text)
|
||||
assert result["overall_confidence"] == 0.0
|
||||
|
||||
def test_garbled_response(self):
|
||||
result = parse_role_confirm_response("I cannot determine the roles")
|
||||
assert result["overall_confidence"] == 0.0
|
||||
assert len(result["confirmed"]) == 0
|
||||
|
||||
def test_confidence_as_string(self):
|
||||
data = json.dumps({"confirmed": [], "overall_confidence": "0.85"})
|
||||
result = parse_role_confirm_response(data)
|
||||
assert result["overall_confidence"] == 0.85
|
||||
|
||||
|
||||
# ── verify_screen_match (OCR-anchored) tests ─────────────────────────
|
||||
|
||||
|
||||
class TestVerifyScreenMatchOcrAnchored:
|
||||
def test_full_match(self):
|
||||
ocr = mock_ocr_client_factory(["Rechercher", "Connexion", "Dashboard"])
|
||||
vlm = mock_vlm_client_factory({
|
||||
"confirmed": [
|
||||
{"index": 1, "role_confirmed": True, "actual_role": "bouton", "confidence": 0.92},
|
||||
],
|
||||
"overall_confidence": 0.92,
|
||||
})
|
||||
result = verify_screen_match(
|
||||
"/tmp/test.png",
|
||||
[{"role": "bouton", "text": "Rechercher"}],
|
||||
ocr_client=ocr,
|
||||
vlm_client=vlm,
|
||||
)
|
||||
assert result.match == True
|
||||
assert result.confidence >= 0.7
|
||||
|
||||
def test_ocr_presence_fail(self):
|
||||
"""OCR doesn't find expected text → mismatch (deterministic, no VLM needed)."""
|
||||
ocr = mock_ocr_client_factory(["Accueil", "Paramètres"])
|
||||
vlm = mock_vlm_client_factory({})
|
||||
result = verify_screen_match(
|
||||
"/tmp/test.png",
|
||||
[{"role": "bouton", "text": "Rechercher"}],
|
||||
ocr_client=ocr,
|
||||
vlm_client=vlm,
|
||||
)
|
||||
assert result.match == False
|
||||
assert "OCR presence" in result.reason
|
||||
assert len(result.mismatches) > 0
|
||||
|
||||
def test_role_not_confirmed(self):
|
||||
"""OCR finds text, VLM says it's a label not a button → mismatch."""
|
||||
ocr = mock_ocr_client_factory(["Rechercher"])
|
||||
vlm = mock_vlm_client_factory({
|
||||
"confirmed": [
|
||||
{"index": 1, "role_confirmed": False, "actual_role": "label", "confidence": 0.6},
|
||||
],
|
||||
"overall_confidence": 0.6,
|
||||
})
|
||||
result = verify_screen_match(
|
||||
"/tmp/test.png",
|
||||
[{"role": "bouton", "text": "Rechercher"}],
|
||||
ocr_client=ocr,
|
||||
vlm_client=vlm,
|
||||
)
|
||||
assert result.match == False
|
||||
|
||||
def test_ocr_error(self):
|
||||
"""OCR engine fails → fail-safe mismatch."""
|
||||
def failing_ocr(image_path):
|
||||
raise RuntimeError("OCR engine down")
|
||||
vlm = mock_vlm_client_factory({})
|
||||
result = verify_screen_match(
|
||||
"/tmp/test.png",
|
||||
[{"role": "bouton", "text": "Rechercher"}],
|
||||
ocr_client=failing_ocr,
|
||||
vlm_client=vlm,
|
||||
)
|
||||
assert result.match == False
|
||||
assert "OCR error" in result.reason
|
||||
|
||||
def test_vlm_error_partial_match(self):
|
||||
"""OCR finds texts, VLM fails → partial match (presence OK, role unknown)."""
|
||||
ocr = mock_ocr_client_factory(["Rechercher"])
|
||||
def failing_vlm(image_path, prompt):
|
||||
raise RuntimeError("VLM service down")
|
||||
result = verify_screen_match(
|
||||
"/tmp/test.png",
|
||||
[{"role": "bouton", "text": "Rechercher"}],
|
||||
ocr_client=ocr,
|
||||
vlm_client=failing_vlm,
|
||||
)
|
||||
# Presence confirmed by OCR → partial match, confidence=0.5
|
||||
assert result.match == True
|
||||
assert result.confidence == 0.5
|
||||
assert "VLM role confirm failed" in result.reason
|
||||
|
||||
def test_no_expected_elements(self):
|
||||
ocr = mock_ocr_client_factory(["Login"])
|
||||
vlm = mock_vlm_client_factory({})
|
||||
result = verify_screen_match("/tmp/test.png", [], ocr_client=ocr, vlm_client=vlm)
|
||||
assert result.match == True
|
||||
assert result.confidence == 1.0
|
||||
|
||||
def test_describe_match(self):
|
||||
result = ScreenMatchResult(match=True, confidence=0.92)
|
||||
assert "OK" in result.describe()
|
||||
|
||||
def test_describe_mismatch(self):
|
||||
result = ScreenMatchResult(
|
||||
match=False, confidence=0.3,
|
||||
mismatches=["bouton: Rechercher"],
|
||||
)
|
||||
assert "mismatch" in result.describe()
|
||||
|
||||
def test_multiple_elements_mixed(self):
|
||||
"""2 elements: 1 found+role OK, 1 not found in OCR → mismatch."""
|
||||
ocr = mock_ocr_client_factory(["Connexion"])
|
||||
vlm = mock_vlm_client_factory({
|
||||
"confirmed": [
|
||||
{"index": 1, "role_confirmed": True, "actual_role": "bouton", "confidence": 0.9},
|
||||
],
|
||||
"overall_confidence": 0.9,
|
||||
})
|
||||
result = verify_screen_match(
|
||||
"/tmp/test.png",
|
||||
[
|
||||
{"role": "bouton", "text": "Connexion"},
|
||||
{"role": "champ", "text": "Nom Patient"},
|
||||
],
|
||||
ocr_client=ocr,
|
||||
vlm_client=vlm,
|
||||
)
|
||||
assert result.match == False # "Nom Patient" not found by OCR
|
||||
|
||||
def test_fuzzy_ocr_match(self):
|
||||
"""OCR reads 'Rechércher' (accent), expected 'Rechercher' → still found."""
|
||||
ocr = mock_ocr_client_factory(["Rechércher"])
|
||||
vlm = mock_vlm_client_factory({
|
||||
"confirmed": [
|
||||
{"index": 1, "role_confirmed": True, "actual_role": "bouton", "confidence": 0.9},
|
||||
],
|
||||
"overall_confidence": 0.9,
|
||||
})
|
||||
result = verify_screen_match(
|
||||
"/tmp/test.png",
|
||||
[{"role": "bouton", "text": "Rechercher"}],
|
||||
ocr_client=ocr,
|
||||
vlm_client=vlm,
|
||||
)
|
||||
assert result.match == True
|
||||
|
||||
def test_no_text_elements_trivially_match(self):
|
||||
"""Elements without text key → no presence check needed → trivially OK."""
|
||||
ocr = mock_ocr_client_factory(["Dashboard"])
|
||||
vlm = mock_vlm_client_factory({})
|
||||
result = verify_screen_match(
|
||||
"/tmp/test.png",
|
||||
[{"role": "page"}],
|
||||
ocr_client=ocr,
|
||||
vlm_client=vlm,
|
||||
)
|
||||
assert result.match == True
|
||||
|
||||
|
||||
# ── verify_before / verify_after tests ────────────────────────────────
|
||||
|
||||
|
||||
class TestVerifyBeforeAfter:
|
||||
def test_verify_before_match(self):
|
||||
ocr = mock_ocr_client_factory(["Login", "Password", "Connexion"])
|
||||
vlm = mock_vlm_client_factory({
|
||||
"confirmed": [
|
||||
{"index": 1, "role_confirmed": True, "actual_role": "champ", "confidence": 0.85},
|
||||
],
|
||||
"overall_confidence": 0.85,
|
||||
})
|
||||
result = verify_before(
|
||||
"/tmp/login.png",
|
||||
[{"role": "champ", "text": "Login"}],
|
||||
ocr_client=ocr,
|
||||
vlm_client=vlm,
|
||||
context="page login",
|
||||
)
|
||||
assert result.match == True
|
||||
|
||||
def test_verify_after_higher_threshold(self):
|
||||
"""verify_after uses min_confidence=0.8. VLM returns 0.75 → mismatch."""
|
||||
ocr = mock_ocr_client_factory(["Dashboard"])
|
||||
vlm = mock_vlm_client_factory({
|
||||
"confirmed": [
|
||||
{"index": 1, "role_confirmed": True, "actual_role": "page", "confidence": 0.75},
|
||||
],
|
||||
"overall_confidence": 0.75,
|
||||
})
|
||||
result = verify_after(
|
||||
"/tmp/dashboard.png",
|
||||
[{"role": "page", "text": "Dashboard"}],
|
||||
ocr_client=ocr,
|
||||
vlm_client=vlm,
|
||||
)
|
||||
# 0.75 < 0.8 threshold → role mismatch
|
||||
assert result.match == False
|
||||
|
||||
def test_verify_after_passes_at_0_8(self):
|
||||
ocr = mock_ocr_client_factory(["Dashboard"])
|
||||
vlm = mock_vlm_client_factory({
|
||||
"confirmed": [
|
||||
{"index": 1, "role_confirmed": True, "actual_role": "page", "confidence": 0.85},
|
||||
],
|
||||
"overall_confidence": 0.85,
|
||||
})
|
||||
result = verify_after(
|
||||
"/tmp/dashboard.png",
|
||||
[{"role": "page", "text": "Dashboard"}],
|
||||
ocr_client=ocr,
|
||||
vlm_client=vlm,
|
||||
)
|
||||
assert result.match == True
|
||||
|
||||
def test_verify_before_ocr_missing(self):
|
||||
"""Pre-action: expected text not on screen → mismatch (can't proceed)."""
|
||||
ocr = mock_ocr_client_factory(["Accueil"])
|
||||
vlm = mock_vlm_client_factory({})
|
||||
result = verify_before(
|
||||
"/tmp/page.png",
|
||||
[{"role": "bouton", "text": "Connexion"}],
|
||||
ocr_client=ocr,
|
||||
vlm_client=vlm,
|
||||
context="pre-login",
|
||||
)
|
||||
assert result.match == False
|
||||
assert "OCR presence" in result.reason
|
||||
|
||||
|
||||
# ── OcrPresenceResult dataclass tests ─────────────────────────────────
|
||||
|
||||
|
||||
class TestOcrPresenceResult:
|
||||
def test_presence_ratio_all_found(self):
|
||||
result = OcrPresenceResult(
|
||||
found_texts={"Login": "Login", "Password": "Password"},
|
||||
missing=[],
|
||||
all_found=True,
|
||||
)
|
||||
assert result.presence_ratio == 1.0
|
||||
|
||||
def test_presence_ratio_half_found(self):
|
||||
result = OcrPresenceResult(
|
||||
found_texts={"Login": "Login", "Password": ""},
|
||||
missing=["champ: Password"],
|
||||
all_found=False,
|
||||
)
|
||||
assert result.presence_ratio == 0.5
|
||||
|
||||
def test_presence_ratio_empty(self):
|
||||
result = OcrPresenceResult(
|
||||
found_texts={},
|
||||
missing=[],
|
||||
all_found=True,
|
||||
)
|
||||
assert result.presence_ratio == 1.0
|
||||
Reference in New Issue
Block a user