f9a0531325
- core/navigation/ : visual_verifier (presence=OCR, role=VLM ancre sur tokens), grounding (OCR-anchor first, VLM fallback, cache coords valide par la vue), visual_login (verify_before/after, DETTE-023), action_resolver (pont runtime) - api_stream/replay_engine : dispatch action navigate server-side, never-fail -> needs_review, import depuis core.navigation (boot 5005 garanti) - 131 tests verts (wiring boot, e2e handler, unit modules) Chantier Qwen 01-02/07/2026, revue croisee Claude (plan deploy v2). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
337 lines
14 KiB
Python
337 lines
14 KiB
Python
"""Tests for core/navigation/visual_login.py — login form resolution + verification."""
|
|
|
|
import json
|
|
import pytest
|
|
from core.navigation.visual_login import (
|
|
LoginFormConfig,
|
|
LoginResolution,
|
|
dpi_urgences_login_config,
|
|
verify_login_visible,
|
|
verify_login_success,
|
|
resolve_login_form,
|
|
_ocr_detailed_to_simple,
|
|
)
|
|
from core.navigation.grounding import (
|
|
CoordsCache,
|
|
GroundedElement,
|
|
OcrTokenInfo,
|
|
OcrDetailedClient,
|
|
)
|
|
from core.navigation.visual_verifier import (
|
|
ScreenMatchResult,
|
|
VlmClient,
|
|
OcrClient,
|
|
)
|
|
|
|
|
|
# ── Mock factories ─────────────────────────────────────────────────────
|
|
|
|
|
|
def mock_ocr_detailed_client_factory(tokens: list):
|
|
"""Factory for mock OcrDetailedClient."""
|
|
def client(image_path: str) -> list:
|
|
return tokens
|
|
return client
|
|
|
|
|
|
def mock_ocr_simple_client_factory(tokens: list):
|
|
"""Factory for mock OcrClient (text-only)."""
|
|
def client(image_path: str) -> list:
|
|
return tokens
|
|
return client
|
|
|
|
|
|
def mock_vlm_client_factory(response_json: dict):
|
|
"""Factory for mock VlmClient."""
|
|
def client(image_path: str, prompt: str) -> str:
|
|
return json.dumps(response_json)
|
|
return client
|
|
|
|
|
|
# ── Default config tests ───────────────────────────────────────────────
|
|
|
|
|
|
class TestDpiUrgencesLoginConfig:
|
|
def test_default_config(self):
|
|
config = dpi_urgences_login_config()
|
|
assert config.login_field["role"] == "champ"
|
|
assert config.login_field["text"] == "Login"
|
|
assert config.password_field["text"] == "Mot de passe"
|
|
assert config.submit_button["text"] == "Connexion"
|
|
assert len(config.success_elements) >= 1
|
|
assert config.context != ""
|
|
|
|
def test_config_fields_are_dicts(self):
|
|
config = dpi_urgences_login_config()
|
|
assert isinstance(config.login_field, dict)
|
|
assert isinstance(config.password_field, dict)
|
|
assert isinstance(config.submit_button, dict)
|
|
|
|
|
|
# ── _ocr_detailed_to_simple tests ────────────────────────────────────
|
|
|
|
|
|
class TestOcrDetailedToSimple:
|
|
def test_conversion(self):
|
|
tokens = [
|
|
OcrTokenInfo(text="Login", bbox=(100, 50, 200, 90)),
|
|
OcrTokenInfo(text="Password", bbox=(100, 100, 200, 140)),
|
|
]
|
|
detailed = mock_ocr_detailed_client_factory(tokens)
|
|
simple = _ocr_detailed_to_simple(detailed)
|
|
result = simple("/tmp/test.png")
|
|
assert result == ["Login", "Password"]
|
|
|
|
def test_empty_tokens(self):
|
|
detailed = mock_ocr_detailed_client_factory([])
|
|
simple = _ocr_detailed_to_simple(detailed)
|
|
result = simple("/tmp/test.png")
|
|
assert result == []
|
|
|
|
|
|
# ── verify_login_visible tests ────────────────────────────────────────
|
|
|
|
|
|
class TestVerifyLoginVisible:
|
|
def test_form_visible(self):
|
|
"""All 3 fields found by OCR + roles confirmed → match."""
|
|
config = LoginFormConfig(
|
|
login_field={"role": "champ", "text": "Login"},
|
|
password_field={"role": "champ", "text": "Mot de passe"},
|
|
submit_button={"role": "bouton", "text": "Connexion"},
|
|
context="DPI login",
|
|
)
|
|
ocr = mock_ocr_simple_client_factory(["Login", "Mot de passe", "Connexion"])
|
|
vlm = mock_vlm_client_factory({
|
|
"confirmed": [
|
|
{"index": 1, "role_confirmed": True, "actual_role": "champ", "confidence": 0.9},
|
|
{"index": 2, "role_confirmed": True, "actual_role": "champ", "confidence": 0.9},
|
|
{"index": 3, "role_confirmed": True, "actual_role": "bouton", "confidence": 0.9},
|
|
],
|
|
"overall_confidence": 0.9,
|
|
})
|
|
result = verify_login_visible("/tmp/login.png", config, ocr, vlm)
|
|
assert result.match == True
|
|
|
|
def test_form_missing_button(self):
|
|
"""Connexion button not found by OCR → mismatch."""
|
|
config = LoginFormConfig(
|
|
login_field={"role": "champ", "text": "Login"},
|
|
password_field={"role": "champ", "text": "Mot de passe"},
|
|
submit_button={"role": "bouton", "text": "Connexion"},
|
|
)
|
|
ocr = mock_ocr_simple_client_factory(["Login", "Mot de passe"]) # missing Connexion
|
|
vlm = mock_vlm_client_factory({})
|
|
result = verify_login_visible("/tmp/login.png", config, ocr, vlm)
|
|
assert result.match == False
|
|
|
|
def test_form_wrong_role(self):
|
|
"""OCR finds text but VLM says button is a label → mismatch."""
|
|
config = LoginFormConfig(
|
|
login_field={"role": "champ", "text": "Login"},
|
|
password_field={"role": "champ", "text": "Mot de passe"},
|
|
submit_button={"role": "bouton", "text": "Connexion"},
|
|
)
|
|
ocr = mock_ocr_simple_client_factory(["Login", "Mot de passe", "Connexion"])
|
|
vlm = mock_vlm_client_factory({
|
|
"confirmed": [
|
|
{"index": 1, "role_confirmed": True, "actual_role": "champ", "confidence": 0.9},
|
|
{"index": 2, "role_confirmed": True, "actual_role": "champ", "confidence": 0.9},
|
|
{"index": 3, "role_confirmed": False, "actual_role": "label", "confidence": 0.5},
|
|
],
|
|
"overall_confidence": 0.5,
|
|
})
|
|
result = verify_login_visible("/tmp/login.png", config, ocr, vlm)
|
|
assert result.match == False
|
|
|
|
|
|
# ── verify_login_success tests ────────────────────────────────────────
|
|
|
|
|
|
class TestVerifyLoginSuccess:
|
|
def test_dashboard_visible(self):
|
|
"""Dashboard found by OCR + role confirmed → success."""
|
|
config = LoginFormConfig(
|
|
login_field={"role": "champ", "text": "Login"},
|
|
password_field={"role": "champ", "text": "Mot de passe"},
|
|
submit_button={"role": "bouton", "text": "Connexion"},
|
|
success_elements=[{"role": "page", "text": "Dashboard"}],
|
|
)
|
|
ocr = mock_ocr_simple_client_factory(["Dashboard", "Accueil"])
|
|
vlm = mock_vlm_client_factory({
|
|
"confirmed": [
|
|
{"index": 1, "role_confirmed": True, "actual_role": "page", "confidence": 0.92},
|
|
],
|
|
"overall_confidence": 0.92,
|
|
})
|
|
result = verify_login_success("/tmp/dashboard.png", config, ocr, vlm)
|
|
assert result.match == True
|
|
|
|
def test_no_success_elements(self):
|
|
"""Config has no success_elements → can't verify."""
|
|
config = LoginFormConfig(
|
|
login_field={"role": "champ", "text": "Login"},
|
|
password_field={"role": "champ", "text": "Mot de passe"},
|
|
submit_button={"role": "bouton", "text": "Connexion"},
|
|
success_elements=[], # empty!
|
|
)
|
|
ocr = mock_ocr_simple_client_factory(["Dashboard"])
|
|
vlm = mock_vlm_client_factory({})
|
|
result = verify_login_success("/tmp/page.png", config, ocr, vlm)
|
|
assert result.match == False
|
|
assert "no success_elements" in result.reason
|
|
|
|
def test_still_on_login_page(self):
|
|
"""After login, still seeing login form → mismatch."""
|
|
config = LoginFormConfig(
|
|
login_field={"role": "champ", "text": "Login"},
|
|
password_field={"role": "champ", "text": "Mot de passe"},
|
|
submit_button={"role": "bouton", "text": "Connexion"},
|
|
success_elements=[{"role": "page", "text": "Dashboard"}],
|
|
)
|
|
# OCR sees login form texts, not Dashboard
|
|
ocr = mock_ocr_simple_client_factory(["Login", "Mot de passe", "Connexion"])
|
|
vlm = mock_vlm_client_factory({})
|
|
result = verify_login_success("/tmp/still_login.png", config, ocr, vlm)
|
|
assert result.match == False
|
|
|
|
|
|
# ── resolve_login_form tests ──────────────────────────────────────────
|
|
|
|
|
|
class TestResolveLoginForm:
|
|
def test_all_fields_ocr_anchor(self):
|
|
"""All 3 fields found by OCR with bbox → full resolution."""
|
|
config = LoginFormConfig(
|
|
login_field={"role": "champ", "text": "Login"},
|
|
password_field={"role": "champ", "text": "Mot de passe"},
|
|
submit_button={"role": "bouton", "text": "Connexion"},
|
|
)
|
|
ocr = mock_ocr_detailed_client_factory([
|
|
OcrTokenInfo(text="Login", bbox=(100, 50, 250, 90)),
|
|
OcrTokenInfo(text="Mot de passe", bbox=(100, 100, 250, 140)),
|
|
OcrTokenInfo(text="Connexion", bbox=(100, 150, 250, 190)),
|
|
])
|
|
vlm = mock_vlm_client_factory({})
|
|
result = resolve_login_form("/tmp/login.png", config, ocr, vlm)
|
|
assert result.all_resolved == True
|
|
assert result.login_field is not None
|
|
assert result.login_field.method == "ocr_anchor"
|
|
assert result.password_field is not None
|
|
assert result.submit_button is not None
|
|
assert result.method == "ocr_anchor"
|
|
|
|
def test_partial_ocr_vlm_fallback(self):
|
|
"""Login + password by OCR, button by VLM → mixed method."""
|
|
config = LoginFormConfig(
|
|
login_field={"role": "champ", "text": "Login"},
|
|
password_field={"role": "champ", "text": "Password"},
|
|
submit_button={"role": "bouton", "text": "Connexion"},
|
|
)
|
|
ocr = mock_ocr_detailed_client_factory([
|
|
OcrTokenInfo(text="Login", bbox=(100, 50, 250, 90)),
|
|
OcrTokenInfo(text="Password", bbox=(100, 100, 250, 140)),
|
|
# Connexion not in OCR → VLM fallback
|
|
])
|
|
vlm = mock_vlm_client_factory({
|
|
"found": True,
|
|
"bbox": [0.2, 0.4, 0.4, 0.5],
|
|
"confidence": 0.85,
|
|
})
|
|
result = resolve_login_form("/tmp/login.png", config, ocr, vlm)
|
|
assert result.all_resolved == True
|
|
assert result.login_field.method == "ocr_anchor"
|
|
assert result.submit_button.method == "vlm_grounder"
|
|
assert result.method == "mixed"
|
|
|
|
def test_incomplete_resolution(self):
|
|
"""Button not found by OCR or VLM → incomplete."""
|
|
config = LoginFormConfig(
|
|
login_field={"role": "champ", "text": "Login"},
|
|
password_field={"role": "champ", "text": "Password"},
|
|
submit_button={"role": "bouton", "text": "Connexion"},
|
|
)
|
|
ocr = mock_ocr_detailed_client_factory([
|
|
OcrTokenInfo(text="Login", bbox=(100, 50, 250, 90)),
|
|
OcrTokenInfo(text="Password", bbox=(100, 100, 250, 140)),
|
|
])
|
|
vlm = mock_vlm_client_factory({"found": False, "bbox": [], "confidence": 0.0})
|
|
result = resolve_login_form("/tmp/login.png", config, ocr, vlm)
|
|
assert result.all_resolved == False
|
|
assert result.submit_button is None
|
|
|
|
def test_cache_hit(self):
|
|
"""All fields cached → returned directly."""
|
|
cache = CoordsCache()
|
|
cache.put("champ:login", (100, 50, 250, 90), (175, 70), "ocr_anchor")
|
|
cache.put("champ:mot de passe", (100, 100, 250, 140), (175, 120), "ocr_anchor")
|
|
cache.put("bouton:connexion", (100, 150, 250, 190), (175, 170), "ocr_anchor")
|
|
|
|
config = LoginFormConfig(
|
|
login_field={"role": "champ", "text": "Login"},
|
|
password_field={"role": "champ", "text": "Mot de passe"},
|
|
submit_button={"role": "bouton", "text": "Connexion"},
|
|
)
|
|
ocr = mock_ocr_detailed_client_factory([])
|
|
vlm = mock_vlm_client_factory({})
|
|
result = resolve_login_form(
|
|
"/tmp/login.png", config, ocr, vlm, coords_cache=cache,
|
|
)
|
|
assert result.all_resolved == True
|
|
assert result.method == "cache"
|
|
assert result.login_field.center == (175, 70)
|
|
|
|
def test_with_dpi_default_config(self):
|
|
"""Full flow with dpi_urgences_login_config."""
|
|
config = dpi_urgences_login_config()
|
|
ocr = mock_ocr_detailed_client_factory([
|
|
OcrTokenInfo(text="Login", bbox=(100, 50, 250, 90)),
|
|
OcrTokenInfo(text="Mot de passe", bbox=(100, 100, 250, 140)),
|
|
OcrTokenInfo(text="Connexion", bbox=(100, 150, 250, 190)),
|
|
])
|
|
vlm = mock_vlm_client_factory({})
|
|
result = resolve_login_form("/tmp/login.png", config, ocr, vlm)
|
|
assert result.all_resolved == True
|
|
|
|
|
|
# ── LoginResolution describe tests ────────────────────────────────────
|
|
|
|
|
|
class TestLoginResolutionDescribe:
|
|
def test_all_resolved(self):
|
|
resolution = LoginResolution(
|
|
login_field=GroundedElement(
|
|
role="champ", text="Login",
|
|
bbox=(100, 50, 250, 90), center=(175, 70),
|
|
confidence=0.9, method="ocr_anchor",
|
|
),
|
|
password_field=GroundedElement(
|
|
role="champ", text="Mot de passe",
|
|
bbox=(100, 100, 250, 140), center=(175, 120),
|
|
confidence=0.9, method="ocr_anchor",
|
|
),
|
|
submit_button=GroundedElement(
|
|
role="bouton", text="Connexion",
|
|
bbox=(100, 150, 250, 190), center=(175, 170),
|
|
confidence=0.9, method="ocr_anchor",
|
|
),
|
|
all_resolved=True,
|
|
method="ocr_anchor",
|
|
)
|
|
desc = resolution.describe()
|
|
assert "OK" in desc
|
|
assert "login@" in desc
|
|
assert "button@" in desc
|
|
|
|
def test_incomplete(self):
|
|
resolution = LoginResolution(
|
|
login_field=None,
|
|
password_field=None,
|
|
submit_button=None,
|
|
all_resolved=False,
|
|
method="",
|
|
)
|
|
desc = resolution.describe()
|
|
assert "INCOMPLETE" in desc
|
|
assert "NOT FOUND" in desc
|