"""Tests for core/navigation/visual_login.py — login form resolution + verification.""" import json import pytest from core.navigation.visual_login import ( LoginFormConfig, LoginResolution, dpi_urgences_login_config, verify_login_visible, verify_login_success, resolve_login_form, _ocr_detailed_to_simple, ) from core.navigation.grounding import ( CoordsCache, GroundedElement, OcrTokenInfo, OcrDetailedClient, ) from core.navigation.visual_verifier import ( ScreenMatchResult, VlmClient, OcrClient, ) # ── Mock factories ───────────────────────────────────────────────────── def mock_ocr_detailed_client_factory(tokens: list): """Factory for mock OcrDetailedClient.""" def client(image_path: str) -> list: return tokens return client def mock_ocr_simple_client_factory(tokens: list): """Factory for mock OcrClient (text-only).""" def client(image_path: str) -> list: return tokens return client def mock_vlm_client_factory(response_json: dict): """Factory for mock VlmClient.""" def client(image_path: str, prompt: str) -> str: return json.dumps(response_json) return client # ── Default config tests ─────────────────────────────────────────────── class TestDpiUrgencesLoginConfig: def test_default_config(self): config = dpi_urgences_login_config() assert config.login_field["role"] == "champ" assert config.login_field["text"] == "Login" assert config.password_field["text"] == "Mot de passe" assert config.submit_button["text"] == "Connexion" assert len(config.success_elements) >= 1 assert config.context != "" def test_config_fields_are_dicts(self): config = dpi_urgences_login_config() assert isinstance(config.login_field, dict) assert isinstance(config.password_field, dict) assert isinstance(config.submit_button, dict) # ── _ocr_detailed_to_simple tests ──────────────────────────────────── class TestOcrDetailedToSimple: def test_conversion(self): tokens = [ OcrTokenInfo(text="Login", bbox=(100, 50, 200, 90)), OcrTokenInfo(text="Password", bbox=(100, 100, 200, 140)), ] detailed = mock_ocr_detailed_client_factory(tokens) simple = _ocr_detailed_to_simple(detailed) result = simple("/tmp/test.png") assert result == ["Login", "Password"] def test_empty_tokens(self): detailed = mock_ocr_detailed_client_factory([]) simple = _ocr_detailed_to_simple(detailed) result = simple("/tmp/test.png") assert result == [] # ── verify_login_visible tests ──────────────────────────────────────── class TestVerifyLoginVisible: def test_form_visible(self): """All 3 fields found by OCR + roles confirmed → match.""" config = LoginFormConfig( login_field={"role": "champ", "text": "Login"}, password_field={"role": "champ", "text": "Mot de passe"}, submit_button={"role": "bouton", "text": "Connexion"}, context="DPI login", ) ocr = mock_ocr_simple_client_factory(["Login", "Mot de passe", "Connexion"]) vlm = mock_vlm_client_factory({ "confirmed": [ {"index": 1, "role_confirmed": True, "actual_role": "champ", "confidence": 0.9}, {"index": 2, "role_confirmed": True, "actual_role": "champ", "confidence": 0.9}, {"index": 3, "role_confirmed": True, "actual_role": "bouton", "confidence": 0.9}, ], "overall_confidence": 0.9, }) result = verify_login_visible("/tmp/login.png", config, ocr, vlm) assert result.match == True def test_form_missing_button(self): """Connexion button not found by OCR → mismatch.""" config = LoginFormConfig( login_field={"role": "champ", "text": "Login"}, password_field={"role": "champ", "text": "Mot de passe"}, submit_button={"role": "bouton", "text": "Connexion"}, ) ocr = mock_ocr_simple_client_factory(["Login", "Mot de passe"]) # missing Connexion vlm = mock_vlm_client_factory({}) result = verify_login_visible("/tmp/login.png", config, ocr, vlm) assert result.match == False def test_form_wrong_role(self): """OCR finds text but VLM says button is a label → mismatch.""" config = LoginFormConfig( login_field={"role": "champ", "text": "Login"}, password_field={"role": "champ", "text": "Mot de passe"}, submit_button={"role": "bouton", "text": "Connexion"}, ) ocr = mock_ocr_simple_client_factory(["Login", "Mot de passe", "Connexion"]) vlm = mock_vlm_client_factory({ "confirmed": [ {"index": 1, "role_confirmed": True, "actual_role": "champ", "confidence": 0.9}, {"index": 2, "role_confirmed": True, "actual_role": "champ", "confidence": 0.9}, {"index": 3, "role_confirmed": False, "actual_role": "label", "confidence": 0.5}, ], "overall_confidence": 0.5, }) result = verify_login_visible("/tmp/login.png", config, ocr, vlm) assert result.match == False # ── verify_login_success tests ──────────────────────────────────────── class TestVerifyLoginSuccess: def test_dashboard_visible(self): """Dashboard found by OCR + role confirmed → success.""" config = LoginFormConfig( login_field={"role": "champ", "text": "Login"}, password_field={"role": "champ", "text": "Mot de passe"}, submit_button={"role": "bouton", "text": "Connexion"}, success_elements=[{"role": "page", "text": "Dashboard"}], ) ocr = mock_ocr_simple_client_factory(["Dashboard", "Accueil"]) vlm = mock_vlm_client_factory({ "confirmed": [ {"index": 1, "role_confirmed": True, "actual_role": "page", "confidence": 0.92}, ], "overall_confidence": 0.92, }) result = verify_login_success("/tmp/dashboard.png", config, ocr, vlm) assert result.match == True def test_no_success_elements(self): """Config has no success_elements → can't verify.""" config = LoginFormConfig( login_field={"role": "champ", "text": "Login"}, password_field={"role": "champ", "text": "Mot de passe"}, submit_button={"role": "bouton", "text": "Connexion"}, success_elements=[], # empty! ) ocr = mock_ocr_simple_client_factory(["Dashboard"]) vlm = mock_vlm_client_factory({}) result = verify_login_success("/tmp/page.png", config, ocr, vlm) assert result.match == False assert "no success_elements" in result.reason def test_still_on_login_page(self): """After login, still seeing login form → mismatch.""" config = LoginFormConfig( login_field={"role": "champ", "text": "Login"}, password_field={"role": "champ", "text": "Mot de passe"}, submit_button={"role": "bouton", "text": "Connexion"}, success_elements=[{"role": "page", "text": "Dashboard"}], ) # OCR sees login form texts, not Dashboard ocr = mock_ocr_simple_client_factory(["Login", "Mot de passe", "Connexion"]) vlm = mock_vlm_client_factory({}) result = verify_login_success("/tmp/still_login.png", config, ocr, vlm) assert result.match == False # ── resolve_login_form tests ────────────────────────────────────────── class TestResolveLoginForm: def test_all_fields_ocr_anchor(self): """All 3 fields found by OCR with bbox → full resolution.""" config = LoginFormConfig( login_field={"role": "champ", "text": "Login"}, password_field={"role": "champ", "text": "Mot de passe"}, submit_button={"role": "bouton", "text": "Connexion"}, ) ocr = mock_ocr_detailed_client_factory([ OcrTokenInfo(text="Login", bbox=(100, 50, 250, 90)), OcrTokenInfo(text="Mot de passe", bbox=(100, 100, 250, 140)), OcrTokenInfo(text="Connexion", bbox=(100, 150, 250, 190)), ]) vlm = mock_vlm_client_factory({}) result = resolve_login_form("/tmp/login.png", config, ocr, vlm) assert result.all_resolved == True assert result.login_field is not None assert result.login_field.method == "ocr_anchor" assert result.password_field is not None assert result.submit_button is not None assert result.method == "ocr_anchor" def test_partial_ocr_vlm_fallback(self): """Login + password by OCR, button by VLM → mixed method.""" config = LoginFormConfig( login_field={"role": "champ", "text": "Login"}, password_field={"role": "champ", "text": "Password"}, submit_button={"role": "bouton", "text": "Connexion"}, ) ocr = mock_ocr_detailed_client_factory([ OcrTokenInfo(text="Login", bbox=(100, 50, 250, 90)), OcrTokenInfo(text="Password", bbox=(100, 100, 250, 140)), # Connexion not in OCR → VLM fallback ]) vlm = mock_vlm_client_factory({ "found": True, "bbox": [0.2, 0.4, 0.4, 0.5], "confidence": 0.85, }) result = resolve_login_form("/tmp/login.png", config, ocr, vlm) assert result.all_resolved == True assert result.login_field.method == "ocr_anchor" assert result.submit_button.method == "vlm_grounder" assert result.method == "mixed" def test_incomplete_resolution(self): """Button not found by OCR or VLM → incomplete.""" config = LoginFormConfig( login_field={"role": "champ", "text": "Login"}, password_field={"role": "champ", "text": "Password"}, submit_button={"role": "bouton", "text": "Connexion"}, ) ocr = mock_ocr_detailed_client_factory([ OcrTokenInfo(text="Login", bbox=(100, 50, 250, 90)), OcrTokenInfo(text="Password", bbox=(100, 100, 250, 140)), ]) vlm = mock_vlm_client_factory({"found": False, "bbox": [], "confidence": 0.0}) result = resolve_login_form("/tmp/login.png", config, ocr, vlm) assert result.all_resolved == False assert result.submit_button is None def test_cache_hit(self): """All fields cached → returned directly.""" cache = CoordsCache() cache.put("champ:login", (100, 50, 250, 90), (175, 70), "ocr_anchor") cache.put("champ:mot de passe", (100, 100, 250, 140), (175, 120), "ocr_anchor") cache.put("bouton:connexion", (100, 150, 250, 190), (175, 170), "ocr_anchor") config = LoginFormConfig( login_field={"role": "champ", "text": "Login"}, password_field={"role": "champ", "text": "Mot de passe"}, submit_button={"role": "bouton", "text": "Connexion"}, ) ocr = mock_ocr_detailed_client_factory([]) vlm = mock_vlm_client_factory({}) result = resolve_login_form( "/tmp/login.png", config, ocr, vlm, coords_cache=cache, ) assert result.all_resolved == True assert result.method == "cache" assert result.login_field.center == (175, 70) def test_with_dpi_default_config(self): """Full flow with dpi_urgences_login_config.""" config = dpi_urgences_login_config() ocr = mock_ocr_detailed_client_factory([ OcrTokenInfo(text="Login", bbox=(100, 50, 250, 90)), OcrTokenInfo(text="Mot de passe", bbox=(100, 100, 250, 140)), OcrTokenInfo(text="Connexion", bbox=(100, 150, 250, 190)), ]) vlm = mock_vlm_client_factory({}) result = resolve_login_form("/tmp/login.png", config, ocr, vlm) assert result.all_resolved == True # ── LoginResolution describe tests ──────────────────────────────────── class TestLoginResolutionDescribe: def test_all_resolved(self): resolution = LoginResolution( login_field=GroundedElement( role="champ", text="Login", bbox=(100, 50, 250, 90), center=(175, 70), confidence=0.9, method="ocr_anchor", ), password_field=GroundedElement( role="champ", text="Mot de passe", bbox=(100, 100, 250, 140), center=(175, 120), confidence=0.9, method="ocr_anchor", ), submit_button=GroundedElement( role="bouton", text="Connexion", bbox=(100, 150, 250, 190), center=(175, 170), confidence=0.9, method="ocr_anchor", ), all_resolved=True, method="ocr_anchor", ) desc = resolution.describe() assert "OK" in desc assert "login@" in desc assert "button@" in desc def test_incomplete(self): resolution = LoginResolution( login_field=None, password_field=None, submit_button=None, all_resolved=False, method="", ) desc = resolution.describe() assert "INCOMPLETE" in desc assert "NOT FOUND" in desc