rpa_vision_v3/tests/unit/test_replay_memory.py

from types import SimpleNamespace

from agent_v0.server_v1 import replay_memory
from core.learning.target_memory_store import TargetMemoryStore


class _DummyStore:
    def __init__(self, fp):
        self._fp = fp

    def lookup(self, screen_sig, spec_shim):
        return self._fp


def test_memory_lookup_uses_window_relative_coords_when_available(monkeypatch):
    fp = SimpleNamespace(
        bbox=(0.566016, 0.400625, 0.0, 0.0),
        etype="position_fallback",
        confidence=0.2,
    )
    monkeypatch.setattr(replay_memory, "get_memory_store", lambda: _DummyStore(fp))

    result = replay_memory.memory_lookup(
        window_title="Rechercher",
        target_spec={
            "by_text": "Bloc-notes",
            "window_capture": {
                "click_relative": [681, 448],
                "window_size": [1287, 1407],
            },
        },
    )

    assert result is not None
    assert result["method"] == "memory_position_fallback"
    assert result["x_pct"] == 681 / 1287
    assert result["y_pct"] == 448 / 1407


def test_memory_lookup_keeps_bbox_coords_without_window_capture(monkeypatch):
    fp = SimpleNamespace(
        bbox=(0.566016, 0.400625, 0.0, 0.0),
        etype="position_fallback",
        confidence=0.2,
    )
    monkeypatch.setattr(replay_memory, "get_memory_store", lambda: _DummyStore(fp))

    result = replay_memory.memory_lookup(
        window_title="Rechercher",
        target_spec={"by_text": "Bloc-notes"},
    )

    assert result is not None
    assert result["x_pct"] == 0.566016
    assert result["y_pct"] == 0.400625


def test_memory_lookup_keeps_learned_visual_coords_with_window_capture(monkeypatch):
    fp = SimpleNamespace(
        bbox=(0.402734375, 0.578125, 0.0, 0.0),
        etype="anchor_template",
        confidence=0.99,
    )
    monkeypatch.setattr(replay_memory, "get_memory_store", lambda: _DummyStore(fp))

    result = replay_memory.memory_lookup(
        window_title="*test – Bloc-notes",
        target_spec={
            "by_text": "Enregistrer",
            "by_role": "yolo",
            "context_hints": {
                "expected_window_before": "*test – Bloc-notes",
                "interaction": "toolbar_save_button",
            },
            "window_capture": {
                "click_relative": [860, 634],
                "window_size": [1920, 1116],
            },
        },
    )

    assert result is not None
    assert result["method"] == "memory_anchor_template"
    assert result["x_pct"] == 0.402734375
    assert result["y_pct"] == 0.578125


def test_memory_lookup_skips_window_transition_even_if_record_exists(monkeypatch):
    fp = SimpleNamespace(
        bbox=(0.5, 0.8, 0.0, 0.0),
        etype="grounding_vlm",
        confidence=0.85,
    )
    monkeypatch.setattr(replay_memory, "get_memory_store", lambda: _DummyStore(fp))

    result = replay_memory.memory_lookup(
        window_title="*test – Bloc-notes",
        target_spec={
            "by_text": "Enregistrer",
            "by_role": "button",
            "context_hints": {
                "expected_window_before": "*test – Bloc-notes",
                "expected_window_after": "Enregistrer sous",
                "requires_window_transition": True,
            },
        },
    )

    assert result is None


def test_memory_lookup_rejects_generic_button_without_context(monkeypatch):
    fp = SimpleNamespace(
        bbox=(0.5, 0.8, 0.0, 0.0),
        etype="grounding_vlm",
        confidence=0.85,
    )
    monkeypatch.setattr(replay_memory, "get_memory_store", lambda: _DummyStore(fp))

    result = replay_memory.memory_lookup(
        window_title="*test – Bloc-notes",
        target_spec={"by_text": "Enregistrer", "by_role": "button"},
    )

    assert result is None


def test_memory_lookup_allows_generic_button_with_context(monkeypatch):
    fp = SimpleNamespace(
        bbox=(0.5, 0.8, 0.0, 0.0),
        etype="grounding_vlm",
        confidence=0.85,
    )
    monkeypatch.setattr(replay_memory, "get_memory_store", lambda: _DummyStore(fp))

    result = replay_memory.memory_lookup(
        window_title="Enregistrer sous",
        target_spec={
            "by_text": "Enregistrer",
            "by_role": "button",
            "window_title": "Enregistrer sous",
            "context_hints": {
                "expected_window_before": "Enregistrer sous",
                "interaction": "save_dialog_primary_button",
            },
        },
    )

    assert result is not None
    assert result["method"] == "memory_grounding_vlm"


def test_target_spec_hash_distinguishes_same_text_with_different_spatial_hints(tmp_path):
    store = TargetMemoryStore(base_path=str(tmp_path / "learning"))

    spec_left = replay_memory._TargetSpecLike(
        {
            "by_text": "Enregistrer",
            "by_role": "yolo",
            "vlm_description": "Dans la fenêtre '*test – Bloc-notes', l'élément cliqué se trouve au milieu au centre de l'écran",
            "window_capture": {
                "click_relative": [860, 634],
                "window_size": [1920, 1116],
            },
            "som_element": {
                "bbox_norm": [0.40234375, 0.701875, 0.46640625, 0.74125],
                "center_norm": [0.434375, 0.72125],
            },
        }
    )
    spec_right = replay_memory._TargetSpecLike(
        {
            "by_text": "Enregistrer",
            "by_role": "yolo",
            "vlm_description": "Dans la fenêtre '*test – Bloc-notes', l'élément cliqué se trouve au milieu au centre de l'écran",
            "window_capture": {
                "click_relative": [1491, 38],
                "window_size": [1920, 1116],
            },
            "som_element": {
                "bbox_norm": [0.697265625, 0.335625, 0.715625, 0.3625],
                "center_norm": [0.70625, 0.34875],
            },
        }
    )

    assert store._hash_target_spec(spec_left) != store._hash_target_spec(spec_right)