snapshot: WIP 5j replay reliability (B1 watchdog + dialog handlers + grounding drift)

Snapshot avant correction du blocage relance Léa (3 incidents 24h: SSH refusé,
polls morts ×2). Point de rollback stable.

Contenu:
- agent_v1/core/executor.py: 5 patchs dialog handling (saveas drift, close_tab
  hotkey fallback, confirm_save Unicode apostrophe, foreground dialog
  recontextualization, runtime_dialog in-loop) + helpers normalize_window_hint,
  requires_post_verify_window_transition
- agent_v1/core/grounding.py: garde drift template fix (fallback_x/y plumbed)
- server_v1/replay_watchdog.py (NEW): orphan watchdog B1, scan 10s timeout 30s
- server_v1/api_stream.py: dispatched_action plumbing, watchdog lifespan,
  metrics endpoint
- server_v1/replay_engine.py: _schedule_retry préserve original_action +
  dispatched_action
- stream_processor.py: gardes _infer_tab_switch_target (no false switch_tab
  on save_as dialog open) + _attach_expected_window_before
- tests/integration: test_replay_watchdog.py (8 cas), test_stream_processor.py
- tests/unit: test_executor_verify_window_guard.py (start_button, close_tab,
  runtime_dialog, post_verify, transition fallbacks)

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Dom
2026-05-24 16:48:37 +02:00
parent 5ea4960e65
commit 7df51d2c79
47 changed files with 9811 additions and 451 deletions

View File

@@ -0,0 +1,118 @@
from types import SimpleNamespace
from agent_v0.server_v1 import replay_memory
from core.learning.target_memory_store import TargetMemoryStore
class _DummyStore:
def __init__(self, fp):
self._fp = fp
def lookup(self, screen_sig, spec_shim):
return self._fp
def test_memory_lookup_uses_window_relative_coords_when_available(monkeypatch):
fp = SimpleNamespace(
bbox=(0.566016, 0.400625, 0.0, 0.0),
etype="position_fallback",
confidence=0.2,
)
monkeypatch.setattr(replay_memory, "get_memory_store", lambda: _DummyStore(fp))
result = replay_memory.memory_lookup(
window_title="Rechercher",
target_spec={
"by_text": "Bloc-notes",
"window_capture": {
"click_relative": [681, 448],
"window_size": [1287, 1407],
},
},
)
assert result is not None
assert result["method"] == "memory_position_fallback"
assert result["x_pct"] == 681 / 1287
assert result["y_pct"] == 448 / 1407
def test_memory_lookup_keeps_bbox_coords_without_window_capture(monkeypatch):
fp = SimpleNamespace(
bbox=(0.566016, 0.400625, 0.0, 0.0),
etype="position_fallback",
confidence=0.2,
)
monkeypatch.setattr(replay_memory, "get_memory_store", lambda: _DummyStore(fp))
result = replay_memory.memory_lookup(
window_title="Rechercher",
target_spec={"by_text": "Bloc-notes"},
)
assert result is not None
assert result["x_pct"] == 0.566016
assert result["y_pct"] == 0.400625
def test_memory_lookup_keeps_learned_visual_coords_with_window_capture(monkeypatch):
fp = SimpleNamespace(
bbox=(0.402734375, 0.578125, 0.0, 0.0),
etype="anchor_template",
confidence=0.99,
)
monkeypatch.setattr(replay_memory, "get_memory_store", lambda: _DummyStore(fp))
result = replay_memory.memory_lookup(
window_title="*test Bloc-notes",
target_spec={
"by_text": "Enregistrer",
"by_role": "yolo",
"window_capture": {
"click_relative": [860, 634],
"window_size": [1920, 1116],
},
},
)
assert result is not None
assert result["method"] == "memory_anchor_template"
assert result["x_pct"] == 0.402734375
assert result["y_pct"] == 0.578125
def test_target_spec_hash_distinguishes_same_text_with_different_spatial_hints(tmp_path):
store = TargetMemoryStore(base_path=str(tmp_path / "learning"))
spec_left = replay_memory._TargetSpecLike(
{
"by_text": "Enregistrer",
"by_role": "yolo",
"vlm_description": "Dans la fenêtre '*test Bloc-notes', l'élément cliqué se trouve au milieu au centre de l'écran",
"window_capture": {
"click_relative": [860, 634],
"window_size": [1920, 1116],
},
"som_element": {
"bbox_norm": [0.40234375, 0.701875, 0.46640625, 0.74125],
"center_norm": [0.434375, 0.72125],
},
}
)
spec_right = replay_memory._TargetSpecLike(
{
"by_text": "Enregistrer",
"by_role": "yolo",
"vlm_description": "Dans la fenêtre '*test Bloc-notes', l'élément cliqué se trouve au milieu au centre de l'écran",
"window_capture": {
"click_relative": [1491, 38],
"window_size": [1920, 1116],
},
"som_element": {
"bbox_norm": [0.697265625, 0.335625, 0.715625, 0.3625],
"center_norm": [0.70625, 0.34875],
},
}
)
assert store._hash_target_spec(spec_left) != store._hash_target_spec(spec_right)