Snapshot avant correction du blocage relance Léa (3 incidents 24h: SSH refusé, polls morts ×2). Point de rollback stable. Contenu: - agent_v1/core/executor.py: 5 patchs dialog handling (saveas drift, close_tab hotkey fallback, confirm_save Unicode apostrophe, foreground dialog recontextualization, runtime_dialog in-loop) + helpers normalize_window_hint, requires_post_verify_window_transition - agent_v1/core/grounding.py: garde drift template fix (fallback_x/y plumbed) - server_v1/replay_watchdog.py (NEW): orphan watchdog B1, scan 10s timeout 30s - server_v1/api_stream.py: dispatched_action plumbing, watchdog lifespan, metrics endpoint - server_v1/replay_engine.py: _schedule_retry préserve original_action + dispatched_action - stream_processor.py: gardes _infer_tab_switch_target (no false switch_tab on save_as dialog open) + _attach_expected_window_before - tests/integration: test_replay_watchdog.py (8 cas), test_stream_processor.py - tests/unit: test_executor_verify_window_guard.py (start_button, close_tab, runtime_dialog, post_verify, transition fallbacks) Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
486 lines
20 KiB
Python
486 lines
20 KiB
Python
"""Garde dimensions monitor — agent_v0/agent_v1/vision/capturer.py
|
||
|
||
Contexte (démo GHT 19 mai 2026) : `mss.monitors[1]` peut retourner
|
||
intermittemment des dimensions tronquées (cas observé : 2560×60 au lieu
|
||
de 2560×1600). Toute capture utilisant ces dims pour normaliser des
|
||
coordonnées empoisonne ensuite la mémoire persistante (`TargetMemoryStore`).
|
||
|
||
Ce module teste la garde qui doit :
|
||
- détecter une dimension aberrante avant capture
|
||
- retenter (mss peut avoir un cache stale)
|
||
- tomber en fallback sur un autre monitor physique si dispo
|
||
- abandonner explicitement (logs WARNING/ERROR) sans empoisonner
|
||
|
||
Périmètre : capturer.py uniquement (pas executor, pas replay).
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import logging
|
||
from pathlib import Path
|
||
from unittest.mock import MagicMock, patch
|
||
|
||
import pytest
|
||
from PIL import Image
|
||
|
||
|
||
def _make_mock_mss(monitors_sequence):
|
||
"""Construit un mock `mss.mss()` qui renvoie successivement les listes
|
||
`monitors` fournies. Permet de simuler retry / changement de dims
|
||
entre deux appels.
|
||
|
||
Args:
|
||
monitors_sequence: liste de listes-de-monitors. Chaque entrée
|
||
représente l'état renvoyé par `sct.monitors` à un appel
|
||
successif de `mss.mss()`. La dernière entrée est réutilisée
|
||
si plus d'appels ont lieu.
|
||
|
||
Returns:
|
||
Un mock utilisable comme `patch(..., side_effect=mock)` côté `mss.mss`.
|
||
"""
|
||
call_counter = {"n": 0}
|
||
instances = []
|
||
|
||
def factory():
|
||
idx = min(call_counter["n"], len(monitors_sequence) - 1)
|
||
call_counter["n"] += 1
|
||
instance = MagicMock(name=f"mss_instance_{idx}")
|
||
instance.monitors = monitors_sequence[idx]
|
||
|
||
# grab() renvoie un objet avec size + bgra pour passer dans PIL
|
||
grab_result = MagicMock()
|
||
# On simule un buffer cohérent avec les dims du monitor sain
|
||
m = monitors_sequence[idx][1] if len(monitors_sequence[idx]) > 1 else {}
|
||
w = m.get("width", 100)
|
||
h = m.get("height", 100)
|
||
grab_result.size = (w, h)
|
||
# Une image saine ne doit pas être entièrement noire, sinon le nouveau
|
||
# fail-closed black-frame la rejetterait.
|
||
grab_result.bgra = b"\x80\x80\x80\x00" * (w * h)
|
||
instance.grab = MagicMock(return_value=grab_result)
|
||
|
||
# context manager
|
||
cm = MagicMock(name=f"mss_cm_{idx}")
|
||
cm.__enter__ = MagicMock(return_value=instance)
|
||
cm.__exit__ = MagicMock(return_value=False)
|
||
instances.append((cm, instance))
|
||
return cm
|
||
|
||
factory.instances = instances
|
||
return factory
|
||
|
||
|
||
def _vision_capturer(tmp_path):
|
||
"""Import paresseux pour permettre au patch d'opérer avant le import."""
|
||
from agent_v0.agent_v1.vision.capturer import VisionCapturer
|
||
return VisionCapturer(str(tmp_path))
|
||
|
||
|
||
def _solid_img(color: tuple[int, int, int], size=(320, 240)) -> Image.Image:
|
||
"""Image unie simple pour piloter les tests de fallback noir."""
|
||
return Image.new("RGB", size, color)
|
||
|
||
|
||
# ============================================================================
|
||
# Test 1 — Dim aberrante (height=60) refusée : capture_full_context renvoie ""
|
||
# ============================================================================
|
||
|
||
|
||
def test_capture_full_context_returns_empty_when_monitor_height_aberrant(
|
||
tmp_path: Path, caplog: pytest.LogCaptureFixture
|
||
):
|
||
"""Cas démo GHT : mss.monitors[1] = 2560×60 (au lieu de 2560×1600).
|
||
|
||
La capture doit refuser de produire un PNG basé sur ces dims (sinon
|
||
toute coord normalisée derrière sera fausse d'un facteur ~27×).
|
||
Retour attendu : chaîne vide (comme le contrat existant en cas
|
||
d'erreur).
|
||
"""
|
||
aberrant_monitors = [
|
||
{"left": 0, "top": 0, "width": 2560, "height": 1660}, # composite
|
||
{"left": 0, "top": 0, "width": 2560, "height": 60}, # PRIMAIRE aberrant
|
||
]
|
||
factory = _make_mock_mss([aberrant_monitors])
|
||
|
||
with patch("agent_v0.agent_v1.vision.capturer.mss.mss", side_effect=factory), \
|
||
patch("agent_v0.agent_v1.vision.capturer.time.sleep"):
|
||
caplog.set_level(logging.WARNING, logger="agent_v0.agent_v1.vision.capturer")
|
||
cap = _vision_capturer(tmp_path)
|
||
result = cap.capture_full_context("test_aberrant")
|
||
|
||
assert result == "", (
|
||
f"Capture devrait retourner '' sur dim aberrante, got {result!r}"
|
||
)
|
||
|
||
# Sanity : aucun grab() ne doit avoir été appelé sur un monitor aberrant.
|
||
# Tous les mss instances créés ne doivent JAMAIS avoir appelé grab().
|
||
for _cm, instance in factory.instances:
|
||
instance.grab.assert_not_called()
|
||
|
||
|
||
# ============================================================================
|
||
# Test 2 — Le log WARNING doit citer la dim observée (debuggabilité)
|
||
# ============================================================================
|
||
|
||
|
||
def test_aberrant_monitor_logs_warning_with_observed_dimensions(
|
||
tmp_path: Path, caplog: pytest.LogCaptureFixture
|
||
):
|
||
"""L'opérateur doit pouvoir diagnostiquer la cause depuis les logs sans
|
||
rejouer la session. Le WARNING doit contenir les dims aberrantes vues.
|
||
"""
|
||
aberrant_monitors = [
|
||
{"left": 0, "top": 0, "width": 2560, "height": 1660},
|
||
{"left": 0, "top": 0, "width": 2560, "height": 60},
|
||
]
|
||
factory = _make_mock_mss([aberrant_monitors])
|
||
|
||
with patch("agent_v0.agent_v1.vision.capturer.mss.mss", side_effect=factory), \
|
||
patch("agent_v0.agent_v1.vision.capturer.time.sleep"):
|
||
caplog.set_level(logging.WARNING, logger="agent_v0.agent_v1.vision.capturer")
|
||
cap = _vision_capturer(tmp_path)
|
||
cap.capture_full_context("test")
|
||
|
||
warnings = [r for r in caplog.records if r.levelno == logging.WARNING]
|
||
assert warnings, "Au moins un WARNING attendu sur dim aberrante"
|
||
msg = " ".join(r.getMessage() for r in warnings)
|
||
assert "2560" in msg, f"Largeur observée doit apparaître dans le WARNING : {msg!r}"
|
||
assert "60" in msg, f"Hauteur observée doit apparaître dans le WARNING : {msg!r}"
|
||
|
||
|
||
# ============================================================================
|
||
# Test 3 — Retry : un 1er appel aberrant suivi d'un appel sain produit la capture
|
||
# ============================================================================
|
||
|
||
|
||
def test_capture_retries_when_first_monitor_query_is_aberrant(
|
||
tmp_path: Path, caplog: pytest.LogCaptureFixture
|
||
):
|
||
"""Le bug observé est intermittent (mss peut avoir un cache stale). Si on
|
||
retente immédiatement, le second appel renvoie souvent les vraies dims.
|
||
La capture doit donc retenter et réussir quand le second appel est sain.
|
||
"""
|
||
aberrant_then_ok = [
|
||
# 1er appel : aberrant
|
||
[
|
||
{"left": 0, "top": 0, "width": 2560, "height": 1660},
|
||
{"left": 0, "top": 0, "width": 2560, "height": 60},
|
||
],
|
||
# 2e appel : OK
|
||
[
|
||
{"left": 0, "top": 0, "width": 2560, "height": 1660},
|
||
{"left": 0, "top": 0, "width": 2560, "height": 1600},
|
||
],
|
||
]
|
||
factory = _make_mock_mss(aberrant_then_ok)
|
||
|
||
with patch("agent_v0.agent_v1.vision.capturer.mss.mss", side_effect=factory), \
|
||
patch("agent_v0.agent_v1.vision.capturer.time.sleep"):
|
||
caplog.set_level(logging.WARNING, logger="agent_v0.agent_v1.vision.capturer")
|
||
cap = _vision_capturer(tmp_path)
|
||
result = cap.capture_full_context("test_retry", force=True)
|
||
|
||
assert result, (
|
||
f"Capture doit réussir après retry sur dims saines, got {result!r}"
|
||
)
|
||
assert Path(result).exists(), "Le PNG doit être physiquement créé"
|
||
|
||
# Au moins 2 appels mss.mss() : le premier (aberrant) + le retry
|
||
assert len(factory.instances) >= 2, (
|
||
f"Au moins 2 appels mss.mss() attendus (retry), vu {len(factory.instances)}"
|
||
)
|
||
|
||
|
||
# ============================================================================
|
||
# Test 4 — Fallback : monitors[1] aberrant mais monitors[2] sain → capture OK
|
||
# ============================================================================
|
||
|
||
|
||
def test_capture_falls_back_to_secondary_monitor_when_primary_aberrant(
|
||
tmp_path: Path, caplog: pytest.LogCaptureFixture
|
||
):
|
||
"""Cas multi-écrans : monitors[1] cassé en permanence, monitors[2] sain.
|
||
La capture doit utiliser monitors[2] et logger un WARNING fallback.
|
||
"""
|
||
monitors_with_fallback = [
|
||
{"left": 0, "top": 0, "width": 2560, "height": 1660}, # composite
|
||
{"left": 0, "top": 0, "width": 2560, "height": 60}, # primaire cassé
|
||
{"left": 2560, "top": 0, "width": 1920, "height": 1080}, # secondaire sain
|
||
]
|
||
# Même état renvoyé à tous les appels (cas stationnaire, pas intermittent)
|
||
factory = _make_mock_mss([monitors_with_fallback])
|
||
|
||
with patch("agent_v0.agent_v1.vision.capturer.mss.mss", side_effect=factory), \
|
||
patch("agent_v0.agent_v1.vision.capturer.time.sleep"):
|
||
caplog.set_level(logging.WARNING, logger="agent_v0.agent_v1.vision.capturer")
|
||
cap = _vision_capturer(tmp_path)
|
||
result = cap.capture_full_context("test_fallback", force=True)
|
||
|
||
assert result, f"Capture doit réussir via monitor[2], got {result!r}"
|
||
msg = " ".join(r.getMessage() for r in caplog.records)
|
||
assert "fallback" in msg.lower(), (
|
||
f"Un log doit signaler le fallback monitor : {msg!r}"
|
||
)
|
||
|
||
|
||
# ============================================================================
|
||
# Test 5 — capture_dual bénéficie aussi de la garde
|
||
# ============================================================================
|
||
|
||
|
||
def test_capture_dual_returns_empty_dict_when_monitor_aberrant(tmp_path: Path):
|
||
"""capture_dual (3 captures simultanées) ne doit pas non plus produire
|
||
de PNG sur dim aberrante : c'est la même source d'empoisonnement.
|
||
"""
|
||
aberrant_monitors = [
|
||
{"left": 0, "top": 0, "width": 2560, "height": 1660},
|
||
{"left": 0, "top": 0, "width": 2560, "height": 60},
|
||
]
|
||
factory = _make_mock_mss([aberrant_monitors])
|
||
|
||
with patch("agent_v0.agent_v1.vision.capturer.mss.mss", side_effect=factory), \
|
||
patch("agent_v0.agent_v1.vision.capturer.time.sleep"):
|
||
cap = _vision_capturer(tmp_path)
|
||
result = cap.capture_dual(x=100, y=200, screenshot_id="shot_dual")
|
||
|
||
assert result == {}, (
|
||
f"capture_dual doit retourner {{}} sur dim aberrante, got {result!r}"
|
||
)
|
||
|
||
|
||
# ============================================================================
|
||
# Test 6 — capture_active_window bénéficie aussi de la garde
|
||
# ============================================================================
|
||
|
||
|
||
def test_capture_active_window_returns_none_when_monitor_aberrant(tmp_path: Path):
|
||
"""capture_active_window (standalone, sans full_img fourni) doit aussi
|
||
refuser de capturer sur monitor aberrant.
|
||
"""
|
||
aberrant_monitors = [
|
||
{"left": 0, "top": 0, "width": 2560, "height": 1660},
|
||
{"left": 0, "top": 0, "width": 2560, "height": 60},
|
||
]
|
||
factory = _make_mock_mss([aberrant_monitors])
|
||
|
||
# Mocker get_active_window_rect pour qu'il renvoie une fenêtre valide
|
||
# (sinon le test sort prématurément avant d'atteindre le grab).
|
||
fake_rect = {
|
||
"rect": [100, 100, 800, 600],
|
||
"size": [700, 500],
|
||
"title": "Test Window",
|
||
"app_name": "test_app",
|
||
}
|
||
|
||
with patch("agent_v0.agent_v1.vision.capturer.mss.mss", side_effect=factory), \
|
||
patch("agent_v0.agent_v1.vision.capturer.time.sleep"), \
|
||
patch(
|
||
"agent_v0.agent_v1.window_info_crossplatform.get_active_window_rect",
|
||
return_value=fake_rect,
|
||
):
|
||
cap = _vision_capturer(tmp_path)
|
||
result = cap.capture_active_window(x=200, y=300, screenshot_id="shot_win")
|
||
|
||
assert result is None, (
|
||
f"capture_active_window doit retourner None sur dim aberrante, got {result!r}"
|
||
)
|
||
|
||
|
||
# ============================================================================
|
||
# Test 7 — Non-régression : dim normale produit toujours un PNG
|
||
# ============================================================================
|
||
|
||
|
||
def test_capture_full_context_succeeds_on_normal_dimensions(tmp_path: Path):
|
||
"""Sanity check : la garde ne casse pas le chemin nominal."""
|
||
normal_monitors = [
|
||
{"left": 0, "top": 0, "width": 2560, "height": 1660},
|
||
{"left": 0, "top": 0, "width": 2560, "height": 1600},
|
||
]
|
||
factory = _make_mock_mss([normal_monitors])
|
||
|
||
with patch("agent_v0.agent_v1.vision.capturer.mss.mss", side_effect=factory), \
|
||
patch("agent_v0.agent_v1.vision.capturer.time.sleep"):
|
||
cap = _vision_capturer(tmp_path)
|
||
result = cap.capture_full_context("test_normal", force=True)
|
||
|
||
assert result, f"Capture nominale doit produire un PNG, got {result!r}"
|
||
assert Path(result).exists(), "PNG doit exister sur disque"
|
||
# Un seul appel mss.mss() attendu en cas normal (pas de retry)
|
||
assert len(factory.instances) == 1, (
|
||
f"Un seul appel mss.mss() attendu sur dims saines, vu {len(factory.instances)}"
|
||
)
|
||
|
||
|
||
# ============================================================================
|
||
# Test 8 — fail-closed : capture_dual refuse le fallback monitor secondaire
|
||
# ============================================================================
|
||
|
||
|
||
def test_capture_dual_fails_closed_when_only_secondary_monitor_sane(
|
||
tmp_path: Path, caplog: pytest.LogCaptureFixture
|
||
):
|
||
"""capture_dual reçoit des coords (x, y) en système écran composite.
|
||
Si on capture monitors[2] (offset 2560, 0), le crop calculé via
|
||
img.crop((x, y, ...)) pointe à la mauvaise zone car les coords ne
|
||
sont pas traduites. Plutôt que de produire une image décalée
|
||
silencieusement, on refuse le fallback secondaire pour cette méthode.
|
||
"""
|
||
monitors_with_fallback = [
|
||
{"left": 0, "top": 0, "width": 2560, "height": 1660},
|
||
{"left": 0, "top": 0, "width": 2560, "height": 60}, # primary cassé
|
||
{"left": 2560, "top": 0, "width": 1920, "height": 1080}, # secondary sain
|
||
]
|
||
factory = _make_mock_mss([monitors_with_fallback])
|
||
|
||
with patch("agent_v0.agent_v1.vision.capturer.mss.mss", side_effect=factory), \
|
||
patch("agent_v0.agent_v1.vision.capturer.time.sleep"):
|
||
caplog.set_level(logging.WARNING, logger="agent_v0.agent_v1.vision.capturer")
|
||
cap = _vision_capturer(tmp_path)
|
||
result = cap.capture_dual(x=300, y=400, screenshot_id="shot_dual_fb")
|
||
|
||
assert result == {}, (
|
||
f"capture_dual doit fail-closed sur fallback secondaire, got {result!r}"
|
||
)
|
||
msg = " ".join(r.getMessage() for r in caplog.records).lower()
|
||
assert "fallback" in msg or "secondaire" in msg or "refus" in msg, (
|
||
f"Un log doit expliquer le refus du fallback pour coords : {msg!r}"
|
||
)
|
||
|
||
|
||
# ============================================================================
|
||
# Test 9 — fail-closed : capture_active_window refuse le fallback secondaire
|
||
# ============================================================================
|
||
|
||
|
||
def test_capture_active_window_fails_closed_when_only_secondary_monitor_sane(
|
||
tmp_path: Path,
|
||
):
|
||
"""Même raison que test 8 : capture_active_window cropperait depuis l'image
|
||
de monitors[2] avec un win_rect en coords globales → zone fausse.
|
||
"""
|
||
monitors_with_fallback = [
|
||
{"left": 0, "top": 0, "width": 2560, "height": 1660},
|
||
{"left": 0, "top": 0, "width": 2560, "height": 60},
|
||
{"left": 2560, "top": 0, "width": 1920, "height": 1080},
|
||
]
|
||
factory = _make_mock_mss([monitors_with_fallback])
|
||
fake_rect = {
|
||
"rect": [100, 100, 800, 600], # coords globales dans monitors[1]
|
||
"size": [700, 500],
|
||
"title": "Test Window",
|
||
"app_name": "test_app",
|
||
}
|
||
|
||
with patch("agent_v0.agent_v1.vision.capturer.mss.mss", side_effect=factory), \
|
||
patch("agent_v0.agent_v1.vision.capturer.time.sleep"), \
|
||
patch(
|
||
"agent_v0.agent_v1.window_info_crossplatform.get_active_window_rect",
|
||
return_value=fake_rect,
|
||
):
|
||
cap = _vision_capturer(tmp_path)
|
||
result = cap.capture_active_window(x=200, y=300, screenshot_id="shot_win_fb")
|
||
|
||
assert result is None, (
|
||
f"capture_active_window doit fail-closed sur fallback secondaire, got {result!r}"
|
||
)
|
||
|
||
|
||
# ============================================================================
|
||
# Test 10 — mss noir : fallback ImageGrab
|
||
# ============================================================================
|
||
|
||
|
||
def test_capture_screen_image_falls_back_to_imagegrab_when_mss_is_black():
|
||
"""Un frame mss noir ne doit plus être accepté silencieusement.
|
||
|
||
Si ImageGrab fournit une image exploitable, elle doit être retenue.
|
||
"""
|
||
from agent_v0.agent_v1.vision import capturer
|
||
|
||
black_img = _solid_img((0, 0, 0))
|
||
fallback_img = _solid_img((210, 180, 90))
|
||
monitor = {"left": 0, "top": 0, "width": 320, "height": 240}
|
||
|
||
with patch.object(
|
||
capturer, "_acquire_safe_grab", return_value=(monitor, black_img)
|
||
), patch.object(
|
||
capturer,
|
||
"_capture_via_imagegrab",
|
||
return_value=(monitor, fallback_img, {
|
||
"backend": "imagegrab",
|
||
"luma": {"mean": 180.0, "stddev": 0.0, "min": 180, "max": 180},
|
||
}),
|
||
):
|
||
out_monitor, out_img, meta = capturer.capture_screen_image()
|
||
|
||
assert out_monitor == monitor
|
||
assert out_img is fallback_img
|
||
assert meta["backend"] == "imagegrab"
|
||
|
||
|
||
# ============================================================================
|
||
# Test 11 — capture_dual dégradé : conserver window_capture
|
||
# ============================================================================
|
||
|
||
|
||
def test_capture_dual_keeps_window_capture_when_fullscreen_is_unavailable(
|
||
tmp_path: Path,
|
||
):
|
||
"""Même sans full/crop, la capture fenêtre doit survivre.
|
||
|
||
Cela permet au serveur de conserver un contexte utile plutôt que de
|
||
travailler sur un écran noir.
|
||
"""
|
||
fake_window = {
|
||
"window_image": str(tmp_path / "window_only.png"),
|
||
"window_title": "Bloc-notes",
|
||
"app_name": "notepad.exe",
|
||
"window_rect": [100, 100, 800, 600],
|
||
"window_size": [700, 500],
|
||
"click_in_window": [42, 24],
|
||
"click_inside_window": True,
|
||
}
|
||
|
||
cap = _vision_capturer(tmp_path)
|
||
with patch(
|
||
"agent_v0.agent_v1.vision.capturer.capture_screen_image",
|
||
return_value=(None, None, {"backend": "mss_black"}),
|
||
), patch.object(cap, "capture_active_window", return_value=fake_window):
|
||
result = cap.capture_dual(x=200, y=300, screenshot_id="shot_dual")
|
||
|
||
assert "full" not in result
|
||
assert "crop" not in result
|
||
assert result["window_capture"] == fake_window
|
||
|
||
|
||
# ============================================================================
|
||
# Test 12 — non-régression : capture_full_context PEUT utiliser le fallback
|
||
# ============================================================================
|
||
|
||
|
||
def test_capture_full_context_still_uses_secondary_fallback(
|
||
tmp_path: Path, caplog: pytest.LogCaptureFixture
|
||
):
|
||
"""capture_full_context (heartbeat) ne porte pas de coords client : un
|
||
écran sain quelconque suffit. Le fallback secondaire reste autorisé.
|
||
Sinon le heartbeat tomberait dès qu'un monitor est cassé en permanence.
|
||
"""
|
||
monitors_with_fallback = [
|
||
{"left": 0, "top": 0, "width": 2560, "height": 1660},
|
||
{"left": 0, "top": 0, "width": 2560, "height": 60},
|
||
{"left": 2560, "top": 0, "width": 1920, "height": 1080},
|
||
]
|
||
factory = _make_mock_mss([monitors_with_fallback])
|
||
|
||
with patch("agent_v0.agent_v1.vision.capturer.mss.mss", side_effect=factory), \
|
||
patch("agent_v0.agent_v1.vision.capturer.time.sleep"):
|
||
caplog.set_level(logging.WARNING, logger="agent_v0.agent_v1.vision.capturer")
|
||
cap = _vision_capturer(tmp_path)
|
||
result = cap.capture_full_context("test_heartbeat_fb", force=True)
|
||
|
||
assert result, (
|
||
f"capture_full_context doit accepter fallback (heartbeat sans coords), got {result!r}"
|
||
)
|
||
assert Path(result).exists()
|