fix(agent): capture JPEG+downscale (allege CPU/disque, frequence intacte) + robustesse chemin _background/shots
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Dom
320
tests/unit/test_capturer_capture_io_format.py
Normal file
320
tests/unit/test_capturer_capture_io_format.py
Normal file
@@ -0,0 +1,320 @@
|
||||
"""Politique de format des captures + robustesse du répertoire shots.
|
||||
|
||||
Deux corrections testées ici (agent_v0/agent_v1/vision) :
|
||||
|
||||
1. FORMAT (allègement) : `capturer.py` doit déléguer l'écriture à
|
||||
`capture_io.save_capture`, qui applique la politique :
|
||||
- crop → PNG lossless (cible de grounding qwen3-vl)
|
||||
- full/window/context → JPEG q85
|
||||
- heartbeat → JPEG downscalé (largeur max ~1280)
|
||||
Aujourd'hui tout était sauvé en `img.save(path, "PNG", quality=...)`
|
||||
(le `quality` est ignoré par PNG → PNG lossless plein écran, ~90 Go).
|
||||
|
||||
2. BUG chemin (poste Émilie) : ``[Errno 2] No such file or directory:
|
||||
..._background/shots/context...``. Le répertoire `shots/` est créé une
|
||||
seule fois dans `__init__`, mais l'auto-cleanup (`SessionStorage`,
|
||||
`shutil.rmtree`) peut supprimer tout le dossier de session `_background`.
|
||||
Les sauvegardes suivantes doivent recréer le répertoire cible
|
||||
(`os.makedirs(dir, exist_ok=True)`) avant chaque écriture.
|
||||
|
||||
Tests 100% mockés : aucune vraie capture écran (mss est patché).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from PIL import Image
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers (repris du style de test_capturer_monitor_guard.py)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_mock_mss(monitors):
|
||||
"""Mock `mss.mss()` renvoyant un monitor sain unique (image grise unie)."""
|
||||
|
||||
def factory():
|
||||
instance = MagicMock()
|
||||
instance.monitors = monitors
|
||||
grab_result = MagicMock()
|
||||
m = monitors[1] if len(monitors) > 1 else monitors[0]
|
||||
w, h = m["width"], m["height"]
|
||||
grab_result.size = (w, h)
|
||||
grab_result.bgra = b"\x80\x80\x80\x00" * (w * h)
|
||||
instance.grab = MagicMock(return_value=grab_result)
|
||||
cm = MagicMock()
|
||||
cm.__enter__ = MagicMock(return_value=instance)
|
||||
cm.__exit__ = MagicMock(return_value=False)
|
||||
return cm
|
||||
|
||||
return factory
|
||||
|
||||
|
||||
_NORMAL_MONITORS = [
|
||||
{"left": 0, "top": 0, "width": 800, "height": 600}, # composite
|
||||
{"left": 0, "top": 0, "width": 800, "height": 600}, # primaire sain
|
||||
]
|
||||
|
||||
|
||||
def _vision_capturer(tmp_path):
|
||||
from agent_v0.agent_v1.vision.capturer import VisionCapturer
|
||||
|
||||
return VisionCapturer(str(tmp_path))
|
||||
|
||||
|
||||
def _patch_mss():
|
||||
"""Contexte : mss patché + time.sleep no-op + pas de floutage.
|
||||
|
||||
Le floutage est désactivé pour isoler la politique d'écriture (le blur
|
||||
ouvre/modifie l'image mais n'impacte pas le format de sortie ; on le coupe
|
||||
pour rester déterministe).
|
||||
"""
|
||||
return (
|
||||
patch(
|
||||
"agent_v0.agent_v1.vision.capturer.mss.mss",
|
||||
side_effect=_make_mock_mss(_NORMAL_MONITORS),
|
||||
),
|
||||
patch("agent_v0.agent_v1.vision.capturer.time.sleep"),
|
||||
patch("agent_v0.agent_v1.vision.capturer.BLUR_SENSITIVE", False),
|
||||
)
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# PARTIE A — Politique save_capture (unité capture_io)
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
def test_save_capture_crop_stays_png(tmp_path: Path):
|
||||
from agent_v0.agent_v1.vision import capture_io
|
||||
|
||||
img = Image.new("RGB", (80, 80), (10, 20, 30))
|
||||
out = capture_io.save_capture(img, str(tmp_path / "shot_crop"), "crop")
|
||||
|
||||
assert out.endswith(".png"), f"crop doit rester PNG, got {out!r}"
|
||||
assert Path(out).exists()
|
||||
with Image.open(out) as reopened:
|
||||
assert reopened.format == "PNG"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kind", ["full", "window", "context"])
|
||||
def test_save_capture_context_kinds_are_jpeg(tmp_path: Path, kind: str):
|
||||
from agent_v0.agent_v1.vision import capture_io
|
||||
|
||||
img = Image.new("RGB", (640, 480), (120, 130, 140))
|
||||
out = capture_io.save_capture(img, str(tmp_path / f"shot_{kind}"), kind)
|
||||
|
||||
assert out.endswith(".jpg"), f"{kind} doit être JPEG, got {out!r}"
|
||||
assert Path(out).exists()
|
||||
with Image.open(out) as reopened:
|
||||
assert reopened.format == "JPEG"
|
||||
|
||||
|
||||
def test_save_capture_heartbeat_is_downscaled_jpeg(tmp_path: Path):
|
||||
from agent_v0.agent_v1.vision import capture_io
|
||||
|
||||
# Image large (2560) → doit être réduite à HEARTBEAT_MAX_WIDTH.
|
||||
img = Image.new("RGB", (2560, 1440), (50, 60, 70))
|
||||
out = capture_io.save_capture(img, str(tmp_path / "hb"), "heartbeat")
|
||||
|
||||
assert out.endswith(".jpg")
|
||||
with Image.open(out) as reopened:
|
||||
assert reopened.format == "JPEG"
|
||||
assert reopened.width == capture_io.HEARTBEAT_MAX_WIDTH, (
|
||||
f"heartbeat doit être downscalé à {capture_io.HEARTBEAT_MAX_WIDTH}, "
|
||||
f"got {reopened.width}"
|
||||
)
|
||||
# ratio préservé (1440 * 1280/2560 = 720)
|
||||
assert reopened.height == 720
|
||||
|
||||
|
||||
def test_save_capture_heartbeat_smaller_than_max_is_not_upscaled(tmp_path: Path):
|
||||
from agent_v0.agent_v1.vision import capture_io
|
||||
|
||||
img = Image.new("RGB", (640, 360), (1, 2, 3))
|
||||
out = capture_io.save_capture(img, str(tmp_path / "hb_small"), "heartbeat")
|
||||
with Image.open(out) as reopened:
|
||||
assert reopened.width == 640, "no-op si déjà plus petit que le max"
|
||||
|
||||
|
||||
def test_save_capture_heartbeat_downscale_reduces_pixel_count(tmp_path: Path):
|
||||
"""Preuve de l'allègement heartbeat par la mesure objective du code :
|
||||
le downscale réduit le nombre de pixels (2560×1440 → 1280×720 = /4 surface).
|
||||
On mesure la géométrie de sortie (déterministe), pas le poids d'un JPEG
|
||||
synthétique (qui dépend de libjpeg et n'est pas représentatif d'un vrai
|
||||
écran)."""
|
||||
from agent_v0.agent_v1.vision import capture_io
|
||||
|
||||
src = Image.new("RGB", (2560, 1440))
|
||||
out = capture_io.save_capture(src, str(tmp_path / "hb_measure"), "heartbeat")
|
||||
with Image.open(out) as small:
|
||||
src_pixels = src.width * src.height
|
||||
out_pixels = small.width * small.height
|
||||
assert out_pixels < src_pixels / 3, (
|
||||
f"Le downscale heartbeat doit diviser la surface par ~4 "
|
||||
f"({src_pixels} → {out_pixels})"
|
||||
)
|
||||
|
||||
|
||||
def test_save_capture_rejects_unknown_kind(tmp_path: Path):
|
||||
from agent_v0.agent_v1.vision import capture_io
|
||||
|
||||
img = Image.new("RGB", (10, 10))
|
||||
with pytest.raises(ValueError):
|
||||
capture_io.save_capture(img, str(tmp_path / "x"), "bogus")
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# PARTIE B — Câblage dans capturer.py (format des sorties runtime)
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
def test_capture_full_context_writes_jpeg(tmp_path: Path):
|
||||
"""capture_full_context (context / focus_change / result_of_*) → JPEG."""
|
||||
p1, p2, p3 = _patch_mss()
|
||||
with p1, p2, p3:
|
||||
cap = _vision_capturer(tmp_path)
|
||||
out = cap.capture_full_context("focus_change", force=True)
|
||||
|
||||
assert out, "capture attendue"
|
||||
assert out.endswith(".jpg"), f"context doit être JPEG, got {out!r}"
|
||||
assert Path(out).exists()
|
||||
with Image.open(out) as im:
|
||||
assert im.format == "JPEG"
|
||||
|
||||
|
||||
def test_capture_full_context_heartbeat_is_jpeg(tmp_path: Path):
|
||||
"""Un suffixe 'heartbeat' doit produire un JPEG (downscalé côté politique)."""
|
||||
p1, p2, p3 = _patch_mss()
|
||||
with p1, p2, p3:
|
||||
cap = _vision_capturer(tmp_path)
|
||||
out = cap.capture_full_context("heartbeat", force=True)
|
||||
|
||||
assert out.endswith(".jpg"), f"heartbeat doit être JPEG, got {out!r}"
|
||||
with Image.open(out) as im:
|
||||
assert im.format == "JPEG"
|
||||
|
||||
|
||||
def test_capture_dual_full_is_jpeg_crop_is_png(tmp_path: Path):
|
||||
"""capture_dual : full/window en JPEG, crop en PNG (contrat serveur)."""
|
||||
p1, p2, p3 = _patch_mss()
|
||||
with p1, p2, p3, patch(
|
||||
# Neutraliser la capture fenêtre (dépend d'API OS) pour isoler full+crop
|
||||
"agent_v0.agent_v1.vision.capturer.VisionCapturer.capture_active_window",
|
||||
return_value=None,
|
||||
):
|
||||
cap = _vision_capturer(tmp_path)
|
||||
result = cap.capture_dual(x=100, y=200, screenshot_id="shot42")
|
||||
|
||||
assert "full" in result and "crop" in result
|
||||
assert result["full"].endswith(".jpg"), f"full doit être JPEG, got {result['full']!r}"
|
||||
assert result["crop"].endswith(".png"), f"crop doit rester PNG, got {result['crop']!r}"
|
||||
assert Path(result["full"]).exists()
|
||||
assert Path(result["crop"]).exists()
|
||||
with Image.open(result["full"]) as im:
|
||||
assert im.format == "JPEG"
|
||||
with Image.open(result["crop"]) as im:
|
||||
assert im.format == "PNG"
|
||||
|
||||
|
||||
def test_capture_active_window_writes_jpeg(tmp_path: Path):
|
||||
"""La fenêtre active est une vue contextuelle → JPEG."""
|
||||
p1, p2, p3 = _patch_mss()
|
||||
fake_rect = {
|
||||
"rect": [100, 100, 500, 400],
|
||||
"size": [400, 300],
|
||||
"title": "Bloc-notes",
|
||||
"app_name": "notepad.exe",
|
||||
}
|
||||
full_img = Image.new("RGB", (800, 600), (90, 90, 90))
|
||||
with p1, p2, p3, patch(
|
||||
"agent_v0.agent_v1.window_info_crossplatform.get_active_window_rect",
|
||||
return_value=fake_rect,
|
||||
):
|
||||
cap = _vision_capturer(tmp_path)
|
||||
result = cap.capture_active_window(
|
||||
x=200, y=200, screenshot_id="shotW", full_img=full_img
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
assert result["window_image"].endswith(".jpg"), (
|
||||
f"window doit être JPEG, got {result['window_image']!r}"
|
||||
)
|
||||
with Image.open(result["window_image"]) as im:
|
||||
assert im.format == "JPEG"
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# PARTIE C — BUG chemin : shots/ recréé si supprimé par l'auto-cleanup
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
def test_capture_full_context_recreates_shots_dir_after_rmtree(tmp_path: Path):
|
||||
"""Reproduction du bug poste Émilie.
|
||||
|
||||
L'auto-cleanup (`SessionStorage.shutil.rmtree`) supprime tout le dossier
|
||||
de session `_background` (donc `shots/`). Une capture ultérieure ne doit
|
||||
PAS lever `[Errno 2] No such file or directory` : le répertoire cible
|
||||
doit être recréé avant l'écriture.
|
||||
"""
|
||||
p1, p2, p3 = _patch_mss()
|
||||
with p1, p2, p3:
|
||||
cap = _vision_capturer(tmp_path)
|
||||
|
||||
# Simule l'auto-cleanup : la session entière est purgée après ACK.
|
||||
shutil.rmtree(cap.shots_dir)
|
||||
assert not Path(cap.shots_dir).exists()
|
||||
|
||||
out = cap.capture_full_context("context_after_purge", force=True)
|
||||
|
||||
assert out, "La capture doit réussir même après purge du dossier shots"
|
||||
assert Path(out).exists(), "Le fichier doit être physiquement écrit"
|
||||
assert Path(cap.shots_dir).exists(), "shots/ doit avoir été recréé"
|
||||
|
||||
|
||||
def test_capture_dual_recreates_shots_dir_after_rmtree(tmp_path: Path):
|
||||
"""capture_dual doit aussi survivre à la purge du dossier shots."""
|
||||
p1, p2, p3 = _patch_mss()
|
||||
with p1, p2, p3, patch(
|
||||
"agent_v0.agent_v1.vision.capturer.VisionCapturer.capture_active_window",
|
||||
return_value=None,
|
||||
):
|
||||
cap = _vision_capturer(tmp_path)
|
||||
shutil.rmtree(cap.shots_dir)
|
||||
|
||||
result = cap.capture_dual(x=50, y=60, screenshot_id="shot_purge")
|
||||
|
||||
assert result.get("full") and result.get("crop"), (
|
||||
"capture_dual doit produire full+crop même après purge"
|
||||
)
|
||||
assert Path(result["full"]).exists()
|
||||
assert Path(result["crop"]).exists()
|
||||
|
||||
|
||||
def test_capture_active_window_recreates_shots_dir_after_rmtree(tmp_path: Path):
|
||||
"""capture_active_window (crop fenêtre depuis full fourni) survit à la purge."""
|
||||
p1, p2, p3 = _patch_mss()
|
||||
fake_rect = {
|
||||
"rect": [10, 10, 210, 210],
|
||||
"size": [200, 200],
|
||||
"title": "W",
|
||||
"app_name": "w.exe",
|
||||
}
|
||||
full_img = Image.new("RGB", (400, 400), (70, 70, 70))
|
||||
with p1, p2, p3, patch(
|
||||
"agent_v0.agent_v1.window_info_crossplatform.get_active_window_rect",
|
||||
return_value=fake_rect,
|
||||
):
|
||||
cap = _vision_capturer(tmp_path)
|
||||
shutil.rmtree(cap.shots_dir)
|
||||
|
||||
result = cap.capture_active_window(
|
||||
x=50, y=50, screenshot_id="shotW_purge", full_img=full_img
|
||||
)
|
||||
|
||||
assert result is not None, "capture fenêtre doit réussir après purge"
|
||||
assert Path(result["window_image"]).exists()
|
||||
Reference in New Issue
Block a user