Dom/rpa_vision_v3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(p0): secure agent revocation and R6 worker queue

Browse Source
This commit is contained in:
Dom
2026-06-02 15:52:35 +02:00
parent 2dd306724c
commit 7a1a5cb6fd
11 changed files with 2869 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files

459
tests/integration/test_stream_processor.py
View File

@@ -11,6 +11,7 @@ import shutil
import sys
import tempfile
import threading
import types
from pathlib import Path
from unittest.mock import MagicMock, patch
@@ -171,6 +172,271 @@ class TestLiveSessionManager:
class TestStreamProcessor:
def test_default_initialization_stays_light(self, temp_dir):
"""Par défaut, l'API HTTP ne charge pas les composants VLM/GPU."""
from agent_v0.server_v1.stream_processor import StreamProcessor
test_processor = StreamProcessor(data_dir=temp_dir)
test_processor._ensure_initialized()
assert test_processor._initialized is True
assert test_processor._screen_analyzer is None
assert test_processor._clip_embedder is None
assert test_processor._faiss_manager is None
def test_enable_vlm_initialization_loads_components(self, temp_dir, monkeypatch):
"""Le worker VLM peut explicitement charger ScreenAnalyzer/CLIP/FAISS."""
from agent_v0.server_v1.stream_processor import StreamProcessor
screen_module = types.ModuleType("core.pipeline.screen_analyzer")
clip_module = types.ModuleType("core.embedding.clip_embedder")
state_module = types.ModuleType("core.embedding.state_embedding_builder")
faiss_module = types.ModuleType("core.embedding.faiss_manager")
class FakeScreenAnalyzer:
def __init__(self, session_id=""):
self.session_id = session_id
class FakeCLIPEmbedder:
pass
class FakeStateEmbeddingBuilder:
def __init__(self, *args, **kwargs):
self.args = args
self.kwargs = kwargs
class FakeFAISSManager:
def __init__(self, *args, **kwargs):
self.args = args
self.kwargs = kwargs
self.index = MagicMock(ntotal=0)
screen_module.ScreenAnalyzer = FakeScreenAnalyzer
clip_module.CLIPEmbedder = FakeCLIPEmbedder
state_module.StateEmbeddingBuilder = FakeStateEmbeddingBuilder
faiss_module.FAISSManager = FakeFAISSManager
monkeypatch.setitem(sys.modules, "core.pipeline.screen_analyzer", screen_module)
monkeypatch.setitem(sys.modules, "core.embedding.clip_embedder", clip_module)
monkeypatch.setitem(sys.modules, "core.embedding.state_embedding_builder", state_module)
monkeypatch.setitem(sys.modules, "core.embedding.faiss_manager", faiss_module)
test_processor = StreamProcessor(data_dir=temp_dir, enable_vlm=True)
test_processor._ensure_initialized()
assert test_processor._initialized is True
assert isinstance(test_processor._screen_analyzer, FakeScreenAnalyzer)
assert isinstance(test_processor._clip_embedder, FakeCLIPEmbedder)
assert isinstance(test_processor._state_embedding_builder, FakeStateEmbeddingBuilder)
assert isinstance(test_processor._faiss_manager, FakeFAISSManager)
def test_enable_vlm_screen_analyzer_failure_does_not_cache_broken_state(
self, temp_dir, monkeypatch, caplog
):
"""N1 anti-poison : en mode VLM, si ScreenAnalyzer échoue à l'init, ne PAS
figer _initialized=True (sinon le worker reste cassé à vie, cf. blocage R6
des 5 jours). Doit logger en critical et permettre un retry au cycle suivant.
"""
import logging
from agent_v0.server_v1.stream_processor import StreamProcessor
screen_module = types.ModuleType("core.pipeline.screen_analyzer")
clip_module = types.ModuleType("core.embedding.clip_embedder")
state_module = types.ModuleType("core.embedding.state_embedding_builder")
faiss_module = types.ModuleType("core.embedding.faiss_manager")
class BrokenScreenAnalyzer:
def __init__(self, session_id=""):
raise RuntimeError("CUDA indisponible au démarrage du worker")
class HealedScreenAnalyzer:
def __init__(self, session_id=""):
self.session_id = session_id
class FakeCLIPEmbedder:
pass
class FakeStateEmbeddingBuilder:
def __init__(self, *args, **kwargs):
pass
class FakeFAISSManager:
def __init__(self, *args, **kwargs):
self.index = MagicMock(ntotal=0)
screen_module.ScreenAnalyzer = BrokenScreenAnalyzer
clip_module.CLIPEmbedder = FakeCLIPEmbedder
state_module.StateEmbeddingBuilder = FakeStateEmbeddingBuilder
faiss_module.FAISSManager = FakeFAISSManager
monkeypatch.setitem(sys.modules, "core.pipeline.screen_analyzer", screen_module)
monkeypatch.setitem(sys.modules, "core.embedding.clip_embedder", clip_module)
monkeypatch.setitem(sys.modules, "core.embedding.state_embedding_builder", state_module)
monkeypatch.setitem(sys.modules, "core.embedding.faiss_manager", faiss_module)
test_processor = StreamProcessor(data_dir=temp_dir, enable_vlm=True)
with caplog.at_level(logging.CRITICAL):
test_processor._ensure_initialized()
# Pas de cache à vie : l'état reste retry-able
assert test_processor._initialized is False
assert test_processor._screen_analyzer is None
assert any(rec.levelno == logging.CRITICAL for rec in caplog.records), (
"un log critical doit signaler le worker VLM dégradé"
)
# Retry au cycle suivant : ScreenAnalyzer réparé → init réussit cette fois
screen_module.ScreenAnalyzer = HealedScreenAnalyzer
test_processor._ensure_initialized()
assert test_processor._initialized is True
assert isinstance(test_processor._screen_analyzer, HealedScreenAnalyzer)
def test_worker_writes_health_file_with_component_status(self, tmp_path, monkeypatch):
"""N2 : le worker écrit _worker_health.json avec le statut des composants
dérivé du processor, le pid, les stats et le statut global."""
from agent_v0.server_v1 import run_worker
data_dir = tmp_path / "data" / "training"
data_dir.mkdir(parents=True)
monkeypatch.setattr(run_worker, "DATA_DIR", data_dir)
worker = run_worker.VLMWorker()
class FakeProc:
_enable_vlm = True
_screen_analyzer = object()
_clip_embedder = object()
_faiss_manager = object()
_state_embedding_builder = object()
worker._processor = FakeProc()
worker._stats["sessions_processed"] = 1
worker._stats["total_screenshots_analyzed"] = 7
worker._write_health("healthy")
health_path = data_dir / "_worker_health.json"
assert health_path.exists()
data = json.loads(health_path.read_text(encoding="utf-8"))
assert data["status"] == "healthy"
assert data["pid"] == os.getpid()
assert data["components"] == {
"screen_analyzer": True,
"clip_embedder": True,
"faiss_manager": True,
"state_embedding_builder": True,
}
assert data["stats"]["sessions_processed"] == 1
assert data["stats"]["total_screenshots_analyzed"] == 7
def test_worker_health_degraded_when_screen_analyzer_missing(self, tmp_path, monkeypatch):
"""N2 : worker VLM dont le ScreenAnalyzer est absent => status 'degraded',
même si l'appelant demande 'healthy'."""
from agent_v0.server_v1 import run_worker
data_dir = tmp_path / "data" / "training"
data_dir.mkdir(parents=True)
monkeypatch.setattr(run_worker, "DATA_DIR", data_dir)
worker = run_worker.VLMWorker()
class DegradedProc:
_enable_vlm = True
_screen_analyzer = None
_clip_embedder = object()
_faiss_manager = object()
_state_embedding_builder = None
worker._processor = DegradedProc()
worker._write_health("healthy")
data = json.loads((data_dir / "_worker_health.json").read_text(encoding="utf-8"))
assert data["status"] == "degraded"
assert data["components"]["screen_analyzer"] is False
def test_worker_health_file_contains_no_patient_data(self, tmp_path, monkeypatch):
"""N2 confidentialité : le health file ne contient que des clés autorisées —
aucune donnée patient (OCR, noms de fichiers screenshots, contenu session)."""
from agent_v0.server_v1 import run_worker
data_dir = tmp_path / "data" / "training"
data_dir.mkdir(parents=True)
monkeypatch.setattr(run_worker, "DATA_DIR", data_dir)
worker = run_worker.VLMWorker()
worker._current_session = "sess_20260529T154427_f95956"
worker._write_health("busy")
data = json.loads((data_dir / "_worker_health.json").read_text(encoding="utf-8"))
allowed_top = {
"pid", "started_at", "last_cycle", "current_session",
"queue_length", "components", "stats", "status",
}
assert set(data.keys()) <= allowed_top, f"clés inattendues: {set(data.keys()) - allowed_top}"
# current_session ne porte que l'identifiant, pas de contenu de session
assert data["current_session"] == "sess_20260529T154427_f95956"
def test_sd_notify_noop_without_socket(self, monkeypatch):
"""N3 : hors systemd (NOTIFY_SOCKET absent), _sd_notify est un no-op
silencieux qui retourne False — jamais d'exception."""
from agent_v0.server_v1 import run_worker
monkeypatch.delenv("NOTIFY_SOCKET", raising=False)
worker = run_worker.VLMWorker()
assert worker._sd_notify("WATCHDOG=1") is False
def test_sd_notify_sends_watchdog_to_socket(self, tmp_path, monkeypatch):
"""N3 : sous systemd, _sd_notify écrit l'état brut dans $NOTIFY_SOCKET."""
import socket
from agent_v0.server_v1 import run_worker
sock_path = str(tmp_path / "notify.sock")
listener = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
listener.bind(sock_path)
listener.settimeout(2)
try:
monkeypatch.setenv("NOTIFY_SOCKET", sock_path)
worker = run_worker.VLMWorker()
assert worker._sd_notify("WATCHDOG=1") is True
received = listener.recv(64)
assert received == b"WATCHDOG=1"
finally:
listener.close()
def test_vlm_worker_uses_training_root_data_dir(self, tmp_path, monkeypatch):
"""Le worker R6 doit produire workflows/embeddings sous data/training."""
from agent_v0.server_v1 import run_worker
data_dir = tmp_path / "data" / "training"
live_sessions_dir = data_dir / "live_sessions"
monkeypatch.setattr(run_worker, "DATA_DIR", data_dir)
monkeypatch.setattr(run_worker, "LIVE_SESSIONS_DIR", live_sessions_dir)
test_worker = run_worker.VLMWorker()
test_processor = test_worker._get_processor()
assert test_processor.data_dir == data_dir
assert test_processor.session_manager._live_sessions_dir == live_sessions_dir
assert test_processor._enable_vlm is True
def test_stream_worker_standalone_uses_training_root_data_dir(self, tmp_path):
"""Le StreamWorker standalone garde aussi data/training comme racine."""
from agent_v0.server_v1.worker_stream import StreamWorker
live_sessions_dir = tmp_path / "data" / "training" / "live_sessions"
test_worker = StreamWorker(live_dir=str(live_sessions_dir))
assert test_worker.processor.data_dir == live_sessions_dir.parent
assert test_worker.processor.session_manager._live_sessions_dir == live_sessions_dir
assert test_worker.processor._enable_vlm is True
def test_process_event(self, processor):
result = processor.process_event("sess_010", {
"type": "mouse_click",
@@ -181,6 +447,49 @@ class TestStreamProcessor:
session = processor.session_manager.get_session("sess_010")
assert session.last_window_info["title"] == "Chrome"
def test_restore_user_events_keeps_key_combo(self, processor, tmp_path):
session_id = "sess_restore_combo"
session_dir = tmp_path / session_id
session_dir.mkdir()
(session_dir / "live_events.jsonl").write_text(
json.dumps({
"session_id": session_id,
"timestamp": 1779900720.0,
"event": {
"type": "key_combo",
"keys": ["win", "s"],
"raw_keys": [
{"action": "release", "kind": "vk", "vk": 83, "char": "s"},
{"action": "release", "kind": "key", "name": "cmd"},
],
"timestamp": 1779900719.5,
"window": {"title": "Rechercher", "app_name": "SearchHost.exe"},
"screenshot_id": "shot_0001",
},
}) + "\n"
+ json.dumps({
"session_id": session_id,
"timestamp": 1779900725.0,
"event": {
"type": "text_input",
"text": "test",
"timestamp": 1779900725.0,
"window": {"title": "Rechercher", "app_name": "SearchHost.exe"},
},
}) + "\n",
encoding="utf-8",
)
processor.session_manager.add_event(session_id, {"type": "text_input", "text": "old"})
processor._restore_user_events(session_id, session_dir)
session = processor.session_manager.get_session(session_id)
assert [event["type"] for event in session.events] == ["key_combo", "text_input"]
assert session.events[0]["keys"] == ["win", "s"]
assert session.events[0]["raw_keys"][0]["vk"] == 83
assert session.events[0]["screenshot_id"] == "shot_0001"
def test_process_crop(self, processor):
result = processor.process_crop("sess_011", "shot_001_crop", "/tmp/crop.png")
assert result["status"] == "crop_stored"
@@ -479,6 +788,156 @@ class TestStreamProcessor:
assert first_hints.get("active_tab_label") == "test"
assert "fermer l'onglet actif 'test'" in first_spec.get("vlm_description", "")
def test_build_replay_save_as_button_gets_semantic_target(
self, tmp_path, monkeypatch,
):
"""Le clic du bouton Enregistrer dans Save As ne doit pas rester
anchor-only/positionnel.
Régression live 2026-05-25 : avec RPA_SKIP_BUILD_VISION, l'action
Save As était seulement décrite par position + crop, puis résolue par
template matching trop haut/gauche. Le builder doit encoder le bouton
primaire stable ``Enregistrer``.
"""
from agent_v0.server_v1 import stream_processor as sp
session_dir = tmp_path / "sess"
(session_dir / "shots").mkdir(parents=True)
monkeypatch.setattr(sp, "_load_crop_for_event", lambda *args, **kwargs: "abc123")
monkeypatch.setattr(
sp,
"enrich_click_from_screenshot",
lambda *args, **kwargs: {
"anchor_image_base64": "abc123",
"by_text": "",
"by_role": "",
"vlm_description": "positionnel",
},
)
monkeypatch.setattr(sp, "_attach_expected_screenshots", lambda *args, **kwargs: None)
monkeypatch.setattr(sp, "_enrich_actions_with_intentions", lambda *args, **kwargs: None)
monkeypatch.setattr(sp, "_unload_gemma4", lambda *args, **kwargs: None)
events = [
{"event": {
"type": "mouse_click",
"timestamp": 1.0,
"pos": [1329, 1265],
"button": "left",
"screenshot_id": "shot_006",
"screen_metadata": {"screen_resolution": [2560, 1600]},
"window": {"title": "Enregistrer sous", "app_name": "Notepad.exe"},
"window_capture": {
"rect": [332, 522, 1613, 1323],
"click_relative": [997, 743],
"window_size": [1281, 801],
"click_inside_window": True,
},
}},
{"event": {
"type": "window_focus_change",
"timestamp": 1.2,
"from": {"title": "Enregistrer sous", "app_name": "Notepad.exe"},
"to": {"title": "*test – Bloc-notes", "app_name": "Notepad.exe"},
}},
]
actions = sp.build_replay_from_raw_events(
events,
session_id="sess_save_as_button",
session_dir=str(session_dir),
)
clicks = [a for a in actions if a.get("type") == "click"]
assert len(clicks) == 1
spec = clicks[0].get("target_spec", {})
hints = spec.get("context_hints") or {}
assert spec.get("by_text") == "Enregistrer"
assert spec.get("by_text_source") == "heuristic"
assert spec.get("by_role") == "button"
assert spec.get("window_title") == "Enregistrer sous"
assert hints.get("interaction") == "save_dialog_primary_button"
assert hints.get("expected_after_window") == "*test – Bloc-notes"
assert clicks[0].get("expected_window_title") == "*test – Bloc-notes"
def test_build_replay_cuts_post_save_out_of_window_click(
self, tmp_path, monkeypatch,
):
"""Le clic hors fenêtre après retour Bloc-notes est parasite.
C'est l'ancienne action finale 17/18 : coordonnées en bas à droite,
``click_inside_window=false``. Elle ne fait pas partie du coeur
"saisir et enregistrer".
"""
from agent_v0.server_v1 import stream_processor as sp
session_dir = tmp_path / "sess"
(session_dir / "shots").mkdir(parents=True)
monkeypatch.setattr(sp, "_load_crop_for_event", lambda *args, **kwargs: "abc123")
monkeypatch.setattr(
sp,
"enrich_click_from_screenshot",
lambda *args, **kwargs: {"anchor_image_base64": "abc123"},
)
monkeypatch.setattr(sp, "_attach_expected_screenshots", lambda *args, **kwargs: None)
monkeypatch.setattr(sp, "_enrich_actions_with_intentions", lambda *args, **kwargs: None)
monkeypatch.setattr(sp, "_unload_gemma4", lambda *args, **kwargs: None)
events = [
{"event": {
"type": "mouse_click",
"timestamp": 1.0,
"pos": [1329, 1265],
"button": "left",
"screenshot_id": "shot_006",
"screen_metadata": {"screen_resolution": [2560, 1600]},
"window": {"title": "Enregistrer sous", "app_name": "Notepad.exe"},
"window_capture": {
"rect": [332, 522, 1613, 1323],
"click_relative": [997, 743],
"window_size": [1281, 801],
"click_inside_window": True,
},
}},
{"event": {
"type": "window_focus_change",
"timestamp": 1.2,
"from": {"title": "Enregistrer sous", "app_name": "Notepad.exe"},
"to": {"title": "*test – Bloc-notes", "app_name": "Notepad.exe"},
}},
{"event": {
"type": "mouse_click",
"timestamp": 1.5,
"pos": [2248, 1577],
"button": "left",
"screenshot_id": "shot_007",
"screen_metadata": {"screen_resolution": [2560, 1600]},
"window": {
"title": "http192.168.1.408765dossier.htmlid=.txt – Bloc-notes",
"app_name": "Notepad.exe",
},
"window_capture": {
"rect": [323, 522, 2243, 1638],
"click_relative": [1925, 1055],
"window_size": [1920, 1116],
"click_inside_window": False,
},
}},
]
actions = sp.build_replay_from_raw_events(
events,
session_id="sess_post_save_cut",
session_dir=str(session_dir),
)
clicks = [a for a in actions if a.get("type") == "click"]
assert len(clicks) == 1
assert clicks[0].get("target_spec", {}).get("by_text") == "Enregistrer"
assert clicks[0].get("expected_window_title") == "*test – Bloc-notes"
# =========================================================================
# StreamWorker