Compare commits
16 Commits
2a1b1ed80e
...
f9a0531325
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f9a0531325 | ||
|
|
ab78ae390a | ||
|
|
e59489e2cd | ||
|
|
86e31ada34 | ||
|
|
94fd93ad19 | ||
|
|
50f34b5727 | ||
|
|
a1b3062991 | ||
|
|
a210e5ee32 | ||
|
|
5d235e49f1 | ||
|
|
e679804cfd | ||
|
|
e57b54a100 | ||
|
|
d34c1f2697 | ||
|
|
61664c9a36 | ||
|
|
9ab5ed4671 | ||
|
|
144a5c288a | ||
|
|
e3f61de4ad |
@@ -27,7 +27,7 @@ if platform.system() == "Windows":
|
|||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
AGENT_VERSION = os.environ.get("RPA_AGENT_VERSION", "1.0.1")
|
AGENT_VERSION = os.environ.get("RPA_AGENT_VERSION", "1.0.2")
|
||||||
|
|
||||||
# Identifiant unique de la machine (utilisé pour le multi-machine)
|
# Identifiant unique de la machine (utilisé pour le multi-machine)
|
||||||
# Configurable via variable d'environnement, sinon auto-généré depuis hostname + OS
|
# Configurable via variable d'environnement, sinon auto-généré depuis hostname + OS
|
||||||
@@ -103,6 +103,16 @@ LOG_SHIP_INTERVAL_S = float(os.environ.get("RPA_LOG_SHIP_INTERVAL_S", "30"))
|
|||||||
AUTO_UPDATE_ENABLED = os.environ.get("RPA_AUTO_UPDATE_ENABLED", "false").lower() in (
|
AUTO_UPDATE_ENABLED = os.environ.get("RPA_AUTO_UPDATE_ENABLED", "false").lower() in (
|
||||||
"true", "1", "yes", "on",
|
"true", "1", "yes", "on",
|
||||||
)
|
)
|
||||||
|
# Intervalle entre deux interrogations serveur pour une MAJ (secondes).
|
||||||
|
# Défaut 1 h : une MAJ n'est jamais urgente ; on interroge peu pour ne pas
|
||||||
|
# charger le réseau clinique. Le check ne fait de toute façon aucun swap.
|
||||||
|
AUTO_UPDATE_INTERVAL_S = float(os.environ.get("RPA_AUTO_UPDATE_INTERVAL_S", "3600"))
|
||||||
|
# Dossier de STAGING des ZIP d'update (jamais les fichiers vivants). Équivalent
|
||||||
|
# de `Lea_next\\`. Sous LOCALAPPDATA en prod Windows, sinon à côté de l'agent.
|
||||||
|
AUTO_UPDATE_STAGING_DIR = os.environ.get(
|
||||||
|
"RPA_AUTO_UPDATE_STAGING_DIR",
|
||||||
|
str(BASE_DIR / "_update_staging"),
|
||||||
|
)
|
||||||
|
|
||||||
# Monitoring
|
# Monitoring
|
||||||
PERF_MONITOR_INTERVAL_S = 30
|
PERF_MONITOR_INTERVAL_S = 30
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ from .config import (
|
|||||||
SESSIONS_ROOT, AGENT_VERSION, SERVER_URL, MACHINE_ID, LOG_RETENTION_DAYS, LOG_FILE,
|
SESSIONS_ROOT, AGENT_VERSION, SERVER_URL, MACHINE_ID, LOG_RETENTION_DAYS, LOG_FILE,
|
||||||
SCREEN_RESOLUTION, DPI_SCALE, OS_THEME, API_TOKEN, MAX_SESSION_DURATION_S,
|
SCREEN_RESOLUTION, DPI_SCALE, OS_THEME, API_TOKEN, MAX_SESSION_DURATION_S,
|
||||||
STREAMING_ENDPOINT, LOG_SHIP_ENABLED, LOG_SHIP_INTERVAL_S,
|
STREAMING_ENDPOINT, LOG_SHIP_ENABLED, LOG_SHIP_INTERVAL_S,
|
||||||
|
AUTO_UPDATE_ENABLED, AUTO_UPDATE_INTERVAL_S, AUTO_UPDATE_STAGING_DIR,
|
||||||
)
|
)
|
||||||
from .core.captor import EventCaptorV1
|
from .core.captor import EventCaptorV1
|
||||||
from .core.executor import ActionExecutorV1
|
from .core.executor import ActionExecutorV1
|
||||||
@@ -158,6 +159,31 @@ class AgentV1:
|
|||||||
threading.Thread(target=self._replay_poll_loop, daemon=True).start()
|
threading.Thread(target=self._replay_poll_loop, daemon=True).start()
|
||||||
threading.Thread(target=self._background_heartbeat_loop, daemon=True).start()
|
threading.Thread(target=self._background_heartbeat_loop, daemon=True).start()
|
||||||
|
|
||||||
|
# DETTE-022 v2 : MAJ silencieuse — boucle de check GATED (défaut OFF).
|
||||||
|
# Interroge le serveur (canary-aware) et télécharge en STAGING ; le swap
|
||||||
|
# réel reste réservé révision humaine (updater.apply_update = stub no-op).
|
||||||
|
# Activable poste par poste via RPA_AUTO_UPDATE_ENABLED, sans rebuild.
|
||||||
|
if AUTO_UPDATE_ENABLED:
|
||||||
|
threading.Thread(
|
||||||
|
target=self._auto_update_loop, daemon=True, name="lea-auto-update"
|
||||||
|
).start()
|
||||||
|
|
||||||
|
# MAJ silencieuse — confirmation de boot post-swap. Si Lea.bat vient
|
||||||
|
# d'appliquer une MAJ (marqueur PENDING_BOOT), on désarme le rollback
|
||||||
|
# après ~90 s de tourne STABLE (liveness LOCALE, indépendante du DGX).
|
||||||
|
# Un quit propre avant 90 s confirme aussi (cf. main()). Seul un vrai
|
||||||
|
# crash laisse PENDING_BOOT → rollback au prochain lancement.
|
||||||
|
if _pending_boot_marker_exists():
|
||||||
|
def _boot_confirm():
|
||||||
|
import os as _os
|
||||||
|
import time as _time
|
||||||
|
_time.sleep(float(_os.environ.get("RPA_BOOT_CONFIRM_DELAY_S", "90")))
|
||||||
|
if self.running:
|
||||||
|
_confirm_boot_ok()
|
||||||
|
threading.Thread(
|
||||||
|
target=_boot_confirm, daemon=True, name="lea-boot-confirm"
|
||||||
|
).start()
|
||||||
|
|
||||||
# Mini-serveur HTTP pour captures a la demande (port 5006)
|
# Mini-serveur HTTP pour captures a la demande (port 5006)
|
||||||
self._capture_server = CaptureServer()
|
self._capture_server = CaptureServer()
|
||||||
self._capture_server.start()
|
self._capture_server.start()
|
||||||
@@ -441,6 +467,67 @@ class AgentV1:
|
|||||||
logger.debug(f"[HEARTBEAT] Erreur: {e}")
|
logger.debug(f"[HEARTBEAT] Erreur: {e}")
|
||||||
time.sleep(5)
|
time.sleep(5)
|
||||||
|
|
||||||
|
def _auto_update_loop(self):
|
||||||
|
"""DETTE-022 v2 — boucle de MAJ silencieuse GATED (défaut OFF).
|
||||||
|
|
||||||
|
Interroge périodiquement le serveur (endpoint canary-aware), et si une
|
||||||
|
MAJ est proposée pour CE poste, la télécharge dans le STAGING après
|
||||||
|
vérif SHA256. Le swap réel N'EST PAS fait ici : `updater.run_update_cycle`
|
||||||
|
s'arrête au staging (apply_update = stub réservé révision humaine + swap
|
||||||
|
hors-process par Lea.bat au prochain démarrage).
|
||||||
|
|
||||||
|
SÉCURITÉ — « au bon moment » : on NE stage PAS pendant un enregistrement
|
||||||
|
ou un replay actif (self.session_id / self._replay_active), pour ne pas
|
||||||
|
perturber le travail utilisateur ni consommer du réseau au mauvais
|
||||||
|
moment. Best-effort : aucune exception ne remonte (ne casse jamais Léa).
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
from .network.updater import run_update_cycle
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("[UPDATE] Module updater indisponible : %s", e)
|
||||||
|
return
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
"[UPDATE] Boucle MAJ silencieuse démarrée (intervalle=%.0fs, "
|
||||||
|
"version=%s) — check seul, swap réservé révision humaine",
|
||||||
|
AUTO_UPDATE_INTERVAL_S, AGENT_VERSION,
|
||||||
|
)
|
||||||
|
|
||||||
|
while self.running:
|
||||||
|
# Découpe l'attente pour réagir vite à l'arrêt.
|
||||||
|
waited = 0.0
|
||||||
|
step = 1.0
|
||||||
|
while self.running and waited < AUTO_UPDATE_INTERVAL_S:
|
||||||
|
time.sleep(step)
|
||||||
|
waited += step
|
||||||
|
if not self.running:
|
||||||
|
break
|
||||||
|
|
||||||
|
# « Au bon moment » : jamais en plein travail (enregistrement/replay).
|
||||||
|
if self.session_id or getattr(self, "_replay_active", False):
|
||||||
|
logger.debug("[UPDATE] Report du check (session/replay active)")
|
||||||
|
continue
|
||||||
|
|
||||||
|
try:
|
||||||
|
result = run_update_cycle(
|
||||||
|
local_version=AGENT_VERSION,
|
||||||
|
machine_id=self.machine_id,
|
||||||
|
staging_dir=AUTO_UPDATE_STAGING_DIR,
|
||||||
|
)
|
||||||
|
status = result.get("status")
|
||||||
|
if status == "staged":
|
||||||
|
logger.info(
|
||||||
|
"[UPDATE] MAJ %s téléchargée en staging (SHA256=%s) — "
|
||||||
|
"swap réservé révision humaine, non appliqué",
|
||||||
|
result.get("target_version"),
|
||||||
|
result.get("sha256_verified"),
|
||||||
|
)
|
||||||
|
elif status not in ("up_to_date", "disabled"):
|
||||||
|
logger.debug("[UPDATE] Cycle: %s", result)
|
||||||
|
except Exception as e:
|
||||||
|
# run_update_cycle est déjà best-effort ; double filet ici.
|
||||||
|
logger.debug("[UPDATE] Erreur boucle MAJ : %s", e)
|
||||||
|
|
||||||
def stop_session(self):
|
def stop_session(self):
|
||||||
# Sauvegarder le session_id avant de l'annuler (pour les logs)
|
# Sauvegarder le session_id avant de l'annuler (pour les logs)
|
||||||
ended_session_id = self.session_id
|
ended_session_id = self.session_id
|
||||||
@@ -607,29 +694,20 @@ class AgentV1:
|
|||||||
def run(self):
|
def run(self):
|
||||||
self.ui.run()
|
self.ui.run()
|
||||||
|
|
||||||
def _headless_keepalive(agent):
|
def _install_signal_handlers(agent, watchdog) -> None:
|
||||||
"""Maintient le main thread vivant quand l'UI tray ne peut pas tourner.
|
"""Installe SIGTERM/SIGINT/SIGBREAK pour un arrêt propre du main thread.
|
||||||
|
|
||||||
Sans cela, ``agent.run()`` retourne immédiatement (pystray échoue quand
|
Met ``agent.running=False`` (les daemon threads s'arrêtent) et réveille
|
||||||
Léa est lancée via SSH sans session interactive Windows), le main thread
|
le watchdog (qui sort de sa boucle de surveillance). Sans session
|
||||||
se termine, et TOUS les daemon threads — y compris ``_replay_poll_loop``
|
interactive (pystray.Icon.stop indisponible), c'est le SEUL moyen
|
||||||
— meurent avec lui. Observé 3 fois en 24h les 24/05 :
|
d'arrêter Léa proprement : ``kill -TERM <pid>`` ou Ctrl+C.
|
||||||
- SSH ``Permission denied`` (1231)
|
|
||||||
- polls morts après relance distante (1620)
|
|
||||||
- polls morts ``replay_sess_506d6fa2`` (1627)
|
|
||||||
|
|
||||||
Le keepalive ne se déclenche QUE si ``agent.run()`` est sorti tout en
|
|
||||||
laissant ``agent.running=True`` (cas anormal). En mode interactif
|
|
||||||
normal, ``pystray.Icon.run()`` ne sort jamais, donc ce code est
|
|
||||||
invisible.
|
|
||||||
"""
|
"""
|
||||||
import signal as _sig
|
import signal as _sig
|
||||||
_stop = threading.Event()
|
|
||||||
|
|
||||||
def _handler(sig, frame):
|
def _handler(sig, frame):
|
||||||
logger.info(f"[MAIN] Signal {sig} recu — arret propre")
|
logger.info(f"[MAIN] Signal {sig} recu — arret propre")
|
||||||
_stop.set()
|
|
||||||
agent.running = False
|
agent.running = False
|
||||||
|
watchdog.stop()
|
||||||
|
|
||||||
for sig_name in ("SIGTERM", "SIGINT", "SIGBREAK"):
|
for sig_name in ("SIGTERM", "SIGINT", "SIGBREAK"):
|
||||||
sig_obj = getattr(_sig, sig_name, None)
|
sig_obj = getattr(_sig, sig_name, None)
|
||||||
@@ -640,33 +718,78 @@ def _headless_keepalive(agent):
|
|||||||
except (ValueError, OSError):
|
except (ValueError, OSError):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
logger.info(
|
|
||||||
"[MAIN] Keepalive headless actif — main thread bloque pour maintenir "
|
def _agent_should_live(agent) -> bool:
|
||||||
"les daemon threads (_replay_poll_loop, heartbeat, capture_server) vivants. "
|
"""Vrai tant que Léa doit vivre : agent actif ET pas de Quitter explicite.
|
||||||
"Pour stopper Lea : kill -TERM <pid> ou Ctrl+C."
|
|
||||||
)
|
Un « Quitter » utilisateur (``ui._quit_requested``) doit stopper le
|
||||||
|
watchdog pour de bon ; une simple déconnexion RDP ne met JAMAIS ce flag
|
||||||
|
→ le tray revient tout seul à la reconnexion.
|
||||||
|
"""
|
||||||
|
if not getattr(agent, "running", False):
|
||||||
|
return False
|
||||||
|
ui = getattr(agent, "ui", None)
|
||||||
|
if ui is not None and getattr(ui, "_quit_requested", False):
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def _pending_boot_marker_exists() -> bool:
|
||||||
|
"""True si Lea.bat a posé PENDING_BOOT (boot post-MAJ à valider)."""
|
||||||
try:
|
try:
|
||||||
_stop.wait()
|
from .network.updater import _resolve_app_dir
|
||||||
except KeyboardInterrupt:
|
return (_resolve_app_dir(None) / "PENDING_BOOT").exists()
|
||||||
pass
|
except Exception:
|
||||||
agent.running = False
|
return False
|
||||||
logger.info("[MAIN] Keepalive termine — agent.running=False, daemon threads vont s'arreter")
|
|
||||||
|
|
||||||
|
def _confirm_boot_ok() -> None:
|
||||||
|
"""Confirme un boot post-MAJ : écrit boot_ok + retire PENDING_BOOT.
|
||||||
|
|
||||||
|
Désarme le rollback de Lea.bat. No-op si pas de PENDING_BOOT (boot normal).
|
||||||
|
Best-effort — ne doit jamais casser l'arrêt/la vie de Léa.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
if not _pending_boot_marker_exists():
|
||||||
|
return
|
||||||
|
from .network import updater
|
||||||
|
updater.write_boot_ok_marker(AGENT_VERSION)
|
||||||
|
logger.info("[MAJ] Boot confirmé (v%s) — rollback désarmé", AGENT_VERSION)
|
||||||
|
except Exception as e: # noqa: BLE001
|
||||||
|
logger.debug("confirm_boot_ok: %s", e)
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
agent = AgentV1()
|
from .ui.session_watchdog import InteractiveSessionWatchdog
|
||||||
try:
|
|
||||||
agent.run()
|
|
||||||
except Exception:
|
|
||||||
logger.exception("[MAIN] agent.run() a leve une exception")
|
|
||||||
|
|
||||||
if getattr(agent, "running", False):
|
agent = AgentV1()
|
||||||
logger.warning(
|
|
||||||
"[MAIN] agent.run() est sorti mais agent.running=True — "
|
# Résilience RDP/Citrix : au lieu de bloquer le main thread pour toujours
|
||||||
"probablement pystray sans session interactive (SSH). "
|
# quand pystray sort (session interactive perdue), on surveille la
|
||||||
"Bascule en keepalive headless."
|
# session et on ré-affiche le tray + le chat à chaque reconnexion.
|
||||||
)
|
# agent.run() (== agent.ui.run()) est ré-entrant : les threads de fond
|
||||||
_headless_keepalive(agent)
|
# ne démarrent qu'une fois, seule l'icône est recréée. Les daemon threads
|
||||||
|
# de capture/heartbeat/replay tournent contre agent.running et restent
|
||||||
|
# uniques — le watchdog n'y touche pas.
|
||||||
|
watchdog = InteractiveSessionWatchdog(
|
||||||
|
run_ui=agent.run,
|
||||||
|
is_running=lambda: _agent_should_live(agent),
|
||||||
|
)
|
||||||
|
_install_signal_handlers(agent, watchdog)
|
||||||
|
|
||||||
|
try:
|
||||||
|
watchdog.run()
|
||||||
|
# Sortie normale du watchdog = quit propre (tray / session) → le boot
|
||||||
|
# était sain : on confirme (couvre un quit AVANT les 90 s, évite un faux
|
||||||
|
# rollback). No-op si ce n'est pas un boot post-MAJ.
|
||||||
|
_confirm_boot_ok()
|
||||||
|
except KeyboardInterrupt:
|
||||||
|
logger.info("[MAIN] Interruption clavier — arret propre")
|
||||||
|
except Exception:
|
||||||
|
logger.exception("[MAIN] Le watchdog de session a leve une exception")
|
||||||
|
finally:
|
||||||
|
agent.running = False
|
||||||
|
logger.info("[MAIN] Sortie — agent.running=False, daemon threads vont s'arreter")
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|||||||
@@ -18,12 +18,11 @@ Ce module ne contient que les parties PURES / testables, sans réseau réel :
|
|||||||
les fichiers vivants. Retourne un plan d'application.
|
les fichiers vivants. Retourne un plan d'application.
|
||||||
- `auto_update_enabled()` : lit le flag (défaut OFF).
|
- `auto_update_enabled()` : lit le flag (défaut OFF).
|
||||||
|
|
||||||
⚠️⚠️ PARTIES DANGEREUSES — RÉSERVÉES RÉVISION HUMAINE ⚠️⚠️
|
⚠️ SWAP — répartition claire des responsabilités :
|
||||||
Le remplacement réel des fichiers (`apply_update`), l'écriture du marker
|
`apply_update` / `write_boot_ok_marker` ci-dessous ne font que l'ARMEMENT côté
|
||||||
rollback (`write_boot_ok_marker`), l'édition de `Lea.bat` et le redémarrage
|
Python (extraction vers `agent_v1_new/` + marqueurs) — ils n'écrasent JAMAIS un
|
||||||
ne sont PAS implémentés ici : ce sont des STUBS no-op explicites. Un agent ne
|
fichier vivant. Le remplacement ATOMIQUE (renames), le redémarrage et le
|
||||||
doit pas écrire de code qui écrase des binaires vivants ou relance un process
|
rollback sont faits HORS-PROCESS par `Lea.bat` au démarrage (revu ligne à ligne).
|
||||||
sans supervision. Les points d'extension sont marqués `# TODO swap supervisé`.
|
|
||||||
|
|
||||||
Pattern d'import / résilience aligné sur `log_shipper.py` (même branche).
|
Pattern d'import / résilience aligné sur `log_shipper.py` (même branche).
|
||||||
|
|
||||||
@@ -33,8 +32,10 @@ Branche feat/push-log-dgx.
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import hashlib
|
import hashlib
|
||||||
|
import json
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
|
import shutil
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Callable, Optional, Tuple
|
from typing import Callable, Optional, Tuple
|
||||||
|
|
||||||
@@ -238,61 +239,243 @@ def download_update(
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
# ===========================================================================
|
# ---------------------------------------------------------------------------
|
||||||
# ⚠️ ZONE DANGEREUSE — STUBS RÉSERVÉS RÉVISION HUMAINE (NE PAS IMPLÉMENTER
|
# Interrogation serveur — checker INJECTABLE (GET /agents/update/check)
|
||||||
# PAR UN AGENT). Points d'extension explicites, no-op pour l'instant.
|
# ---------------------------------------------------------------------------
|
||||||
# ===========================================================================
|
|
||||||
|
|
||||||
def apply_update(prepared: dict) -> dict:
|
def _default_update_checker(local_version: str, machine_id: str):
|
||||||
"""STUB — application réelle de l'update (swap des fichiers).
|
"""Interroge le serveur : y a-t-il une MAJ ? (best-effort, INJECTABLE).
|
||||||
|
|
||||||
Réservé révision humaine : remplacer des fichiers vivants du client et
|
GET SERVER_URL/agents/update/check?current_version=..&machine_id=..
|
||||||
déclencher un swap est trop risqué pour être généré par un agent. La
|
(endpoint gated côté serveur — 503 si RPA_AUTO_UPDATE_SERVER_ENABLED OFF,
|
||||||
mécanique cible (design v2) est :
|
auquel cas on renvoie None : pas de MAJ). Bearer si présent. Pattern aligné
|
||||||
|
sur `log_shipper._default_sender`. INJECTABLE : remplacé par un fake en test.
|
||||||
- code-only : extraire `agent_v1\\` + `lea_ui\\` + `run_agent_v1.py` +
|
|
||||||
`config.py` du ZIP staging, poser un marker `UPDATE_READY`
|
|
||||||
(`update_type=code-only`) ; le swap effectif est fait par `Lea.bat`
|
|
||||||
au prochain démarrage (xcopy ciblé).
|
|
||||||
- full : poser `UPDATE_READY` (`update_type=full`) ; `Lea.bat` fait le
|
|
||||||
backup complet `Lea_prev\\` puis le swap complet.
|
|
||||||
|
|
||||||
# TODO swap supervisé : extraction ZIP + écriture marker UPDATE_READY.
|
|
||||||
# NE PAS écraser les fichiers vivants depuis Python — c'est Lea.bat qui
|
|
||||||
# swappe hors-process. Édition de Lea.bat + restart = hors périmètre agent.
|
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
{applied: False, reason: "réservé révision humaine (swap supervisé)"}
|
Le dict réponse serveur (`should_update` sait le lire), ou None si
|
||||||
|
indisponible / gated / erreur (jamais d'exception ne remonte).
|
||||||
"""
|
"""
|
||||||
logger.info(
|
try:
|
||||||
"apply_update appelé mais NON implémenté (stub réservé révision humaine) : %r",
|
import requests # import tardif
|
||||||
prepared.get("target_version") if isinstance(prepared, dict) else prepared,
|
|
||||||
)
|
headers = {}
|
||||||
|
try:
|
||||||
|
from ..config import SERVER_URL, API_TOKEN
|
||||||
|
|
||||||
|
base = SERVER_URL
|
||||||
|
if API_TOKEN:
|
||||||
|
headers["Authorization"] = f"Bearer {API_TOKEN}"
|
||||||
|
except Exception:
|
||||||
|
base = ""
|
||||||
|
url = f"{base}/agents/update/check"
|
||||||
|
resp = requests.get(
|
||||||
|
url,
|
||||||
|
params={"current_version": local_version, "machine_id": machine_id},
|
||||||
|
headers=headers,
|
||||||
|
timeout=10,
|
||||||
|
allow_redirects=False,
|
||||||
|
)
|
||||||
|
# 503 = endpoint gated OFF côté serveur → pas de MAJ (silencieux).
|
||||||
|
if resp.status_code == 503:
|
||||||
|
return None
|
||||||
|
if not resp.ok:
|
||||||
|
logger.debug("update/check HTTP %s", resp.status_code)
|
||||||
|
return None
|
||||||
|
return resp.json()
|
||||||
|
except Exception as e:
|
||||||
|
logger.debug("update/check indisponible : %s", e)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Orchestrateur GATED — check → décide → download (staging) → stub apply
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def run_update_cycle(
|
||||||
|
local_version: str,
|
||||||
|
machine_id: str,
|
||||||
|
staging_dir,
|
||||||
|
checker: Optional[Callable[[str, str], object]] = None,
|
||||||
|
downloader: Optional[Callable[[str], bytes]] = None,
|
||||||
|
app_dir=None,
|
||||||
|
) -> dict:
|
||||||
|
"""Un cycle complet de MAJ silencieuse — GATED, best-effort, SANS swap.
|
||||||
|
|
||||||
|
Enchaîne :
|
||||||
|
1. GATE `auto_update_enabled()` (RPA_AUTO_UPDATE_ENABLED, défaut OFF) —
|
||||||
|
si OFF, ne fait STRICTEMENT rien (aucun appel réseau).
|
||||||
|
2. `checker(local_version, machine_id)` → réponse serveur (canary-aware).
|
||||||
|
3. `should_update(...)` → plan (double garde semver, jamais de downgrade).
|
||||||
|
4. `download_update(...)` → ZIP dans le STAGING + vérif SHA256. Ne touche
|
||||||
|
JAMAIS les fichiers vivants.
|
||||||
|
5. `apply_update` ARME le swap (extraction `agent_v1_new/` + marqueur
|
||||||
|
UPDATE_READY) mais NE swappe PAS : le remplacement atomique + le
|
||||||
|
redémarrage sont faits par Lea.bat au prochain démarrage. `applied`
|
||||||
|
reste False tant que Léa n'a pas redémarré sur la nouvelle version.
|
||||||
|
|
||||||
|
Jamais d'exception ne remonte (ne doit JAMAIS casser Léa). Retourne un dict
|
||||||
|
d'état pour le diagnostic / le log :
|
||||||
|
status ∈ {disabled, check_failed, up_to_date, download_failed, staged}
|
||||||
|
|
||||||
|
Args:
|
||||||
|
checker : callable `(local_version, machine_id) -> dict|None`
|
||||||
|
INJECTABLE (défaut = HTTP réel vers l'endpoint gated).
|
||||||
|
downloader : callable `(url) -> bytes` INJECTABLE (défaut = HTTP réel).
|
||||||
|
"""
|
||||||
|
if not auto_update_enabled():
|
||||||
|
return {"status": "disabled", "applied": False}
|
||||||
|
|
||||||
|
chk = checker if checker is not None else _default_update_checker
|
||||||
|
|
||||||
|
try:
|
||||||
|
server_response = chk(local_version, machine_id)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("update check a levé : %s", e)
|
||||||
|
return {"status": "check_failed", "applied": False, "error": str(e)}
|
||||||
|
|
||||||
|
plan = should_update(local_version, server_response)
|
||||||
|
if plan is None:
|
||||||
|
return {"status": "up_to_date", "applied": False}
|
||||||
|
|
||||||
|
staged = download_update(plan, staging_dir, downloader=downloader)
|
||||||
|
if not staged.get("ok"):
|
||||||
|
return {
|
||||||
|
"status": "download_failed",
|
||||||
|
"applied": False,
|
||||||
|
"error": staged.get("error"),
|
||||||
|
}
|
||||||
|
|
||||||
|
# Armement du swap : extraction du ZIP vers agent_v1_new\ + marqueur
|
||||||
|
# UPDATE_READY. Le swap ATOMIQUE (renames) et le redémarrage sont faits
|
||||||
|
# HORS-PROCESS par Lea.bat au prochain démarrage — JAMAIS depuis ici
|
||||||
|
# (on n'écrase pas les fichiers d'un Léa en cours d'exécution).
|
||||||
|
armed = apply_update(staged, app_dir=app_dir)
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"applied": False,
|
"status": "armed" if armed.get("armed") else "arm_failed",
|
||||||
"reason": "réservé révision humaine — swap supervisé (Lea.bat), hors périmètre agent",
|
"applied": False, # le swap effectif est fait par Lea.bat, pas ici
|
||||||
|
"armed": bool(armed.get("armed", False)),
|
||||||
|
"target_version": staged.get("target_version"),
|
||||||
|
"update_type": staged.get("update_type"),
|
||||||
|
"staged_zip": staged.get("staged_zip"),
|
||||||
|
"sha256_verified": staged.get("sha256_verified", False),
|
||||||
|
"marker": armed.get("marker"),
|
||||||
|
"error": armed.get("error"),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def write_boot_ok_marker(version: str) -> dict:
|
# ===========================================================================
|
||||||
"""STUB — écriture du marker rollback `boot_ok_{version}` (R1).
|
# SWAP — côté Python : ARMEMENT SEULEMENT (extraction + marqueurs).
|
||||||
|
# Le remplacement ATOMIQUE des fichiers vivants + le redémarrage + le
|
||||||
|
# rollback sont faits HORS-PROCESS par `Lea.bat` au démarrage (renames).
|
||||||
|
# Python n'écrase JAMAIS les fichiers d'un Léa en cours d'exécution.
|
||||||
|
# ===========================================================================
|
||||||
|
|
||||||
Réservé révision humaine : le marker pilote le rollback de Lea.bat au
|
def _resolve_app_dir(app_dir) -> Path:
|
||||||
prochain démarrage. Sa sémantique (health-check ~60s heartbeat DGX +
|
"""Répertoire d'install (contient `agent_v1/`, `run_agent_v1.py`, `Lea.bat`).
|
||||||
session active AVANT écriture) doit être validée à la main pour éviter un
|
|
||||||
faux rollback (cas DGX down ≠ Léa N+1 buguée — cf. design R1, cas edge 3).
|
|
||||||
|
|
||||||
# TODO swap supervisé : écrire `%LOCALAPPDATA%\\Lea\\boot_ok_{version}`
|
INJECTABLE (tests : tmp_path). Défaut = parent du package agent_v1.
|
||||||
# après ~60s de heartbeat DGX sain + session active (main.py startup).
|
"""
|
||||||
|
if app_dir is not None:
|
||||||
|
return Path(app_dir)
|
||||||
|
try:
|
||||||
|
from ..config import BASE_DIR # BASE_DIR = dossier du package agent_v1
|
||||||
|
return Path(BASE_DIR).parent
|
||||||
|
except Exception:
|
||||||
|
return Path(__file__).resolve().parent.parent.parent
|
||||||
|
|
||||||
|
|
||||||
|
def apply_update(prepared: dict, app_dir=None) -> dict:
|
||||||
|
"""ARME le swap : extrait le ZIP staging vers `agent_v1_new/` + marqueur.
|
||||||
|
|
||||||
|
NE swappe PAS et NE redémarre PAS (c'est le rôle de `Lea.bat`). Écrit
|
||||||
|
uniquement à côté des fichiers vivants (dossier neuf + marqueur), donc
|
||||||
|
l'opération est sûre même sur un Léa en cours d'exécution.
|
||||||
|
|
||||||
|
1. Extrait `prepared["staged_zip"]` → `<app_dir>/agent_v1_new/`
|
||||||
|
(nettoyé au préalable ; garde-fou zip-slip).
|
||||||
|
2. Écrit `<app_dir>/UPDATE_READY` (JSON : version, type, chemins) que
|
||||||
|
`Lea.bat` lira au prochain démarrage pour faire le swap atomique.
|
||||||
|
|
||||||
|
Best-effort : aucune exception ne remonte (ne doit jamais casser Léa).
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
{written: False, reason: "..."}
|
succès : {armed: True, applied: False, target_version, update_type,
|
||||||
|
marker, extracted_to}
|
||||||
|
échec : {armed: False, applied: False, error}
|
||||||
"""
|
"""
|
||||||
logger.info(
|
if not isinstance(prepared, dict):
|
||||||
"write_boot_ok_marker appelé mais NON implémenté (stub R1) : version=%s",
|
return {"armed": False, "applied": False, "error": "prepared invalide"}
|
||||||
version,
|
staged_zip = prepared.get("staged_zip")
|
||||||
)
|
target_version = prepared.get("target_version", "unknown")
|
||||||
return {
|
update_type = _normalize_update_type(prepared.get("update_type"))
|
||||||
"written": False,
|
try:
|
||||||
"reason": "réservé révision humaine — marker rollback (health-check), hors périmètre agent",
|
root = _resolve_app_dir(app_dir)
|
||||||
}
|
zip_path = Path(staged_zip) if staged_zip else None
|
||||||
|
if zip_path is None or not zip_path.is_file():
|
||||||
|
return {"armed": False, "applied": False, "error": "ZIP staging introuvable"}
|
||||||
|
|
||||||
|
new_dir = root / "agent_v1_new"
|
||||||
|
if new_dir.exists():
|
||||||
|
shutil.rmtree(new_dir, ignore_errors=True) # nettoie un staging partiel
|
||||||
|
new_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
import zipfile
|
||||||
|
new_root = new_dir.resolve()
|
||||||
|
with zipfile.ZipFile(zip_path) as zf:
|
||||||
|
for name in zf.namelist(): # garde-fou zip-slip (chemins ../)
|
||||||
|
dest = (new_dir / name).resolve()
|
||||||
|
if not str(dest).startswith(str(new_root)):
|
||||||
|
shutil.rmtree(new_dir, ignore_errors=True)
|
||||||
|
return {"armed": False, "applied": False,
|
||||||
|
"error": f"zip-slip refusé : {name}"}
|
||||||
|
zf.extractall(new_dir)
|
||||||
|
|
||||||
|
marker = root / "UPDATE_READY"
|
||||||
|
marker.write_text(json.dumps({
|
||||||
|
"target_version": target_version,
|
||||||
|
"update_type": update_type,
|
||||||
|
"extracted_to": str(new_dir),
|
||||||
|
"staged_zip": str(zip_path),
|
||||||
|
}), encoding="utf-8")
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
"Update ARMÉ : %s (%s) → %s ; swap au prochain démarrage (Lea.bat)",
|
||||||
|
target_version, update_type, new_dir,
|
||||||
|
)
|
||||||
|
return {"armed": True, "applied": False, "target_version": target_version,
|
||||||
|
"update_type": update_type, "marker": str(marker),
|
||||||
|
"extracted_to": str(new_dir)}
|
||||||
|
except Exception as e: # noqa: BLE001
|
||||||
|
logger.warning("apply_update (armement) a échoué : %s", e)
|
||||||
|
return {"armed": False, "applied": False, "error": f"arm_failed: {e}"}
|
||||||
|
|
||||||
|
|
||||||
|
def write_boot_ok_marker(version: str, app_dir=None) -> dict:
|
||||||
|
"""Confirme un boot sain : écrit `boot_ok_{version}` + désarme le rollback.
|
||||||
|
|
||||||
|
Appelé par `main.py` après ~90 s de tourne STABLE (liveness LOCALE,
|
||||||
|
indépendante du DGX — évite un faux rollback quand le réseau est coupé).
|
||||||
|
Retirer `PENDING_BOOT*` dit à `Lea.bat` que la nouvelle version a démarré
|
||||||
|
correctement (sinon, au prochain lancement, Lea.bat rollback vers la
|
||||||
|
version précédente).
|
||||||
|
|
||||||
|
Best-effort : aucune exception ne remonte.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
root = _resolve_app_dir(app_dir)
|
||||||
|
marker = root / f"boot_ok_{version}"
|
||||||
|
marker.write_text("ok", encoding="utf-8")
|
||||||
|
cleared = []
|
||||||
|
for p in root.glob("PENDING_BOOT*"):
|
||||||
|
try:
|
||||||
|
p.unlink()
|
||||||
|
cleared.append(p.name)
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
logger.info("boot_ok écrit (%s) ; PENDING_BOOT retiré : %s",
|
||||||
|
version, cleared or "aucun")
|
||||||
|
return {"written": True, "marker": str(marker), "cleared_pending": cleared}
|
||||||
|
except Exception as e: # noqa: BLE001
|
||||||
|
logger.warning("write_boot_ok_marker a échoué : %s", e)
|
||||||
|
return {"written": False, "error": str(e)}
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ mss>=9.0.1 # Capture d'écran haute performance
|
|||||||
pynput>=1.7.7 # Clavier/Souris Cross-plateforme
|
pynput>=1.7.7 # Clavier/Souris Cross-plateforme
|
||||||
Pillow>=10.0.0 # Crops et processing image
|
Pillow>=10.0.0 # Crops et processing image
|
||||||
requests>=2.31.0 # Streaming réseau
|
requests>=2.31.0 # Streaming réseau
|
||||||
|
httpx>=0.27 # Client HTTP orchestrateur Léa (POST /api/learn/start) — brique conversationnelle
|
||||||
python-socketio[client]>=5.10,<6.0 # Bus feedback Léa 'lea:*' (compat Flask-SocketIO 5.3.x serveur)
|
python-socketio[client]>=5.10,<6.0 # Bus feedback Léa 'lea:*' (compat Flask-SocketIO 5.3.x serveur)
|
||||||
psutil>=5.9.0 # Monitoring CPU/RAM
|
psutil>=5.9.0 # Monitoring CPU/RAM
|
||||||
screeninfo>=0.8 # QW1 — détection des monitors physiques + offsets
|
screeninfo>=0.8 # QW1 — détection des monitors physiques + offsets
|
||||||
|
|||||||
197
agent_v0/agent_v1/ui/session_watchdog.py
Normal file
197
agent_v0/agent_v1/ui/session_watchdog.py
Normal file
@@ -0,0 +1,197 @@
|
|||||||
|
# agent_v1/ui/session_watchdog.py
|
||||||
|
"""Watchdog de session interactive Windows — résilience RDP/Citrix.
|
||||||
|
|
||||||
|
Problème résolu (preuve poste clinique Émilie, 01/07) :
|
||||||
|
09:46:28 [MAIN] agent.run() est sorti mais agent.running=True — probablement
|
||||||
|
pystray sans session interactive (SSH)
|
||||||
|
09:46:28 [MAIN] Keepalive headless actif — main thread bloque...
|
||||||
|
|
||||||
|
Sur les postes cliniques (tous RDP/Citrix), la session interactive
|
||||||
|
disparaît quand l'utilisateur se déconnecte / la session bascule en
|
||||||
|
verrouillage. `pystray.Icon.run()` sort alors immédiatement (plus de
|
||||||
|
bureau interactif `WinSta0\\Default` pour recevoir les entrées et afficher
|
||||||
|
l'icône). L'ancien `_headless_keepalive` bloquait le main thread *pour
|
||||||
|
toujours* : l'icône tray + la fenêtre chat DISPARAISSAIENT et ne
|
||||||
|
revenaient JAMAIS, même après reconnexion RDP. Les soignants croyaient
|
||||||
|
que Léa avait planté (la capture continuait pourtant en fond).
|
||||||
|
|
||||||
|
Solution : un watchdog qui surveille la disponibilité du bureau
|
||||||
|
interactif via `OpenInputDesktop()` (signal Win32 canonique — échoue quand
|
||||||
|
la session est déconnectée/verrouillée, réussit à la reconnexion) et
|
||||||
|
(re)lance l'UI tray dès qu'une session redevient disponible. Les threads
|
||||||
|
de fond (heartbeat, replay poll, capture_server) NE SONT JAMAIS touchés :
|
||||||
|
ils tournent contre `agent.running` et restent uniques. On ne relance
|
||||||
|
JAMAIS un second `AgentV1` — seulement la couche UI (tray + chat).
|
||||||
|
|
||||||
|
État de l'art (recherche 01/07) :
|
||||||
|
- `OpenInputDesktop()` échoue (ERROR_ACCESS_DENIED / ERROR_INVALID_...)
|
||||||
|
quand le processus n'est pas rattaché au windowstation interactif
|
||||||
|
`WinSta0` — c'est exactement le cas quand la session RDP est
|
||||||
|
déconnectée. C'est la méthode fiable recommandée (comparer les
|
||||||
|
*noms* de bureau via GetUserObjectInformation n'apporte rien de plus
|
||||||
|
ici : on a juste besoin d'un booléen « input desktop dispo ? »).
|
||||||
|
- `WTSGetActiveConsoleSessionId` renvoie une pseudo-session même sans
|
||||||
|
login → PAS fiable pour ce besoin.
|
||||||
|
- `pystray.Icon.run()` ne sort jamais en session interactive normale ;
|
||||||
|
il sort immédiatement sinon → c'est notre signal de « session perdue ».
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import platform
|
||||||
|
import threading
|
||||||
|
from typing import Callable, Optional
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# Intervalle de sondage du bureau interactif (secondes).
|
||||||
|
# 3s = compromis : réactif à la reconnexion sans marteler l'API Win32.
|
||||||
|
POLL_INTERVAL_S = 3.0
|
||||||
|
|
||||||
|
|
||||||
|
def is_interactive_desktop_available() -> bool:
|
||||||
|
"""Retourne True si un bureau interactif Windows est disponible.
|
||||||
|
|
||||||
|
Utilise `OpenInputDesktop()` : succès => le windowstation interactif
|
||||||
|
(`WinSta0\\Default`) est accessible et peut afficher un tray. Échec =>
|
||||||
|
session RDP/Citrix déconnectée ou verrouillée sans bureau d'entrée.
|
||||||
|
|
||||||
|
Hors Windows (Linux/dev/tests) : renvoie toujours True (pas de notion
|
||||||
|
de bureau interactif verrouillable ici — on laisse l'UI tourner).
|
||||||
|
Toute erreur d'appel Win32 est traitée comme « indisponible » (prudent)
|
||||||
|
SAUF l'indisponibilité de l'API elle-même (pywin32 absent) → True pour
|
||||||
|
ne pas priver un poste de son tray à cause d'une dépendance manquante.
|
||||||
|
"""
|
||||||
|
if platform.system() != "Windows":
|
||||||
|
return True
|
||||||
|
|
||||||
|
try:
|
||||||
|
import win32con # type: ignore
|
||||||
|
import win32service # type: ignore
|
||||||
|
except Exception:
|
||||||
|
# pywin32 indisponible : on ne peut pas sonder → on suppose dispo
|
||||||
|
# (comportement historique : tenter l'UI plutôt que la bloquer).
|
||||||
|
logger.debug("pywin32 indisponible — sondage bureau interactif ignoré")
|
||||||
|
return True
|
||||||
|
|
||||||
|
hdesk = None
|
||||||
|
try:
|
||||||
|
# DESKTOP_SWITCHDESKTOP (0x0100) = droit minimal, aligné sur l'usage
|
||||||
|
# documenté pour tester la présence du bureau d'entrée.
|
||||||
|
hdesk = win32service.OpenInputDesktop(0, False, win32con.DESKTOP_SWITCHDESKTOP)
|
||||||
|
return hdesk is not None
|
||||||
|
except Exception:
|
||||||
|
# OpenInputDesktop lève quand aucun bureau d'entrée n'est accessible
|
||||||
|
# (session déconnectée / verrouillée). C'est le cas « indisponible ».
|
||||||
|
return False
|
||||||
|
finally:
|
||||||
|
if hdesk is not None:
|
||||||
|
try:
|
||||||
|
# PyHANDLE se ferme via .Close() (pywin32) ; fallback silencieux.
|
||||||
|
hdesk.Close()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class InteractiveSessionWatchdog:
|
||||||
|
"""Surveille la session interactive et (re)lance l'UI tray à la reconnexion.
|
||||||
|
|
||||||
|
Ne détient AUCUN état de capture. Sa seule responsabilité : garantir
|
||||||
|
qu'il existe au plus UN tray vivant à la fois, et le ressusciter quand
|
||||||
|
une session interactive redevient disponible. Les daemon threads de
|
||||||
|
l'agent (heartbeat/replay/capture) sont indépendants et intacts.
|
||||||
|
|
||||||
|
Paramètres :
|
||||||
|
run_ui : callable bloquant qui lance le tray (typiquement
|
||||||
|
``agent.ui.run`` / ``agent.run``). Retourne quand le
|
||||||
|
tray sort (normal en fin de session interactive).
|
||||||
|
is_running : callable -> bool ; True tant que l'agent doit vivre
|
||||||
|
(typiquement ``lambda: agent.running``).
|
||||||
|
is_available : callable -> bool de détection de session (injectable
|
||||||
|
pour les tests). Défaut = is_interactive_desktop_available.
|
||||||
|
poll_interval_s : période de sondage quand la session est absente.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
run_ui: Callable[[], None],
|
||||||
|
is_running: Callable[[], bool],
|
||||||
|
is_available: Optional[Callable[[], bool]] = None,
|
||||||
|
poll_interval_s: float = POLL_INTERVAL_S,
|
||||||
|
) -> None:
|
||||||
|
self._run_ui = run_ui
|
||||||
|
self._is_running = is_running
|
||||||
|
self._is_available = is_available or is_interactive_desktop_available
|
||||||
|
self._poll_interval_s = poll_interval_s
|
||||||
|
self._wake = threading.Event()
|
||||||
|
# Sérialise le lancement de l'UI : jamais deux trays en parallèle.
|
||||||
|
self._ui_lock = threading.Lock()
|
||||||
|
|
||||||
|
def stop(self) -> None:
|
||||||
|
"""Réveille le watchdog pour qu'il réévalue ``is_running`` et sorte."""
|
||||||
|
self._wake.set()
|
||||||
|
|
||||||
|
def _run_ui_once(self) -> None:
|
||||||
|
"""Lance l'UI tray une fois (bloquant) sous verrou, avec garde d'erreur.
|
||||||
|
|
||||||
|
Le verrou empêche formellement qu'un second appel démarre un tray
|
||||||
|
alors qu'un premier tourne encore (invariant « un seul tray »).
|
||||||
|
"""
|
||||||
|
with self._ui_lock:
|
||||||
|
try:
|
||||||
|
self._run_ui()
|
||||||
|
except Exception:
|
||||||
|
# Un crash du tray ne doit jamais tuer le watchdog : on log et
|
||||||
|
# on laisse la boucle décider (retry ou sortie selon is_running).
|
||||||
|
logger.exception("[WATCHDOG] Le tray UI a levé une exception")
|
||||||
|
|
||||||
|
def run(self) -> None:
|
||||||
|
"""Boucle principale (bloque le main thread à la place du keepalive).
|
||||||
|
|
||||||
|
Cycle :
|
||||||
|
1. Attendre qu'un bureau interactif soit disponible.
|
||||||
|
2. (Re)lancer le tray — bloque jusqu'à sa sortie (déconnexion RDP).
|
||||||
|
3. Recommencer tant que ``is_running`` est vrai.
|
||||||
|
|
||||||
|
Ne consomme pas de CPU en boucle serrée : sonde toutes les
|
||||||
|
``poll_interval_s`` via un Event interruptible (réveil immédiat au stop).
|
||||||
|
"""
|
||||||
|
logger.info(
|
||||||
|
"[WATCHDOG] Surveillance session interactive active "
|
||||||
|
"(re-affichage auto du tray + chat à la reconnexion RDP/Citrix)."
|
||||||
|
)
|
||||||
|
first_cycle = True
|
||||||
|
|
||||||
|
while self._is_running():
|
||||||
|
if not self._is_available():
|
||||||
|
# Session absente : sonder périodiquement sans brûler le CPU.
|
||||||
|
if first_cycle:
|
||||||
|
logger.warning(
|
||||||
|
"[WATCHDOG] Aucune session interactive — Léa reste active "
|
||||||
|
"en fond (capture/heartbeat), tray masqué. En attente de "
|
||||||
|
"reconnexion RDP/Citrix pour ré-afficher l'interface."
|
||||||
|
)
|
||||||
|
# Event.wait renvoie True si stop() a été appelé → on sort.
|
||||||
|
if self._wake.wait(timeout=self._poll_interval_s):
|
||||||
|
break
|
||||||
|
first_cycle = False
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Session disponible : (re)lancer le tray.
|
||||||
|
if not first_cycle:
|
||||||
|
logger.info(
|
||||||
|
"[WATCHDOG] Session interactive détectée — ré-affichage du "
|
||||||
|
"tray et de la fenêtre chat de Léa."
|
||||||
|
)
|
||||||
|
first_cycle = False
|
||||||
|
|
||||||
|
# Bloque jusqu'à la sortie du tray (fin de session interactive).
|
||||||
|
self._run_ui_once()
|
||||||
|
|
||||||
|
# Le tray est sorti. Si l'agent doit vivre, on reboucle (le
|
||||||
|
# prochain tour re-sondera la session et re-affichera le tray).
|
||||||
|
if not self._is_running():
|
||||||
|
break
|
||||||
|
|
||||||
|
logger.info("[WATCHDOG] Arrêt de la surveillance de session interactive.")
|
||||||
@@ -137,6 +137,15 @@ class SmartTrayV1:
|
|||||||
self._state_lock = threading.Lock()
|
self._state_lock = threading.Lock()
|
||||||
self._stop_event = threading.Event()
|
self._stop_event = threading.Event()
|
||||||
|
|
||||||
|
# Résilience RDP/Citrix : run() peut être rappelé plusieurs fois par le
|
||||||
|
# watchdog de session (ré-affichage du tray à la reconnexion). Les
|
||||||
|
# threads de fond (connexion, cache workflows, hotkey) et l'accueil ne
|
||||||
|
# doivent démarrer QU'UNE fois — sinon on duplique les threads.
|
||||||
|
self._bg_started = False
|
||||||
|
# Signalé quand l'utilisateur a demandé Quitter : le watchdog ne doit
|
||||||
|
# alors PAS relancer le tray.
|
||||||
|
self._quit_requested = False
|
||||||
|
|
||||||
# Notifications
|
# Notifications
|
||||||
self._notifier = NotificationManager()
|
self._notifier = NotificationManager()
|
||||||
|
|
||||||
@@ -709,6 +718,11 @@ class SmartTrayV1:
|
|||||||
"""Arrete proprement l'agent et quitte."""
|
"""Arrete proprement l'agent et quitte."""
|
||||||
logger.info("Arret demande par l'utilisateur")
|
logger.info("Arret demande par l'utilisateur")
|
||||||
|
|
||||||
|
# Marquer l'arret volontaire : le watchdog de session ne doit PAS
|
||||||
|
# relancer le tray après un Quitter explicite (à distinguer d'une
|
||||||
|
# simple déconnexion RDP où le tray doit revenir tout seul).
|
||||||
|
self._quit_requested = True
|
||||||
|
|
||||||
# Arreter la session si en cours
|
# Arreter la session si en cours
|
||||||
if self.is_recording:
|
if self.is_recording:
|
||||||
self.on_stop()
|
self.on_stop()
|
||||||
@@ -885,17 +899,24 @@ class SmartTrayV1:
|
|||||||
# ------------------------------------------------------------------
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
def run(self) -> None:
|
def run(self) -> None:
|
||||||
"""Demarre le tray, les threads de fond, et entre dans la boucle principale."""
|
"""Demarre (ou ré-affiche) le tray et entre dans la boucle pystray.
|
||||||
# Notification d'accueil — divulgation IA (Article 50, Reglement IA)
|
|
||||||
self._notifier.greet()
|
|
||||||
|
|
||||||
# Enregistrer le hotkey global Ctrl+Shift+L (toggle chat)
|
Ré-entrant : le watchdog de session (session_watchdog.py) rappelle
|
||||||
self._start_hotkey()
|
cette méthode à chaque reconnexion RDP/Citrix pour ré-afficher le
|
||||||
|
tray + la fenêtre chat. Les initialisations one-shot (accueil,
|
||||||
|
hotkey, threads de fond connexion/cache) sont protégées par
|
||||||
|
``_bg_started`` pour ne PAS dupliquer les threads. Seule l'icône
|
||||||
|
pystray est recréée à chaque appel (l'ancienne est morte avec la
|
||||||
|
session précédente).
|
||||||
|
"""
|
||||||
|
self._start_background_once()
|
||||||
|
|
||||||
# Tooltip avec identifiant machine pour le multi-machine
|
# Tooltip avec identifiant machine pour le multi-machine
|
||||||
tray_title = f"Agent V1 - {self.machine_id}"
|
tray_title = f"Agent V1 - {self.machine_id}"
|
||||||
|
|
||||||
# Menu statique — reconstruit via _update_icon() quand l'état change
|
# Menu statique — reconstruit via _update_icon() quand l'état change.
|
||||||
|
# Nouvelle icône à chaque (ré)affichage : l'objet pystray précédent
|
||||||
|
# est invalide une fois sa boucle sortie (session interactive perdue).
|
||||||
self.icon = pystray.Icon(
|
self.icon = pystray.Icon(
|
||||||
"AgentV1",
|
"AgentV1",
|
||||||
self._current_icon(),
|
self._current_icon(),
|
||||||
@@ -903,6 +924,33 @@ class SmartTrayV1:
|
|||||||
menu=pystray.Menu(*self._get_menu_items()),
|
menu=pystray.Menu(*self._get_menu_items()),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Rafraîchir les workflows au (ré)affichage — utile après reconnexion.
|
||||||
|
if self._bg_started and self.server_client is not None:
|
||||||
|
threading.Thread(target=self._fetch_workflows, daemon=True).start()
|
||||||
|
|
||||||
|
# Boucle principale pystray (bloquante). Sort quand la session
|
||||||
|
# interactive disparaît (RDP déconnecté) OU sur _on_quit → le
|
||||||
|
# watchdog décide alors de relancer ou non.
|
||||||
|
logger.info("SmartTrayV1 demarre — entree dans la boucle pystray")
|
||||||
|
self.icon.run()
|
||||||
|
|
||||||
|
def _start_background_once(self) -> None:
|
||||||
|
"""Initialisations one-shot : accueil, hotkey, threads de fond.
|
||||||
|
|
||||||
|
Idempotent : les appels suivants (ré-affichage tray) sont des no-op.
|
||||||
|
Garantit qu'on n'accumule pas de threads connexion/cache à chaque
|
||||||
|
reconnexion RDP.
|
||||||
|
"""
|
||||||
|
if self._bg_started:
|
||||||
|
return
|
||||||
|
self._bg_started = True
|
||||||
|
|
||||||
|
# Notification d'accueil — divulgation IA (Article 50, Reglement IA)
|
||||||
|
self._notifier.greet()
|
||||||
|
|
||||||
|
# Enregistrer le hotkey global Ctrl+Shift+L (toggle chat)
|
||||||
|
self._start_hotkey()
|
||||||
|
|
||||||
# Demarrer le thread de verification connexion
|
# Demarrer le thread de verification connexion
|
||||||
if self.server_client is not None:
|
if self.server_client is not None:
|
||||||
conn_thread = threading.Thread(
|
conn_thread = threading.Thread(
|
||||||
@@ -924,7 +972,3 @@ class SmartTrayV1:
|
|||||||
threading.Thread(
|
threading.Thread(
|
||||||
target=self._fetch_workflows, daemon=True
|
target=self._fetch_workflows, daemon=True
|
||||||
).start()
|
).start()
|
||||||
|
|
||||||
# Boucle principale pystray (bloquante)
|
|
||||||
logger.info("SmartTrayV1 demarre — entree dans la boucle pystray")
|
|
||||||
self.icon.run()
|
|
||||||
|
|||||||
110
agent_v0/agent_v1/vision/capture_io.py
Normal file
110
agent_v0/agent_v1/vision/capture_io.py
Normal file
@@ -0,0 +1,110 @@
|
|||||||
|
"""Politique de sauvegarde des captures — réduction du poids disque.
|
||||||
|
|
||||||
|
Constat : tous les shots étaient sauvés en PNG plein écran lossless
|
||||||
|
(``img.save(path, "PNG", quality=...)`` — PNG ignore ``quality``), d'où
|
||||||
|
~90 Go pour 13 sessions. La majorité de ce poids n'a aucune valeur de
|
||||||
|
grounding (full + full_blurred en doublon, heartbeats plein écran).
|
||||||
|
|
||||||
|
Cette politique distingue le **type** de shot et écrit le format adapté :
|
||||||
|
|
||||||
|
- ``crop`` → PNG lossless. C'est la cible de grounding qwen3-vl ; on
|
||||||
|
préserve chaque pixel (perte JPEG = bruit sur de petites icônes). Le crop
|
||||||
|
fait 80×80 → poids négligeable, aucun intérêt à le dégrader.
|
||||||
|
- ``full`` / ``window`` / ``context`` → JPEG ``quality=SCREENSHOT_QUALITY,
|
||||||
|
optimize=True``. Ce sont des vues contextuelles / humaines : la
|
||||||
|
compression JPEG (~5-10x) est sans impact fonctionnel.
|
||||||
|
- ``heartbeat`` → JPEG **downscalé** (largeur max ``HEARTBEAT_MAX_WIDTH``,
|
||||||
|
ratio préservé). C'est de la *liveness* (le serveur vérifie juste qu'un
|
||||||
|
écran a changé), pas du grounding → la pleine résolution est du gaspillage.
|
||||||
|
|
||||||
|
``save_capture`` retourne le chemin RÉELLEMENT écrit, extension ajustée selon
|
||||||
|
le format. L'appelant doit utiliser ce retour (et non un chemin ``.png``
|
||||||
|
présumé) pour streamer / référencer le bon fichier.
|
||||||
|
|
||||||
|
⚠️ Contrat avec le serveur : l'extension du crop NE DOIT PAS changer (le
|
||||||
|
serveur retrouve le crop par basename via ``vision_info.crop`` — voir
|
||||||
|
``stream_processor._extract_crop_b64`` stratégie 1). C'est pourquoi ``crop``
|
||||||
|
reste PNG. Les full/window/context/heartbeat sont retrouvés par
|
||||||
|
``screenshot_id`` avec extension ``.png`` hardcodée côté serveur, mais le
|
||||||
|
serveur réécrit toujours l'upload sous ``{shot_id}.png`` (le suffixe envoyé
|
||||||
|
sur le fil est ignoré) → changer l'extension LOCALE de ces types est sûr.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
from typing import Iterable
|
||||||
|
|
||||||
|
from PIL import Image
|
||||||
|
|
||||||
|
from ..config import SCREENSHOT_QUALITY
|
||||||
|
|
||||||
|
# Types sauvés en JPEG (vue contextuelle / humaine, pas de grounding pixel).
|
||||||
|
_JPEG_KINDS: frozenset = frozenset({"full", "window", "context"})
|
||||||
|
|
||||||
|
# Largeur max d'un heartbeat downscalé. 1280 px suffit largement pour de la
|
||||||
|
# liveness (détecter qu'un écran a changé) ; on divise le poids d'un 2560 px
|
||||||
|
# par ~4 (surface) avant compression JPEG.
|
||||||
|
HEARTBEAT_MAX_WIDTH = 1280
|
||||||
|
|
||||||
|
|
||||||
|
def _ensure_jpeg_ready(img: Image.Image) -> Image.Image:
|
||||||
|
"""Convertit en RGB si nécessaire (JPEG ne supporte ni alpha ni palette)."""
|
||||||
|
if img.mode in ("RGBA", "LA", "P"):
|
||||||
|
return img.convert("RGB")
|
||||||
|
return img
|
||||||
|
|
||||||
|
|
||||||
|
def _downscale_to_width(img: Image.Image, max_width: int) -> Image.Image:
|
||||||
|
"""Réduit l'image à ``max_width`` en préservant le ratio (no-op si plus petite)."""
|
||||||
|
if img.width <= max_width:
|
||||||
|
return img
|
||||||
|
new_height = max(1, round(img.height * max_width / img.width))
|
||||||
|
return img.resize((max_width, new_height), Image.LANCZOS)
|
||||||
|
|
||||||
|
|
||||||
|
def save_capture(img: Image.Image, path_base: str, kind: str) -> str:
|
||||||
|
"""Sauve ``img`` selon la politique du ``kind`` et retourne le chemin écrit.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
img: image PIL à sauvegarder.
|
||||||
|
path_base: chemin SANS extension (ex.
|
||||||
|
``.../shots/shot_0001_full``). L'extension finale (``.png`` ou
|
||||||
|
``.jpg``) est ajoutée par la politique.
|
||||||
|
kind: type de shot — ``"crop"`` | ``"full"`` | ``"window"`` |
|
||||||
|
``"context"`` | ``"heartbeat"``.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Le chemin RÉELLEMENT écrit, avec la bonne extension.
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
ValueError: si ``kind`` n'est pas reconnu (fail-closed : on refuse
|
||||||
|
d'écrire un fichier dont la politique est indéterminée).
|
||||||
|
"""
|
||||||
|
if kind == "crop":
|
||||||
|
out_path = f"{path_base}.png"
|
||||||
|
img.save(out_path, "PNG")
|
||||||
|
return out_path
|
||||||
|
|
||||||
|
if kind in _JPEG_KINDS:
|
||||||
|
out_path = f"{path_base}.jpg"
|
||||||
|
_ensure_jpeg_ready(img).save(
|
||||||
|
out_path, "JPEG", quality=SCREENSHOT_QUALITY, optimize=True
|
||||||
|
)
|
||||||
|
return out_path
|
||||||
|
|
||||||
|
if kind == "heartbeat":
|
||||||
|
out_path = f"{path_base}.jpg"
|
||||||
|
small = _downscale_to_width(_ensure_jpeg_ready(img), HEARTBEAT_MAX_WIDTH)
|
||||||
|
small.save(out_path, "JPEG", quality=SCREENSHOT_QUALITY)
|
||||||
|
return out_path
|
||||||
|
|
||||||
|
raise ValueError(
|
||||||
|
f"kind de capture inconnu : {kind!r} "
|
||||||
|
f"(attendu: crop, full, window, context, heartbeat)"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def known_kinds() -> Iterable[str]:
|
||||||
|
"""Retourne les ``kind`` supportés (utile pour la validation appelant)."""
|
||||||
|
return ("crop", *sorted(_JPEG_KINDS), "heartbeat")
|
||||||
@@ -18,8 +18,9 @@ import platform
|
|||||||
from typing import Any, Dict, List, Optional, Tuple
|
from typing import Any, Dict, List, Optional, Tuple
|
||||||
from PIL import Image, ImageFilter, ImageStat
|
from PIL import Image, ImageFilter, ImageStat
|
||||||
import mss
|
import mss
|
||||||
from ..config import TARGETED_CROP_SIZE, SCREENSHOT_QUALITY, BLUR_SENSITIVE
|
from ..config import TARGETED_CROP_SIZE, BLUR_SENSITIVE
|
||||||
from .blur_sensitive import blur_sensitive_regions
|
from .blur_sensitive import blur_sensitive_regions
|
||||||
|
from .capture_io import save_capture
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -425,6 +426,18 @@ class VisionCapturer:
|
|||||||
# On ne crée plus self.sct ici car mss n'est pas thread-safe sous Windows
|
# On ne crée plus self.sct ici car mss n'est pas thread-safe sous Windows
|
||||||
self.last_img_hash = None
|
self.last_img_hash = None
|
||||||
|
|
||||||
|
def _ensure_shots_dir(self) -> None:
|
||||||
|
"""Garantit l'existence de `shots/` avant toute écriture.
|
||||||
|
|
||||||
|
Le dossier est créé dans `__init__`, mais l'auto-cleanup de
|
||||||
|
`SessionStorage` (`shutil.rmtree` par âge/taille) peut supprimer tout
|
||||||
|
le dossier de session — y compris la session permanente `_background`.
|
||||||
|
Sans ce garde, la capture suivante lève `[Errno 2] No such file or
|
||||||
|
directory` (bug observé poste Émilie). On recrée donc le répertoire
|
||||||
|
cible juste avant chaque sauvegarde.
|
||||||
|
"""
|
||||||
|
os.makedirs(self.shots_dir, exist_ok=True)
|
||||||
|
|
||||||
def capture_full_context(self, name_suffix: str, force=False) -> str:
|
def capture_full_context(self, name_suffix: str, force=False) -> str:
|
||||||
"""
|
"""
|
||||||
Capture l'écran complet.
|
Capture l'écran complet.
|
||||||
@@ -460,9 +473,15 @@ class VisionCapturer:
|
|||||||
if BLUR_SENSITIVE:
|
if BLUR_SENSITIVE:
|
||||||
blur_sensitive_regions(img)
|
blur_sensitive_regions(img)
|
||||||
|
|
||||||
path = os.path.join(self.shots_dir, f"context_{int(time.time())}_{name_suffix}.png")
|
# Politique d'écriture : les heartbeats sont de la liveness pure
|
||||||
img.save(path, "PNG", quality=SCREENSHOT_QUALITY)
|
# (le serveur vérifie juste qu'un écran a changé) → JPEG downscalé.
|
||||||
return path
|
# Les autres contextes (focus_change, result_of_*) → JPEG q85.
|
||||||
|
kind = "heartbeat" if "heartbeat" in name_suffix else "context"
|
||||||
|
self._ensure_shots_dir()
|
||||||
|
path_base = os.path.join(
|
||||||
|
self.shots_dir, f"context_{int(time.time())}_{name_suffix}"
|
||||||
|
)
|
||||||
|
return save_capture(img, path_base, kind)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Erreur Context Capture: {e}")
|
logger.error(f"Erreur Context Capture: {e}")
|
||||||
return ""
|
return ""
|
||||||
@@ -506,10 +525,10 @@ class VisionCapturer:
|
|||||||
return result
|
return result
|
||||||
return {}
|
return {}
|
||||||
|
|
||||||
full_path = os.path.join(self.shots_dir, f"{screenshot_id}_full.png")
|
full_base = os.path.join(self.shots_dir, f"{screenshot_id}_full")
|
||||||
|
|
||||||
# Capture du Crop (Cœur de l'apprentissage qwen3-vl)
|
# Capture du Crop (Cœur de l'apprentissage qwen3-vl)
|
||||||
crop_path = os.path.join(self.shots_dir, f"{screenshot_id}_crop.png")
|
crop_base = os.path.join(self.shots_dir, f"{screenshot_id}_crop")
|
||||||
w, h = TARGETED_CROP_SIZE
|
w, h = TARGETED_CROP_SIZE
|
||||||
left = max(0, x - w // 2)
|
left = max(0, x - w // 2)
|
||||||
top = max(0, y - h // 2)
|
top = max(0, y - h // 2)
|
||||||
@@ -523,8 +542,11 @@ class VisionCapturer:
|
|||||||
blur_sensitive_regions(img)
|
blur_sensitive_regions(img)
|
||||||
blur_sensitive_regions(crop_img)
|
blur_sensitive_regions(crop_img)
|
||||||
|
|
||||||
img.save(full_path, "PNG", quality=SCREENSHOT_QUALITY)
|
# Politique d'écriture : full = vue contextuelle → JPEG q85 ;
|
||||||
crop_img.save(crop_path, "PNG", quality=SCREENSHOT_QUALITY)
|
# crop = cible de grounding qwen3-vl → PNG lossless (contrat serveur).
|
||||||
|
self._ensure_shots_dir()
|
||||||
|
full_path = save_capture(img, full_base, "full")
|
||||||
|
crop_path = save_capture(crop_img, crop_base, "crop")
|
||||||
|
|
||||||
# Mise à jour du hash pour le prochain heartbeat
|
# Mise à jour du hash pour le prochain heartbeat
|
||||||
self.last_img_hash = self._compute_quick_hash(img)
|
self.last_img_hash = self._compute_quick_hash(img)
|
||||||
@@ -648,11 +670,12 @@ class VisionCapturer:
|
|||||||
if BLUR_SENSITIVE:
|
if BLUR_SENSITIVE:
|
||||||
blur_sensitive_regions(window_img)
|
blur_sensitive_regions(window_img)
|
||||||
|
|
||||||
# Sauvegarde
|
# Sauvegarde — fenêtre = vue contextuelle → JPEG q85 (politique).
|
||||||
window_path = os.path.join(
|
self._ensure_shots_dir()
|
||||||
self.shots_dir, f"{screenshot_id}_window.png"
|
window_base = os.path.join(
|
||||||
|
self.shots_dir, f"{screenshot_id}_window"
|
||||||
)
|
)
|
||||||
window_img.save(window_path, "PNG", quality=SCREENSHOT_QUALITY)
|
window_path = save_capture(window_img, window_base, "window")
|
||||||
|
|
||||||
result = {
|
result = {
|
||||||
"window_image": window_path,
|
"window_image": window_path,
|
||||||
|
|||||||
@@ -436,6 +436,9 @@ from .replay_engine import (
|
|||||||
_notify_error_callback as _notify_error_callback_impl,
|
_notify_error_callback as _notify_error_callback_impl,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Navigate handler — import direct depuis core/navigation (pas via replay_engine)
|
||||||
|
from core.navigation import _handle_navigate_action
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
# Wrappers pour les fonctions replay_engine qui accèdent aux variables globales du module.
|
# Wrappers pour les fonctions replay_engine qui accèdent aux variables globales du module.
|
||||||
@@ -4453,6 +4456,15 @@ async def get_next_action(session_id: str, machine_id: str = "default"):
|
|||||||
),
|
),
|
||||||
timeout=180,
|
timeout=180,
|
||||||
)
|
)
|
||||||
|
elif type_ == "navigate":
|
||||||
|
await asyncio.wait_for(
|
||||||
|
loop.run_in_executor(
|
||||||
|
None,
|
||||||
|
_handle_navigate_action,
|
||||||
|
action, owning_replay, session_id,
|
||||||
|
),
|
||||||
|
timeout=180,
|
||||||
|
)
|
||||||
elif type_ == "t2a_decision":
|
elif type_ == "t2a_decision":
|
||||||
await asyncio.wait_for(
|
await asyncio.wait_for(
|
||||||
loop.run_in_executor(
|
loop.run_in_executor(
|
||||||
@@ -7848,6 +7860,9 @@ async def lea_screen_analyze(payload: _Phase25ScreenRequest, request: Request):
|
|||||||
# client + Lea.bat).
|
# client + Lea.bat).
|
||||||
# =========================================================================
|
# =========================================================================
|
||||||
from .update_check import decide_update as _decide_update # noqa: E402
|
from .update_check import decide_update as _decide_update # noqa: E402
|
||||||
|
from .update_policy import ( # noqa: E402
|
||||||
|
resolve_target_version_from_env as _resolve_target_version_from_env,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _auto_update_server_enabled() -> bool:
|
def _auto_update_server_enabled() -> bool:
|
||||||
@@ -7857,14 +7872,25 @@ def _auto_update_server_enabled() -> bool:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def _latest_agent_version() -> str:
|
def _latest_agent_version(machine_id: Optional[str] = None) -> str:
|
||||||
"""Dernière version d'agent disponible côté serveur.
|
"""Version d'agent cible POUR CE POSTE (canary-aware, DETTE-022 v2).
|
||||||
|
|
||||||
Source de vérité minimale (POC) : variable d'environnement
|
⭐ SÉCURITÉ flotte ⭐ — la version servie est résolue PAR MACHINE via la
|
||||||
RPA_AGENT_LATEST_VERSION. Permet de piloter la fleet sans rebuild. Une
|
politique canary (`update_policy.resolve_target_version_from_env`) : un
|
||||||
évolution future pourra la lire d'un manifeste/DB (cf. design).
|
poste canary (Émilie `lea-4zbgwxty`) reçoit la nouvelle version en premier ;
|
||||||
|
tous les autres restent sur le floor stable. Piloté 100 % par env, sans
|
||||||
|
rebuild :
|
||||||
|
RPA_AGENT_STABLE_VERSION (défaut 1.0.1) — servi à toute la flotte.
|
||||||
|
RPA_AGENT_CANARY_VERSION — servi AUX SEULS postes canary.
|
||||||
|
RPA_AGENT_CANARY_MACHINES — allow-list CSV des machine_id canary.
|
||||||
|
|
||||||
|
Rétrocompat : si `RPA_AGENT_LATEST_VERSION` (ancienne var globale) est
|
||||||
|
positionnée, elle prime — évite toute régression d'un déploiement existant.
|
||||||
"""
|
"""
|
||||||
return os.environ.get("RPA_AGENT_LATEST_VERSION", "1.0.1")
|
legacy = os.environ.get("RPA_AGENT_LATEST_VERSION")
|
||||||
|
if legacy:
|
||||||
|
return legacy
|
||||||
|
return _resolve_target_version_from_env(machine_id)
|
||||||
|
|
||||||
|
|
||||||
@app.get("/api/v1/agents/update/check")
|
@app.get("/api/v1/agents/update/check")
|
||||||
@@ -7877,6 +7903,10 @@ async def check_agent_update(
|
|||||||
|
|
||||||
Réponse : {update_available, latest_version, update_type, url}.
|
Réponse : {update_available, latest_version, update_type, url}.
|
||||||
|
|
||||||
|
La version cible est résolue PAR MACHINE (canary) : voir
|
||||||
|
`_latest_agent_version`. Un poste hors canary ne se voit JAMAIS proposer la
|
||||||
|
version canary (blast radius borné à la liste canary).
|
||||||
|
|
||||||
GATED : si RPA_AUTO_UPDATE_SERVER_ENABLED n'est pas positionné → 503
|
GATED : si RPA_AUTO_UPDATE_SERVER_ENABLED n'est pas positionné → 503
|
||||||
(aucun effet sur le pipeline existant — anti-régression). Auth Bearer
|
(aucun effet sur le pipeline existant — anti-régression). Auth Bearer
|
||||||
requise (dépendance globale `_verify_token`).
|
requise (dépendance globale `_verify_token`).
|
||||||
@@ -7891,7 +7921,7 @@ async def check_agent_update(
|
|||||||
)
|
)
|
||||||
return _decide_update(
|
return _decide_update(
|
||||||
current_version=current_version,
|
current_version=current_version,
|
||||||
latest_version=_latest_agent_version(),
|
latest_version=_latest_agent_version(machine_id),
|
||||||
update_type=update_type,
|
update_type=update_type,
|
||||||
machine_id=machine_id,
|
machine_id=machine_id,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -41,6 +41,7 @@ _ALLOWED_ACTION_TYPES = {
|
|||||||
"extract_text", # OCR serveur sur dernier heartbeat → variable workflow
|
"extract_text", # OCR serveur sur dernier heartbeat → variable workflow
|
||||||
"extract_table", # OCR serveur + filtre regex → liste structurée (boucle)
|
"extract_table", # OCR serveur + filtre regex → liste structurée (boucle)
|
||||||
"extract_dossier", # OCR grille structurée → dossier patient persisté (brique 3)
|
"extract_dossier", # OCR grille structurée → dossier patient persisté (brique 3)
|
||||||
|
"navigate", # Navigation visuelle → coords login/recherche (brique navigation)
|
||||||
"extract_text_scroll", # Marker côté graphe — expansé en sous-actions par _edge_to_normalized_actions
|
"extract_text_scroll", # Marker côté graphe — expansé en sous-actions par _edge_to_normalized_actions
|
||||||
"_concat_text_vars", # Action serveur interne (générée par expansion extract_text_scroll)
|
"_concat_text_vars", # Action serveur interne (générée par expansion extract_text_scroll)
|
||||||
"t2a_decision", # Analyse LLM facturation T2A → variable workflow
|
"t2a_decision", # Analyse LLM facturation T2A → variable workflow
|
||||||
@@ -55,6 +56,7 @@ _SERVER_SIDE_ACTION_TYPES = {
|
|||||||
"extract_text",
|
"extract_text",
|
||||||
"extract_table",
|
"extract_table",
|
||||||
"extract_dossier",
|
"extract_dossier",
|
||||||
|
"navigate",
|
||||||
"t2a_decision",
|
"t2a_decision",
|
||||||
"llm_generate",
|
"llm_generate",
|
||||||
"_concat_text_vars",
|
"_concat_text_vars",
|
||||||
|
|||||||
139
agent_v0/server_v1/update_policy.py
Normal file
139
agent_v0/server_v1/update_policy.py
Normal file
@@ -0,0 +1,139 @@
|
|||||||
|
# agent_v0/server_v1/update_policy.py
|
||||||
|
"""Politique de déploiement CANARY de la MAJ silencieuse Léa (DETTE-022 v2).
|
||||||
|
|
||||||
|
⭐ Brique de SÉCURITÉ centrale ⭐ — 10+ postes cliniques live (Wallerstein).
|
||||||
|
|
||||||
|
Une MAJ ratée peut briquer toute la flotte. La règle non négociable : on ne
|
||||||
|
pousse JAMAIS une nouvelle version sur tous les postes d'un coup. On la déploie
|
||||||
|
d'abord sur UN poste (canary = Émilie `lea-4zbgwxty`), on vérifie, puis on
|
||||||
|
élargit. Ce module résout, PAR MACHINE, la version cible :
|
||||||
|
|
||||||
|
- poste dans la liste canary → `canary_version` (la nouvelle) ;
|
||||||
|
- tous les autres postes → `stable_version` (le floor, inchangé).
|
||||||
|
|
||||||
|
Piloté 100 % par variables d'environnement (config serveur, sans rebuild) :
|
||||||
|
RPA_AGENT_STABLE_VERSION — version servie à toute la flotte (défaut floor).
|
||||||
|
RPA_AGENT_CANARY_VERSION — version servie AUX SEULS postes canary (optionnel).
|
||||||
|
RPA_AGENT_CANARY_MACHINES — allow-list CSV des machine_id canary.
|
||||||
|
|
||||||
|
Promotion = quand le canary est validé, on met RPA_AGENT_STABLE_VERSION à la
|
||||||
|
version canary (toute la flotte suit) et on vide RPA_AGENT_CANARY_MACHINES.
|
||||||
|
Rollback canary = on remet RPA_AGENT_CANARY_VERSION à l'ancienne / on vide la
|
||||||
|
liste : le prochain check ne proposera plus la MAJ (le swap réel côté client
|
||||||
|
reste réservé révision humaine — cf. updater.py).
|
||||||
|
|
||||||
|
Module PUR (aucun import FastAPI, aucune IO) → importable et testable seul
|
||||||
|
(DETTE-013). Le branchement HTTP vit dans api_stream.py.
|
||||||
|
|
||||||
|
Branche feat/push-log-dgx.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
from typing import Optional, Set
|
||||||
|
|
||||||
|
# Réutilise le comparateur semver de la décision (même module serveur, pas de
|
||||||
|
# duplication) : "1.0.2" < "1.0.10" correctement, tolérant aux formats invalides.
|
||||||
|
try: # import relatif quand chargé comme package
|
||||||
|
from .update_check import is_newer
|
||||||
|
except Exception: # chargé par chemin (tests importlib) : import du voisin
|
||||||
|
import importlib.util as _ilu
|
||||||
|
from pathlib import Path as _Path
|
||||||
|
|
||||||
|
_uc_path = _Path(__file__).resolve().parent / "update_check.py"
|
||||||
|
_spec = _ilu.spec_from_file_location("_rpa_update_check_for_policy", _uc_path)
|
||||||
|
_uc = _ilu.module_from_spec(_spec)
|
||||||
|
_spec.loader.exec_module(_uc)
|
||||||
|
is_newer = _uc.is_newer
|
||||||
|
|
||||||
|
|
||||||
|
# Séparateurs tolérés dans l'allow-list canary (CSV, espaces, point-virgule).
|
||||||
|
_CANARY_SEPARATORS = (",", ";")
|
||||||
|
|
||||||
|
|
||||||
|
def parse_canary_machines(raw: Optional[str]) -> Set[str]:
|
||||||
|
"""Parse l'allow-list canary en un ensemble de machine_id.
|
||||||
|
|
||||||
|
Tolérant : virgule / point-virgule / espace comme séparateurs, entrées
|
||||||
|
vides ignorées. `None` ou chaîne vide → ensemble vide (aucun canary).
|
||||||
|
"""
|
||||||
|
if not raw or not isinstance(raw, str):
|
||||||
|
return set()
|
||||||
|
normalized = raw
|
||||||
|
for sep in _CANARY_SEPARATORS:
|
||||||
|
normalized = normalized.replace(sep, " ")
|
||||||
|
return {tok for tok in (t.strip() for t in normalized.split()) if tok}
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_target_version(
|
||||||
|
machine_id: Optional[str],
|
||||||
|
stable_version: str,
|
||||||
|
canary_version: Optional[str],
|
||||||
|
canary_machines: Set[str],
|
||||||
|
) -> str:
|
||||||
|
"""Résout la version cible POUR CE POSTE (cœur canary — sécurité).
|
||||||
|
|
||||||
|
Règles (toutes prudentes par défaut) :
|
||||||
|
1. Poste HORS liste canary → `stable_version` (jamais la nouvelle).
|
||||||
|
2. machine_id absent / liste vide / pas de canary_version → `stable_version`.
|
||||||
|
3. Poste DANS la liste canary ET `canary_version` fournie ET STRICTEMENT
|
||||||
|
plus récente que stable → `canary_version`.
|
||||||
|
4. Garde-fou : si `canary_version` <= `stable_version` (config douteuse,
|
||||||
|
ex. downgrade), on sert quand même `stable_version` (jamais de recul).
|
||||||
|
|
||||||
|
Ne lève jamais. Une version illisible retombe naturellement sur le stable
|
||||||
|
via le comparateur semver tolérant.
|
||||||
|
"""
|
||||||
|
# Cas 1/2 : hors canary → stable.
|
||||||
|
if not machine_id or machine_id not in canary_machines:
|
||||||
|
return stable_version
|
||||||
|
if not canary_version:
|
||||||
|
return stable_version
|
||||||
|
|
||||||
|
# Cas 4 : garde-fou anti-recul — le canary doit être STRICTEMENT plus récent.
|
||||||
|
if not is_newer(canary_version, stable_version):
|
||||||
|
return stable_version
|
||||||
|
|
||||||
|
# Cas 3 : poste canary → nouvelle version.
|
||||||
|
return canary_version
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Lecture de la politique depuis l'environnement (pilotage sans rebuild).
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
# Défaut historique aligné sur AGENT_VERSION client (config.py) et sur le
|
||||||
|
# fallback de _latest_agent_version().
|
||||||
|
_DEFAULT_STABLE_VERSION = "1.0.1"
|
||||||
|
|
||||||
|
|
||||||
|
def stable_version_from_env() -> str:
|
||||||
|
"""Version servie à toute la flotte (floor). Défaut = 1.0.1."""
|
||||||
|
return os.environ.get("RPA_AGENT_STABLE_VERSION", _DEFAULT_STABLE_VERSION)
|
||||||
|
|
||||||
|
|
||||||
|
def canary_version_from_env() -> Optional[str]:
|
||||||
|
"""Version canary (nouvelle), servie aux seuls postes canary. Optionnel."""
|
||||||
|
val = os.environ.get("RPA_AGENT_CANARY_VERSION", "").strip()
|
||||||
|
return val or None
|
||||||
|
|
||||||
|
|
||||||
|
def canary_machines_from_env() -> Set[str]:
|
||||||
|
"""Allow-list canary (machine_id) depuis RPA_AGENT_CANARY_MACHINES."""
|
||||||
|
return parse_canary_machines(os.environ.get("RPA_AGENT_CANARY_MACHINES", ""))
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_target_version_from_env(machine_id: Optional[str]) -> str:
|
||||||
|
"""Raccourci : résout la version cible pour `machine_id` d'après l'env.
|
||||||
|
|
||||||
|
C'est le point d'entrée que l'endpoint serveur appelle. Il isole toute la
|
||||||
|
lecture d'environnement ici (testable en injectant les paramètres via
|
||||||
|
`resolve_target_version`).
|
||||||
|
"""
|
||||||
|
return resolve_target_version(
|
||||||
|
machine_id=machine_id,
|
||||||
|
stable_version=stable_version_from_env(),
|
||||||
|
canary_version=canary_version_from_env(),
|
||||||
|
canary_machines=canary_machines_from_env(),
|
||||||
|
)
|
||||||
119
core/navigation/__init__.py
Normal file
119
core/navigation/__init__.py
Normal file
@@ -0,0 +1,119 @@
|
|||||||
|
"""Navigation brique — login visuel, recherche dossiers, vérification écran.
|
||||||
|
|
||||||
|
Modules :
|
||||||
|
- visual_verifier : verify_before / verify_after chaque action (vision = validateur, OCR-ancré)
|
||||||
|
- grounding : résolution visuelle d'éléments UI (OCR-anchor first, VLM fallback, coords cache)
|
||||||
|
- visual_login : login form resolution + verification (DPI urgences default config)
|
||||||
|
- action_resolver : pont navigation → runtime (coords normalisés, OCR/VLM adapters)
|
||||||
|
|
||||||
|
Pattern d'injection : VlmClient + OcrClient + OcrDetailedClient injectables
|
||||||
|
"""
|
||||||
|
|
||||||
|
from .visual_verifier import verify_screen_match, ScreenMatchResult
|
||||||
|
from .action_resolver import navigate_login, NavigateResult
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"verify_screen_match",
|
||||||
|
"ScreenMatchResult",
|
||||||
|
"navigate_login",
|
||||||
|
"NavigateResult",
|
||||||
|
"_handle_navigate_action",
|
||||||
|
]
|
||||||
|
|
||||||
|
# Handler pour replay_engine — importé par api_stream.py
|
||||||
|
def _handle_navigate_action(
|
||||||
|
action: dict,
|
||||||
|
replay_state: dict,
|
||||||
|
session_id: str,
|
||||||
|
) -> bool:
|
||||||
|
"""Handler serveur pour action navigate (branchement replay_engine).
|
||||||
|
|
||||||
|
Thin wrapper : résout coords du login form et les stocke dans
|
||||||
|
replay_state["variables"] pour les actions type/click suivantes.
|
||||||
|
|
||||||
|
N'échoue jamais le replay — toute erreur → log + needs_review.
|
||||||
|
"""
|
||||||
|
import logging
|
||||||
|
logger = logging.getLogger("navigation._handle_navigate_action")
|
||||||
|
|
||||||
|
params = action.get("parameters") or {}
|
||||||
|
navigate_action = params.get("action", "login")
|
||||||
|
|
||||||
|
# Noms des variables output (configurable)
|
||||||
|
login_var = (params.get("login_coords_var") or "navigate_login_coords").strip()
|
||||||
|
password_var = (params.get("password_coords_var") or "navigate_password_coords").strip()
|
||||||
|
submit_var = (params.get("submit_coords_var") or "navigate_submit_coords").strip()
|
||||||
|
|
||||||
|
variables = replay_state.setdefault("variables", {})
|
||||||
|
|
||||||
|
try:
|
||||||
|
screenshot_path = ""
|
||||||
|
# Résoudre screenshot depuis replay_state
|
||||||
|
if "last_screenshot_path" in replay_state:
|
||||||
|
screenshot_path = replay_state["last_screenshot_path"]
|
||||||
|
elif "last_heartbeat" in replay_state:
|
||||||
|
hb = replay_state["last_heartbeat"]
|
||||||
|
screenshot_path = hb.get("screenshot_path", "") if isinstance(hb, dict) else ""
|
||||||
|
|
||||||
|
if not screenshot_path:
|
||||||
|
logger.warning("navigate: no screenshot for session %s", session_id)
|
||||||
|
variables[login_var] = {"error": "no_screenshot"}
|
||||||
|
return False
|
||||||
|
|
||||||
|
# Dimensions écran (fallback 1920×1080)
|
||||||
|
screen_width = replay_state.get("screen_width", 1920)
|
||||||
|
screen_height = replay_state.get("screen_height", 1080)
|
||||||
|
|
||||||
|
# OCR/VLM clients — lazy import pour éviter circular dependency
|
||||||
|
from core.llm import extract_grid_from_image
|
||||||
|
from core.extraction.vlm_client import make_vllm_client
|
||||||
|
from core.navigation.action_resolver import make_ocr_detailed_from_grid
|
||||||
|
|
||||||
|
ocr_detailed = make_ocr_detailed_from_grid(extract_grid_from_image)
|
||||||
|
vlm_client = make_vllm_client()
|
||||||
|
|
||||||
|
# Config login
|
||||||
|
from core.navigation.visual_login import LoginFormConfig, dpi_urgences_login_config
|
||||||
|
config = dpi_urgences_login_config()
|
||||||
|
if "login_field" in params:
|
||||||
|
config = LoginFormConfig(
|
||||||
|
login_field=params.get("login_field", config.login_field),
|
||||||
|
password_field=params.get("password_field", config.password_field),
|
||||||
|
submit_button=params.get("submit_button", config.submit_button),
|
||||||
|
success_elements=params.get("success_elements", config.success_elements),
|
||||||
|
context=params.get("context", config.context),
|
||||||
|
)
|
||||||
|
|
||||||
|
# Orchestration navigate
|
||||||
|
from core.navigation.action_resolver import navigate_login
|
||||||
|
result = navigate_login(
|
||||||
|
screenshot_path, config=config,
|
||||||
|
ocr_client=ocr_detailed, vlm_client=vlm_client,
|
||||||
|
screen_width=screen_width, screen_height=screen_height,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Stocker coords dans variables (format dict pour substitution)
|
||||||
|
if result.login_coords:
|
||||||
|
variables[login_var] = result.login_coords.to_dict()
|
||||||
|
if result.password_coords:
|
||||||
|
variables[password_var] = result.password_coords.to_dict()
|
||||||
|
if result.submit_coords:
|
||||||
|
variables[submit_var] = result.submit_coords.to_dict()
|
||||||
|
|
||||||
|
variables["navigate_result"] = {
|
||||||
|
"all_resolved": result.all_resolved,
|
||||||
|
"method": result.login_coords.method if result.login_coords else "",
|
||||||
|
"error": result.error,
|
||||||
|
}
|
||||||
|
|
||||||
|
if not result.all_resolved:
|
||||||
|
logger.warning("navigate: incomplete — %s", result.error)
|
||||||
|
return False
|
||||||
|
|
||||||
|
logger.info("navigate: login form resolved OK (method=%s)", result.login_coords.method if result.login_coords else "?")
|
||||||
|
return True
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("navigate: exception (%s) — needs_review", e)
|
||||||
|
variables["navigate_result"] = {"all_resolved": False, "error": str(e)}
|
||||||
|
return False
|
||||||
205
core/navigation/action_resolver.py
Normal file
205
core/navigation/action_resolver.py
Normal file
@@ -0,0 +1,205 @@
|
|||||||
|
"""Action resolver — pont entre modules navigation et runtime replay.
|
||||||
|
|
||||||
|
Orchestre verify → ground → store coords pour le handler replay_engine.
|
||||||
|
Convertit coords pixels → normalisé (x_pct/y_pct) pour le client Agent V1.
|
||||||
|
|
||||||
|
Architecture :
|
||||||
|
- handler replay_engine = thin wrapper (appelle action_resolver)
|
||||||
|
- action_resolver = bridge (adapte OCR/VLM runtime → interfaces navigation)
|
||||||
|
- modules navigation = pure functions (ne connaissent pas le runtime)
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from typing import Any, Callable, Dict, List, Optional, Tuple
|
||||||
|
|
||||||
|
from core.navigation.grounding import (
|
||||||
|
BBox,
|
||||||
|
CoordsCache,
|
||||||
|
GroundedElement,
|
||||||
|
OcrDetailedClient,
|
||||||
|
OcrTokenInfo,
|
||||||
|
ground_element,
|
||||||
|
)
|
||||||
|
from core.navigation.visual_login import (
|
||||||
|
LoginFormConfig,
|
||||||
|
LoginResolution,
|
||||||
|
dpi_urgences_login_config,
|
||||||
|
resolve_login_form,
|
||||||
|
verify_login_visible,
|
||||||
|
verify_login_success,
|
||||||
|
)
|
||||||
|
from core.navigation.visual_verifier import (
|
||||||
|
OcrClient,
|
||||||
|
ScreenMatchResult,
|
||||||
|
VlmClient,
|
||||||
|
)
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
# ── Dataclasses ──────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class NavigateCoords:
|
||||||
|
"""Normalized coords for a grounded element — format Agent V1 client."""
|
||||||
|
|
||||||
|
x_pct: float # center x normalized [0-1]
|
||||||
|
y_pct: float # center y normalized [0-1]
|
||||||
|
bbox_pct: Optional[Tuple[float, float, float, float]] = None # (x1, y1, x2, y2) normalized
|
||||||
|
method: str = "" # grounding method used
|
||||||
|
|
||||||
|
def to_dict(self) -> Dict[str, Any]:
|
||||||
|
d = {"x_pct": self.x_pct, "y_pct": self.y_pct, "method": self.method}
|
||||||
|
if self.bbox_pct:
|
||||||
|
d["bbox_pct"] = list(self.bbox_pct)
|
||||||
|
return d
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class NavigateResult:
|
||||||
|
"""Result of a navigate action — coords for each resolved field."""
|
||||||
|
|
||||||
|
login_coords: Optional[NavigateCoords] = None
|
||||||
|
password_coords: Optional[NavigateCoords] = None
|
||||||
|
submit_coords: Optional[NavigateCoords] = None
|
||||||
|
all_resolved: bool = False
|
||||||
|
pre_verify: Optional[ScreenMatchResult] = None
|
||||||
|
post_verify: Optional[ScreenMatchResult] = None # set later by verify_after
|
||||||
|
error: str = ""
|
||||||
|
|
||||||
|
|
||||||
|
# ── Coordinate conversion ────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def grounded_to_coords(
|
||||||
|
element: GroundedElement,
|
||||||
|
screen_width: int,
|
||||||
|
screen_height: int,
|
||||||
|
) -> NavigateCoords:
|
||||||
|
"""Convert GroundedElement (pixels) to NavigateCoords (normalized pct)."""
|
||||||
|
x_pct = element.center[0] / screen_width if screen_width else 0
|
||||||
|
y_pct = element.center[1] / screen_height if screen_height else 0
|
||||||
|
x1_pct = element.bbox[0] / screen_width if screen_width else 0
|
||||||
|
y1_pct = element.bbox[1] / screen_height if screen_height else 0
|
||||||
|
x2_pct = element.bbox[2] / screen_width if screen_width else 0
|
||||||
|
y2_pct = element.bbox[3] / screen_height if screen_height else 0
|
||||||
|
return NavigateCoords(
|
||||||
|
x_pct=x_pct,
|
||||||
|
y_pct=y_pct,
|
||||||
|
bbox_pct=(x1_pct, y1_pct, x2_pct, y2_pct),
|
||||||
|
method=element.method,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ── OCR adapter ──────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def make_ocr_detailed_from_grid(
|
||||||
|
grid_fn: Callable[[str], List[List[Dict[str, Any]]]],
|
||||||
|
) -> OcrDetailedClient:
|
||||||
|
"""Adapt extract_grid_from_image → OcrDetailedClient (List[OcrTokenInfo]).
|
||||||
|
|
||||||
|
Converts the grid format (list of rows of cells with bbox) into
|
||||||
|
flat OcrTokenInfo list with normalized LTRB bbox.
|
||||||
|
"""
|
||||||
|
from core.extraction.role_mapper import tokens_from_grid
|
||||||
|
|
||||||
|
def client(image_path: str) -> List[OcrTokenInfo]:
|
||||||
|
grid = grid_fn(image_path)
|
||||||
|
ocr_tokens = tokens_from_grid(grid)
|
||||||
|
return [
|
||||||
|
OcrTokenInfo(
|
||||||
|
text=t.text,
|
||||||
|
bbox=t.bbox,
|
||||||
|
confidence=t.confidence,
|
||||||
|
)
|
||||||
|
for t in ocr_tokens
|
||||||
|
]
|
||||||
|
|
||||||
|
return client
|
||||||
|
|
||||||
|
|
||||||
|
def make_ocr_simple_from_detailed(
|
||||||
|
ocr_detailed: OcrDetailedClient,
|
||||||
|
) -> OcrClient:
|
||||||
|
"""Derive text-only OcrClient from OcrDetailedClient."""
|
||||||
|
def client(image_path: str) -> List[str]:
|
||||||
|
return [t.text for t in ocr_detailed(image_path)]
|
||||||
|
return client
|
||||||
|
|
||||||
|
|
||||||
|
# ── Navigate login orchestration ─────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def navigate_login(
|
||||||
|
screenshot_path: str,
|
||||||
|
config: Optional[LoginFormConfig] = None,
|
||||||
|
ocr_client: Optional[OcrDetailedClient] = None,
|
||||||
|
vlm_client: Optional[VlmClient] = None,
|
||||||
|
screen_width: int = 1920,
|
||||||
|
screen_height: int = 1080,
|
||||||
|
coords_cache: Optional[CoordsCache] = None,
|
||||||
|
skip_pre_verify: bool = False,
|
||||||
|
) -> NavigateResult:
|
||||||
|
"""Orchestrate login navigation: verify → ground → convert coords.
|
||||||
|
|
||||||
|
Returns NavigateResult with normalized coords for each field.
|
||||||
|
The handler stores these in replay_state variables for subsequent
|
||||||
|
type/click actions.
|
||||||
|
"""
|
||||||
|
if config is None:
|
||||||
|
config = dpi_urgences_login_config()
|
||||||
|
|
||||||
|
if ocr_client is None or vlm_client is None:
|
||||||
|
return NavigateResult(
|
||||||
|
all_resolved=False,
|
||||||
|
error="ocr_client and vlm_client required",
|
||||||
|
)
|
||||||
|
|
||||||
|
ocr_simple = make_ocr_simple_from_detailed(ocr_client)
|
||||||
|
|
||||||
|
# Step 1: Pre-verification (optional)
|
||||||
|
pre_verify = None
|
||||||
|
if not skip_pre_verify:
|
||||||
|
pre_verify = verify_login_visible(
|
||||||
|
screenshot_path, config, ocr_simple, vlm_client,
|
||||||
|
)
|
||||||
|
if not pre_verify.match:
|
||||||
|
logger.warning("navigate_login: pre-verify failed — %s", pre_verify.describe())
|
||||||
|
return NavigateResult(
|
||||||
|
all_resolved=False,
|
||||||
|
pre_verify=pre_verify,
|
||||||
|
error=f"pre-verify failed: {pre_verify.describe()}",
|
||||||
|
)
|
||||||
|
|
||||||
|
# Step 2: Ground all fields
|
||||||
|
resolution = resolve_login_form(
|
||||||
|
screenshot_path, config, ocr_client, vlm_client,
|
||||||
|
screen_width=screen_width, screen_height=screen_height,
|
||||||
|
coords_cache=coords_cache,
|
||||||
|
)
|
||||||
|
|
||||||
|
if not resolution.all_resolved:
|
||||||
|
logger.warning("navigate_login: incomplete resolution — %s", resolution.describe())
|
||||||
|
return NavigateResult(
|
||||||
|
all_resolved=False,
|
||||||
|
pre_verify=pre_verify,
|
||||||
|
error=f"incomplete resolution: {resolution.describe()}",
|
||||||
|
)
|
||||||
|
|
||||||
|
# Step 3: Convert to normalized coords
|
||||||
|
login_coords = grounded_to_coords(resolution.login_field, screen_width, screen_height) if resolution.login_field else None
|
||||||
|
password_coords = grounded_to_coords(resolution.password_field, screen_width, screen_height) if resolution.password_field else None
|
||||||
|
submit_coords = grounded_to_coords(resolution.submit_button, screen_width, screen_height) if resolution.submit_button else None
|
||||||
|
|
||||||
|
return NavigateResult(
|
||||||
|
login_coords=login_coords,
|
||||||
|
password_coords=password_coords,
|
||||||
|
submit_coords=submit_coords,
|
||||||
|
all_resolved=True,
|
||||||
|
pre_verify=pre_verify,
|
||||||
|
)
|
||||||
375
core/navigation/grounding.py
Normal file
375
core/navigation/grounding.py
Normal file
@@ -0,0 +1,375 @@
|
|||||||
|
"""Grounding — résolution visuelle d'éléments UI → coords (bbox + center).
|
||||||
|
|
||||||
|
Architecture OCR-ancrée (alignée avec visual_verifier) :
|
||||||
|
- STRATÉGIE 1 : OCR-anchor — si le texte cible est trouvé par OCR,
|
||||||
|
utiliser le bbox du token OCR (déterministe, zero hallucination).
|
||||||
|
- STRATÉGIE 2 : VLM grounder — si OCR ne trouve pas le texte,
|
||||||
|
le VLM localise l'élément visuellement (fallback, risque contrôlé).
|
||||||
|
- CACHE coords : mémorise les coords résolues, validées par vision avant usage.
|
||||||
|
Si cached coords fail → re-résolution visuelle.
|
||||||
|
|
||||||
|
Coords = cache local validé par vue (Dom/Claude recadrage 01/07).
|
||||||
|
Vision = source de vérité, coords = shortcut validé.
|
||||||
|
|
||||||
|
BBox format interne : LTRB (x1, y1, x2, y2) pixels absolus —
|
||||||
|
cohérent avec SomElement, OcrToken, DetectedUIElement.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import re
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from typing import Any, Callable, Dict, List, Optional, Tuple
|
||||||
|
|
||||||
|
from core.navigation.visual_verifier import (
|
||||||
|
fuzzy_match,
|
||||||
|
normalize_text,
|
||||||
|
OcrClient,
|
||||||
|
VlmClient,
|
||||||
|
)
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# BBox format: LTRB pixels (x1, y1, x2, y2)
|
||||||
|
BBox = Tuple[int, int, int, int]
|
||||||
|
|
||||||
|
|
||||||
|
# ── Dataclasses ──────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class OcrTokenInfo:
|
||||||
|
"""OCR token with bounding box — for grounding (richer than text-only)."""
|
||||||
|
|
||||||
|
text: str
|
||||||
|
bbox: Optional[BBox] = None # (x1, y1, x2, y2) LTRB pixels
|
||||||
|
confidence: float = 1.0
|
||||||
|
|
||||||
|
|
||||||
|
# Type alias — injectable OCR client returning tokens with bbox
|
||||||
|
# More detailed than visual_verifier's OcrClient (which returns List[str])
|
||||||
|
OcrDetailedClient = Callable[[str], List[OcrTokenInfo]]
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class GroundedElement:
|
||||||
|
"""A UI element grounded on screen with coordinates."""
|
||||||
|
|
||||||
|
role: str
|
||||||
|
text: str
|
||||||
|
bbox: BBox # (x1, y1, x2, y2) LTRB pixels
|
||||||
|
center: Tuple[int, int] # (cx, cy) — click target
|
||||||
|
confidence: float
|
||||||
|
method: str # "ocr_anchor" or "vlm_grounder" or "cache"
|
||||||
|
source_ocr_text: str = "" # actual OCR text that matched (for fuzzy)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class CoordsCacheEntry:
|
||||||
|
"""Cached coordinates for a UI element."""
|
||||||
|
|
||||||
|
element_key: str # "role:text"
|
||||||
|
bbox: BBox
|
||||||
|
center: Tuple[int, int]
|
||||||
|
method: str # how it was originally resolved
|
||||||
|
validation_count: int = 0
|
||||||
|
|
||||||
|
|
||||||
|
class CoordsCache:
|
||||||
|
"""In-memory cache of grounded coordinates.
|
||||||
|
|
||||||
|
Entries are validated by vision before use (verify_after).
|
||||||
|
If cached coords fail verification → invalidate + re-resolve.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self._entries: Dict[str, CoordsCacheEntry] = {}
|
||||||
|
|
||||||
|
def get(self, element_key: str) -> Optional[CoordsCacheEntry]:
|
||||||
|
return self._entries.get(element_key)
|
||||||
|
|
||||||
|
def put(
|
||||||
|
self,
|
||||||
|
element_key: str,
|
||||||
|
bbox: BBox,
|
||||||
|
center: Tuple[int, int],
|
||||||
|
method: str,
|
||||||
|
) -> None:
|
||||||
|
entry = self._entries.get(element_key)
|
||||||
|
if entry:
|
||||||
|
entry.bbox = bbox
|
||||||
|
entry.center = center
|
||||||
|
entry.method = method
|
||||||
|
entry.validation_count += 1
|
||||||
|
else:
|
||||||
|
self._entries[element_key] = CoordsCacheEntry(
|
||||||
|
element_key=element_key,
|
||||||
|
bbox=bbox,
|
||||||
|
center=center,
|
||||||
|
method=method,
|
||||||
|
validation_count=1,
|
||||||
|
)
|
||||||
|
|
||||||
|
def invalidate(self, element_key: str) -> None:
|
||||||
|
self._entries.pop(element_key, None)
|
||||||
|
|
||||||
|
def clear(self) -> None:
|
||||||
|
self._entries.clear()
|
||||||
|
|
||||||
|
def keys(self) -> List[str]:
|
||||||
|
return list(self._entries.keys())
|
||||||
|
|
||||||
|
|
||||||
|
# ── Helper functions ─────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def bbox_center(bbox: BBox) -> Tuple[int, int]:
|
||||||
|
"""Compute center point from LTRB bbox."""
|
||||||
|
x1, y1, x2, y2 = bbox
|
||||||
|
return ((x1 + x2) // 2, (y1 + y2) // 2)
|
||||||
|
|
||||||
|
|
||||||
|
def make_element_key(role: str, text: str) -> str:
|
||||||
|
"""Create a stable cache key from role + text."""
|
||||||
|
return f"{role}:{normalize_text(text)}"
|
||||||
|
|
||||||
|
|
||||||
|
# ── OCR-anchored grounding (deterministic) ───────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def ocr_anchor_ground(
|
||||||
|
ocr_tokens: List[OcrTokenInfo],
|
||||||
|
target: Dict[str, Any],
|
||||||
|
fuzzy_threshold: float = 0.8,
|
||||||
|
) -> Optional[GroundedElement]:
|
||||||
|
"""Ground an element using OCR tokens with bbox (deterministic).
|
||||||
|
|
||||||
|
Finds the target text in OCR tokens via fuzzy match.
|
||||||
|
Returns GroundedElement with bbox from the matching OCR token.
|
||||||
|
"""
|
||||||
|
target_text = target.get("text", "")
|
||||||
|
target_role = target.get("role", "?")
|
||||||
|
|
||||||
|
if not target_text:
|
||||||
|
return None
|
||||||
|
|
||||||
|
for token in ocr_tokens:
|
||||||
|
if fuzzy_match(target_text, token.text, threshold=fuzzy_threshold):
|
||||||
|
if token.bbox is None:
|
||||||
|
continue # token found but no bbox → can't ground
|
||||||
|
|
||||||
|
return GroundedElement(
|
||||||
|
role=target_role,
|
||||||
|
text=target_text,
|
||||||
|
bbox=token.bbox,
|
||||||
|
center=bbox_center(token.bbox),
|
||||||
|
confidence=token.confidence,
|
||||||
|
method="ocr_anchor",
|
||||||
|
source_ocr_text=token.text,
|
||||||
|
)
|
||||||
|
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
# ── VLM grounder (fallback) ─────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def build_grounder_prompt(
|
||||||
|
target: Dict[str, Any],
|
||||||
|
context: str = "",
|
||||||
|
) -> str:
|
||||||
|
"""Build VLM prompt for locating a UI element on screen.
|
||||||
|
|
||||||
|
Asks for bounding box in normalized coordinates [0-1].
|
||||||
|
"""
|
||||||
|
role = target.get("role", "?")
|
||||||
|
text = target.get("text", "")
|
||||||
|
extra = target.get("extra", "")
|
||||||
|
|
||||||
|
prompt = (
|
||||||
|
"You are a UI element locator. Find the specified element on this "
|
||||||
|
"screenshot and return its bounding box.\n"
|
||||||
|
)
|
||||||
|
if context:
|
||||||
|
prompt += f"Context: {context}\n"
|
||||||
|
prompt += f"Target element: {role} with text \"{text}\""
|
||||||
|
if extra:
|
||||||
|
prompt += f" ({extra})"
|
||||||
|
prompt += (
|
||||||
|
"\n\nRespond in JSON format:\n"
|
||||||
|
"{\"found\": true/false, "
|
||||||
|
"\"bbox\": [x1_norm, y1_norm, x2_norm, y2_norm], "
|
||||||
|
"\"confidence\": 0.0-1.0, "
|
||||||
|
"\"description\": \"...\"}\n"
|
||||||
|
"bbox coordinates are normalized [0.0-1.0] relative to image dimensions "
|
||||||
|
"(x1=left, y1=top, x2=right, y2=bottom). "
|
||||||
|
"Only return found=true if you can clearly locate the element."
|
||||||
|
)
|
||||||
|
return prompt
|
||||||
|
|
||||||
|
|
||||||
|
def parse_grounder_response(
|
||||||
|
vlm_text: str,
|
||||||
|
screen_width: int,
|
||||||
|
screen_height: int,
|
||||||
|
target: Dict[str, Any],
|
||||||
|
) -> Optional[GroundedElement]:
|
||||||
|
"""Parse VLM grounder response into GroundedElement.
|
||||||
|
|
||||||
|
Converts normalized bbox [0-1] to absolute pixels.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
data = json.loads(vlm_text)
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
json_match = re.search(r"\{[\s\S]*\}", vlm_text)
|
||||||
|
if json_match:
|
||||||
|
try:
|
||||||
|
data = json.loads(json_match.group())
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
logger.warning("grounding: VLM response not parseable as JSON")
|
||||||
|
return None
|
||||||
|
else:
|
||||||
|
return None
|
||||||
|
|
||||||
|
if not data.get("found", False):
|
||||||
|
return None
|
||||||
|
|
||||||
|
bbox_norm = data.get("bbox", [])
|
||||||
|
if not isinstance(bbox_norm, list) or len(bbox_norm) != 4:
|
||||||
|
logger.warning("grounding: invalid bbox format from VLM")
|
||||||
|
return None
|
||||||
|
|
||||||
|
# Convert normalized [0-1] to absolute pixels
|
||||||
|
try:
|
||||||
|
x1 = int(float(bbox_norm[0]) * screen_width)
|
||||||
|
y1 = int(float(bbox_norm[1]) * screen_height)
|
||||||
|
x2 = int(float(bbox_norm[2]) * screen_width)
|
||||||
|
y2 = int(float(bbox_norm[3]) * screen_height)
|
||||||
|
except (ValueError, TypeError):
|
||||||
|
logger.warning("grounding: bbox values not numeric")
|
||||||
|
return None
|
||||||
|
|
||||||
|
# Clamp to screen bounds
|
||||||
|
x1 = max(0, min(x1, screen_width))
|
||||||
|
y1 = max(0, min(y1, screen_height))
|
||||||
|
x2 = max(x1, min(x2, screen_width))
|
||||||
|
y2 = max(y1, min(y2, screen_height))
|
||||||
|
|
||||||
|
confidence = data.get("confidence", 0.5)
|
||||||
|
if isinstance(confidence, str):
|
||||||
|
try:
|
||||||
|
confidence = float(confidence)
|
||||||
|
except ValueError:
|
||||||
|
confidence = 0.5
|
||||||
|
|
||||||
|
bbox_abs: BBox = (x1, y1, x2, y2)
|
||||||
|
|
||||||
|
return GroundedElement(
|
||||||
|
role=target.get("role", "?"),
|
||||||
|
text=target.get("text", ""),
|
||||||
|
bbox=bbox_abs,
|
||||||
|
center=bbox_center(bbox_abs),
|
||||||
|
confidence=confidence,
|
||||||
|
method="vlm_grounder",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ── Core grounding function (composition) ───────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def ground_element(
|
||||||
|
screenshot_path: str,
|
||||||
|
target: Dict[str, Any],
|
||||||
|
ocr_client: OcrDetailedClient,
|
||||||
|
vlm_client: VlmClient,
|
||||||
|
screen_width: int = 1920,
|
||||||
|
screen_height: int = 1080,
|
||||||
|
coords_cache: Optional[CoordsCache] = None,
|
||||||
|
context: str = "",
|
||||||
|
fuzzy_threshold: float = 0.8,
|
||||||
|
) -> Optional[GroundedElement]:
|
||||||
|
"""Ground a UI element on screen — OCR-anchor first, VLM fallback.
|
||||||
|
|
||||||
|
Resolution strategy:
|
||||||
|
1. Cache: if cached coords exist → return cached (validated separately)
|
||||||
|
2. OCR-anchor: deterministic, zero hallucination
|
||||||
|
3. VLM grounder: fallback when OCR can't find the text
|
||||||
|
|
||||||
|
Args:
|
||||||
|
screenshot_path: path to screenshot image
|
||||||
|
target: {"role": "bouton", "text": "Connexion"} — element to find
|
||||||
|
ocr_client: injectable OCR client returning List[OcrTokenInfo]
|
||||||
|
vlm_client: injectable VLM client (image_path, prompt) -> text
|
||||||
|
screen_width/height: screen dimensions for pixel conversion
|
||||||
|
coords_cache: optional CoordsCache for memoization
|
||||||
|
context: optional context (e.g. "page login DPI")
|
||||||
|
fuzzy_threshold: fuzzy match threshold for OCR anchoring
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
GroundedElement with bbox + center, or None if not found
|
||||||
|
"""
|
||||||
|
target_text = target.get("text", "")
|
||||||
|
target_role = target.get("role", "?")
|
||||||
|
element_key = make_element_key(target_role, target_text)
|
||||||
|
|
||||||
|
# Step 0: Check cache
|
||||||
|
if coords_cache:
|
||||||
|
cached = coords_cache.get(element_key)
|
||||||
|
if cached:
|
||||||
|
cached.validation_count += 1
|
||||||
|
logger.info("grounding: using cached coords for %s", element_key)
|
||||||
|
return GroundedElement(
|
||||||
|
role=target_role,
|
||||||
|
text=target_text,
|
||||||
|
bbox=cached.bbox,
|
||||||
|
center=cached.center,
|
||||||
|
confidence=1.0, # cached = previously validated
|
||||||
|
method="cache",
|
||||||
|
)
|
||||||
|
|
||||||
|
# Step 1: OCR-anchor (deterministic)
|
||||||
|
try:
|
||||||
|
ocr_tokens = ocr_client(screenshot_path)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("grounding: OCR call failed (%s)", e)
|
||||||
|
ocr_tokens = []
|
||||||
|
|
||||||
|
ocr_result = ocr_anchor_ground(ocr_tokens, target, fuzzy_threshold)
|
||||||
|
|
||||||
|
if ocr_result:
|
||||||
|
if coords_cache:
|
||||||
|
coords_cache.put(element_key, ocr_result.bbox, ocr_result.center, "ocr_anchor")
|
||||||
|
logger.info(
|
||||||
|
"grounding: OCR-anchor found '%s' (matched OCR='%s', conf=%.2f)",
|
||||||
|
target_text, ocr_result.source_ocr_text, ocr_result.confidence,
|
||||||
|
)
|
||||||
|
return ocr_result
|
||||||
|
|
||||||
|
# Step 2: VLM grounder (fallback)
|
||||||
|
if not target_text:
|
||||||
|
logger.warning("grounding: no text for target, VLM grounder needs text")
|
||||||
|
return None
|
||||||
|
|
||||||
|
prompt = build_grounder_prompt(target, context)
|
||||||
|
|
||||||
|
try:
|
||||||
|
vlm_text = vlm_client(screenshot_path, prompt)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("grounding: VLM grounder call failed (%s)", e)
|
||||||
|
return None
|
||||||
|
|
||||||
|
vlm_result = parse_grounder_response(vlm_text, screen_width, screen_height, target)
|
||||||
|
|
||||||
|
if vlm_result:
|
||||||
|
if coords_cache:
|
||||||
|
coords_cache.put(element_key, vlm_result.bbox, vlm_result.center, "vlm_grounder")
|
||||||
|
logger.info(
|
||||||
|
"grounding: VLM grounder found '%s' (conf=%.2f)",
|
||||||
|
target_text, vlm_result.confidence,
|
||||||
|
)
|
||||||
|
return vlm_result
|
||||||
|
|
||||||
|
logger.warning("grounding: element '%s' not found by OCR or VLM", target_text)
|
||||||
|
return None
|
||||||
227
core/navigation/visual_login.py
Normal file
227
core/navigation/visual_login.py
Normal file
@@ -0,0 +1,227 @@
|
|||||||
|
"""Visual login — résolution + vérification du formulaire de login par grounding.
|
||||||
|
|
||||||
|
Architecture (alignée visual_verifier + grounding) :
|
||||||
|
- verify_before : formulaire login visible (champs + bouton présents)
|
||||||
|
- resolve_login_form : ground chaque champ (login, password, bouton) → coords
|
||||||
|
- verify_after : dashboard/accueil visible (post-login)
|
||||||
|
- Chaque étape encadrée par vision (DETTE-023 couvert)
|
||||||
|
|
||||||
|
Coords = cache local validé par vue (Dom/Claude recadrage).
|
||||||
|
Le runtime exécute les actions (type/click) — ce module résout + valide.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from typing import Any, Callable, Dict, List, Optional, Tuple
|
||||||
|
|
||||||
|
from core.navigation.grounding import (
|
||||||
|
BBox,
|
||||||
|
CoordsCache,
|
||||||
|
GroundedElement,
|
||||||
|
OcrDetailedClient,
|
||||||
|
OcrTokenInfo,
|
||||||
|
ground_element,
|
||||||
|
)
|
||||||
|
from core.navigation.visual_verifier import (
|
||||||
|
OcrClient,
|
||||||
|
ScreenMatchResult,
|
||||||
|
VlmClient,
|
||||||
|
verify_before,
|
||||||
|
verify_after,
|
||||||
|
)
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
# ── Dataclasses ──────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class LoginFormConfig:
|
||||||
|
"""Configuration for a login form — what to look for."""
|
||||||
|
|
||||||
|
login_field: Dict[str, Any] # {"role": "champ", "text": "Login"}
|
||||||
|
password_field: Dict[str, Any] # {"role": "champ", "text": "Mot de passe"}
|
||||||
|
submit_button: Dict[str, Any] # {"role": "bouton", "text": "Connexion"}
|
||||||
|
success_elements: List[Dict[str, Any]] = field(default_factory=list)
|
||||||
|
context: str = "" # e.g. "DPI urgences"
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class LoginResolution:
|
||||||
|
"""Result of login form resolution — grounded coords for each field."""
|
||||||
|
|
||||||
|
login_field: Optional[GroundedElement] = None
|
||||||
|
password_field: Optional[GroundedElement] = None
|
||||||
|
submit_button: Optional[GroundedElement] = None
|
||||||
|
all_resolved: bool = False
|
||||||
|
method: str = "" # "ocr_anchor", "vlm_grounder", "mixed", "cache"
|
||||||
|
|
||||||
|
def describe(self) -> str:
|
||||||
|
parts = []
|
||||||
|
if self.login_field:
|
||||||
|
parts.append(f"login@{self.login_field.center} ({self.login_field.method})")
|
||||||
|
else:
|
||||||
|
parts.append("login: NOT FOUND")
|
||||||
|
if self.password_field:
|
||||||
|
parts.append(f"password@{self.password_field.center} ({self.password_field.method})")
|
||||||
|
else:
|
||||||
|
parts.append("password: NOT FOUND")
|
||||||
|
if self.submit_button:
|
||||||
|
parts.append(f"button@{self.submit_button.center} ({self.submit_button.method})")
|
||||||
|
else:
|
||||||
|
parts.append("button: NOT FOUND")
|
||||||
|
status = "OK" if self.all_resolved else "INCOMPLETE"
|
||||||
|
return f"Login resolution [{status}]: " + ", ".join(parts)
|
||||||
|
|
||||||
|
|
||||||
|
# ── Default configs ──────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def dpi_urgences_login_config() -> LoginFormConfig:
|
||||||
|
"""Default config for DPI urgences login form."""
|
||||||
|
return LoginFormConfig(
|
||||||
|
login_field={"role": "champ", "text": "Login", "extra": "champ identifiant"},
|
||||||
|
password_field={"role": "champ", "text": "Mot de passe", "extra": "champ password"},
|
||||||
|
submit_button={"role": "bouton", "text": "Connexion", "extra": "bouton submit"},
|
||||||
|
success_elements=[
|
||||||
|
{"role": "page", "text": "Accueil"},
|
||||||
|
{"role": "page", "text": "Dashboard"},
|
||||||
|
],
|
||||||
|
context="DPI urgences — page login",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ── Helper ───────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def _ocr_detailed_to_simple(ocr_detailed: OcrDetailedClient) -> OcrClient:
|
||||||
|
"""Convert OcrDetailedClient (text+bbox) to OcrClient (text-only) for verification."""
|
||||||
|
def client(image_path: str) -> List[str]:
|
||||||
|
return [t.text for t in ocr_detailed(image_path)]
|
||||||
|
return client
|
||||||
|
|
||||||
|
|
||||||
|
# ── Core functions ───────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def verify_login_visible(
|
||||||
|
screenshot_path: str,
|
||||||
|
config: LoginFormConfig,
|
||||||
|
ocr_client: OcrClient,
|
||||||
|
vlm_client: VlmClient,
|
||||||
|
) -> ScreenMatchResult:
|
||||||
|
"""Verify login form is visible on screen (pre-condition).
|
||||||
|
|
||||||
|
Checks that login field, password field, and submit button are present.
|
||||||
|
Uses OCR-anchored verification (deterministic presence, VLM role).
|
||||||
|
"""
|
||||||
|
expected = [
|
||||||
|
config.login_field,
|
||||||
|
config.password_field,
|
||||||
|
config.submit_button,
|
||||||
|
]
|
||||||
|
return verify_before(
|
||||||
|
screenshot_path, expected, ocr_client, vlm_client,
|
||||||
|
context=config.context,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def verify_login_success(
|
||||||
|
screenshot_path: str,
|
||||||
|
config: LoginFormConfig,
|
||||||
|
ocr_client: OcrClient,
|
||||||
|
vlm_client: VlmClient,
|
||||||
|
) -> ScreenMatchResult:
|
||||||
|
"""Verify dashboard/accueil visible after login (post-condition).
|
||||||
|
|
||||||
|
Higher threshold (verify_after = 0.8) — false positive = Léa proceeds wrong.
|
||||||
|
"""
|
||||||
|
if not config.success_elements:
|
||||||
|
# No success criteria defined → can't verify
|
||||||
|
return ScreenMatchResult(
|
||||||
|
match=False,
|
||||||
|
confidence=0.0,
|
||||||
|
reason="no success_elements defined in config",
|
||||||
|
)
|
||||||
|
return verify_after(
|
||||||
|
screenshot_path, config.success_elements, ocr_client, vlm_client,
|
||||||
|
context=f"POST-LOGIN: {config.context}",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_login_form(
|
||||||
|
screenshot_path: str,
|
||||||
|
config: LoginFormConfig,
|
||||||
|
ocr_client: OcrDetailedClient,
|
||||||
|
vlm_client: VlmClient,
|
||||||
|
screen_width: int = 1920,
|
||||||
|
screen_height: int = 1080,
|
||||||
|
coords_cache: Optional[CoordsCache] = None,
|
||||||
|
) -> LoginResolution:
|
||||||
|
"""Ground all login form elements → coords for runtime action.
|
||||||
|
|
||||||
|
Resolution strategy per element:
|
||||||
|
1. Cache hit → return cached coords (validated separately)
|
||||||
|
2. OCR-anchor → deterministic bbox from OCR token
|
||||||
|
3. VLM grounder → fallback visual grounding
|
||||||
|
|
||||||
|
Returns LoginResolution with grounded coords for each field.
|
||||||
|
Runtime uses these coords to type/click.
|
||||||
|
"""
|
||||||
|
login_el = ground_element(
|
||||||
|
screenshot_path, config.login_field,
|
||||||
|
ocr_client=ocr_client, vlm_client=vlm_client,
|
||||||
|
screen_width=screen_width, screen_height=screen_height,
|
||||||
|
coords_cache=coords_cache, context=config.context,
|
||||||
|
)
|
||||||
|
|
||||||
|
password_el = ground_element(
|
||||||
|
screenshot_path, config.password_field,
|
||||||
|
ocr_client=ocr_client, vlm_client=vlm_client,
|
||||||
|
screen_width=screen_width, screen_height=screen_height,
|
||||||
|
coords_cache=coords_cache, context=config.context,
|
||||||
|
)
|
||||||
|
|
||||||
|
button_el = ground_element(
|
||||||
|
screenshot_path, config.submit_button,
|
||||||
|
ocr_client=ocr_client, vlm_client=vlm_client,
|
||||||
|
screen_width=screen_width, screen_height=screen_height,
|
||||||
|
coords_cache=coords_cache, context=config.context,
|
||||||
|
)
|
||||||
|
|
||||||
|
all_resolved = login_el is not None and password_el is not None and button_el is not None
|
||||||
|
|
||||||
|
# Determine overall method
|
||||||
|
methods = []
|
||||||
|
if login_el:
|
||||||
|
methods.append(login_el.method)
|
||||||
|
if password_el:
|
||||||
|
methods.append(password_el.method)
|
||||||
|
if button_el:
|
||||||
|
methods.append(button_el.method)
|
||||||
|
|
||||||
|
unique_methods = set(methods)
|
||||||
|
if len(unique_methods) == 1:
|
||||||
|
method = unique_methods.pop()
|
||||||
|
elif len(unique_methods) > 1:
|
||||||
|
method = "mixed"
|
||||||
|
else:
|
||||||
|
method = ""
|
||||||
|
|
||||||
|
resolution = LoginResolution(
|
||||||
|
login_field=login_el,
|
||||||
|
password_field=password_el,
|
||||||
|
submit_button=button_el,
|
||||||
|
all_resolved=all_resolved,
|
||||||
|
method=method,
|
||||||
|
)
|
||||||
|
|
||||||
|
if all_resolved:
|
||||||
|
logger.info("resolve_login_form: %s", resolution.describe())
|
||||||
|
else:
|
||||||
|
logger.warning("resolve_login_form: incomplete — %s", resolution.describe())
|
||||||
|
|
||||||
|
return resolution
|
||||||
408
core/navigation/visual_verifier.py
Normal file
408
core/navigation/visual_verifier.py
Normal file
@@ -0,0 +1,408 @@
|
|||||||
|
"""Visual verifier — verify_before / verify_after avec ancrage OCR.
|
||||||
|
|
||||||
|
Architecture OCR-ancrée (challenge Claude 01/07, gate-vert 30/06) :
|
||||||
|
- PRESENCE = tokens OCR (déterministe, pas d'hallucination possible)
|
||||||
|
- RÔLE = VLM confirmation (semantic, ancré sur tokens OCR trouvés)
|
||||||
|
- VLM ne décide JAMAIS de la présence d'un élément
|
||||||
|
- Faux positif impossible par construction ; faux négatif = retry acceptable
|
||||||
|
|
||||||
|
Pattern d'injection : OcrClient + VlmClient injectables (tests sans réseau).
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import re
|
||||||
|
import unicodedata
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from difflib import SequenceMatcher
|
||||||
|
from typing import Any, Callable, Dict, List, Optional
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# Type aliases — injectable callables for offline testing
|
||||||
|
VlmClient = Callable[[str, str], str] # (image_path, prompt) -> text
|
||||||
|
OcrClient = Callable[[str], List[str]] # (image_path) -> list of OCR text strings
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class ScreenMatchResult:
|
||||||
|
"""Result of a screen verification check."""
|
||||||
|
|
||||||
|
match: bool
|
||||||
|
confidence: float = 0.0
|
||||||
|
reason: str = ""
|
||||||
|
observed_elements: List[Dict[str, Any]] = field(default_factory=list)
|
||||||
|
expected_elements: List[Dict[str, Any]] = field(default_factory=list)
|
||||||
|
mismatches: List[str] = field(default_factory=list)
|
||||||
|
|
||||||
|
def describe(self) -> str:
|
||||||
|
if self.match:
|
||||||
|
return f"Screen match OK (conf={self.confidence:.2f})"
|
||||||
|
parts = [f"Screen mismatch (conf={self.confidence:.2f})"]
|
||||||
|
if self.mismatches:
|
||||||
|
parts.append("missing: " + ", ".join(self.mismatches))
|
||||||
|
if self.reason:
|
||||||
|
parts.append(self.reason)
|
||||||
|
return " | ".join(parts)
|
||||||
|
|
||||||
|
|
||||||
|
# ── Text normalization (pure functions) ────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def normalize_text(text: str) -> str:
|
||||||
|
"""Normalize text for fuzzy matching: lowercase, strip accents, collapse whitespace."""
|
||||||
|
text = text.lower().strip()
|
||||||
|
# Strip accents: é→e, è→e, ê→e, à→a, etc.
|
||||||
|
text = unicodedata.normalize("NFKD", text)
|
||||||
|
text = "".join(c for c in text if not unicodedata.combining(c))
|
||||||
|
# Collapse whitespace
|
||||||
|
text = re.sub(r"\s+", " ", text)
|
||||||
|
return text
|
||||||
|
|
||||||
|
|
||||||
|
def fuzzy_match(expected: str, observed: str, threshold: float = 0.8) -> bool:
|
||||||
|
"""Check if observed text fuzzy-matches expected text.
|
||||||
|
|
||||||
|
Three strategies (any wins):
|
||||||
|
1. Exact match after normalization
|
||||||
|
2. Substring containment (either direction)
|
||||||
|
3. SequenceMatcher ratio >= threshold
|
||||||
|
"""
|
||||||
|
norm_expected = normalize_text(expected)
|
||||||
|
norm_observed = normalize_text(observed)
|
||||||
|
|
||||||
|
if norm_expected == norm_observed:
|
||||||
|
return True
|
||||||
|
|
||||||
|
if norm_expected in norm_observed or norm_observed in norm_expected:
|
||||||
|
return True
|
||||||
|
|
||||||
|
ratio = SequenceMatcher(None, norm_expected, norm_observed).ratio()
|
||||||
|
return ratio >= threshold
|
||||||
|
|
||||||
|
|
||||||
|
# ── OCR presence check (deterministic, no VLM) ──────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class OcrPresenceResult:
|
||||||
|
"""Result of OCR-based presence check."""
|
||||||
|
|
||||||
|
found_texts: Dict[str, str] = field(default_factory=dict)
|
||||||
|
missing: List[str] = field(default_factory=list)
|
||||||
|
all_found: bool = False
|
||||||
|
|
||||||
|
@property
|
||||||
|
def presence_ratio(self) -> float:
|
||||||
|
if not self.found_texts:
|
||||||
|
return 1.0
|
||||||
|
found_count = sum(1 for v in self.found_texts.values() if v != "")
|
||||||
|
return found_count / len(self.found_texts)
|
||||||
|
|
||||||
|
|
||||||
|
def ocr_presence_check(
|
||||||
|
ocr_tokens: List[str],
|
||||||
|
expected_elements: List[Dict[str, Any]],
|
||||||
|
fuzzy_threshold: float = 0.8,
|
||||||
|
) -> OcrPresenceResult:
|
||||||
|
"""Check presence of expected texts against OCR tokens (deterministic).
|
||||||
|
|
||||||
|
Pure function — no VLM call, zero hallucination risk.
|
||||||
|
"""
|
||||||
|
found_texts: Dict[str, str] = {}
|
||||||
|
missing: List[str] = []
|
||||||
|
|
||||||
|
for el in expected_elements:
|
||||||
|
expected_text = el.get("text", "")
|
||||||
|
if not expected_text:
|
||||||
|
found_texts[""] = ""
|
||||||
|
continue
|
||||||
|
|
||||||
|
matched_ocr = ""
|
||||||
|
for token in ocr_tokens:
|
||||||
|
if fuzzy_match(expected_text, token, threshold=fuzzy_threshold):
|
||||||
|
matched_ocr = token
|
||||||
|
break
|
||||||
|
|
||||||
|
if matched_ocr:
|
||||||
|
found_texts[expected_text] = matched_ocr
|
||||||
|
else:
|
||||||
|
found_texts[expected_text] = ""
|
||||||
|
missing.append(f"{el.get('role', '?')}: {expected_text}")
|
||||||
|
|
||||||
|
all_found = len(missing) == 0
|
||||||
|
return OcrPresenceResult(
|
||||||
|
found_texts=found_texts,
|
||||||
|
missing=missing,
|
||||||
|
all_found=all_found,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ── VLM role confirmation (semantic, anchored on found OCR texts) ────
|
||||||
|
|
||||||
|
|
||||||
|
def build_role_confirm_prompt(
|
||||||
|
found_elements: List[Dict[str, Any]],
|
||||||
|
expected_elements: List[Dict[str, Any]],
|
||||||
|
context: str = "",
|
||||||
|
) -> str:
|
||||||
|
"""Build VLM prompt for role confirmation of OCR-found elements.
|
||||||
|
|
||||||
|
VLM receives found texts and confirms their ROLE only — never presence.
|
||||||
|
"""
|
||||||
|
found_lines = []
|
||||||
|
for i, el in enumerate(found_elements):
|
||||||
|
matched_ocr = el.get("matched_ocr", "")
|
||||||
|
expected_role = el.get("expected_role", "?")
|
||||||
|
line = f"{i+1}. Text \"{matched_ocr}\" — expected role: {expected_role}"
|
||||||
|
found_lines.append(line)
|
||||||
|
|
||||||
|
found_block = "\n".join(found_lines)
|
||||||
|
|
||||||
|
prompt = (
|
||||||
|
"You are a screen role validator. OCR has confirmed these texts are "
|
||||||
|
"present on the screen. Your job is ONLY to confirm their ROLE — "
|
||||||
|
"do NOT re-declare whether they are present.\n"
|
||||||
|
)
|
||||||
|
if context:
|
||||||
|
prompt += f"Context: {context}\n"
|
||||||
|
prompt += (
|
||||||
|
f"Found texts with expected roles:\n{found_block}\n\n"
|
||||||
|
"Respond in JSON format:\n"
|
||||||
|
"{\"confirmed\": [{\"index\": 1, \"role_confirmed\": true/false, "
|
||||||
|
"\"actual_role\": \"...\", \"confidence\": 0.0-1.0}], "
|
||||||
|
"\"overall_confidence\": 0.0-1.0}\n"
|
||||||
|
"Only confirm role_confirmed=true if the text clearly plays the "
|
||||||
|
"expected role (e.g., a button, not just a label with the same text)."
|
||||||
|
)
|
||||||
|
return prompt
|
||||||
|
|
||||||
|
|
||||||
|
def parse_role_confirm_response(vlm_text: str) -> Dict[str, Any]:
|
||||||
|
"""Parse VLM role confirmation JSON response."""
|
||||||
|
try:
|
||||||
|
data = json.loads(vlm_text)
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
json_match = re.search(r"\{[\s\S]*\}", vlm_text)
|
||||||
|
if json_match:
|
||||||
|
try:
|
||||||
|
data = json.loads(json_match.group())
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
logger.warning("role_confirm: VLM response not parseable as JSON")
|
||||||
|
return {"confirmed": [], "overall_confidence": 0.0}
|
||||||
|
else:
|
||||||
|
return {"confirmed": [], "overall_confidence": 0.0}
|
||||||
|
|
||||||
|
confirmed = data.get("confirmed", [])
|
||||||
|
overall_conf = data.get("overall_confidence", 0.0)
|
||||||
|
if isinstance(overall_conf, str):
|
||||||
|
try:
|
||||||
|
overall_conf = float(overall_conf)
|
||||||
|
except ValueError:
|
||||||
|
overall_conf = 0.0
|
||||||
|
|
||||||
|
return {
|
||||||
|
"confirmed": confirmed,
|
||||||
|
"overall_confidence": float(overall_conf),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# ── Core verification (OCR-anchored composition) ────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def verify_screen_match(
|
||||||
|
screenshot_path: str,
|
||||||
|
expected_elements: List[Dict[str, Any]],
|
||||||
|
ocr_client: OcrClient,
|
||||||
|
vlm_client: VlmClient,
|
||||||
|
context: str = "",
|
||||||
|
min_confidence: float = 0.7,
|
||||||
|
) -> ScreenMatchResult:
|
||||||
|
"""Verify screen state with OCR-anchored presence + VLM role confirmation.
|
||||||
|
|
||||||
|
Step 1: OCR screenshot → tokens → deterministic presence check
|
||||||
|
Step 2: VLM confirms role of found elements (not presence!)
|
||||||
|
|
||||||
|
Eliminates VLM self-report hallucination for presence checks.
|
||||||
|
"""
|
||||||
|
if not expected_elements:
|
||||||
|
return ScreenMatchResult(
|
||||||
|
match=True,
|
||||||
|
confidence=1.0,
|
||||||
|
reason="no expected elements to verify",
|
||||||
|
)
|
||||||
|
|
||||||
|
# Step 1: OCR presence check (deterministic)
|
||||||
|
try:
|
||||||
|
ocr_tokens = ocr_client(screenshot_path)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("verify_screen_match: OCR call failed (%s)", e)
|
||||||
|
return ScreenMatchResult(
|
||||||
|
match=False,
|
||||||
|
confidence=0.0,
|
||||||
|
reason=f"OCR error: {e}",
|
||||||
|
expected_elements=expected_elements,
|
||||||
|
)
|
||||||
|
|
||||||
|
presence = ocr_presence_check(ocr_tokens, expected_elements)
|
||||||
|
|
||||||
|
if not presence.all_found:
|
||||||
|
observed = []
|
||||||
|
for el in expected_elements:
|
||||||
|
text = el.get("text", "")
|
||||||
|
matched = presence.found_texts.get(text, "")
|
||||||
|
observed.append({
|
||||||
|
"role": el.get("role", "?"),
|
||||||
|
"expected_text": text,
|
||||||
|
"matched_ocr": matched,
|
||||||
|
"found": matched != "",
|
||||||
|
})
|
||||||
|
return ScreenMatchResult(
|
||||||
|
match=False,
|
||||||
|
confidence=presence.presence_ratio,
|
||||||
|
reason="OCR presence check: some texts not found",
|
||||||
|
observed_elements=observed,
|
||||||
|
expected_elements=expected_elements,
|
||||||
|
mismatches=presence.missing,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Step 2: VLM role confirmation (only for found elements)
|
||||||
|
found_elements = []
|
||||||
|
for el in expected_elements:
|
||||||
|
text = el.get("text", "")
|
||||||
|
matched_ocr = presence.found_texts.get(text, "")
|
||||||
|
if text and matched_ocr:
|
||||||
|
found_elements.append({
|
||||||
|
"text": text,
|
||||||
|
"expected_role": el.get("role", "?"),
|
||||||
|
"matched_ocr": matched_ocr,
|
||||||
|
})
|
||||||
|
|
||||||
|
if not found_elements:
|
||||||
|
# All elements had no text → presence trivially OK
|
||||||
|
return ScreenMatchResult(
|
||||||
|
match=True,
|
||||||
|
confidence=1.0,
|
||||||
|
reason="no text-based elements to verify",
|
||||||
|
expected_elements=expected_elements,
|
||||||
|
)
|
||||||
|
|
||||||
|
prompt = build_role_confirm_prompt(found_elements, expected_elements, context)
|
||||||
|
|
||||||
|
try:
|
||||||
|
vlm_text = vlm_client(screenshot_path, prompt)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("verify_screen_match: VLM role confirm failed (%s)", e)
|
||||||
|
observed = []
|
||||||
|
for el in expected_elements:
|
||||||
|
text = el.get("text", "")
|
||||||
|
observed.append({
|
||||||
|
"role": el.get("role", "?"),
|
||||||
|
"expected_text": text,
|
||||||
|
"matched_ocr": presence.found_texts.get(text, ""),
|
||||||
|
"found": True,
|
||||||
|
"role_confirmed": False,
|
||||||
|
"role_confidence": 0.0,
|
||||||
|
})
|
||||||
|
return ScreenMatchResult(
|
||||||
|
match=True,
|
||||||
|
confidence=0.5,
|
||||||
|
reason=f"OCR presence OK, VLM role confirm failed: {e}",
|
||||||
|
observed_elements=observed,
|
||||||
|
expected_elements=expected_elements,
|
||||||
|
)
|
||||||
|
|
||||||
|
parsed = parse_role_confirm_response(vlm_text)
|
||||||
|
overall_conf = parsed.get("overall_confidence", 0.0)
|
||||||
|
confirmed = parsed.get("confirmed", [])
|
||||||
|
|
||||||
|
observed = []
|
||||||
|
role_mismatches = []
|
||||||
|
for i, el in enumerate(expected_elements):
|
||||||
|
text = el.get("text", "")
|
||||||
|
expected_role = el.get("role", "?")
|
||||||
|
matched_ocr = presence.found_texts.get(text, "")
|
||||||
|
|
||||||
|
role_entry = None
|
||||||
|
for c in confirmed:
|
||||||
|
if c.get("index") == i + 1:
|
||||||
|
role_entry = c
|
||||||
|
break
|
||||||
|
|
||||||
|
role_confirmed = False
|
||||||
|
actual_role = ""
|
||||||
|
role_confidence = 0.0
|
||||||
|
|
||||||
|
if role_entry:
|
||||||
|
role_confirmed = role_entry.get("role_confirmed", False)
|
||||||
|
actual_role = role_entry.get("actual_role", "")
|
||||||
|
role_confidence = role_entry.get("confidence", 0.0)
|
||||||
|
if isinstance(role_confidence, str):
|
||||||
|
try:
|
||||||
|
role_confidence = float(role_confidence)
|
||||||
|
except ValueError:
|
||||||
|
role_confidence = 0.0
|
||||||
|
|
||||||
|
observed.append({
|
||||||
|
"role": expected_role,
|
||||||
|
"expected_text": text,
|
||||||
|
"matched_ocr": matched_ocr,
|
||||||
|
"found": True,
|
||||||
|
"role_confirmed": role_confirmed,
|
||||||
|
"actual_role": actual_role,
|
||||||
|
"role_confidence": role_confidence,
|
||||||
|
})
|
||||||
|
|
||||||
|
if not role_confirmed or role_confidence < min_confidence:
|
||||||
|
role_mismatches.append(
|
||||||
|
f"{expected_role}: {text} (actual={actual_role}, conf={role_confidence:.2f})"
|
||||||
|
)
|
||||||
|
|
||||||
|
is_match = len(role_mismatches) == 0 and overall_conf >= min_confidence
|
||||||
|
|
||||||
|
return ScreenMatchResult(
|
||||||
|
match=is_match,
|
||||||
|
confidence=overall_conf,
|
||||||
|
reason=f"OCR presence: {presence.presence_ratio:.0%}, VLM role: {overall_conf:.2f}",
|
||||||
|
observed_elements=observed,
|
||||||
|
expected_elements=expected_elements,
|
||||||
|
mismatches=presence.missing + role_mismatches,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def verify_before(
|
||||||
|
screenshot_path: str,
|
||||||
|
expected_elements: List[Dict[str, Any]],
|
||||||
|
ocr_client: OcrClient,
|
||||||
|
vlm_client: VlmClient,
|
||||||
|
context: str = "",
|
||||||
|
) -> ScreenMatchResult:
|
||||||
|
"""Verify screen state BEFORE an action (OCR-anchored).
|
||||||
|
|
||||||
|
Checks pre-conditions: expected texts present + roles correct.
|
||||||
|
min_confidence=0.7 — some tolerance for pre-action verification.
|
||||||
|
"""
|
||||||
|
return verify_screen_match(
|
||||||
|
screenshot_path, expected_elements, ocr_client, vlm_client,
|
||||||
|
context=f"PRE-ACTION: {context}", min_confidence=0.7,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def verify_after(
|
||||||
|
screenshot_path: str,
|
||||||
|
expected_elements: List[Dict[str, Any]],
|
||||||
|
ocr_client: OcrClient,
|
||||||
|
vlm_client: VlmClient,
|
||||||
|
context: str = "",
|
||||||
|
) -> ScreenMatchResult:
|
||||||
|
"""Verify screen state AFTER an action (OCR-anchored).
|
||||||
|
|
||||||
|
Checks post-conditions with higher threshold (0.8).
|
||||||
|
False positive = Léa proceeds on wrong assumption → stricter gate.
|
||||||
|
"""
|
||||||
|
return verify_screen_match(
|
||||||
|
screenshot_path, expected_elements, ocr_client, vlm_client,
|
||||||
|
context=f"POST-ACTION: {context}", min_confidence=0.8,
|
||||||
|
)
|
||||||
@@ -208,7 +208,11 @@ REQUIRED=(
|
|||||||
"Lea/python-embed/Lib/site-packages/mss"
|
"Lea/python-embed/Lib/site-packages/mss"
|
||||||
"Lea/python-embed/Lib/site-packages/win32"
|
"Lea/python-embed/Lib/site-packages/win32"
|
||||||
"Lea/python-embed/Lib/site-packages/socketio"
|
"Lea/python-embed/Lib/site-packages/socketio"
|
||||||
)
|
"Lea/python-embed/Lib/site-packages/httpx"
|
||||||
|
"Lea/python-embed/Lib/site-packages/httpcore"
|
||||||
|
"Lea/python-embed/Lib/site-packages/h11"
|
||||||
|
"Lea/python-embed/Lib/site-packages/anyio"
|
||||||
|
"Lea/python-embed/Lib/site-packages/typing_extensions.py"
|
||||||
MISSING=()
|
MISSING=()
|
||||||
for f in "${REQUIRED[@]}"; do
|
for f in "${REQUIRED[@]}"; do
|
||||||
[[ -e "$ASSEMBLY_DIR/$f" ]] || MISSING+=("$f")
|
[[ -e "$ASSEMBLY_DIR/$f" ]] || MISSING+=("$f")
|
||||||
|
|||||||
@@ -23,7 +23,7 @@
|
|||||||
; ============================================================
|
; ============================================================
|
||||||
|
|
||||||
#define MyAppName "Lea"
|
#define MyAppName "Lea"
|
||||||
#define MyAppVersion "1.0.1"
|
#define MyAppVersion "1.0.2"
|
||||||
#define MyAppPublisher "AIVANOV"
|
#define MyAppPublisher "AIVANOV"
|
||||||
#define MyAppURL "https://lea.labs.laurinebazin.design"
|
#define MyAppURL "https://lea.labs.laurinebazin.design"
|
||||||
#define MyAppExeName "Lea.bat"
|
#define MyAppExeName "Lea.bat"
|
||||||
@@ -182,6 +182,7 @@ var
|
|||||||
TokenPage: TInputQueryWizardPage;
|
TokenPage: TInputQueryWizardPage;
|
||||||
MachineIdValue: string;
|
MachineIdValue: string;
|
||||||
ConfigFilePath: string;
|
ConfigFilePath: string;
|
||||||
|
ExistingMachineId: string;
|
||||||
|
|
||||||
// --------------------------------------------------------------------
|
// --------------------------------------------------------------------
|
||||||
// Helper : ajoute des guillemets autour d'une chaine
|
// Helper : ajoute des guillemets autour d'une chaine
|
||||||
@@ -267,6 +268,72 @@ end;
|
|||||||
// --------------------------------------------------------------------
|
// --------------------------------------------------------------------
|
||||||
procedure LoadConfigFromCommandLine(); forward;
|
procedure LoadConfigFromCommandLine(); forward;
|
||||||
|
|
||||||
|
// --------------------------------------------------------------------
|
||||||
|
// UPGRADE — trouve le dossier d'une install Lea existante (config.txt present)
|
||||||
|
// --------------------------------------------------------------------
|
||||||
|
function FindExistingInstallDir(): string;
|
||||||
|
var
|
||||||
|
Candidates: array[0..1] of string;
|
||||||
|
I: Integer;
|
||||||
|
begin
|
||||||
|
Result := '';
|
||||||
|
Candidates[0] := ExpandConstant('{localappdata}\Programs\Lea');
|
||||||
|
Candidates[1] := ExpandConstant('{autopf}\Lea');
|
||||||
|
for I := 0 to 1 do
|
||||||
|
begin
|
||||||
|
if FileExists(Candidates[I] + '\config.txt') then
|
||||||
|
begin
|
||||||
|
Result := Candidates[I];
|
||||||
|
Exit;
|
||||||
|
end;
|
||||||
|
end;
|
||||||
|
end;
|
||||||
|
|
||||||
|
// --------------------------------------------------------------------
|
||||||
|
// UPGRADE — lit le config.txt existant : pre-remplit le wizard avec la
|
||||||
|
// VRAIE conf du poste (serveur/token/user) et MEMORISE le machine_id pour
|
||||||
|
// le PRESERVER (ne pas regenerer une nouvelle identite fleet).
|
||||||
|
// --------------------------------------------------------------------
|
||||||
|
procedure LoadExistingConfig();
|
||||||
|
var
|
||||||
|
Dir, ConfPath: string;
|
||||||
|
Lines: TArrayOfString;
|
||||||
|
I, EqPos: Integer;
|
||||||
|
Line, Key, Value: string;
|
||||||
|
begin
|
||||||
|
ExistingMachineId := '';
|
||||||
|
Dir := FindExistingInstallDir();
|
||||||
|
if Dir = '' then Exit; // install neuve -> comportement par defaut
|
||||||
|
|
||||||
|
ConfPath := Dir + '\config.txt';
|
||||||
|
if LoadStringsFromFile(ConfPath, Lines) then
|
||||||
|
begin
|
||||||
|
for I := 0 to GetArrayLength(Lines) - 1 do
|
||||||
|
begin
|
||||||
|
Line := Trim(Lines[I]);
|
||||||
|
if (Length(Line) = 0) or (Line[1] = '#') then Continue;
|
||||||
|
EqPos := Pos('=', Line);
|
||||||
|
if EqPos = 0 then Continue;
|
||||||
|
Key := Trim(Copy(Line, 1, EqPos - 1));
|
||||||
|
Value := Trim(Copy(Line, EqPos + 1, Length(Line)));
|
||||||
|
|
||||||
|
if Key = 'RPA_SERVER_URL' then TokenPage.Values[0] := Value
|
||||||
|
else if Key = 'RPA_API_TOKEN' then TokenPage.Values[1] := Value
|
||||||
|
else if Key = 'RPA_USER_NAME' then EnrollmentPage.Values[0] := Value
|
||||||
|
else if Key = 'RPA_USER_EMAIL' then EnrollmentPage.Values[1] := Value
|
||||||
|
else if Key = 'RPA_USER_ID' then EnrollmentPage.Values[2] := Value
|
||||||
|
else if Key = 'RPA_MACHINE_ID' then ExistingMachineId := Value;
|
||||||
|
end;
|
||||||
|
end;
|
||||||
|
|
||||||
|
// Fallback : machine_id.txt si absent du config.txt
|
||||||
|
if (ExistingMachineId = '') and FileExists(Dir + '\machine_id.txt') then
|
||||||
|
begin
|
||||||
|
if LoadStringsFromFile(Dir + '\machine_id.txt', Lines) and (GetArrayLength(Lines) > 0) then
|
||||||
|
ExistingMachineId := Trim(Lines[0]);
|
||||||
|
end;
|
||||||
|
end;
|
||||||
|
|
||||||
// --------------------------------------------------------------------
|
// --------------------------------------------------------------------
|
||||||
// Initialisation : cree les pages custom d'enrollment
|
// Initialisation : cree les pages custom d'enrollment
|
||||||
// --------------------------------------------------------------------
|
// --------------------------------------------------------------------
|
||||||
@@ -301,7 +368,11 @@ begin
|
|||||||
TokenPage.Values[0] := SERVER_URL_DEFAULT;
|
TokenPage.Values[0] := SERVER_URL_DEFAULT;
|
||||||
TokenPage.Values[1] := DEFAULT_TOKEN;
|
TokenPage.Values[1] := DEFAULT_TOKEN;
|
||||||
|
|
||||||
// Si un fichier /CONFIG= est passe en ligne de commande, pre-remplir
|
// UPGRADE : si une install existe, pre-remplir avec SA config (pas les
|
||||||
|
// defauts) et memoriser son machine_id pour le preserver.
|
||||||
|
LoadExistingConfig();
|
||||||
|
|
||||||
|
// Si un fichier /CONFIG= est passe en ligne de commande, pre-remplir (prioritaire)
|
||||||
LoadConfigFromCommandLine();
|
LoadConfigFromCommandLine();
|
||||||
end;
|
end;
|
||||||
|
|
||||||
@@ -508,6 +579,54 @@ begin
|
|||||||
DeleteFile(PsFile);
|
DeleteFile(PsFile);
|
||||||
end;
|
end;
|
||||||
|
|
||||||
|
// --------------------------------------------------------------------
|
||||||
|
// UPGRADE — AVANT la copie des fichiers : tuer une Lea en cours (via le
|
||||||
|
// PID du lock) pour liberer les DLL de python-embed. Evite une install
|
||||||
|
// partielle / "reboot required". Ne tue QUE le PID du lock (jamais tous
|
||||||
|
// les pythonw du poste).
|
||||||
|
// --------------------------------------------------------------------
|
||||||
|
function PrepareToInstall(var NeedsRestart: Boolean): String;
|
||||||
|
var
|
||||||
|
AppDir, LockPath, BackupDir, SessionsDir: string;
|
||||||
|
Lines: TArrayOfString;
|
||||||
|
ResultCode: Integer;
|
||||||
|
begin
|
||||||
|
Result := '';
|
||||||
|
AppDir := ExpandConstant('{app}');
|
||||||
|
|
||||||
|
// 1) Tuer une Lea en cours (via le PID du lock) pour liberer les DLL
|
||||||
|
// python-embed. Ne tue QUE ce PID, jamais tous les pythonw du poste.
|
||||||
|
LockPath := AppDir + '\lea_agent.lock';
|
||||||
|
if FileExists(LockPath) then
|
||||||
|
begin
|
||||||
|
if LoadStringsFromFile(LockPath, Lines) and (GetArrayLength(Lines) > 0) then
|
||||||
|
Exec('taskkill.exe', '/F /PID ' + Trim(Lines[0]), '', SW_HIDE, ewWaitUntilTerminated, ResultCode);
|
||||||
|
DeleteFile(LockPath);
|
||||||
|
Sleep(1500);
|
||||||
|
end;
|
||||||
|
|
||||||
|
// UPGRADE uniquement (install existante detectee via config.txt).
|
||||||
|
if FileExists(AppDir + '\config.txt') then
|
||||||
|
begin
|
||||||
|
// 2) BACKUP (rollback) : copie code+config vers <app>_backup, HORS
|
||||||
|
// python-embed / sessions / logs (leger, rapide). Filet si la nouvelle
|
||||||
|
// version deconne : Julien restaure ce dossier.
|
||||||
|
BackupDir := AppDir + '_backup';
|
||||||
|
Exec(ExpandConstant('{cmd}'),
|
||||||
|
'/c rmdir /s /q "' + BackupDir + '" 2>nul & robocopy "' + AppDir + '" "' + BackupDir +
|
||||||
|
'" /E /XD python-embed sessions logs __pycache__ /XF *.pyc /R:1 /W:1 /NFL /NDL /NJH /NJS /NP >nul 2>&1',
|
||||||
|
'', SW_HIDE, ewWaitUntilTerminated, ResultCode);
|
||||||
|
|
||||||
|
// 3) PURGE des captures accumulees (donnees d'apprentissage internes, non
|
||||||
|
// exploitables cote clinique) : libere le disque. Le fix capture JPEG
|
||||||
|
// evite que la saturation reprenne. Les logs (compliance 180j) restent.
|
||||||
|
SessionsDir := AppDir + '\agent_v1\sessions';
|
||||||
|
if DirExists(SessionsDir) then
|
||||||
|
Exec(ExpandConstant('{cmd}'),
|
||||||
|
'/c rmdir /s /q "' + SessionsDir + '"', '', SW_HIDE, ewWaitUntilTerminated, ResultCode);
|
||||||
|
end;
|
||||||
|
end;
|
||||||
|
|
||||||
// --------------------------------------------------------------------
|
// --------------------------------------------------------------------
|
||||||
// Hook : actions apres copie des fichiers (ssPostInstall)
|
// Hook : actions apres copie des fichiers (ssPostInstall)
|
||||||
// --------------------------------------------------------------------
|
// --------------------------------------------------------------------
|
||||||
@@ -515,8 +634,11 @@ procedure CurStepChanged(CurStep: TSetupStep);
|
|||||||
begin
|
begin
|
||||||
if CurStep = ssInstall then
|
if CurStep = ssInstall then
|
||||||
begin
|
begin
|
||||||
// Genere le machine_id AVANT la copie des fichiers
|
// UPGRADE : preserver l'identite existante ; sinon en generer une neuve.
|
||||||
MachineIdValue := GenerateMachineId();
|
if ExistingMachineId <> '' then
|
||||||
|
MachineIdValue := ExistingMachineId
|
||||||
|
else
|
||||||
|
MachineIdValue := GenerateMachineId();
|
||||||
end;
|
end;
|
||||||
|
|
||||||
if CurStep = ssPostInstall then
|
if CurStep = ssPostInstall then
|
||||||
|
|||||||
@@ -81,16 +81,29 @@ cd deploy/installer
|
|||||||
wget https://www.python.org/ftp/python/3.12.8/python-3.12.8-embed-amd64.zip
|
wget https://www.python.org/ftp/python/3.12.8/python-3.12.8-embed-amd64.zip
|
||||||
mkdir python-3.12-embed
|
mkdir python-3.12-embed
|
||||||
unzip python-3.12.8-embed-amd64.zip -d python-3.12-embed/
|
unzip python-3.12.8-embed-amd64.zip -d python-3.12-embed/
|
||||||
|
|
||||||
|
# IMPORTANT : l'embed doit contenir TOUTES les dependances HORS LIGNE.
|
||||||
|
# Le runtime client ne fait AUCUN pip/reseau (POC clinique). On installe donc
|
||||||
|
# les dependances une fois dans l'embed, puis on le commit/reutilise tel quel :
|
||||||
|
python312._pth # decommenter 'import site'
|
||||||
|
python -m pip install --target python-3.12-embed/Lib/site-packages \
|
||||||
|
-r ../lea_package/requirements_agent.txt
|
||||||
|
# => doit inclure httpx (+ httpcore, h11) pour l'orchestrateur Lea (POST /api/learn/start).
|
||||||
```
|
```
|
||||||
|
|
||||||
Le staging copie automatiquement ce dossier si present. Le composant
|
Le staging copie automatiquement ce dossier si present. Le composant
|
||||||
"pythonembed" devient alors selectionnable dans l'installeur.
|
"pythonembed" devient alors selectionnable dans l'installeur.
|
||||||
|
|
||||||
Le script `configure_embed.ps1` :
|
Le script `configure_embed.ps1` (execute a l'installation, sur le poste) :
|
||||||
1. Patche `python312._pth` pour activer `import site`
|
1. Patche `python312._pth` pour activer `import site`
|
||||||
2. Installe `pip` via `get-pip.py`
|
2. VERIFIE que les dependances sont deja embarquees (offline, aucun pip/reseau) —
|
||||||
3. Installe `requirements_agent.txt`
|
`socketio, tkinter, mss, pynput, pystray, plyer, requests, httpx, PIL, win32api` ;
|
||||||
4. Reecrit `Lea.bat` pour pointer sur `python-embed\pythonw.exe`
|
si une dependance manque, l'installation echoue explicitement.
|
||||||
|
3. Reecrit `Lea.bat` pour pointer sur `python-embed\pythonw.exe`
|
||||||
|
|
||||||
|
> Note : `build_installer.sh` et `build_package_full.sh` valident aussi la presence
|
||||||
|
> des paquets (dont `httpx`, `httpcore`, `h11`) dans `Lib/site-packages/` avant de
|
||||||
|
> produire le paquet — un embed incomplet interrompt le build cote Linux.
|
||||||
|
|
||||||
## Installation silencieuse (deploiement de masse)
|
## Installation silencieuse (deploiement de masse)
|
||||||
|
|
||||||
|
|||||||
@@ -154,6 +154,8 @@ REQUIRED_EMBED=(
|
|||||||
"Lib/site-packages/pystray" "Lib/site-packages/plyer"
|
"Lib/site-packages/pystray" "Lib/site-packages/plyer"
|
||||||
"Lib/site-packages/requests" "Lib/site-packages/PIL"
|
"Lib/site-packages/requests" "Lib/site-packages/PIL"
|
||||||
"Lib/site-packages/win32"
|
"Lib/site-packages/win32"
|
||||||
|
"Lib/site-packages/httpx" "Lib/site-packages/httpcore" "Lib/site-packages/h11"
|
||||||
|
"Lib/site-packages/anyio" "Lib/site-packages/typing_extensions.py"
|
||||||
)
|
)
|
||||||
MISSING_EMBED=()
|
MISSING_EMBED=()
|
||||||
for f in "${REQUIRED_EMBED[@]}"; do
|
for f in "${REQUIRED_EMBED[@]}"; do
|
||||||
|
|||||||
@@ -25,3 +25,5 @@ USER_ID=
|
|||||||
# Connexion serveur (remplacer les valeurs CONFIGURE_ME avant utilisation)
|
# Connexion serveur (remplacer les valeurs CONFIGURE_ME avant utilisation)
|
||||||
SERVER_URL=CONFIGURE_ME
|
SERVER_URL=CONFIGURE_ME
|
||||||
API_TOKEN=CONFIGURE_ME
|
API_TOKEN=CONFIGURE_ME
|
||||||
|
|
||||||
|
AGENT_VERSION=1.0.2
|
||||||
|
|||||||
@@ -44,7 +44,7 @@ if ($PthFile) {
|
|||||||
# L'embed DOIT contenir toutes les dependances runtime.
|
# L'embed DOIT contenir toutes les dependances runtime.
|
||||||
# AUCUN pip, AUCUN reseau : si une dependance manque -> echec explicite.
|
# AUCUN pip, AUCUN reseau : si une dependance manque -> echec explicite.
|
||||||
# ---------------------------------------------------------------
|
# ---------------------------------------------------------------
|
||||||
$RequiredModules = @('socketio','tkinter','mss','pynput','pystray','plyer','requests','PIL','win32api')
|
$RequiredModules = @('socketio','tkinter','mss','pynput','pystray','plyer','requests','httpx','PIL','win32api')
|
||||||
$Missing = @()
|
$Missing = @()
|
||||||
foreach ($m in $RequiredModules) {
|
foreach ($m in $RequiredModules) {
|
||||||
& $PythonExe -c "import $m" 2>$null
|
& $PythonExe -c "import $m" 2>$null
|
||||||
@@ -76,6 +76,29 @@ if exist "lea_agent.lock" (
|
|||||||
timeout /t 2 >nul
|
timeout /t 2 >nul
|
||||||
)
|
)
|
||||||
|
|
||||||
|
:: MAJ SILENCIEUSE — swap atomique + rollback (renames uniquement)
|
||||||
|
if exist "PENDING_BOOT" (
|
||||||
|
echo [MAJ] Boot precedent non confirme : retour a la version precedente.
|
||||||
|
if exist "agent_v1_prev" (
|
||||||
|
if exist "agent_v1_echec" rmdir /s /q "agent_v1_echec" >nul 2>&1
|
||||||
|
if exist "agent_v1" move "agent_v1" "agent_v1_echec" >nul 2>&1
|
||||||
|
move "agent_v1_prev" "agent_v1" >nul 2>&1
|
||||||
|
)
|
||||||
|
del /f /q "PENDING_BOOT" >nul 2>&1
|
||||||
|
) else if exist "UPDATE_READY" (
|
||||||
|
if exist "agent_v1_new" (
|
||||||
|
echo [MAJ] Application de la mise a jour...
|
||||||
|
if exist "agent_v1" (
|
||||||
|
if exist "agent_v1_prev" rmdir /s /q "agent_v1_prev" >nul 2>&1
|
||||||
|
move "agent_v1" "agent_v1_prev" >nul 2>&1
|
||||||
|
)
|
||||||
|
move "agent_v1_new" "agent_v1" >nul 2>&1
|
||||||
|
move "UPDATE_READY" "PENDING_BOOT" >nul 2>&1
|
||||||
|
) else (
|
||||||
|
del /f /q "UPDATE_READY" >nul 2>&1
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
if exist "config.txt" (
|
if exist "config.txt" (
|
||||||
for /f "usebackq eol=# tokens=1,* delims==" %%a in ("config.txt") do (
|
for /f "usebackq eol=# tokens=1,* delims==" %%a in ("config.txt") do (
|
||||||
if not "%%a"=="" if not "%%b"=="" set "%%a=%%b"
|
if not "%%a"=="" if not "%%b"=="" set "%%a=%%b"
|
||||||
|
|||||||
@@ -20,6 +20,35 @@ if exist "lea_agent.lock" (
|
|||||||
timeout /t 2 >nul
|
timeout /t 2 >nul
|
||||||
)
|
)
|
||||||
|
|
||||||
|
:: ---------------------------------------------------------------
|
||||||
|
:: MAJ SILENCIEUSE — swap atomique + rollback (hors-process)
|
||||||
|
:: L'ancienne instance est fermee ci-dessus : agent_v1\ est libre.
|
||||||
|
:: Renames uniquement (quasi-atomiques), jamais d'ecrasement fichier par fichier.
|
||||||
|
:: ---------------------------------------------------------------
|
||||||
|
if exist "PENDING_BOOT" (
|
||||||
|
:: Le boot precedent n'a JAMAIS confirme (crash) -> ROLLBACK version precedente
|
||||||
|
echo [MAJ] Boot precedent non confirme : retour a la version precedente.
|
||||||
|
if exist "agent_v1_prev" (
|
||||||
|
if exist "agent_v1_echec" rmdir /s /q "agent_v1_echec" >nul 2>&1
|
||||||
|
if exist "agent_v1" move "agent_v1" "agent_v1_echec" >nul 2>&1
|
||||||
|
move "agent_v1_prev" "agent_v1" >nul 2>&1
|
||||||
|
)
|
||||||
|
del /f /q "PENDING_BOOT" >nul 2>&1
|
||||||
|
) else if exist "UPDATE_READY" (
|
||||||
|
:: Une MAJ est armee (agent_v1_new pret) -> SWAP
|
||||||
|
if exist "agent_v1_new" (
|
||||||
|
echo [MAJ] Application de la mise a jour...
|
||||||
|
if exist "agent_v1" (
|
||||||
|
if exist "agent_v1_prev" rmdir /s /q "agent_v1_prev" >nul 2>&1
|
||||||
|
move "agent_v1" "agent_v1_prev" >nul 2>&1
|
||||||
|
)
|
||||||
|
move "agent_v1_new" "agent_v1" >nul 2>&1
|
||||||
|
move "UPDATE_READY" "PENDING_BOOT" >nul 2>&1
|
||||||
|
) else (
|
||||||
|
del /f /q "UPDATE_READY" >nul 2>&1
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
:: ---------------------------------------------------------------
|
:: ---------------------------------------------------------------
|
||||||
:: Verifier que l'installation a ete faite
|
:: Verifier que l'installation a ete faite
|
||||||
:: ---------------------------------------------------------------
|
:: ---------------------------------------------------------------
|
||||||
|
|||||||
@@ -36,5 +36,15 @@ RPA_MACHINE_ID=CONFIGURE_ME
|
|||||||
RPA_USER_LABEL=CONFIGURE_ME
|
RPA_USER_LABEL=CONFIGURE_ME
|
||||||
|
|
||||||
# --- Parametres avances (ne pas modifier sauf indication) ---
|
# --- Parametres avances (ne pas modifier sauf indication) ---
|
||||||
|
RPA_AGENT_VERSION=1.0.2
|
||||||
RPA_BLUR_SENSITIVE=false
|
RPA_BLUR_SENSITIVE=false
|
||||||
RPA_LOG_RETENTION_DAYS=180
|
RPA_LOG_RETENTION_DAYS=180
|
||||||
|
|
||||||
|
# --- MAJ silencieuse (DETTE-022 v2) — DESACTIVEE par defaut ---
|
||||||
|
# Deploiement CANARY : on active d'ABORD ce flag sur le SEUL poste pilote
|
||||||
|
# (Emilie), on verifie, puis on elargit. Le poste interroge le serveur et
|
||||||
|
# telecharge la MAJ en staging ; le remplacement reel des fichiers reste manuel
|
||||||
|
# / supervise (reserve revision humaine). Decommenter pour activer ce poste :
|
||||||
|
# RPA_AUTO_UPDATE_ENABLED=true
|
||||||
|
# Intervalle d'interrogation serveur en secondes (defaut 3600 = 1h) :
|
||||||
|
# RPA_AUTO_UPDATE_INTERVAL_S=3600
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ mss>=9.0.1 # Capture d'ecran haute performance
|
|||||||
pynput>=1.7.7 # Clavier/Souris
|
pynput>=1.7.7 # Clavier/Souris
|
||||||
Pillow>=10.0.0 # Traitement image (crops, compression)
|
Pillow>=10.0.0 # Traitement image (crops, compression)
|
||||||
requests>=2.31.0 # Communication serveur
|
requests>=2.31.0 # Communication serveur
|
||||||
|
httpx>=0.27 # Client HTTP orchestrateur Lea (POST /api/learn/start) - brique conversationnelle
|
||||||
psutil>=5.9.0 # Monitoring CPU/RAM
|
psutil>=5.9.0 # Monitoring CPU/RAM
|
||||||
pystray>=0.19.5 # Icone systray
|
pystray>=0.19.5 # Icone systray
|
||||||
plyer>=2.1.0 # Notifications toast natives
|
plyer>=2.1.0 # Notifications toast natives
|
||||||
|
|||||||
193
docs/DESIGN_MAJ_SILENCIEUSE_CANARY_2026-07-01.md
Normal file
193
docs/DESIGN_MAJ_SILENCIEUSE_CANARY_2026-07-01.md
Normal file
@@ -0,0 +1,193 @@
|
|||||||
|
# DESIGN — MAJ silencieuse du client Léa + déploiement CANARY (DETTE-022 v2)
|
||||||
|
|
||||||
|
Date : 2026-07-01
|
||||||
|
Branche : `feat/push-log-dgx`
|
||||||
|
Statut : **premier draft fonctionnel — GATED OFF partout, aucun swap réel, revue supervisée Dom requise avant toute activation**
|
||||||
|
|
||||||
|
> ⚠️ RIEN N'A ÉTÉ DÉPLOYÉ. Aucun SSH poste, aucune action fleet. Ce document +
|
||||||
|
> le code de la branche sont un livrable de conception/implémentation pour revue.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. Problème
|
||||||
|
|
||||||
|
Pousser des correctifs au client Léa sur ~19 postes cliniques live (Wallerstein)
|
||||||
|
**sans** patch manuel DSI et **sans** déranger les TIM en plein travail. Contrainte
|
||||||
|
absolue : une MAJ ratée peut **briquer toute la flotte**. Le mécanisme doit donc
|
||||||
|
être **conservateur** : canary lent + rollback béton plutôt que rapide et risqué.
|
||||||
|
|
||||||
|
## 2. État de départ (stub commit `813b33b47`) — ce qui existait déjà
|
||||||
|
|
||||||
|
Le noyau était plus avancé qu'un simple squelette. Déjà présent et **testé (vert)** :
|
||||||
|
|
||||||
|
| Brique | Fichier | Rôle |
|
||||||
|
|---|---|---|
|
||||||
|
| Décision serveur PURE | `agent_v0/server_v1/update_check.py` | `parse_version`/`is_newer` (semver correct : `1.0.2 < 1.0.10`), `decide_update()`, `build_download_url()` |
|
||||||
|
| Endpoint serveur gated | `agent_v0/server_v1/api_stream.py:7843+` | `GET /api/v1/agents/update/check` — **503 si `RPA_AUTO_UPDATE_SERVER_ENABLED` OFF**, Bearer requis |
|
||||||
|
| Noyau client PUR | `agent_v0/agent_v1/network/updater.py` | `auto_update_enabled()` (flag `RPA_AUTO_UPDATE_ENABLED`, défaut OFF), `should_update()` (double garde anti-downgrade), `download_update()` (staging + SHA256, ne touche jamais les fichiers vivants) |
|
||||||
|
| **Stubs dangereux (no-op)** | `updater.py:246+` | `apply_update()` / `write_boot_ok_marker()` — **réservés révision humaine** (swap fichiers, édition `Lea.bat`, restart) |
|
||||||
|
| Version agent | `agent_v0/agent_v1/config.py:30` | `AGENT_VERSION = os.environ.get("RPA_AGENT_VERSION", "1.0.1")` (amorcé `105ade959`) |
|
||||||
|
| Tests | `tests/unit/test_update_check_server.py`, `tests/unit/test_agent_v1_updater.py`, `tests/integration/test_update_check_endpoint.py` | R2/R3 verts |
|
||||||
|
|
||||||
|
### Ce qui MANQUAIT (comblé par ce draft)
|
||||||
|
|
||||||
|
1. **Aucune logique canary** : `decide_update` recevait `machine_id` mais l'ignorait pour choisir la version. La version cible était une seule var globale `RPA_AGENT_LATEST_VERSION` → une MAJ partait sur **toute** la flotte d'un coup. **C'est le trou de sécurité n°1.**
|
||||||
|
2. **Le noyau client n'était pas wiré** : `updater.py` n'était appelé nulle part. `main.py` ne l'importait pas. Aucun caller HTTP de `/agents/update/check`.
|
||||||
|
3. **Pas d'orchestrateur** reliant check → décide → download (staging) côté client.
|
||||||
|
|
||||||
|
## 3. Fleet / versioning existant (réutilisé, pas réinventé)
|
||||||
|
|
||||||
|
- Registre SQLite `enrolled_agents` (`agent_v0/server_v1/agent_registry.py:105`) : colonne `version` + `last_seen_at` par `machine_id`. Le dashboard Fleet (`web_dashboard/templates/index.html:2247`) affiche déjà la version par poste.
|
||||||
|
- **Limite connue** : `version` n'est écrite qu'à l'`enroll` (installateur), pas rafraîchie par le heartbeat runtime. Le serveur connaît donc la version *installée*, pas forcément la *version vive*. → **inventaire de version = amélioration future** (voir §8), non bloquante pour le canary (le canary est piloté par une allow-list de `machine_id`, pas par l'inventaire).
|
||||||
|
|
||||||
|
## 4. Design retenu (et pourquoi)
|
||||||
|
|
||||||
|
Aligné sur l'état de l'art self-update desktop 2025 (canary / blue-green / A-B swap + watchdog rollback + intégrité + version) — sources en fin de doc.
|
||||||
|
|
||||||
|
### 4.1 CANARY côté serveur — la keystone de sécurité (IMPLÉMENTÉ)
|
||||||
|
|
||||||
|
Nouveau module PUR `agent_v0/server_v1/update_policy.py`. Il résout la version cible
|
||||||
|
**PAR MACHINE** :
|
||||||
|
|
||||||
|
- poste dans l'allow-list canary → `canary_version` (la nouvelle) ;
|
||||||
|
- tous les autres postes → `stable_version` (le floor, inchangé).
|
||||||
|
|
||||||
|
Piloté 100 % par **variables d'environnement serveur** (aucun rebuild, aucune
|
||||||
|
DSI) :
|
||||||
|
|
||||||
|
```
|
||||||
|
RPA_AGENT_STABLE_VERSION # version servie à TOUTE la flotte (défaut 1.0.1)
|
||||||
|
RPA_AGENT_CANARY_VERSION # version servie AUX SEULS postes canary (optionnel)
|
||||||
|
RPA_AGENT_CANARY_MACHINES # allow-list CSV des machine_id canary
|
||||||
|
```
|
||||||
|
|
||||||
|
Garde-fous du résolveur (tous prudents par défaut) :
|
||||||
|
- machine_id absent / liste vide / pas de `canary_version` → **stable** ;
|
||||||
|
- `canary_version` doit être **strictement plus récente** que `stable` (sinon on sert stable — jamais de recul) ;
|
||||||
|
- ne lève jamais ; version illisible → retombe sur stable via le comparateur semver tolérant.
|
||||||
|
|
||||||
|
Wiring : `_latest_agent_version(machine_id)` dans `api_stream.py` appelle
|
||||||
|
`resolve_target_version_from_env(machine_id)`. **Rétrocompat** : si l'ancienne
|
||||||
|
`RPA_AGENT_LATEST_VERSION` est positionnée, elle prime (pas de régression d'un
|
||||||
|
déploiement existant).
|
||||||
|
|
||||||
|
**Effet** : la 1.0.2 ne peut PAS fuiter hors de la liste canary. Blast radius =
|
||||||
|
la liste. On démarre la liste = `lea-4zbgwxty` (Émilie) seul.
|
||||||
|
|
||||||
|
**Promotion** = quand le canary est validé : `RPA_AGENT_STABLE_VERSION=<canary>`
|
||||||
|
+ vider `RPA_AGENT_CANARY_MACHINES` → toute la flotte suit.
|
||||||
|
**Rollback canary** = vider `RPA_AGENT_CANARY_MACHINES` / remettre l'ancienne
|
||||||
|
`RPA_AGENT_CANARY_VERSION` → le prochain check ne propose plus rien.
|
||||||
|
|
||||||
|
### 4.2 Orchestrateur client (IMPLÉMENTÉ, GATED, sans swap)
|
||||||
|
|
||||||
|
`updater.run_update_cycle(local_version, machine_id, staging_dir, checker?, downloader?)` :
|
||||||
|
|
||||||
|
1. **GATE** `auto_update_enabled()` (`RPA_AUTO_UPDATE_ENABLED`, défaut OFF) — si OFF, ne fait **strictement rien**, aucun appel réseau ;
|
||||||
|
2. `checker(...)` → réponse serveur (défaut = `_default_update_checker` : GET vers l'endpoint gated, Bearer, 503→None, jamais d'exception) ;
|
||||||
|
3. `should_update(...)` → plan (double garde semver anti-downgrade) ;
|
||||||
|
4. `download_update(...)` → ZIP en **staging** + vérif **SHA256** (fichiers vivants jamais touchés) ;
|
||||||
|
5. `apply_update(staged)` = **stub no-op** → résultat `applied: False`. **Le swap réel n'est PAS fait par du code d'agent.**
|
||||||
|
|
||||||
|
Statuts retournés (diagnostic/log) : `disabled | check_failed | up_to_date | download_failed | staged`. Best-effort total : aucune exception ne remonte (ne casse jamais Léa).
|
||||||
|
|
||||||
|
### 4.3 Wiring runtime (IMPLÉMENTÉ, GATED)
|
||||||
|
|
||||||
|
`main.py` : thread daemon `_auto_update_loop`, démarré **uniquement si**
|
||||||
|
`AUTO_UPDATE_ENABLED`, à côté des boucles permanentes existantes (même pattern
|
||||||
|
que le log shipper). Sécurité « **au bon moment** » : on ne stage PAS pendant un
|
||||||
|
enregistrement (`self.session_id`) ou un replay actif (`self._replay_active`) —
|
||||||
|
pas de perturbation du travail TIM. Intervalle `RPA_AUTO_UPDATE_INTERVAL_S`
|
||||||
|
(défaut **3600 s / 1 h** : une MAJ n'est jamais urgente).
|
||||||
|
|
||||||
|
### 4.4 Intégrité + version
|
||||||
|
|
||||||
|
- **Intégrité** : SHA256 vérifié dans `download_update` (déjà présent) ; mismatch → rejet + staging propre.
|
||||||
|
- **Version** : `AGENT_VERSION` envoyée à chaque check (`current_version`) ; le serveur choisit la cible par machine.
|
||||||
|
- **Signature (à ajouter, §8)** : SHA256 seul protège de la corruption, pas de l'usurpation. Recommandation : signer le manifeste (le SHA256 vient d'un canal authentifié — l'endpoint Bearer — donc chaîne acceptable pour le POC ; signature détachée = durcissement futur).
|
||||||
|
|
||||||
|
### 4.5 Swap atomique + rollback (SPEC — réservé révision humaine, PAS codé par agent)
|
||||||
|
|
||||||
|
Le swap réel reste dans les stubs `apply_update` / `write_boot_ok_marker` et
|
||||||
|
dans `Lea.bat`. **Un agent ne doit pas écrire de code qui écrase des binaires
|
||||||
|
vivants ni relance un process.** Spec cible pour la revue humaine :
|
||||||
|
|
||||||
|
- **A-B / staging** : le ZIP est extrait dans `Lea_next\`. Au **prochain démarrage**, `Lea.bat` (hors-process) : backup `Lea\`→`Lea_prev\`, swap `Lea_next\`→`Lea\`, lance la nouvelle version.
|
||||||
|
- **Watchdog rollback** : la nouvelle version doit écrire un marker `boot_ok_<version>` **après** ~60 s de heartbeat DGX sain + session OK. Si `Lea.bat` ne trouve pas le marker au démarrage suivant (crash au boot), il restaure `Lea_prev\` automatiquement. Cible « rollback latency » < 90 s (état de l'art).
|
||||||
|
- **Cas edge** (documenté dans les stubs) : DGX down ≠ Léa N+1 buguée — le health-check doit distinguer les deux pour éviter un faux rollback.
|
||||||
|
|
||||||
|
## 5. Fichiers touchés (cette branche)
|
||||||
|
|
||||||
|
**Ajouts**
|
||||||
|
- `agent_v0/server_v1/update_policy.py` — canary PUR (résolveur par machine + lecture env).
|
||||||
|
- `tests/unit/test_update_policy_canary.py` — TDD canary (résolveur + env).
|
||||||
|
|
||||||
|
**Modifs**
|
||||||
|
- `agent_v0/server_v1/api_stream.py` — `_latest_agent_version(machine_id)` canary-aware (rétrocompat legacy) + docstring endpoint.
|
||||||
|
- `agent_v0/agent_v1/network/updater.py` — `_default_update_checker()` + `run_update_cycle()` (orchestrateur gated, sans swap).
|
||||||
|
- `agent_v0/agent_v1/config.py` — `AUTO_UPDATE_INTERVAL_S`, `AUTO_UPDATE_STAGING_DIR`.
|
||||||
|
- `agent_v0/agent_v1/main.py` — thread `_auto_update_loop` gated + import config.
|
||||||
|
- `tests/unit/test_agent_v1_updater.py` — TDD `run_update_cycle` (gate off, up-to-date, staged, sha mismatch, checker raise).
|
||||||
|
- `tests/integration/test_update_check_endpoint.py` — TDD canary HTTP (poste canary vs hors-canary).
|
||||||
|
- `deploy/lea_package/config.txt` — flags client MAJ documentés (commentés, OFF).
|
||||||
|
|
||||||
|
**Intacts (réservés révision humaine)** : `updater.apply_update`, `updater.write_boot_ok_marker`, `Lea.bat`.
|
||||||
|
|
||||||
|
## 6. Matrice des flags (tout OFF par défaut)
|
||||||
|
|
||||||
|
| Flag | Côté | Défaut | Effet |
|
||||||
|
|---|---|---|---|
|
||||||
|
| `RPA_AUTO_UPDATE_SERVER_ENABLED` | serveur | OFF (503) | active l'endpoint de décision |
|
||||||
|
| `RPA_AGENT_STABLE_VERSION` | serveur | `1.0.1` | version floor de toute la flotte |
|
||||||
|
| `RPA_AGENT_CANARY_VERSION` | serveur | — | nouvelle version, postes canary seulement |
|
||||||
|
| `RPA_AGENT_CANARY_MACHINES` | serveur | — | allow-list CSV canary |
|
||||||
|
| `RPA_AGENT_LATEST_VERSION` (legacy) | serveur | — | si set, prime sur le canary (rétrocompat) |
|
||||||
|
| `RPA_AUTO_UPDATE_ENABLED` | client | OFF | active la boucle de check + staging |
|
||||||
|
| `RPA_AUTO_UPDATE_INTERVAL_S` | client | `3600` | intervalle de check |
|
||||||
|
|
||||||
|
## 7. Plan de déploiement CANARY (étapes + critères GO / ROLLBACK)
|
||||||
|
|
||||||
|
> Prérequis avant TOUTE étape : la mécanique de **swap réel** (§4.5) doit avoir
|
||||||
|
> été implémentée et revue par un humain. Tant qu'elle est en stub, ce plan ne
|
||||||
|
> fait que **stager** un ZIP (aucun poste ne change réellement de version) — ce
|
||||||
|
> qui est déjà utile pour valider la chaîne check/download/intégrité à vide.
|
||||||
|
|
||||||
|
**Étape 0 — Serveur seul (aucun poste touché)**
|
||||||
|
- Action : `RPA_AUTO_UPDATE_SERVER_ENABLED=true`, `RPA_AGENT_STABLE_VERSION=1.0.1`, PAS de canary encore.
|
||||||
|
- GO si : `GET /agents/update/check` répond 200 pour un `machine_id` quelconque avec `update_available:false`. Aucun poste n'a la MAJ activée côté client.
|
||||||
|
- ROLLBACK : repasser le flag serveur OFF.
|
||||||
|
|
||||||
|
**Étape 1 — Canary Émilie, staging seul**
|
||||||
|
- Action serveur : `RPA_AGENT_CANARY_VERSION=<nouvelle>`, `RPA_AGENT_CANARY_MACHINES=lea-4zbgwxty`.
|
||||||
|
- Action poste Émilie (config.txt) : `RPA_AUTO_UPDATE_ENABLED=true`.
|
||||||
|
- GO si : dans les logs d'Émilie (remontés par le push-log DGX), `[UPDATE] MAJ <v> téléchargée en staging (SHA256=True)`, ZIP présent dans le staging, `applied:False`, Léa continue de tourner normalement (session/replay non perturbés). Vérifier qu'AUCUN autre poste ne reçoit `update_available:true`.
|
||||||
|
- ROLLBACK : vider `RPA_AGENT_CANARY_MACHINES` (le check ne propose plus rien). Aucun impact : rien n'avait été appliqué.
|
||||||
|
|
||||||
|
**Étape 2 — Canary Émilie, swap réel (après implémentation humaine du §4.5)**
|
||||||
|
- GO si : après redémarrage, Émilie tourne la nouvelle version (`AGENT_VERSION` remontée), marker `boot_ok` écrit, heartbeat DGX sain > 24 h, zéro régression fonctionnelle (enregistrement + replay OK).
|
||||||
|
- ROLLBACK : automatique par watchdog `Lea.bat` si pas de `boot_ok` au boot ; manuel = restaurer `Lea_prev\` + vider la liste canary.
|
||||||
|
|
||||||
|
**Étape 3 — Élargissement progressif (rings)**
|
||||||
|
- Ajouter 2-3 postes à `RPA_AGENT_CANARY_MACHINES`, attendre 48 h par palier.
|
||||||
|
- GO/ROLLBACK : mêmes critères qu'étape 2, par palier.
|
||||||
|
|
||||||
|
**Étape 4 — Promotion générale**
|
||||||
|
- `RPA_AGENT_STABLE_VERSION=<nouvelle>` + vider `RPA_AGENT_CANARY_MACHINES`.
|
||||||
|
- Toute la flotte converge au rythme de son intervalle de check.
|
||||||
|
- ROLLBACK flotte : remettre `RPA_AGENT_STABLE_VERSION` à l'ancienne (les postes ne redescendent pas seuls — le swap-down reste une opération supervisée ; les nouveaux checks ne proposeront plus la MAJ).
|
||||||
|
|
||||||
|
## 8. Améliorations futures (hors périmètre de ce draft)
|
||||||
|
|
||||||
|
1. **Swap réel + watchdog rollback** (§4.5) — la brique manquante n°1, révision humaine.
|
||||||
|
2. **Inventaire de version vive** : rafraîchir `enrolled_agents.version` au heartbeat (le serveur saurait exactement quelle version tourne où — utile pour piloter le canary depuis le dashboard).
|
||||||
|
3. **Signature détachée** du manifeste (durcissement au-delà du SHA256 sur canal Bearer).
|
||||||
|
4. **Endpoint de download versionné** : aujourd'hui `/api/fleet/download/<machine_id>` (dashboard) sert l'installateur complet et **ignore `?type=&version=`** ; il faudra qu'il serve le vrai payload `code-only` incrémental attendu par le contrat d'URL.
|
||||||
|
5. **Auto-report du résultat de swap** (succès/rollback) au serveur pour un tableau de bord canary.
|
||||||
|
|
||||||
|
## 9. Sources (état de l'art self-update desktop / canary 2025)
|
||||||
|
|
||||||
|
- [Rollback Strategies for Enterprise: 2025 Best Practices — sparkco.ai](https://sparkco.ai/blog/rollback-strategies-for-enterprise-2025-best-practices)
|
||||||
|
- [Canary Deployment with Auto-Rollback for AI Agents — antigravitylab.net](https://antigravitylab.net/en/articles/agents/antigravity-ai-agent-canary-deployment-burn-rate-slo)
|
||||||
|
- [awesome-agentic-patterns — canary rollout & automatic rollback](https://github.com/nibzard/awesome-agentic-patterns/blob/main/patterns/canary-rollout-and-automatic-rollback-for-agent-policy-changes.md)
|
||||||
|
- [What is Canary Testing — aqua-cloud.io](https://aqua-cloud.io/canary-testing/)
|
||||||
|
- [Rollback Automation Best Practices for CI/CD — hokstadconsulting.com](https://hokstadconsulting.com/blog/rollback-automation-best-practices-for-ci-cd)
|
||||||
@@ -83,3 +83,43 @@ class TestUpdateCheckEndpointEnabled:
|
|||||||
"/api/v1/agents/update/check?current_version=1.0.1",
|
"/api/v1/agents/update/check?current_version=1.0.1",
|
||||||
)
|
)
|
||||||
assert resp.status_code == 401
|
assert resp.status_code == 401
|
||||||
|
|
||||||
|
|
||||||
|
class TestUpdateCheckCanary:
|
||||||
|
"""Canary : seul le poste canary se voit proposer la nouvelle version.
|
||||||
|
|
||||||
|
On n'utilise PAS RPA_AGENT_LATEST_VERSION (var legacy globale) : on pilote
|
||||||
|
la version cible via la politique canary (stable + canary + allow-list).
|
||||||
|
"""
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def _enable_canary(self, monkeypatch):
|
||||||
|
monkeypatch.setenv("RPA_AUTO_UPDATE_SERVER_ENABLED", "true")
|
||||||
|
# Legacy OFF pour que la politique canary pilote la décision.
|
||||||
|
monkeypatch.delenv("RPA_AGENT_LATEST_VERSION", raising=False)
|
||||||
|
monkeypatch.setenv("RPA_AGENT_STABLE_VERSION", "1.0.1")
|
||||||
|
monkeypatch.setenv("RPA_AGENT_CANARY_VERSION", "1.0.2")
|
||||||
|
monkeypatch.setenv("RPA_AGENT_CANARY_MACHINES", "lea-4zbgwxty")
|
||||||
|
|
||||||
|
def test_poste_canary_recoit_la_nouvelle_version(self, client):
|
||||||
|
resp = client.get(
|
||||||
|
"/api/v1/agents/update/check"
|
||||||
|
"?current_version=1.0.1&machine_id=lea-4zbgwxty",
|
||||||
|
headers=_auth_headers(),
|
||||||
|
)
|
||||||
|
assert resp.status_code == 200
|
||||||
|
body = resp.json()
|
||||||
|
assert body["update_available"] is True
|
||||||
|
assert body["latest_version"] == "1.0.2"
|
||||||
|
|
||||||
|
def test_poste_hors_canary_reste_a_jour_sur_stable(self, client):
|
||||||
|
# Poste NON canary, déjà en 1.0.1 = stable → pas de MAJ (blast radius
|
||||||
|
# borné : la 1.0.2 ne fuite pas hors de la liste canary).
|
||||||
|
resp = client.get(
|
||||||
|
"/api/v1/agents/update/check"
|
||||||
|
"?current_version=1.0.1&machine_id=un-autre-poste",
|
||||||
|
headers=_auth_headers(),
|
||||||
|
)
|
||||||
|
assert resp.status_code == 200
|
||||||
|
body = resp.json()
|
||||||
|
assert body["update_available"] is False
|
||||||
|
|||||||
205
tests/unit/test_action_resolver.py
Normal file
205
tests/unit/test_action_resolver.py
Normal file
@@ -0,0 +1,205 @@
|
|||||||
|
"""Tests for core/navigation/action_resolver.py — coordinate conversion + OCR adapters."""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import pytest
|
||||||
|
from core.navigation.action_resolver import (
|
||||||
|
NavigateCoords,
|
||||||
|
NavigateResult,
|
||||||
|
grounded_to_coords,
|
||||||
|
make_ocr_simple_from_detailed,
|
||||||
|
navigate_login,
|
||||||
|
)
|
||||||
|
from core.navigation.grounding import (
|
||||||
|
CoordsCache,
|
||||||
|
GroundedElement,
|
||||||
|
OcrTokenInfo,
|
||||||
|
OcrDetailedClient,
|
||||||
|
)
|
||||||
|
from core.navigation.visual_verifier import VlmClient
|
||||||
|
|
||||||
|
|
||||||
|
# ── Mock factories ─────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def mock_ocr_detailed_client_factory(tokens: list):
|
||||||
|
def client(image_path: str) -> list:
|
||||||
|
return tokens
|
||||||
|
return client
|
||||||
|
|
||||||
|
|
||||||
|
def mock_vlm_client_factory(response_json: dict):
|
||||||
|
def client(image_path: str, prompt: str) -> str:
|
||||||
|
return json.dumps(response_json)
|
||||||
|
return client
|
||||||
|
|
||||||
|
|
||||||
|
# ── grounded_to_coords tests ───────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestGroundedToCoords:
|
||||||
|
def test_basic_conversion(self):
|
||||||
|
el = GroundedElement(
|
||||||
|
role="bouton", text="Connexion",
|
||||||
|
bbox=(200, 50, 400, 100), center=(300, 75),
|
||||||
|
confidence=0.9, method="ocr_anchor",
|
||||||
|
)
|
||||||
|
coords = grounded_to_coords(el, 1920, 1080)
|
||||||
|
assert coords.x_pct == pytest.approx(300 / 1920, abs=0.01)
|
||||||
|
assert coords.y_pct == pytest.approx(75 / 1080, abs=0.01)
|
||||||
|
assert coords.method == "ocr_anchor"
|
||||||
|
assert coords.bbox_pct is not None
|
||||||
|
|
||||||
|
def test_to_dict(self):
|
||||||
|
coords = NavigateCoords(x_pct=0.15, y_pct=0.07, method="ocr_anchor")
|
||||||
|
d = coords.to_dict()
|
||||||
|
assert d["x_pct"] == 0.15
|
||||||
|
assert d["y_pct"] == 0.07
|
||||||
|
assert d["method"] == "ocr_anchor"
|
||||||
|
|
||||||
|
def test_to_dict_with_bbox(self):
|
||||||
|
coords = NavigateCoords(
|
||||||
|
x_pct=0.15, y_pct=0.07,
|
||||||
|
bbox_pct=(0.10, 0.05, 0.20, 0.09),
|
||||||
|
method="vlm_grounder",
|
||||||
|
)
|
||||||
|
d = coords.to_dict()
|
||||||
|
assert "bbox_pct" in d
|
||||||
|
assert len(d["bbox_pct"]) == 4
|
||||||
|
|
||||||
|
|
||||||
|
# ── make_ocr_simple_from_detailed tests ────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestMakeOcrSimpleFromDetailed:
|
||||||
|
def test_conversion(self):
|
||||||
|
tokens = [
|
||||||
|
OcrTokenInfo(text="Login", bbox=(100, 50, 250, 90)),
|
||||||
|
OcrTokenInfo(text="Password", bbox=(100, 100, 250, 140)),
|
||||||
|
]
|
||||||
|
detailed = mock_ocr_detailed_client_factory(tokens)
|
||||||
|
simple = make_ocr_simple_from_detailed(detailed)
|
||||||
|
result = simple("/tmp/test.png")
|
||||||
|
assert result == ["Login", "Password"]
|
||||||
|
|
||||||
|
def test_empty_tokens(self):
|
||||||
|
detailed = mock_ocr_detailed_client_factory([])
|
||||||
|
simple = make_ocr_simple_from_detailed(detailed)
|
||||||
|
result = simple("/tmp/test.png")
|
||||||
|
assert result == []
|
||||||
|
|
||||||
|
|
||||||
|
# ── navigate_login tests ───────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestNavigateLogin:
|
||||||
|
def test_full_success(self):
|
||||||
|
"""All fields grounded → NavigateResult with coords."""
|
||||||
|
ocr = mock_ocr_detailed_client_factory([
|
||||||
|
OcrTokenInfo(text="Login", bbox=(100, 50, 250, 90), confidence=0.95),
|
||||||
|
OcrTokenInfo(text="Mot de passe", bbox=(100, 100, 250, 140), confidence=0.95),
|
||||||
|
OcrTokenInfo(text="Connexion", bbox=(100, 150, 250, 190), confidence=0.95),
|
||||||
|
])
|
||||||
|
vlm = mock_vlm_client_factory({
|
||||||
|
"confirmed": [
|
||||||
|
{"index": 1, "role_confirmed": True, "actual_role": "champ", "confidence": 0.9},
|
||||||
|
{"index": 2, "role_confirmed": True, "actual_role": "champ", "confidence": 0.9},
|
||||||
|
{"index": 3, "role_confirmed": True, "actual_role": "bouton", "confidence": 0.9},
|
||||||
|
],
|
||||||
|
"overall_confidence": 0.9,
|
||||||
|
})
|
||||||
|
result = navigate_login(
|
||||||
|
"/tmp/login.png",
|
||||||
|
ocr_client=ocr, vlm_client=vlm,
|
||||||
|
skip_pre_verify=True,
|
||||||
|
)
|
||||||
|
assert result.all_resolved == True
|
||||||
|
assert result.login_coords is not None
|
||||||
|
assert result.password_coords is not None
|
||||||
|
assert result.submit_coords is not None
|
||||||
|
assert result.submit_coords.x_pct > 0
|
||||||
|
assert result.submit_coords.y_pct > 0
|
||||||
|
|
||||||
|
def test_no_clients_error(self):
|
||||||
|
"""Missing OCR/VLM clients → error."""
|
||||||
|
result = navigate_login("/tmp/login.png", ocr_client=None, vlm_client=None)
|
||||||
|
assert result.all_resolved == False
|
||||||
|
assert "required" in result.error
|
||||||
|
|
||||||
|
def test_pre_verify_fail(self):
|
||||||
|
"""Pre-verify fails → early abort."""
|
||||||
|
ocr = mock_ocr_detailed_client_factory([
|
||||||
|
OcrTokenInfo(text="Accueil", bbox=(0, 0, 100, 40)),
|
||||||
|
])
|
||||||
|
vlm = mock_vlm_client_factory({})
|
||||||
|
result = navigate_login(
|
||||||
|
"/tmp/page.png",
|
||||||
|
ocr_client=ocr, vlm_client=vlm,
|
||||||
|
skip_pre_verify=False,
|
||||||
|
)
|
||||||
|
assert result.all_resolved == False
|
||||||
|
assert result.pre_verify is not None
|
||||||
|
assert result.pre_verify.match == False
|
||||||
|
|
||||||
|
def test_skip_pre_verify(self):
|
||||||
|
"""Skip pre-verify → proceed to grounding even if form incomplete."""
|
||||||
|
ocr = mock_ocr_detailed_client_factory([
|
||||||
|
OcrTokenInfo(text="Login", bbox=(100, 50, 250, 90)),
|
||||||
|
OcrTokenInfo(text="Mot de passe", bbox=(100, 100, 250, 140)),
|
||||||
|
OcrTokenInfo(text="Connexion", bbox=(100, 150, 250, 190)),
|
||||||
|
])
|
||||||
|
vlm = mock_vlm_client_factory({})
|
||||||
|
result = navigate_login(
|
||||||
|
"/tmp/login.png",
|
||||||
|
ocr_client=ocr, vlm_client=vlm,
|
||||||
|
skip_pre_verify=True,
|
||||||
|
)
|
||||||
|
assert result.pre_verify is None # skipped
|
||||||
|
assert result.all_resolved == True
|
||||||
|
|
||||||
|
|
||||||
|
# ── NavigateResult dataclass tests ─────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestNavigateResult:
|
||||||
|
def test_default(self):
|
||||||
|
result = NavigateResult()
|
||||||
|
assert result.all_resolved == False
|
||||||
|
assert result.login_coords is None
|
||||||
|
assert result.error == ""
|
||||||
|
|
||||||
|
def test_with_coords(self):
|
||||||
|
result = NavigateResult(
|
||||||
|
login_coords=NavigateCoords(x_pct=0.15, y_pct=0.07, method="ocr_anchor"),
|
||||||
|
all_resolved=True,
|
||||||
|
)
|
||||||
|
assert result.login_coords.x_pct == 0.15
|
||||||
|
|
||||||
|
|
||||||
|
# ── Import validation ──────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestImportValidation:
|
||||||
|
def test_action_resolver_imports(self):
|
||||||
|
"""Verify action_resolver module imports cleanly."""
|
||||||
|
from core.navigation.action_resolver import (
|
||||||
|
NavigateCoords,
|
||||||
|
NavigateResult,
|
||||||
|
grounded_to_coords,
|
||||||
|
make_ocr_detailed_from_grid,
|
||||||
|
make_ocr_simple_from_detailed,
|
||||||
|
navigate_login,
|
||||||
|
)
|
||||||
|
assert NavigateCoords is not None
|
||||||
|
assert NavigateResult is not None
|
||||||
|
|
||||||
|
def test_navigation_package_handler(self):
|
||||||
|
"""Verify _handle_navigate_action is importable from package."""
|
||||||
|
from core.navigation import _handle_navigate_action
|
||||||
|
assert callable(_handle_navigate_action)
|
||||||
|
|
||||||
|
def test_navigation_package_exports(self):
|
||||||
|
"""Verify package __all__ includes navigate exports."""
|
||||||
|
import core.navigation as nav
|
||||||
|
assert "navigate_login" in nav.__all__
|
||||||
|
assert "NavigateResult" in nav.__all__
|
||||||
|
assert "_handle_navigate_action" in nav.__all__
|
||||||
248
tests/unit/test_agent_v1_session_watchdog.py
Normal file
248
tests/unit/test_agent_v1_session_watchdog.py
Normal file
@@ -0,0 +1,248 @@
|
|||||||
|
"""Tests du watchdog de session interactive (résilience RDP/Citrix).
|
||||||
|
|
||||||
|
Vérifie que :
|
||||||
|
- Le tray est ré-affiché à la reconnexion RDP (run_ui rappelé).
|
||||||
|
- Un seul tray tourne à la fois (invariant « un seul tray »).
|
||||||
|
- Les threads de fond de l'agent (heartbeat/replay) ne sont JAMAIS
|
||||||
|
relancés par le watchdog (il ne relance QUE l'UI).
|
||||||
|
- Un Quitter explicite arrête le watchdog (pas de résurrection du tray).
|
||||||
|
- Le détecteur de session Windows tombe en marche (True) hors Windows.
|
||||||
|
|
||||||
|
Aucune vraie UI : run_ui et is_available sont des callables mockés.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import sys
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
|
# Mocker les libs GUI/Win32 avant tout import du module sous test.
|
||||||
|
sys.modules.setdefault("pynput", MagicMock())
|
||||||
|
sys.modules.setdefault("pynput.mouse", MagicMock())
|
||||||
|
sys.modules.setdefault("pynput.keyboard", MagicMock())
|
||||||
|
sys.modules.setdefault("pystray", MagicMock())
|
||||||
|
|
||||||
|
from agent_v0.agent_v1.ui.session_watchdog import ( # noqa: E402
|
||||||
|
InteractiveSessionWatchdog,
|
||||||
|
is_interactive_desktop_available,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Détection de session
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def test_detection_hors_windows_renvoie_true(monkeypatch):
|
||||||
|
"""Hors Windows (dev/tests Linux) : bureau toujours 'disponible'."""
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"agent_v0.agent_v1.ui.session_watchdog.platform.system",
|
||||||
|
lambda: "Linux",
|
||||||
|
)
|
||||||
|
assert is_interactive_desktop_available() is True
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Boucle du watchdog
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def test_relance_ui_a_la_reconnexion():
|
||||||
|
"""Session absente puis présente => le tray est (ré)affiché une fois dispo.
|
||||||
|
|
||||||
|
Scénario : la 1re sonde dit 'indisponible' (RDP déconnecté), la 2e dit
|
||||||
|
'disponible' (reconnexion) => run_ui doit être appelé exactement une fois,
|
||||||
|
puis le watchdog s'arrête.
|
||||||
|
"""
|
||||||
|
availability = iter([False, True])
|
||||||
|
run_ui_calls = []
|
||||||
|
|
||||||
|
def _run_ui():
|
||||||
|
run_ui_calls.append(time.time())
|
||||||
|
|
||||||
|
# L'agent vit jusqu'à ce que le tray ait été affiché une fois.
|
||||||
|
state = {"alive": True}
|
||||||
|
|
||||||
|
def _is_running():
|
||||||
|
# Après le premier affichage du tray, l'agent s'arrête.
|
||||||
|
return state["alive"] and len(run_ui_calls) == 0
|
||||||
|
|
||||||
|
def _is_available():
|
||||||
|
return next(availability)
|
||||||
|
|
||||||
|
wd = InteractiveSessionWatchdog(
|
||||||
|
run_ui=_run_ui,
|
||||||
|
is_running=_is_running,
|
||||||
|
is_available=_is_available,
|
||||||
|
poll_interval_s=0.01, # sonde très rapide pour le test
|
||||||
|
)
|
||||||
|
wd.run()
|
||||||
|
|
||||||
|
# Le tray a été (ré)affiché exactement une fois après la reconnexion.
|
||||||
|
assert len(run_ui_calls) == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_reaffichage_apres_chaque_deconnexion():
|
||||||
|
"""Deux cycles connexion→déconnexion => tray relancé à chaque reconnexion."""
|
||||||
|
run_ui_calls = []
|
||||||
|
|
||||||
|
# is_available toujours True ; le tray 'sort' immédiatement (déconnexion).
|
||||||
|
def _run_ui():
|
||||||
|
run_ui_calls.append(1)
|
||||||
|
|
||||||
|
def _is_running():
|
||||||
|
# Vivre pour 2 affichages de tray, puis arrêter.
|
||||||
|
return len(run_ui_calls) < 2
|
||||||
|
|
||||||
|
wd = InteractiveSessionWatchdog(
|
||||||
|
run_ui=_run_ui,
|
||||||
|
is_running=_is_running,
|
||||||
|
is_available=lambda: True,
|
||||||
|
poll_interval_s=0.01,
|
||||||
|
)
|
||||||
|
wd.run()
|
||||||
|
|
||||||
|
assert len(run_ui_calls) == 2
|
||||||
|
|
||||||
|
|
||||||
|
def test_un_seul_tray_a_la_fois():
|
||||||
|
"""L'invariant 'un seul tray' : run_ui n'est jamais réentrant en parallèle."""
|
||||||
|
concurrent = {"count": 0, "max": 0}
|
||||||
|
lock = threading.Lock()
|
||||||
|
|
||||||
|
def _run_ui():
|
||||||
|
with lock:
|
||||||
|
concurrent["count"] += 1
|
||||||
|
concurrent["max"] = max(concurrent["max"], concurrent["count"])
|
||||||
|
time.sleep(0.02) # simule un tray qui tourne un peu
|
||||||
|
with lock:
|
||||||
|
concurrent["count"] -= 1
|
||||||
|
|
||||||
|
calls = {"n": 0}
|
||||||
|
|
||||||
|
def _is_running():
|
||||||
|
calls["n"] += 1
|
||||||
|
return calls["n"] <= 3 # 3 cycles de tray
|
||||||
|
|
||||||
|
wd = InteractiveSessionWatchdog(
|
||||||
|
run_ui=_run_ui,
|
||||||
|
is_running=_is_running,
|
||||||
|
is_available=lambda: True,
|
||||||
|
poll_interval_s=0.01,
|
||||||
|
)
|
||||||
|
wd.run()
|
||||||
|
|
||||||
|
# Jamais deux trays simultanés.
|
||||||
|
assert concurrent["max"] == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_stop_reveille_le_watchdog_en_attente():
|
||||||
|
"""stop() sort immédiatement la boucle quand la session est absente."""
|
||||||
|
run_ui_calls = []
|
||||||
|
|
||||||
|
wd = InteractiveSessionWatchdog(
|
||||||
|
run_ui=lambda: run_ui_calls.append(1),
|
||||||
|
is_running=lambda: True,
|
||||||
|
is_available=lambda: False, # jamais de session => reste en attente
|
||||||
|
poll_interval_s=60, # long : seul stop() peut débloquer
|
||||||
|
)
|
||||||
|
|
||||||
|
t = threading.Thread(target=wd.run)
|
||||||
|
t.start()
|
||||||
|
time.sleep(0.05) # laisser entrer dans l'attente
|
||||||
|
wd.stop()
|
||||||
|
t.join(timeout=2)
|
||||||
|
|
||||||
|
assert not t.is_alive() # le watchdog est bien sorti
|
||||||
|
assert run_ui_calls == [] # aucun tray (jamais de session dispo)
|
||||||
|
|
||||||
|
|
||||||
|
def test_crash_du_tray_ne_tue_pas_le_watchdog():
|
||||||
|
"""Une exception dans run_ui est absorbée ; le watchdog reste maître."""
|
||||||
|
calls = {"n": 0}
|
||||||
|
|
||||||
|
def _run_ui():
|
||||||
|
calls["n"] += 1
|
||||||
|
raise RuntimeError("tray HS")
|
||||||
|
|
||||||
|
def _is_running():
|
||||||
|
return calls["n"] < 2 # tolérer 2 crashs puis sortir
|
||||||
|
|
||||||
|
wd = InteractiveSessionWatchdog(
|
||||||
|
run_ui=_run_ui,
|
||||||
|
is_running=_is_running,
|
||||||
|
is_available=lambda: True,
|
||||||
|
poll_interval_s=0.01,
|
||||||
|
)
|
||||||
|
# Ne doit PAS lever : le crash est loggé, pas propagé.
|
||||||
|
wd.run()
|
||||||
|
|
||||||
|
assert calls["n"] == 2
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Intégration avec main._agent_should_live (Quitter vs déconnexion)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def test_tray_run_reentrant_ne_relance_pas_les_threads_de_fond(monkeypatch):
|
||||||
|
"""SmartTrayV1.run() appelé 2x (reconnexion RDP) : threads de fond 1x seulement.
|
||||||
|
|
||||||
|
On vérifie que `_start_background_once` est idempotent : les threads
|
||||||
|
connexion/cache et l'accueil ne démarrent qu'au premier affichage, mais
|
||||||
|
une nouvelle icône pystray est recréée à chaque appel (ré-affichage).
|
||||||
|
"""
|
||||||
|
import threading as _threading
|
||||||
|
from agent_v0.agent_v1.ui import smart_tray as smart_tray_mod
|
||||||
|
|
||||||
|
tray = smart_tray_mod.SmartTrayV1.__new__(smart_tray_mod.SmartTrayV1)
|
||||||
|
tray._bg_started = False
|
||||||
|
tray.machine_id = "poste_test"
|
||||||
|
tray.server_client = None # pas de threads réseau => simplifie
|
||||||
|
tray.icon = None
|
||||||
|
|
||||||
|
greet_calls = {"n": 0}
|
||||||
|
hotkey_calls = {"n": 0}
|
||||||
|
icons_created = {"n": 0}
|
||||||
|
|
||||||
|
tray._notifier = MagicMock()
|
||||||
|
tray._notifier.greet.side_effect = lambda: greet_calls.__setitem__("n", greet_calls["n"] + 1)
|
||||||
|
monkeypatch.setattr(tray, "_start_hotkey", lambda: hotkey_calls.__setitem__("n", hotkey_calls["n"] + 1))
|
||||||
|
monkeypatch.setattr(tray, "_current_icon", lambda: object())
|
||||||
|
monkeypatch.setattr(tray, "_get_menu_items", lambda: [])
|
||||||
|
|
||||||
|
# Icône pystray factice : run() ne bloque pas (simule une sortie immédiate).
|
||||||
|
class _FakeIcon:
|
||||||
|
def __init__(self, *a, **k):
|
||||||
|
icons_created["n"] += 1
|
||||||
|
|
||||||
|
def run(self):
|
||||||
|
return None
|
||||||
|
|
||||||
|
monkeypatch.setattr(smart_tray_mod.pystray, "Icon", _FakeIcon)
|
||||||
|
monkeypatch.setattr(smart_tray_mod.pystray, "Menu", lambda *a, **k: None)
|
||||||
|
|
||||||
|
# Deux affichages successifs (déconnexion → reconnexion).
|
||||||
|
tray.run()
|
||||||
|
tray.run()
|
||||||
|
|
||||||
|
# Accueil + hotkey : une seule fois (one-shot).
|
||||||
|
assert greet_calls["n"] == 1
|
||||||
|
assert hotkey_calls["n"] == 1
|
||||||
|
# Mais une nouvelle icône à chaque affichage (le tray revient bien).
|
||||||
|
assert icons_created["n"] == 2
|
||||||
|
|
||||||
|
|
||||||
|
def test_agent_should_live_distingue_quit_et_deconnexion():
|
||||||
|
"""Quitter explicite arrête le watchdog ; une déconnexion RDP non."""
|
||||||
|
from types import SimpleNamespace
|
||||||
|
from agent_v0.agent_v1.main import _agent_should_live
|
||||||
|
|
||||||
|
# Agent actif, tray sans quit demandé => doit vivre (déconnexion RDP OK).
|
||||||
|
agent = SimpleNamespace(running=True, ui=SimpleNamespace(_quit_requested=False))
|
||||||
|
assert _agent_should_live(agent) is True
|
||||||
|
|
||||||
|
# Quitter explicite => ne doit plus vivre (pas de résurrection).
|
||||||
|
agent.ui._quit_requested = True
|
||||||
|
assert _agent_should_live(agent) is False
|
||||||
|
|
||||||
|
# agent.running=False => ne vit plus (arrêt global).
|
||||||
|
agent2 = SimpleNamespace(running=False, ui=SimpleNamespace(_quit_requested=False))
|
||||||
|
assert _agent_should_live(agent2) is False
|
||||||
@@ -206,20 +206,203 @@ class TestDownloadUpdate:
|
|||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Stubs réservés à la révision humaine — DOIVENT être no-op explicites
|
# apply_update — ARMEMENT du swap (extraction agent_v1_new + marqueur).
|
||||||
|
# NE swappe PAS et NE touche PAS les fichiers vivants (Lea.bat le fait au boot).
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
class TestDangerousPartsAreStubs:
|
def _make_zip(path, entries):
|
||||||
def test_apply_update_est_un_stub_non_implemente(self, mod, tmp_path):
|
"""Fabrique un ZIP {nom: contenu} pour les tests."""
|
||||||
# Le swap réel est réservé révision humaine : le stub NE TOUCHE RIEN
|
import zipfile
|
||||||
# et signale qu'il n'est pas implémenté.
|
with zipfile.ZipFile(path, "w") as zf:
|
||||||
result = mod.apply_update(
|
for name, content in entries.items():
|
||||||
{"target_version": "1.0.2", "update_type": "code-only",
|
zf.writestr(name, content)
|
||||||
"staged_zip": str(tmp_path / "x.zip")}
|
return path
|
||||||
)
|
|
||||||
assert result["applied"] is False
|
|
||||||
assert "human" in result["reason"].lower() or "supervis" in result["reason"].lower()
|
|
||||||
|
|
||||||
def test_write_boot_ok_marker_est_un_stub(self, mod):
|
|
||||||
result = mod.write_boot_ok_marker("1.0.2")
|
class TestApplyUpdateArm:
|
||||||
assert result["written"] is False
|
def test_arme_extrait_et_pose_marqueur(self, mod, tmp_path):
|
||||||
|
app = tmp_path / "app"; app.mkdir()
|
||||||
|
z = _make_zip(tmp_path / "u.zip", {"main.py": "v2", "sub/x.py": "y"})
|
||||||
|
res = mod.apply_update(
|
||||||
|
{"target_version": "1.0.2", "update_type": "code-only", "staged_zip": str(z)},
|
||||||
|
app_dir=app,
|
||||||
|
)
|
||||||
|
assert res["armed"] is True and res["applied"] is False
|
||||||
|
new_dir = app / "agent_v1_new"
|
||||||
|
assert (new_dir / "main.py").read_text() == "v2"
|
||||||
|
assert (new_dir / "sub" / "x.py").read_text() == "y"
|
||||||
|
import json as _j
|
||||||
|
data = _j.loads((app / "UPDATE_READY").read_text())
|
||||||
|
assert data["target_version"] == "1.0.2"
|
||||||
|
assert data["update_type"] == "code-only"
|
||||||
|
|
||||||
|
def test_ne_touche_pas_le_agent_v1_vivant(self, mod, tmp_path):
|
||||||
|
app = tmp_path / "app"; (app / "agent_v1").mkdir(parents=True)
|
||||||
|
live = app / "agent_v1" / "sentinelle.txt"
|
||||||
|
live.write_text("VERSION_VIVANTE")
|
||||||
|
z = _make_zip(tmp_path / "u.zip", {"main.py": "v2"})
|
||||||
|
mod.apply_update(
|
||||||
|
{"target_version": "1.0.2", "update_type": "code-only", "staged_zip": str(z)},
|
||||||
|
app_dir=app,
|
||||||
|
)
|
||||||
|
assert live.read_text() == "VERSION_VIVANTE" # swap différé à Lea.bat
|
||||||
|
|
||||||
|
def test_zip_introuvable_pas_de_crash_ni_marqueur(self, mod, tmp_path):
|
||||||
|
app = tmp_path / "app"; app.mkdir()
|
||||||
|
res = mod.apply_update(
|
||||||
|
{"target_version": "1.0.2", "update_type": "code-only",
|
||||||
|
"staged_zip": str(tmp_path / "absent.zip")},
|
||||||
|
app_dir=app,
|
||||||
|
)
|
||||||
|
assert res["armed"] is False and "error" in res
|
||||||
|
assert not (app / "UPDATE_READY").exists()
|
||||||
|
|
||||||
|
def test_relance_nettoie_agent_v1_new_precedent(self, mod, tmp_path):
|
||||||
|
app = tmp_path / "app"; app.mkdir()
|
||||||
|
stale = app / "agent_v1_new"; stale.mkdir()
|
||||||
|
(stale / "vieux.txt").write_text("obsolete")
|
||||||
|
z = _make_zip(tmp_path / "u.zip", {"main.py": "v2"})
|
||||||
|
mod.apply_update(
|
||||||
|
{"target_version": "1.0.3", "update_type": "code-only", "staged_zip": str(z)},
|
||||||
|
app_dir=app,
|
||||||
|
)
|
||||||
|
assert not (app / "agent_v1_new" / "vieux.txt").exists()
|
||||||
|
assert (app / "agent_v1_new" / "main.py").read_text() == "v2"
|
||||||
|
|
||||||
|
def test_zip_slip_refuse(self, mod, tmp_path):
|
||||||
|
app = tmp_path / "app"; app.mkdir()
|
||||||
|
z = _make_zip(tmp_path / "evil.zip", {"../evil.py": "pwn"})
|
||||||
|
res = mod.apply_update(
|
||||||
|
{"target_version": "1.0.2", "update_type": "code-only", "staged_zip": str(z)},
|
||||||
|
app_dir=app,
|
||||||
|
)
|
||||||
|
assert res["armed"] is False
|
||||||
|
assert not (app / "evil.py").exists()
|
||||||
|
|
||||||
|
|
||||||
|
class TestWriteBootOkMarker:
|
||||||
|
def test_ecrit_boot_ok_et_desarme_pending(self, mod, tmp_path):
|
||||||
|
app = tmp_path / "app"; app.mkdir()
|
||||||
|
(app / "PENDING_BOOT_1.0.2").write_text("x")
|
||||||
|
res = mod.write_boot_ok_marker("1.0.2", app_dir=app)
|
||||||
|
assert res["written"] is True
|
||||||
|
assert (app / "boot_ok_1.0.2").exists()
|
||||||
|
assert not (app / "PENDING_BOOT_1.0.2").exists()
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# run_update_cycle — orchestrateur GATED (check → décide → stage → stub apply)
|
||||||
|
# AUCUN réseau réel, AUCUN swap réel : checker/downloader INJECTABLES, le swap
|
||||||
|
# reste un stub no-op (réservé révision humaine).
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class TestRunUpdateCycle:
|
||||||
|
def _checker(self, response):
|
||||||
|
"""Fabrique un checker injectable qui renvoie `response`."""
|
||||||
|
def _c(local_version, machine_id):
|
||||||
|
return response
|
||||||
|
return _c
|
||||||
|
|
||||||
|
def test_gate_off_ne_fait_rien(self, mod, tmp_path, monkeypatch):
|
||||||
|
# Flag OFF (défaut) : le cycle ne doit RIEN faire (pas d'appel réseau).
|
||||||
|
monkeypatch.delenv("RPA_AUTO_UPDATE_ENABLED", raising=False)
|
||||||
|
called = {"n": 0}
|
||||||
|
|
||||||
|
def _checker(local_version, machine_id):
|
||||||
|
called["n"] += 1
|
||||||
|
return {"update_available": True, "latest_version": "9.9.9",
|
||||||
|
"url": "http://x", "sha256": None}
|
||||||
|
|
||||||
|
result = mod.run_update_cycle(
|
||||||
|
local_version="1.0.1",
|
||||||
|
machine_id="pc-1",
|
||||||
|
staging_dir=tmp_path,
|
||||||
|
checker=_checker,
|
||||||
|
downloader=lambda u: b"x",
|
||||||
|
)
|
||||||
|
assert result["status"] == "disabled"
|
||||||
|
assert called["n"] == 0 # aucun appel réseau quand OFF
|
||||||
|
|
||||||
|
def test_a_jour_ne_stage_rien(self, mod, tmp_path, monkeypatch):
|
||||||
|
monkeypatch.setenv("RPA_AUTO_UPDATE_ENABLED", "true")
|
||||||
|
result = mod.run_update_cycle(
|
||||||
|
local_version="1.0.1",
|
||||||
|
machine_id="pc-1",
|
||||||
|
staging_dir=tmp_path,
|
||||||
|
checker=self._checker(
|
||||||
|
{"update_available": False, "latest_version": "1.0.1"}
|
||||||
|
),
|
||||||
|
downloader=lambda u: b"should-not-be-called",
|
||||||
|
)
|
||||||
|
assert result["status"] == "up_to_date"
|
||||||
|
assert list(tmp_path.glob("*.zip")) == []
|
||||||
|
|
||||||
|
def test_maj_dispo_arme_le_swap_mais_ne_swappe_pas(
|
||||||
|
self, mod, tmp_path, monkeypatch
|
||||||
|
):
|
||||||
|
monkeypatch.setenv("RPA_AUTO_UPDATE_ENABLED", "true")
|
||||||
|
# payload = un VRAI ZIP (le download le stage, apply_update l'extrait)
|
||||||
|
import io, zipfile
|
||||||
|
buf = io.BytesIO()
|
||||||
|
with zipfile.ZipFile(buf, "w") as zf:
|
||||||
|
zf.writestr("main.py", "code v1.0.2")
|
||||||
|
payload = buf.getvalue()
|
||||||
|
sha = hashlib.sha256(payload).hexdigest()
|
||||||
|
app = tmp_path / "app"; app.mkdir()
|
||||||
|
|
||||||
|
result = mod.run_update_cycle(
|
||||||
|
local_version="1.0.1",
|
||||||
|
machine_id="pc-1",
|
||||||
|
staging_dir=tmp_path,
|
||||||
|
checker=self._checker({
|
||||||
|
"update_available": True,
|
||||||
|
"latest_version": "1.0.2",
|
||||||
|
"update_type": "code-only",
|
||||||
|
"url": "http://srv/dl?version=1.0.2",
|
||||||
|
"sha256": sha,
|
||||||
|
}),
|
||||||
|
downloader=lambda u: payload,
|
||||||
|
app_dir=app,
|
||||||
|
)
|
||||||
|
# Téléchargé + vérifié + ARMÉ (agent_v1_new + UPDATE_READY), mais PAS
|
||||||
|
# swappé : le remplacement atomique est fait par Lea.bat au reboot.
|
||||||
|
assert result["status"] == "armed"
|
||||||
|
assert result["target_version"] == "1.0.2"
|
||||||
|
assert result["sha256_verified"] is True
|
||||||
|
assert result["applied"] is False
|
||||||
|
assert (app / "UPDATE_READY").exists()
|
||||||
|
assert (app / "agent_v1_new" / "main.py").read_text() == "code v1.0.2"
|
||||||
|
|
||||||
|
def test_sha256_mismatch_ne_stage_pas(self, mod, tmp_path, monkeypatch):
|
||||||
|
monkeypatch.setenv("RPA_AUTO_UPDATE_ENABLED", "true")
|
||||||
|
result = mod.run_update_cycle(
|
||||||
|
local_version="1.0.1",
|
||||||
|
machine_id="pc-1",
|
||||||
|
staging_dir=tmp_path,
|
||||||
|
checker=self._checker({
|
||||||
|
"update_available": True,
|
||||||
|
"latest_version": "1.0.2",
|
||||||
|
"update_type": "code-only",
|
||||||
|
"url": "http://x",
|
||||||
|
"sha256": "0" * 64,
|
||||||
|
}),
|
||||||
|
downloader=lambda u: b"corrupted",
|
||||||
|
)
|
||||||
|
assert result["status"] == "download_failed"
|
||||||
|
assert list(tmp_path.glob("*.zip")) == []
|
||||||
|
|
||||||
|
def test_checker_qui_leve_pas_de_crash(self, mod, tmp_path, monkeypatch):
|
||||||
|
monkeypatch.setenv("RPA_AUTO_UPDATE_ENABLED", "true")
|
||||||
|
|
||||||
|
def _boom(local_version, machine_id):
|
||||||
|
raise RuntimeError("serveur down / 503")
|
||||||
|
|
||||||
|
result = mod.run_update_cycle(
|
||||||
|
local_version="1.0.1",
|
||||||
|
machine_id="pc-1",
|
||||||
|
staging_dir=tmp_path,
|
||||||
|
checker=_boom,
|
||||||
|
downloader=lambda u: b"x",
|
||||||
|
)
|
||||||
|
# Best-effort : jamais d'exception ne remonte (ne casse pas Léa).
|
||||||
|
assert result["status"] == "check_failed"
|
||||||
|
|||||||
320
tests/unit/test_capturer_capture_io_format.py
Normal file
320
tests/unit/test_capturer_capture_io_format.py
Normal file
@@ -0,0 +1,320 @@
|
|||||||
|
"""Politique de format des captures + robustesse du répertoire shots.
|
||||||
|
|
||||||
|
Deux corrections testées ici (agent_v0/agent_v1/vision) :
|
||||||
|
|
||||||
|
1. FORMAT (allègement) : `capturer.py` doit déléguer l'écriture à
|
||||||
|
`capture_io.save_capture`, qui applique la politique :
|
||||||
|
- crop → PNG lossless (cible de grounding qwen3-vl)
|
||||||
|
- full/window/context → JPEG q85
|
||||||
|
- heartbeat → JPEG downscalé (largeur max ~1280)
|
||||||
|
Aujourd'hui tout était sauvé en `img.save(path, "PNG", quality=...)`
|
||||||
|
(le `quality` est ignoré par PNG → PNG lossless plein écran, ~90 Go).
|
||||||
|
|
||||||
|
2. BUG chemin (poste Émilie) : ``[Errno 2] No such file or directory:
|
||||||
|
..._background/shots/context...``. Le répertoire `shots/` est créé une
|
||||||
|
seule fois dans `__init__`, mais l'auto-cleanup (`SessionStorage`,
|
||||||
|
`shutil.rmtree`) peut supprimer tout le dossier de session `_background`.
|
||||||
|
Les sauvegardes suivantes doivent recréer le répertoire cible
|
||||||
|
(`os.makedirs(dir, exist_ok=True)`) avant chaque écriture.
|
||||||
|
|
||||||
|
Tests 100% mockés : aucune vraie capture écran (mss est patché).
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import shutil
|
||||||
|
from pathlib import Path
|
||||||
|
from unittest.mock import MagicMock, patch
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from PIL import Image
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Helpers (repris du style de test_capturer_monitor_guard.py)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _make_mock_mss(monitors):
|
||||||
|
"""Mock `mss.mss()` renvoyant un monitor sain unique (image grise unie)."""
|
||||||
|
|
||||||
|
def factory():
|
||||||
|
instance = MagicMock()
|
||||||
|
instance.monitors = monitors
|
||||||
|
grab_result = MagicMock()
|
||||||
|
m = monitors[1] if len(monitors) > 1 else monitors[0]
|
||||||
|
w, h = m["width"], m["height"]
|
||||||
|
grab_result.size = (w, h)
|
||||||
|
grab_result.bgra = b"\x80\x80\x80\x00" * (w * h)
|
||||||
|
instance.grab = MagicMock(return_value=grab_result)
|
||||||
|
cm = MagicMock()
|
||||||
|
cm.__enter__ = MagicMock(return_value=instance)
|
||||||
|
cm.__exit__ = MagicMock(return_value=False)
|
||||||
|
return cm
|
||||||
|
|
||||||
|
return factory
|
||||||
|
|
||||||
|
|
||||||
|
_NORMAL_MONITORS = [
|
||||||
|
{"left": 0, "top": 0, "width": 800, "height": 600}, # composite
|
||||||
|
{"left": 0, "top": 0, "width": 800, "height": 600}, # primaire sain
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _vision_capturer(tmp_path):
|
||||||
|
from agent_v0.agent_v1.vision.capturer import VisionCapturer
|
||||||
|
|
||||||
|
return VisionCapturer(str(tmp_path))
|
||||||
|
|
||||||
|
|
||||||
|
def _patch_mss():
|
||||||
|
"""Contexte : mss patché + time.sleep no-op + pas de floutage.
|
||||||
|
|
||||||
|
Le floutage est désactivé pour isoler la politique d'écriture (le blur
|
||||||
|
ouvre/modifie l'image mais n'impacte pas le format de sortie ; on le coupe
|
||||||
|
pour rester déterministe).
|
||||||
|
"""
|
||||||
|
return (
|
||||||
|
patch(
|
||||||
|
"agent_v0.agent_v1.vision.capturer.mss.mss",
|
||||||
|
side_effect=_make_mock_mss(_NORMAL_MONITORS),
|
||||||
|
),
|
||||||
|
patch("agent_v0.agent_v1.vision.capturer.time.sleep"),
|
||||||
|
patch("agent_v0.agent_v1.vision.capturer.BLUR_SENSITIVE", False),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ===========================================================================
|
||||||
|
# PARTIE A — Politique save_capture (unité capture_io)
|
||||||
|
# ===========================================================================
|
||||||
|
|
||||||
|
|
||||||
|
def test_save_capture_crop_stays_png(tmp_path: Path):
|
||||||
|
from agent_v0.agent_v1.vision import capture_io
|
||||||
|
|
||||||
|
img = Image.new("RGB", (80, 80), (10, 20, 30))
|
||||||
|
out = capture_io.save_capture(img, str(tmp_path / "shot_crop"), "crop")
|
||||||
|
|
||||||
|
assert out.endswith(".png"), f"crop doit rester PNG, got {out!r}"
|
||||||
|
assert Path(out).exists()
|
||||||
|
with Image.open(out) as reopened:
|
||||||
|
assert reopened.format == "PNG"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("kind", ["full", "window", "context"])
|
||||||
|
def test_save_capture_context_kinds_are_jpeg(tmp_path: Path, kind: str):
|
||||||
|
from agent_v0.agent_v1.vision import capture_io
|
||||||
|
|
||||||
|
img = Image.new("RGB", (640, 480), (120, 130, 140))
|
||||||
|
out = capture_io.save_capture(img, str(tmp_path / f"shot_{kind}"), kind)
|
||||||
|
|
||||||
|
assert out.endswith(".jpg"), f"{kind} doit être JPEG, got {out!r}"
|
||||||
|
assert Path(out).exists()
|
||||||
|
with Image.open(out) as reopened:
|
||||||
|
assert reopened.format == "JPEG"
|
||||||
|
|
||||||
|
|
||||||
|
def test_save_capture_heartbeat_is_downscaled_jpeg(tmp_path: Path):
|
||||||
|
from agent_v0.agent_v1.vision import capture_io
|
||||||
|
|
||||||
|
# Image large (2560) → doit être réduite à HEARTBEAT_MAX_WIDTH.
|
||||||
|
img = Image.new("RGB", (2560, 1440), (50, 60, 70))
|
||||||
|
out = capture_io.save_capture(img, str(tmp_path / "hb"), "heartbeat")
|
||||||
|
|
||||||
|
assert out.endswith(".jpg")
|
||||||
|
with Image.open(out) as reopened:
|
||||||
|
assert reopened.format == "JPEG"
|
||||||
|
assert reopened.width == capture_io.HEARTBEAT_MAX_WIDTH, (
|
||||||
|
f"heartbeat doit être downscalé à {capture_io.HEARTBEAT_MAX_WIDTH}, "
|
||||||
|
f"got {reopened.width}"
|
||||||
|
)
|
||||||
|
# ratio préservé (1440 * 1280/2560 = 720)
|
||||||
|
assert reopened.height == 720
|
||||||
|
|
||||||
|
|
||||||
|
def test_save_capture_heartbeat_smaller_than_max_is_not_upscaled(tmp_path: Path):
|
||||||
|
from agent_v0.agent_v1.vision import capture_io
|
||||||
|
|
||||||
|
img = Image.new("RGB", (640, 360), (1, 2, 3))
|
||||||
|
out = capture_io.save_capture(img, str(tmp_path / "hb_small"), "heartbeat")
|
||||||
|
with Image.open(out) as reopened:
|
||||||
|
assert reopened.width == 640, "no-op si déjà plus petit que le max"
|
||||||
|
|
||||||
|
|
||||||
|
def test_save_capture_heartbeat_downscale_reduces_pixel_count(tmp_path: Path):
|
||||||
|
"""Preuve de l'allègement heartbeat par la mesure objective du code :
|
||||||
|
le downscale réduit le nombre de pixels (2560×1440 → 1280×720 = /4 surface).
|
||||||
|
On mesure la géométrie de sortie (déterministe), pas le poids d'un JPEG
|
||||||
|
synthétique (qui dépend de libjpeg et n'est pas représentatif d'un vrai
|
||||||
|
écran)."""
|
||||||
|
from agent_v0.agent_v1.vision import capture_io
|
||||||
|
|
||||||
|
src = Image.new("RGB", (2560, 1440))
|
||||||
|
out = capture_io.save_capture(src, str(tmp_path / "hb_measure"), "heartbeat")
|
||||||
|
with Image.open(out) as small:
|
||||||
|
src_pixels = src.width * src.height
|
||||||
|
out_pixels = small.width * small.height
|
||||||
|
assert out_pixels < src_pixels / 3, (
|
||||||
|
f"Le downscale heartbeat doit diviser la surface par ~4 "
|
||||||
|
f"({src_pixels} → {out_pixels})"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_save_capture_rejects_unknown_kind(tmp_path: Path):
|
||||||
|
from agent_v0.agent_v1.vision import capture_io
|
||||||
|
|
||||||
|
img = Image.new("RGB", (10, 10))
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
capture_io.save_capture(img, str(tmp_path / "x"), "bogus")
|
||||||
|
|
||||||
|
|
||||||
|
# ===========================================================================
|
||||||
|
# PARTIE B — Câblage dans capturer.py (format des sorties runtime)
|
||||||
|
# ===========================================================================
|
||||||
|
|
||||||
|
|
||||||
|
def test_capture_full_context_writes_jpeg(tmp_path: Path):
|
||||||
|
"""capture_full_context (context / focus_change / result_of_*) → JPEG."""
|
||||||
|
p1, p2, p3 = _patch_mss()
|
||||||
|
with p1, p2, p3:
|
||||||
|
cap = _vision_capturer(tmp_path)
|
||||||
|
out = cap.capture_full_context("focus_change", force=True)
|
||||||
|
|
||||||
|
assert out, "capture attendue"
|
||||||
|
assert out.endswith(".jpg"), f"context doit être JPEG, got {out!r}"
|
||||||
|
assert Path(out).exists()
|
||||||
|
with Image.open(out) as im:
|
||||||
|
assert im.format == "JPEG"
|
||||||
|
|
||||||
|
|
||||||
|
def test_capture_full_context_heartbeat_is_jpeg(tmp_path: Path):
|
||||||
|
"""Un suffixe 'heartbeat' doit produire un JPEG (downscalé côté politique)."""
|
||||||
|
p1, p2, p3 = _patch_mss()
|
||||||
|
with p1, p2, p3:
|
||||||
|
cap = _vision_capturer(tmp_path)
|
||||||
|
out = cap.capture_full_context("heartbeat", force=True)
|
||||||
|
|
||||||
|
assert out.endswith(".jpg"), f"heartbeat doit être JPEG, got {out!r}"
|
||||||
|
with Image.open(out) as im:
|
||||||
|
assert im.format == "JPEG"
|
||||||
|
|
||||||
|
|
||||||
|
def test_capture_dual_full_is_jpeg_crop_is_png(tmp_path: Path):
|
||||||
|
"""capture_dual : full/window en JPEG, crop en PNG (contrat serveur)."""
|
||||||
|
p1, p2, p3 = _patch_mss()
|
||||||
|
with p1, p2, p3, patch(
|
||||||
|
# Neutraliser la capture fenêtre (dépend d'API OS) pour isoler full+crop
|
||||||
|
"agent_v0.agent_v1.vision.capturer.VisionCapturer.capture_active_window",
|
||||||
|
return_value=None,
|
||||||
|
):
|
||||||
|
cap = _vision_capturer(tmp_path)
|
||||||
|
result = cap.capture_dual(x=100, y=200, screenshot_id="shot42")
|
||||||
|
|
||||||
|
assert "full" in result and "crop" in result
|
||||||
|
assert result["full"].endswith(".jpg"), f"full doit être JPEG, got {result['full']!r}"
|
||||||
|
assert result["crop"].endswith(".png"), f"crop doit rester PNG, got {result['crop']!r}"
|
||||||
|
assert Path(result["full"]).exists()
|
||||||
|
assert Path(result["crop"]).exists()
|
||||||
|
with Image.open(result["full"]) as im:
|
||||||
|
assert im.format == "JPEG"
|
||||||
|
with Image.open(result["crop"]) as im:
|
||||||
|
assert im.format == "PNG"
|
||||||
|
|
||||||
|
|
||||||
|
def test_capture_active_window_writes_jpeg(tmp_path: Path):
|
||||||
|
"""La fenêtre active est une vue contextuelle → JPEG."""
|
||||||
|
p1, p2, p3 = _patch_mss()
|
||||||
|
fake_rect = {
|
||||||
|
"rect": [100, 100, 500, 400],
|
||||||
|
"size": [400, 300],
|
||||||
|
"title": "Bloc-notes",
|
||||||
|
"app_name": "notepad.exe",
|
||||||
|
}
|
||||||
|
full_img = Image.new("RGB", (800, 600), (90, 90, 90))
|
||||||
|
with p1, p2, p3, patch(
|
||||||
|
"agent_v0.agent_v1.window_info_crossplatform.get_active_window_rect",
|
||||||
|
return_value=fake_rect,
|
||||||
|
):
|
||||||
|
cap = _vision_capturer(tmp_path)
|
||||||
|
result = cap.capture_active_window(
|
||||||
|
x=200, y=200, screenshot_id="shotW", full_img=full_img
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result is not None
|
||||||
|
assert result["window_image"].endswith(".jpg"), (
|
||||||
|
f"window doit être JPEG, got {result['window_image']!r}"
|
||||||
|
)
|
||||||
|
with Image.open(result["window_image"]) as im:
|
||||||
|
assert im.format == "JPEG"
|
||||||
|
|
||||||
|
|
||||||
|
# ===========================================================================
|
||||||
|
# PARTIE C — BUG chemin : shots/ recréé si supprimé par l'auto-cleanup
|
||||||
|
# ===========================================================================
|
||||||
|
|
||||||
|
|
||||||
|
def test_capture_full_context_recreates_shots_dir_after_rmtree(tmp_path: Path):
|
||||||
|
"""Reproduction du bug poste Émilie.
|
||||||
|
|
||||||
|
L'auto-cleanup (`SessionStorage.shutil.rmtree`) supprime tout le dossier
|
||||||
|
de session `_background` (donc `shots/`). Une capture ultérieure ne doit
|
||||||
|
PAS lever `[Errno 2] No such file or directory` : le répertoire cible
|
||||||
|
doit être recréé avant l'écriture.
|
||||||
|
"""
|
||||||
|
p1, p2, p3 = _patch_mss()
|
||||||
|
with p1, p2, p3:
|
||||||
|
cap = _vision_capturer(tmp_path)
|
||||||
|
|
||||||
|
# Simule l'auto-cleanup : la session entière est purgée après ACK.
|
||||||
|
shutil.rmtree(cap.shots_dir)
|
||||||
|
assert not Path(cap.shots_dir).exists()
|
||||||
|
|
||||||
|
out = cap.capture_full_context("context_after_purge", force=True)
|
||||||
|
|
||||||
|
assert out, "La capture doit réussir même après purge du dossier shots"
|
||||||
|
assert Path(out).exists(), "Le fichier doit être physiquement écrit"
|
||||||
|
assert Path(cap.shots_dir).exists(), "shots/ doit avoir été recréé"
|
||||||
|
|
||||||
|
|
||||||
|
def test_capture_dual_recreates_shots_dir_after_rmtree(tmp_path: Path):
|
||||||
|
"""capture_dual doit aussi survivre à la purge du dossier shots."""
|
||||||
|
p1, p2, p3 = _patch_mss()
|
||||||
|
with p1, p2, p3, patch(
|
||||||
|
"agent_v0.agent_v1.vision.capturer.VisionCapturer.capture_active_window",
|
||||||
|
return_value=None,
|
||||||
|
):
|
||||||
|
cap = _vision_capturer(tmp_path)
|
||||||
|
shutil.rmtree(cap.shots_dir)
|
||||||
|
|
||||||
|
result = cap.capture_dual(x=50, y=60, screenshot_id="shot_purge")
|
||||||
|
|
||||||
|
assert result.get("full") and result.get("crop"), (
|
||||||
|
"capture_dual doit produire full+crop même après purge"
|
||||||
|
)
|
||||||
|
assert Path(result["full"]).exists()
|
||||||
|
assert Path(result["crop"]).exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_capture_active_window_recreates_shots_dir_after_rmtree(tmp_path: Path):
|
||||||
|
"""capture_active_window (crop fenêtre depuis full fourni) survit à la purge."""
|
||||||
|
p1, p2, p3 = _patch_mss()
|
||||||
|
fake_rect = {
|
||||||
|
"rect": [10, 10, 210, 210],
|
||||||
|
"size": [200, 200],
|
||||||
|
"title": "W",
|
||||||
|
"app_name": "w.exe",
|
||||||
|
}
|
||||||
|
full_img = Image.new("RGB", (400, 400), (70, 70, 70))
|
||||||
|
with p1, p2, p3, patch(
|
||||||
|
"agent_v0.agent_v1.window_info_crossplatform.get_active_window_rect",
|
||||||
|
return_value=fake_rect,
|
||||||
|
):
|
||||||
|
cap = _vision_capturer(tmp_path)
|
||||||
|
shutil.rmtree(cap.shots_dir)
|
||||||
|
|
||||||
|
result = cap.capture_active_window(
|
||||||
|
x=50, y=50, screenshot_id="shotW_purge", full_img=full_img
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result is not None, "capture fenêtre doit réussir après purge"
|
||||||
|
assert Path(result["window_image"]).exists()
|
||||||
406
tests/unit/test_grounding.py
Normal file
406
tests/unit/test_grounding.py
Normal file
@@ -0,0 +1,406 @@
|
|||||||
|
"""Tests for core/navigation/grounding.py — OCR-anchored grounding + VLM fallback + coords cache."""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import pytest
|
||||||
|
from core.navigation.grounding import (
|
||||||
|
OcrTokenInfo,
|
||||||
|
GroundedElement,
|
||||||
|
CoordsCacheEntry,
|
||||||
|
CoordsCache,
|
||||||
|
bbox_center,
|
||||||
|
make_element_key,
|
||||||
|
ocr_anchor_ground,
|
||||||
|
build_grounder_prompt,
|
||||||
|
parse_grounder_response,
|
||||||
|
ground_element,
|
||||||
|
)
|
||||||
|
from core.navigation.visual_verifier import normalize_text
|
||||||
|
|
||||||
|
|
||||||
|
# ── Mock factories ─────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def mock_ocr_detailed_client_factory(tokens: list):
|
||||||
|
"""Factory for mock OcrDetailedClient returning List[OcrTokenInfo]."""
|
||||||
|
def client(image_path: str) -> list:
|
||||||
|
return tokens
|
||||||
|
return client
|
||||||
|
|
||||||
|
|
||||||
|
def mock_vlm_client_factory(response_json: dict):
|
||||||
|
"""Factory for mock VlmClient returning given JSON."""
|
||||||
|
def client(image_path: str, prompt: str) -> str:
|
||||||
|
return json.dumps(response_json)
|
||||||
|
return client
|
||||||
|
|
||||||
|
|
||||||
|
# ── bbox_center tests ──────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestBboxCenter:
|
||||||
|
def test_basic(self):
|
||||||
|
assert bbox_center((100, 200, 300, 400)) == (200, 300)
|
||||||
|
|
||||||
|
def test_zero_origin(self):
|
||||||
|
assert bbox_center((0, 0, 100, 100)) == (50, 50)
|
||||||
|
|
||||||
|
def test_symmetric(self):
|
||||||
|
assert bbox_center((10, 10, 20, 20)) == (15, 15)
|
||||||
|
|
||||||
|
|
||||||
|
# ── make_element_key tests ─────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestMakeElementKey:
|
||||||
|
def test_basic(self):
|
||||||
|
key = make_element_key("bouton", "Rechercher")
|
||||||
|
assert key == "bouton:rechercher"
|
||||||
|
|
||||||
|
def test_normalized(self):
|
||||||
|
key = make_element_key("champ", "Nom Prénom")
|
||||||
|
assert "nom" in key and "prenom" in key
|
||||||
|
|
||||||
|
def test_consistent(self):
|
||||||
|
# Same element always produces same key
|
||||||
|
assert make_element_key("bouton", "Connexion") == make_element_key("bouton", "CONNEXION")
|
||||||
|
|
||||||
|
|
||||||
|
# ── ocr_anchor_ground tests ────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestOcrAnchorGround:
|
||||||
|
def test_exact_match(self):
|
||||||
|
tokens = [OcrTokenInfo(text="Rechercher", bbox=(100, 50, 250, 90), confidence=0.95)]
|
||||||
|
result = ocr_anchor_ground(tokens, {"role": "bouton", "text": "Rechercher"})
|
||||||
|
assert result is not None
|
||||||
|
assert result.method == "ocr_anchor"
|
||||||
|
assert result.bbox == (100, 50, 250, 90)
|
||||||
|
assert result.center == (175, 70)
|
||||||
|
assert result.confidence == 0.95
|
||||||
|
|
||||||
|
def test_fuzzy_match(self):
|
||||||
|
tokens = [OcrTokenInfo(text="Rechércher", bbox=(100, 50, 250, 90))]
|
||||||
|
result = ocr_anchor_ground(tokens, {"role": "bouton", "text": "Rechercher"})
|
||||||
|
assert result is not None
|
||||||
|
assert result.source_ocr_text == "Rechércher"
|
||||||
|
|
||||||
|
def test_no_match(self):
|
||||||
|
tokens = [OcrTokenInfo(text="Accueil", bbox=(100, 50, 250, 90))]
|
||||||
|
result = ocr_anchor_ground(tokens, {"role": "bouton", "text": "Rechercher"})
|
||||||
|
assert result is None
|
||||||
|
|
||||||
|
def test_token_without_bbox(self):
|
||||||
|
tokens = [OcrTokenInfo(text="Rechercher", bbox=None)]
|
||||||
|
result = ocr_anchor_ground(tokens, {"role": "bouton", "text": "Rechercher"})
|
||||||
|
assert result is None # found text but no bbox → can't ground
|
||||||
|
|
||||||
|
def test_no_text_target(self):
|
||||||
|
tokens = [OcrTokenInfo(text="Dashboard", bbox=(0, 0, 1920, 1080))]
|
||||||
|
result = ocr_anchor_ground(tokens, {"role": "page"}) # no text key
|
||||||
|
assert result is None # no text to match
|
||||||
|
|
||||||
|
def test_multiple_tokens_first_match(self):
|
||||||
|
tokens = [
|
||||||
|
OcrTokenInfo(text="Accueil", bbox=(0, 0, 100, 40)),
|
||||||
|
OcrTokenInfo(text="Connexion", bbox=(200, 50, 350, 90)),
|
||||||
|
]
|
||||||
|
result = ocr_anchor_ground(tokens, {"role": "bouton", "text": "Connexion"})
|
||||||
|
assert result is not None
|
||||||
|
assert result.bbox == (200, 50, 350, 90)
|
||||||
|
|
||||||
|
|
||||||
|
# ── build_grounder_prompt tests ────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestBuildGrounderPrompt:
|
||||||
|
def test_basic_prompt(self):
|
||||||
|
prompt = build_grounder_prompt({"role": "bouton", "text": "Connexion"})
|
||||||
|
assert "bouton" in prompt
|
||||||
|
assert "Connexion" in prompt
|
||||||
|
assert "bbox" in prompt
|
||||||
|
|
||||||
|
def test_with_context(self):
|
||||||
|
prompt = build_grounder_prompt(
|
||||||
|
{"role": "champ", "text": "Login"},
|
||||||
|
context="page login DPI",
|
||||||
|
)
|
||||||
|
assert "page login DPI" in prompt
|
||||||
|
|
||||||
|
def test_with_extra(self):
|
||||||
|
prompt = build_grounder_prompt(
|
||||||
|
{"role": "champ", "text": "IPP", "extra": "colonne gauche"},
|
||||||
|
)
|
||||||
|
assert "colonne gauche" in prompt
|
||||||
|
|
||||||
|
|
||||||
|
# ── parse_grounder_response tests ──────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestParseGrounderResponse:
|
||||||
|
def test_valid_response(self):
|
||||||
|
vlm_text = json.dumps({
|
||||||
|
"found": True,
|
||||||
|
"bbox": [0.1, 0.2, 0.3, 0.4],
|
||||||
|
"confidence": 0.92,
|
||||||
|
"description": "login button",
|
||||||
|
})
|
||||||
|
result = parse_grounder_response(vlm_text, 1920, 1080, {"role": "bouton", "text": "Connexion"})
|
||||||
|
assert result is not None
|
||||||
|
assert result.method == "vlm_grounder"
|
||||||
|
assert result.bbox == (192, 216, 576, 432) # 0.1*1920, 0.2*1080, 0.3*1920, 0.4*1080
|
||||||
|
assert result.confidence == 0.92
|
||||||
|
|
||||||
|
def test_not_found(self):
|
||||||
|
vlm_text = json.dumps({"found": False, "bbox": [], "confidence": 0.0})
|
||||||
|
result = parse_grounder_response(vlm_text, 1920, 1080, {"role": "bouton", "text": "Connexion"})
|
||||||
|
assert result is None
|
||||||
|
|
||||||
|
def test_json_in_markdown(self):
|
||||||
|
vlm_text = "```json\n{\"found\": true, \"bbox\": [0.5, 0.5, 0.6, 0.6], \"confidence\": 0.8}\n```"
|
||||||
|
result = parse_grounder_response(vlm_text, 1920, 1080, {"role": "bouton", "text": "Connexion"})
|
||||||
|
assert result is not None
|
||||||
|
|
||||||
|
def test_garbled_response(self):
|
||||||
|
result = parse_grounder_response("I cannot find the element", 1920, 1080, {"role": "bouton", "text": "Connexion"})
|
||||||
|
assert result is None
|
||||||
|
|
||||||
|
def test_invalid_bbox_format(self):
|
||||||
|
vlm_text = json.dumps({"found": True, "bbox": [0.1, 0.2], "confidence": 0.8})
|
||||||
|
result = parse_grounder_response(vlm_text, 1920, 1080, {"role": "bouton", "text": "Connexion"})
|
||||||
|
assert result is None # bbox must have 4 values
|
||||||
|
|
||||||
|
def test_confidence_as_string(self):
|
||||||
|
vlm_text = json.dumps({"found": True, "bbox": [0.1, 0.2, 0.3, 0.4], "confidence": "0.85"})
|
||||||
|
result = parse_grounder_response(vlm_text, 1920, 1080, {"role": "bouton", "text": "Connexion"})
|
||||||
|
assert result is not None
|
||||||
|
assert result.confidence == 0.85
|
||||||
|
|
||||||
|
def test_bbox_clamped_to_screen(self):
|
||||||
|
vlm_text = json.dumps({"found": True, "bbox": [-0.1, -0.1, 1.5, 1.5], "confidence": 0.7})
|
||||||
|
result = parse_grounder_response(vlm_text, 1920, 1080, {"role": "bouton", "text": "Connexion"})
|
||||||
|
assert result is not None
|
||||||
|
assert result.bbox[0] >= 0
|
||||||
|
assert result.bbox[1] >= 0
|
||||||
|
assert result.bbox[2] <= 1920
|
||||||
|
assert result.bbox[3] <= 1080
|
||||||
|
|
||||||
|
|
||||||
|
# ── ground_element (composition) tests ─────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestGroundElement:
|
||||||
|
def test_ocr_anchor_success(self):
|
||||||
|
"""OCR finds text with bbox → grounded via OCR (deterministic)."""
|
||||||
|
ocr = mock_ocr_detailed_client_factory([
|
||||||
|
OcrTokenInfo(text="Connexion", bbox=(200, 50, 350, 90), confidence=0.95),
|
||||||
|
])
|
||||||
|
vlm = mock_vlm_client_factory({})
|
||||||
|
result = ground_element(
|
||||||
|
"/tmp/login.png",
|
||||||
|
{"role": "bouton", "text": "Connexion"},
|
||||||
|
ocr_client=ocr,
|
||||||
|
vlm_client=vlm,
|
||||||
|
)
|
||||||
|
assert result is not None
|
||||||
|
assert result.method == "ocr_anchor"
|
||||||
|
assert result.bbox == (200, 50, 350, 90)
|
||||||
|
|
||||||
|
def test_vlm_fallback(self):
|
||||||
|
"""OCR doesn't find text → VLM grounder succeeds."""
|
||||||
|
ocr = mock_ocr_detailed_client_factory([
|
||||||
|
OcrTokenInfo(text="Accueil", bbox=(0, 0, 100, 40)),
|
||||||
|
])
|
||||||
|
vlm = mock_vlm_client_factory({
|
||||||
|
"found": True,
|
||||||
|
"bbox": [0.2, 0.3, 0.4, 0.5],
|
||||||
|
"confidence": 0.85,
|
||||||
|
})
|
||||||
|
result = ground_element(
|
||||||
|
"/tmp/login.png",
|
||||||
|
{"role": "bouton", "text": "Connexion"},
|
||||||
|
ocr_client=ocr,
|
||||||
|
vlm_client=vlm,
|
||||||
|
)
|
||||||
|
assert result is not None
|
||||||
|
assert result.method == "vlm_grounder"
|
||||||
|
|
||||||
|
def test_not_found_any_method(self):
|
||||||
|
"""Both OCR and VLM fail → None."""
|
||||||
|
ocr = mock_ocr_detailed_client_factory([OcrTokenInfo(text="Accueil", bbox=(0, 0, 100, 40))])
|
||||||
|
vlm = mock_vlm_client_factory({"found": False, "bbox": [], "confidence": 0.0})
|
||||||
|
result = ground_element(
|
||||||
|
"/tmp/login.png",
|
||||||
|
{"role": "bouton", "text": "Connexion"},
|
||||||
|
ocr_client=ocr,
|
||||||
|
vlm_client=vlm,
|
||||||
|
)
|
||||||
|
assert result is None
|
||||||
|
|
||||||
|
def test_ocr_error_vlm_fallback(self):
|
||||||
|
"""OCR engine fails → VLM fallback."""
|
||||||
|
def failing_ocr(image_path):
|
||||||
|
raise RuntimeError("OCR engine down")
|
||||||
|
vlm = mock_vlm_client_factory({
|
||||||
|
"found": True,
|
||||||
|
"bbox": [0.2, 0.3, 0.4, 0.5],
|
||||||
|
"confidence": 0.8,
|
||||||
|
})
|
||||||
|
result = ground_element(
|
||||||
|
"/tmp/login.png",
|
||||||
|
{"role": "bouton", "text": "Connexion"},
|
||||||
|
ocr_client=failing_ocr,
|
||||||
|
vlm_client=vlm,
|
||||||
|
)
|
||||||
|
assert result is not None
|
||||||
|
assert result.method == "vlm_grounder"
|
||||||
|
|
||||||
|
def test_vlm_error_ocr_success(self):
|
||||||
|
"""VLM fails but OCR succeeds → OCR anchor used."""
|
||||||
|
ocr = mock_ocr_detailed_client_factory([
|
||||||
|
OcrTokenInfo(text="Connexion", bbox=(200, 50, 350, 90)),
|
||||||
|
])
|
||||||
|
def failing_vlm(image_path, prompt):
|
||||||
|
raise RuntimeError("VLM down")
|
||||||
|
result = ground_element(
|
||||||
|
"/tmp/login.png",
|
||||||
|
{"role": "bouton", "text": "Connexion"},
|
||||||
|
ocr_client=ocr,
|
||||||
|
vlm_client=failing_vlm,
|
||||||
|
)
|
||||||
|
assert result is not None
|
||||||
|
assert result.method == "ocr_anchor"
|
||||||
|
|
||||||
|
def test_both_fail(self):
|
||||||
|
"""OCR + VLM both fail → None."""
|
||||||
|
def failing_ocr(image_path):
|
||||||
|
raise RuntimeError("OCR down")
|
||||||
|
def failing_vlm(image_path, prompt):
|
||||||
|
raise RuntimeError("VLM down")
|
||||||
|
result = ground_element(
|
||||||
|
"/tmp/login.png",
|
||||||
|
{"role": "bouton", "text": "Connexion"},
|
||||||
|
ocr_client=failing_ocr,
|
||||||
|
vlm_client=failing_vlm,
|
||||||
|
)
|
||||||
|
assert result is None
|
||||||
|
|
||||||
|
def test_no_text_target(self):
|
||||||
|
"""Target without text → VLM grounder skipped, None."""
|
||||||
|
ocr = mock_ocr_detailed_client_factory([])
|
||||||
|
vlm = mock_vlm_client_factory({})
|
||||||
|
result = ground_element(
|
||||||
|
"/tmp/page.png",
|
||||||
|
{"role": "page"},
|
||||||
|
ocr_client=ocr,
|
||||||
|
vlm_client=vlm,
|
||||||
|
)
|
||||||
|
assert result is None
|
||||||
|
|
||||||
|
def test_cache_hit(self):
|
||||||
|
"""Cached coords exist → returned directly."""
|
||||||
|
cache = CoordsCache()
|
||||||
|
cache.put("bouton:connexion", (200, 50, 350, 90), (275, 70), "ocr_anchor")
|
||||||
|
|
||||||
|
ocr = mock_ocr_detailed_client_factory([])
|
||||||
|
vlm = mock_vlm_client_factory({})
|
||||||
|
result = ground_element(
|
||||||
|
"/tmp/login.png",
|
||||||
|
{"role": "bouton", "text": "Connexion"},
|
||||||
|
ocr_client=ocr,
|
||||||
|
vlm_client=vlm,
|
||||||
|
coords_cache=cache,
|
||||||
|
)
|
||||||
|
assert result is not None
|
||||||
|
assert result.method == "cache"
|
||||||
|
assert result.bbox == (200, 50, 350, 90)
|
||||||
|
|
||||||
|
def test_cache_stored_on_ocr_anchor(self):
|
||||||
|
"""OCR anchor result → stored in cache."""
|
||||||
|
cache = CoordsCache()
|
||||||
|
ocr = mock_ocr_detailed_client_factory([
|
||||||
|
OcrTokenInfo(text="Connexion", bbox=(200, 50, 350, 90)),
|
||||||
|
])
|
||||||
|
vlm = mock_vlm_client_factory({})
|
||||||
|
ground_element(
|
||||||
|
"/tmp/login.png",
|
||||||
|
{"role": "bouton", "text": "Connexion"},
|
||||||
|
ocr_client=ocr,
|
||||||
|
vlm_client=vlm,
|
||||||
|
coords_cache=cache,
|
||||||
|
)
|
||||||
|
cached = cache.get("bouton:connexion")
|
||||||
|
assert cached is not None
|
||||||
|
assert cached.bbox == (200, 50, 350, 90)
|
||||||
|
assert cached.method == "ocr_anchor"
|
||||||
|
|
||||||
|
def test_cache_stored_on_vlm_grounder(self):
|
||||||
|
"""VLM grounder result → stored in cache."""
|
||||||
|
cache = CoordsCache()
|
||||||
|
ocr = mock_ocr_detailed_client_factory([])
|
||||||
|
vlm = mock_vlm_client_factory({
|
||||||
|
"found": True,
|
||||||
|
"bbox": [0.2, 0.3, 0.4, 0.5],
|
||||||
|
"confidence": 0.85,
|
||||||
|
})
|
||||||
|
ground_element(
|
||||||
|
"/tmp/login.png",
|
||||||
|
{"role": "bouton", "text": "Connexion"},
|
||||||
|
ocr_client=ocr,
|
||||||
|
vlm_client=vlm,
|
||||||
|
coords_cache=cache,
|
||||||
|
)
|
||||||
|
cached = cache.get("bouton:connexion")
|
||||||
|
assert cached is not None
|
||||||
|
assert cached.method == "vlm_grounder"
|
||||||
|
|
||||||
|
|
||||||
|
# ── CoordsCache tests ──────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestCoordsCache:
|
||||||
|
def test_put_and_get(self):
|
||||||
|
cache = CoordsCache()
|
||||||
|
cache.put("bouton:connexion", (200, 50, 350, 90), (275, 70), "ocr_anchor")
|
||||||
|
entry = cache.get("bouton:connexion")
|
||||||
|
assert entry is not None
|
||||||
|
assert entry.bbox == (200, 50, 350, 90)
|
||||||
|
|
||||||
|
def test_get_missing(self):
|
||||||
|
cache = CoordsCache()
|
||||||
|
assert cache.get("bouton:connexion") is None
|
||||||
|
|
||||||
|
def test_invalidate(self):
|
||||||
|
cache = CoordsCache()
|
||||||
|
cache.put("bouton:connexion", (200, 50, 350, 90), (275, 70), "ocr_anchor")
|
||||||
|
cache.invalidate("bouton:connexion")
|
||||||
|
assert cache.get("bouton:connexion") is None
|
||||||
|
|
||||||
|
def test_clear(self):
|
||||||
|
cache = CoordsCache()
|
||||||
|
cache.put("a", (0, 0, 10, 10), (5, 5), "ocr_anchor")
|
||||||
|
cache.put("b", (0, 0, 20, 20), (10, 10), "vlm_grounder")
|
||||||
|
cache.clear()
|
||||||
|
assert cache.get("a") is None
|
||||||
|
assert cache.get("b") is None
|
||||||
|
|
||||||
|
def test_keys(self):
|
||||||
|
cache = CoordsCache()
|
||||||
|
cache.put("a", (0, 0, 10, 10), (5, 5), "ocr_anchor")
|
||||||
|
cache.put("b", (0, 0, 20, 20), (10, 10), "vlm_grounder")
|
||||||
|
assert sorted(cache.keys()) == ["a", "b"]
|
||||||
|
|
||||||
|
def test_update_existing(self):
|
||||||
|
cache = CoordsCache()
|
||||||
|
cache.put("bouton:connexion", (200, 50, 350, 90), (275, 70), "ocr_anchor")
|
||||||
|
cache.put("bouton:connexion", (300, 60, 400, 100), (350, 80), "vlm_grounder")
|
||||||
|
entry = cache.get("bouton:connexion")
|
||||||
|
assert entry is not None
|
||||||
|
assert entry.bbox == (300, 60, 400, 100) # updated
|
||||||
|
assert entry.validation_count == 2
|
||||||
|
|
||||||
|
def test_validation_count_increments(self):
|
||||||
|
cache = CoordsCache()
|
||||||
|
cache.put("a", (0, 0, 10, 10), (5, 5), "ocr_anchor")
|
||||||
|
assert cache.get("a").validation_count == 1
|
||||||
|
cache.put("a", (0, 0, 10, 10), (5, 5), "ocr_anchor")
|
||||||
|
assert cache.get("a").validation_count == 2
|
||||||
151
tests/unit/test_navigate_handler_e2e.py
Normal file
151
tests/unit/test_navigate_handler_e2e.py
Normal file
@@ -0,0 +1,151 @@
|
|||||||
|
"""End-to-end mocked test for navigate action handler — 3 edge-case scenarios.
|
||||||
|
|
||||||
|
Tests the _handle_navigate_action handler with mocked OCR/VLM, verifying:
|
||||||
|
- Nominal: all resolved, coords populated in variables
|
||||||
|
- OCR miss + VLM fail: no phantom coords, all_resolved=False
|
||||||
|
- No screenshot: error="no_screenshot", False return
|
||||||
|
|
||||||
|
NOTE: The handler uses lazy imports inside its body. Mock targets must be
|
||||||
|
at the source module (core.navigation.action_resolver.navigate_login) rather
|
||||||
|
than the package-level re-export (core.navigation.navigate_login).
|
||||||
|
"""
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from unittest.mock import patch, MagicMock
|
||||||
|
from core.navigation.action_resolver import NavigateCoords, NavigateResult
|
||||||
|
from core.navigation import _handle_navigate_action
|
||||||
|
|
||||||
|
|
||||||
|
def _patch_all_deps(navigate_login_result=None, navigate_login_side_effect=None):
|
||||||
|
"""Return stacked patches for handler's lazy imports + navigate_login."""
|
||||||
|
nl_mock = MagicMock(return_value=navigate_login_result) if navigate_login_result else None
|
||||||
|
if navigate_login_side_effect:
|
||||||
|
nl_mock = MagicMock(side_effect=navigate_login_side_effect)
|
||||||
|
|
||||||
|
return (
|
||||||
|
patch("core.llm.extract_grid_from_image", return_value=[]),
|
||||||
|
patch("core.extraction.vlm_client.make_vllm_client", return_value=MagicMock()),
|
||||||
|
patch("core.navigation.action_resolver.make_ocr_detailed_from_grid",
|
||||||
|
return_value=MagicMock(return_value=[])),
|
||||||
|
patch("core.navigation.action_resolver.navigate_login", nl_mock),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class TestNominalCase:
|
||||||
|
"""All fields grounded → coords populated, all_resolved=True."""
|
||||||
|
|
||||||
|
def test_nominal_coords_populated(self):
|
||||||
|
mock_result = NavigateResult(
|
||||||
|
login_coords=NavigateCoords(x_pct=0.15, y_pct=0.07, method="ocr_anchor"),
|
||||||
|
password_coords=NavigateCoords(x_pct=0.15, y_pct=0.25, method="ocr_anchor"),
|
||||||
|
submit_coords=NavigateCoords(x_pct=0.50, y_pct=0.35, method="ocr_anchor"),
|
||||||
|
all_resolved=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
action = {"parameters": {"action": "login"}}
|
||||||
|
replay_state = {
|
||||||
|
"last_screenshot_path": "/tmp/login_screen.png",
|
||||||
|
"screen_width": 1920,
|
||||||
|
"screen_height": 1080,
|
||||||
|
}
|
||||||
|
|
||||||
|
p1, p2, p3, p4 = _patch_all_deps(navigate_login_result=mock_result)
|
||||||
|
with p1, p2, p3, p4:
|
||||||
|
result = _handle_navigate_action(action, replay_state, "test-session")
|
||||||
|
|
||||||
|
assert result is True
|
||||||
|
vars_ = replay_state["variables"]
|
||||||
|
assert "navigate_login_coords" in vars_
|
||||||
|
assert vars_["navigate_login_coords"]["x_pct"] == 0.15
|
||||||
|
assert "navigate_password_coords" in vars_
|
||||||
|
assert "navigate_submit_coords" in vars_
|
||||||
|
assert vars_["navigate_result"]["all_resolved"] is True
|
||||||
|
|
||||||
|
|
||||||
|
class TestOcrMissVlmFail:
|
||||||
|
"""OCR misses target + VLM grounder also fails → no phantom coords."""
|
||||||
|
|
||||||
|
def test_no_phantom_coords_on_failure(self):
|
||||||
|
mock_result = NavigateResult(
|
||||||
|
login_coords=None,
|
||||||
|
password_coords=None,
|
||||||
|
submit_coords=None,
|
||||||
|
all_resolved=False,
|
||||||
|
error="grounding failed — no login form elements found",
|
||||||
|
)
|
||||||
|
|
||||||
|
action = {"parameters": {"action": "login"}}
|
||||||
|
replay_state = {
|
||||||
|
"last_screenshot_path": "/tmp/no_login_form.png",
|
||||||
|
"screen_width": 1920,
|
||||||
|
"screen_height": 1080,
|
||||||
|
}
|
||||||
|
|
||||||
|
p1, p2, p3, p4 = _patch_all_deps(navigate_login_result=mock_result)
|
||||||
|
with p1, p2, p3, p4:
|
||||||
|
result = _handle_navigate_action(action, replay_state, "test-session")
|
||||||
|
|
||||||
|
assert result is False
|
||||||
|
vars_ = replay_state["variables"]
|
||||||
|
# No coords keys should be present (coords are None → not stored)
|
||||||
|
assert "navigate_login_coords" not in vars_
|
||||||
|
assert "navigate_password_coords" not in vars_
|
||||||
|
assert "navigate_submit_coords" not in vars_
|
||||||
|
# Error must be non-empty
|
||||||
|
assert vars_["navigate_result"]["all_resolved"] is False
|
||||||
|
assert "grounding failed" in vars_["navigate_result"]["error"]
|
||||||
|
|
||||||
|
|
||||||
|
class TestNoScreenshot:
|
||||||
|
"""No screenshot in replay_state → error="no_screenshot", False."""
|
||||||
|
|
||||||
|
def test_no_screenshot_error(self):
|
||||||
|
action = {"parameters": {"action": "login"}}
|
||||||
|
replay_state = {} # No screenshot at all
|
||||||
|
|
||||||
|
result = _handle_navigate_action(action, replay_state, "test-session")
|
||||||
|
|
||||||
|
assert result is False
|
||||||
|
vars_ = replay_state["variables"]
|
||||||
|
assert vars_["navigate_login_coords"]["error"] == "no_screenshot"
|
||||||
|
|
||||||
|
def test_empty_screenshot_path(self):
|
||||||
|
action = {"parameters": {"action": "login"}}
|
||||||
|
replay_state = {"last_screenshot_path": ""}
|
||||||
|
|
||||||
|
result = _handle_navigate_action(action, replay_state, "test-session")
|
||||||
|
|
||||||
|
assert result is False
|
||||||
|
vars_ = replay_state["variables"]
|
||||||
|
assert vars_["navigate_login_coords"]["error"] == "no_screenshot"
|
||||||
|
|
||||||
|
|
||||||
|
class TestNeverFailReplay:
|
||||||
|
"""Handler must never raise — even on malformed input, returns False."""
|
||||||
|
|
||||||
|
def test_missing_parameters(self):
|
||||||
|
action = {} # No "parameters" key
|
||||||
|
replay_state = {"last_screenshot_path": "/tmp/x.png"}
|
||||||
|
|
||||||
|
mock_result = NavigateResult(all_resolved=False, error="no params")
|
||||||
|
p1, p2, p3, p4 = _patch_all_deps(navigate_login_result=mock_result)
|
||||||
|
with p1, p2, p3, p4:
|
||||||
|
result = _handle_navigate_action(action, replay_state, "test-session")
|
||||||
|
assert result is False
|
||||||
|
|
||||||
|
def test_exception_in_inner_call(self):
|
||||||
|
action = {"parameters": {"action": "login"}}
|
||||||
|
replay_state = {
|
||||||
|
"last_screenshot_path": "/tmp/login.png",
|
||||||
|
"screen_width": 1920,
|
||||||
|
"screen_height": 1080,
|
||||||
|
}
|
||||||
|
|
||||||
|
p1, p2, p3, p4 = _patch_all_deps(navigate_login_side_effect=RuntimeError("boom"))
|
||||||
|
with p1, p2, p3, p4:
|
||||||
|
result = _handle_navigate_action(action, replay_state, "test-session")
|
||||||
|
|
||||||
|
assert result is False
|
||||||
|
vars_ = replay_state["variables"]
|
||||||
|
assert vars_["navigate_result"]["all_resolved"] is False
|
||||||
|
assert "boom" in vars_["navigate_result"]["error"]
|
||||||
62
tests/unit/test_navigate_wiring.py
Normal file
62
tests/unit/test_navigate_wiring.py
Normal file
@@ -0,0 +1,62 @@
|
|||||||
|
"""Boot non-regression test for navigate wiring — catches import/regression bugs.
|
||||||
|
|
||||||
|
This test would have caught the ImportError where _handle_navigate_action
|
||||||
|
was incorrectly imported from replay_engine instead of core/navigation.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
|
||||||
|
class TestApiStreamImports:
|
||||||
|
"""(1) api_stream must import without error."""
|
||||||
|
|
||||||
|
def test_import_api_stream(self):
|
||||||
|
from agent_v0.server_v1 import api_stream
|
||||||
|
assert api_stream is not None
|
||||||
|
|
||||||
|
|
||||||
|
class TestAllowedActionTypes:
|
||||||
|
"""(2) 'navigate' must be in both _ALLOWED and _SERVER_SIDE."""
|
||||||
|
|
||||||
|
def test_navigate_in_allowed(self):
|
||||||
|
from agent_v0.server_v1.replay_engine import _ALLOWED_ACTION_TYPES
|
||||||
|
assert "navigate" in _ALLOWED_ACTION_TYPES
|
||||||
|
|
||||||
|
def test_navigate_in_server_side(self):
|
||||||
|
from agent_v0.server_v1.replay_engine import _SERVER_SIDE_ACTION_TYPES
|
||||||
|
assert "navigate" in _SERVER_SIDE_ACTION_TYPES
|
||||||
|
|
||||||
|
|
||||||
|
class TestNavigateHandlerCallable:
|
||||||
|
"""(3) _handle_navigate_action must be callable with correct signature."""
|
||||||
|
|
||||||
|
def test_handler_imported_from_core_navigation(self):
|
||||||
|
from core.navigation import _handle_navigate_action
|
||||||
|
assert callable(_handle_navigate_action)
|
||||||
|
|
||||||
|
def test_handler_imported_in_api_stream(self):
|
||||||
|
from agent_v0.server_v1 import api_stream
|
||||||
|
handler = api_stream._handle_navigate_action
|
||||||
|
assert callable(handler)
|
||||||
|
|
||||||
|
def test_handler_signature(self):
|
||||||
|
"""Signature: (action: dict, replay_state: dict, session_id: str) -> bool."""
|
||||||
|
from core.navigation import _handle_navigate_action
|
||||||
|
import inspect
|
||||||
|
sig = inspect.signature(_handle_navigate_action)
|
||||||
|
params = list(sig.parameters.keys())
|
||||||
|
assert params == ["action", "replay_state", "session_id"]
|
||||||
|
assert sig.return_annotation == bool
|
||||||
|
|
||||||
|
|
||||||
|
class TestDispatchBlockExists:
|
||||||
|
"""Verify the navigate dispatch block is wired in api_stream."""
|
||||||
|
|
||||||
|
def test_navigate_dispatch_reference(self):
|
||||||
|
"""Source must contain the navigate dispatch elif block."""
|
||||||
|
import agent_v0.server_v1.api_stream as mod
|
||||||
|
source = inspect.getsource(mod)
|
||||||
|
assert "type_ == \"navigate\"" in source
|
||||||
|
|
||||||
|
|
||||||
|
import inspect
|
||||||
162
tests/unit/test_update_policy_canary.py
Normal file
162
tests/unit/test_update_policy_canary.py
Normal file
@@ -0,0 +1,162 @@
|
|||||||
|
"""TDD — DETTE-022 v2 : CANARY server-side pour la MAJ silencieuse Léa.
|
||||||
|
|
||||||
|
Périmètre testé ICI = logique PURE de la POLITIQUE de déploiement canary,
|
||||||
|
testable sans démarrer le serveur (DETTE-013 : on N'IMPORTE PAS `api_stream`
|
||||||
|
— on charge `update_policy.py` par chemin, comme test_update_check_server).
|
||||||
|
|
||||||
|
Objectif SÉCURITÉ (10+ postes cliniques live) : une MAJ ne doit JAMAIS
|
||||||
|
partir sur toute la flotte d'un coup. Le canary résout la version cible
|
||||||
|
*par machine* :
|
||||||
|
|
||||||
|
- un poste dans la liste canary reçoit la version `canary` (Émilie d'abord) ;
|
||||||
|
- tous les autres restent sur la version `stable` (floor) tant que le canary
|
||||||
|
n'est pas promu.
|
||||||
|
|
||||||
|
`resolve_target_version(machine_id, ...)` est la brique PURE ; `decide_update`
|
||||||
|
côté serveur l'appelle pour choisir la version cible avant de comparer.
|
||||||
|
|
||||||
|
Le NOYAU dangereux (swap fichiers / Lea.bat / restart) reste HORS périmètre.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import importlib.util
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
_MOD_PATH = (
|
||||||
|
Path(__file__).resolve().parents[2]
|
||||||
|
/ "agent_v0" / "server_v1" / "update_policy.py"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _load_module():
|
||||||
|
spec = importlib.util.spec_from_file_location("rpa_update_policy", _MOD_PATH)
|
||||||
|
mod = importlib.util.module_from_spec(spec)
|
||||||
|
spec.loader.exec_module(mod)
|
||||||
|
return mod
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def mod():
|
||||||
|
return _load_module()
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# parse_canary_machines — liste d'allow-list (CSV / espaces tolérés)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class TestParseCanaryMachines:
|
||||||
|
def test_liste_csv(self, mod):
|
||||||
|
assert mod.parse_canary_machines("lea-4zbgwxty") == {"lea-4zbgwxty"}
|
||||||
|
assert mod.parse_canary_machines("a,b,c") == {"a", "b", "c"}
|
||||||
|
|
||||||
|
def test_espaces_et_vides_toleres(self, mod):
|
||||||
|
assert mod.parse_canary_machines(" a , b , ") == {"a", "b"}
|
||||||
|
assert mod.parse_canary_machines("") == set()
|
||||||
|
assert mod.parse_canary_machines(None) == set()
|
||||||
|
|
||||||
|
def test_supporte_separateurs_espace_et_point_virgule(self, mod):
|
||||||
|
# Tolérant : virgule, point-virgule, espace comme séparateurs.
|
||||||
|
assert mod.parse_canary_machines("a; b c") == {"a", "b", "c"}
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# resolve_target_version — LE cœur canary (sécurité)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class TestResolveTargetVersion:
|
||||||
|
def test_machine_canary_recoit_version_canary(self, mod):
|
||||||
|
# Émilie (canary) reçoit la nouvelle version en premier.
|
||||||
|
target = mod.resolve_target_version(
|
||||||
|
machine_id="lea-4zbgwxty",
|
||||||
|
stable_version="1.0.1",
|
||||||
|
canary_version="1.0.2",
|
||||||
|
canary_machines={"lea-4zbgwxty"},
|
||||||
|
)
|
||||||
|
assert target == "1.0.2"
|
||||||
|
|
||||||
|
def test_machine_hors_canary_reste_sur_stable(self, mod):
|
||||||
|
# Tous les autres postes restent sur la version stable (floor).
|
||||||
|
target = mod.resolve_target_version(
|
||||||
|
machine_id="lea-autre-poste",
|
||||||
|
stable_version="1.0.1",
|
||||||
|
canary_version="1.0.2",
|
||||||
|
canary_machines={"lea-4zbgwxty"},
|
||||||
|
)
|
||||||
|
assert target == "1.0.1"
|
||||||
|
|
||||||
|
def test_pas_de_canary_configure_tout_le_monde_stable(self, mod):
|
||||||
|
# Aucun canary défini → personne ne monte (défaut ultra-prudent).
|
||||||
|
target = mod.resolve_target_version(
|
||||||
|
machine_id="lea-4zbgwxty",
|
||||||
|
stable_version="1.0.1",
|
||||||
|
canary_version="1.0.2",
|
||||||
|
canary_machines=set(),
|
||||||
|
)
|
||||||
|
assert target == "1.0.1"
|
||||||
|
|
||||||
|
def test_canary_version_absente_retombe_sur_stable(self, mod):
|
||||||
|
# Si canary_version n'est pas fournie, même un poste canary reste stable.
|
||||||
|
target = mod.resolve_target_version(
|
||||||
|
machine_id="lea-4zbgwxty",
|
||||||
|
stable_version="1.0.1",
|
||||||
|
canary_version=None,
|
||||||
|
canary_machines={"lea-4zbgwxty"},
|
||||||
|
)
|
||||||
|
assert target == "1.0.1"
|
||||||
|
|
||||||
|
def test_machine_id_none_reste_stable(self, mod):
|
||||||
|
# machine_id inconnu / non fourni → jamais canary (prudence).
|
||||||
|
target = mod.resolve_target_version(
|
||||||
|
machine_id=None,
|
||||||
|
stable_version="1.0.1",
|
||||||
|
canary_version="1.0.2",
|
||||||
|
canary_machines={"lea-4zbgwxty"},
|
||||||
|
)
|
||||||
|
assert target == "1.0.1"
|
||||||
|
|
||||||
|
def test_canary_ne_downgrade_jamais_en_dessous_de_stable(self, mod):
|
||||||
|
# GARDE-FOU : si le canary_version est PLUS ANCIEN que stable (erreur
|
||||||
|
# de config), on NE descend PAS le poste canary — on sert stable.
|
||||||
|
target = mod.resolve_target_version(
|
||||||
|
machine_id="lea-4zbgwxty",
|
||||||
|
stable_version="1.0.5",
|
||||||
|
canary_version="1.0.2", # plus ancien → config douteuse
|
||||||
|
canary_machines={"lea-4zbgwxty"},
|
||||||
|
)
|
||||||
|
assert target == "1.0.5"
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Lecture depuis l'environnement (pilotage sans rebuild) — défauts prudents
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class TestEnvPolicy:
|
||||||
|
def test_defauts_prudents_aucune_maj(self, mod, monkeypatch):
|
||||||
|
# Aucune var positionnée → stable par défaut, pas de canary.
|
||||||
|
for var in (
|
||||||
|
"RPA_AGENT_STABLE_VERSION",
|
||||||
|
"RPA_AGENT_CANARY_VERSION",
|
||||||
|
"RPA_AGENT_CANARY_MACHINES",
|
||||||
|
):
|
||||||
|
monkeypatch.delenv(var, raising=False)
|
||||||
|
assert mod.stable_version_from_env() == "1.0.1"
|
||||||
|
assert mod.canary_version_from_env() is None
|
||||||
|
assert mod.canary_machines_from_env() == set()
|
||||||
|
# Un poste quelconque reste sur stable.
|
||||||
|
assert mod.resolve_target_version_from_env("lea-4zbgwxty") == "1.0.1"
|
||||||
|
|
||||||
|
def test_canary_actif_via_env_seul_le_poste_canary_monte(self, mod, monkeypatch):
|
||||||
|
monkeypatch.setenv("RPA_AGENT_STABLE_VERSION", "1.0.1")
|
||||||
|
monkeypatch.setenv("RPA_AGENT_CANARY_VERSION", "1.0.2")
|
||||||
|
monkeypatch.setenv("RPA_AGENT_CANARY_MACHINES", "lea-4zbgwxty")
|
||||||
|
assert mod.resolve_target_version_from_env("lea-4zbgwxty") == "1.0.2"
|
||||||
|
assert mod.resolve_target_version_from_env("autre-poste") == "1.0.1"
|
||||||
|
|
||||||
|
def test_promotion_toute_la_flotte_suit(self, mod, monkeypatch):
|
||||||
|
# Promotion : on met stable = version canary, on vide la liste canary.
|
||||||
|
monkeypatch.setenv("RPA_AGENT_STABLE_VERSION", "1.0.2")
|
||||||
|
monkeypatch.delenv("RPA_AGENT_CANARY_VERSION", raising=False)
|
||||||
|
monkeypatch.delenv("RPA_AGENT_CANARY_MACHINES", raising=False)
|
||||||
|
assert mod.resolve_target_version_from_env("autre-poste") == "1.0.2"
|
||||||
|
assert mod.resolve_target_version_from_env("lea-4zbgwxty") == "1.0.2"
|
||||||
336
tests/unit/test_visual_login.py
Normal file
336
tests/unit/test_visual_login.py
Normal file
@@ -0,0 +1,336 @@
|
|||||||
|
"""Tests for core/navigation/visual_login.py — login form resolution + verification."""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import pytest
|
||||||
|
from core.navigation.visual_login import (
|
||||||
|
LoginFormConfig,
|
||||||
|
LoginResolution,
|
||||||
|
dpi_urgences_login_config,
|
||||||
|
verify_login_visible,
|
||||||
|
verify_login_success,
|
||||||
|
resolve_login_form,
|
||||||
|
_ocr_detailed_to_simple,
|
||||||
|
)
|
||||||
|
from core.navigation.grounding import (
|
||||||
|
CoordsCache,
|
||||||
|
GroundedElement,
|
||||||
|
OcrTokenInfo,
|
||||||
|
OcrDetailedClient,
|
||||||
|
)
|
||||||
|
from core.navigation.visual_verifier import (
|
||||||
|
ScreenMatchResult,
|
||||||
|
VlmClient,
|
||||||
|
OcrClient,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ── Mock factories ─────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def mock_ocr_detailed_client_factory(tokens: list):
|
||||||
|
"""Factory for mock OcrDetailedClient."""
|
||||||
|
def client(image_path: str) -> list:
|
||||||
|
return tokens
|
||||||
|
return client
|
||||||
|
|
||||||
|
|
||||||
|
def mock_ocr_simple_client_factory(tokens: list):
|
||||||
|
"""Factory for mock OcrClient (text-only)."""
|
||||||
|
def client(image_path: str) -> list:
|
||||||
|
return tokens
|
||||||
|
return client
|
||||||
|
|
||||||
|
|
||||||
|
def mock_vlm_client_factory(response_json: dict):
|
||||||
|
"""Factory for mock VlmClient."""
|
||||||
|
def client(image_path: str, prompt: str) -> str:
|
||||||
|
return json.dumps(response_json)
|
||||||
|
return client
|
||||||
|
|
||||||
|
|
||||||
|
# ── Default config tests ───────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestDpiUrgencesLoginConfig:
|
||||||
|
def test_default_config(self):
|
||||||
|
config = dpi_urgences_login_config()
|
||||||
|
assert config.login_field["role"] == "champ"
|
||||||
|
assert config.login_field["text"] == "Login"
|
||||||
|
assert config.password_field["text"] == "Mot de passe"
|
||||||
|
assert config.submit_button["text"] == "Connexion"
|
||||||
|
assert len(config.success_elements) >= 1
|
||||||
|
assert config.context != ""
|
||||||
|
|
||||||
|
def test_config_fields_are_dicts(self):
|
||||||
|
config = dpi_urgences_login_config()
|
||||||
|
assert isinstance(config.login_field, dict)
|
||||||
|
assert isinstance(config.password_field, dict)
|
||||||
|
assert isinstance(config.submit_button, dict)
|
||||||
|
|
||||||
|
|
||||||
|
# ── _ocr_detailed_to_simple tests ────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestOcrDetailedToSimple:
|
||||||
|
def test_conversion(self):
|
||||||
|
tokens = [
|
||||||
|
OcrTokenInfo(text="Login", bbox=(100, 50, 200, 90)),
|
||||||
|
OcrTokenInfo(text="Password", bbox=(100, 100, 200, 140)),
|
||||||
|
]
|
||||||
|
detailed = mock_ocr_detailed_client_factory(tokens)
|
||||||
|
simple = _ocr_detailed_to_simple(detailed)
|
||||||
|
result = simple("/tmp/test.png")
|
||||||
|
assert result == ["Login", "Password"]
|
||||||
|
|
||||||
|
def test_empty_tokens(self):
|
||||||
|
detailed = mock_ocr_detailed_client_factory([])
|
||||||
|
simple = _ocr_detailed_to_simple(detailed)
|
||||||
|
result = simple("/tmp/test.png")
|
||||||
|
assert result == []
|
||||||
|
|
||||||
|
|
||||||
|
# ── verify_login_visible tests ────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestVerifyLoginVisible:
|
||||||
|
def test_form_visible(self):
|
||||||
|
"""All 3 fields found by OCR + roles confirmed → match."""
|
||||||
|
config = LoginFormConfig(
|
||||||
|
login_field={"role": "champ", "text": "Login"},
|
||||||
|
password_field={"role": "champ", "text": "Mot de passe"},
|
||||||
|
submit_button={"role": "bouton", "text": "Connexion"},
|
||||||
|
context="DPI login",
|
||||||
|
)
|
||||||
|
ocr = mock_ocr_simple_client_factory(["Login", "Mot de passe", "Connexion"])
|
||||||
|
vlm = mock_vlm_client_factory({
|
||||||
|
"confirmed": [
|
||||||
|
{"index": 1, "role_confirmed": True, "actual_role": "champ", "confidence": 0.9},
|
||||||
|
{"index": 2, "role_confirmed": True, "actual_role": "champ", "confidence": 0.9},
|
||||||
|
{"index": 3, "role_confirmed": True, "actual_role": "bouton", "confidence": 0.9},
|
||||||
|
],
|
||||||
|
"overall_confidence": 0.9,
|
||||||
|
})
|
||||||
|
result = verify_login_visible("/tmp/login.png", config, ocr, vlm)
|
||||||
|
assert result.match == True
|
||||||
|
|
||||||
|
def test_form_missing_button(self):
|
||||||
|
"""Connexion button not found by OCR → mismatch."""
|
||||||
|
config = LoginFormConfig(
|
||||||
|
login_field={"role": "champ", "text": "Login"},
|
||||||
|
password_field={"role": "champ", "text": "Mot de passe"},
|
||||||
|
submit_button={"role": "bouton", "text": "Connexion"},
|
||||||
|
)
|
||||||
|
ocr = mock_ocr_simple_client_factory(["Login", "Mot de passe"]) # missing Connexion
|
||||||
|
vlm = mock_vlm_client_factory({})
|
||||||
|
result = verify_login_visible("/tmp/login.png", config, ocr, vlm)
|
||||||
|
assert result.match == False
|
||||||
|
|
||||||
|
def test_form_wrong_role(self):
|
||||||
|
"""OCR finds text but VLM says button is a label → mismatch."""
|
||||||
|
config = LoginFormConfig(
|
||||||
|
login_field={"role": "champ", "text": "Login"},
|
||||||
|
password_field={"role": "champ", "text": "Mot de passe"},
|
||||||
|
submit_button={"role": "bouton", "text": "Connexion"},
|
||||||
|
)
|
||||||
|
ocr = mock_ocr_simple_client_factory(["Login", "Mot de passe", "Connexion"])
|
||||||
|
vlm = mock_vlm_client_factory({
|
||||||
|
"confirmed": [
|
||||||
|
{"index": 1, "role_confirmed": True, "actual_role": "champ", "confidence": 0.9},
|
||||||
|
{"index": 2, "role_confirmed": True, "actual_role": "champ", "confidence": 0.9},
|
||||||
|
{"index": 3, "role_confirmed": False, "actual_role": "label", "confidence": 0.5},
|
||||||
|
],
|
||||||
|
"overall_confidence": 0.5,
|
||||||
|
})
|
||||||
|
result = verify_login_visible("/tmp/login.png", config, ocr, vlm)
|
||||||
|
assert result.match == False
|
||||||
|
|
||||||
|
|
||||||
|
# ── verify_login_success tests ────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestVerifyLoginSuccess:
|
||||||
|
def test_dashboard_visible(self):
|
||||||
|
"""Dashboard found by OCR + role confirmed → success."""
|
||||||
|
config = LoginFormConfig(
|
||||||
|
login_field={"role": "champ", "text": "Login"},
|
||||||
|
password_field={"role": "champ", "text": "Mot de passe"},
|
||||||
|
submit_button={"role": "bouton", "text": "Connexion"},
|
||||||
|
success_elements=[{"role": "page", "text": "Dashboard"}],
|
||||||
|
)
|
||||||
|
ocr = mock_ocr_simple_client_factory(["Dashboard", "Accueil"])
|
||||||
|
vlm = mock_vlm_client_factory({
|
||||||
|
"confirmed": [
|
||||||
|
{"index": 1, "role_confirmed": True, "actual_role": "page", "confidence": 0.92},
|
||||||
|
],
|
||||||
|
"overall_confidence": 0.92,
|
||||||
|
})
|
||||||
|
result = verify_login_success("/tmp/dashboard.png", config, ocr, vlm)
|
||||||
|
assert result.match == True
|
||||||
|
|
||||||
|
def test_no_success_elements(self):
|
||||||
|
"""Config has no success_elements → can't verify."""
|
||||||
|
config = LoginFormConfig(
|
||||||
|
login_field={"role": "champ", "text": "Login"},
|
||||||
|
password_field={"role": "champ", "text": "Mot de passe"},
|
||||||
|
submit_button={"role": "bouton", "text": "Connexion"},
|
||||||
|
success_elements=[], # empty!
|
||||||
|
)
|
||||||
|
ocr = mock_ocr_simple_client_factory(["Dashboard"])
|
||||||
|
vlm = mock_vlm_client_factory({})
|
||||||
|
result = verify_login_success("/tmp/page.png", config, ocr, vlm)
|
||||||
|
assert result.match == False
|
||||||
|
assert "no success_elements" in result.reason
|
||||||
|
|
||||||
|
def test_still_on_login_page(self):
|
||||||
|
"""After login, still seeing login form → mismatch."""
|
||||||
|
config = LoginFormConfig(
|
||||||
|
login_field={"role": "champ", "text": "Login"},
|
||||||
|
password_field={"role": "champ", "text": "Mot de passe"},
|
||||||
|
submit_button={"role": "bouton", "text": "Connexion"},
|
||||||
|
success_elements=[{"role": "page", "text": "Dashboard"}],
|
||||||
|
)
|
||||||
|
# OCR sees login form texts, not Dashboard
|
||||||
|
ocr = mock_ocr_simple_client_factory(["Login", "Mot de passe", "Connexion"])
|
||||||
|
vlm = mock_vlm_client_factory({})
|
||||||
|
result = verify_login_success("/tmp/still_login.png", config, ocr, vlm)
|
||||||
|
assert result.match == False
|
||||||
|
|
||||||
|
|
||||||
|
# ── resolve_login_form tests ──────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestResolveLoginForm:
|
||||||
|
def test_all_fields_ocr_anchor(self):
|
||||||
|
"""All 3 fields found by OCR with bbox → full resolution."""
|
||||||
|
config = LoginFormConfig(
|
||||||
|
login_field={"role": "champ", "text": "Login"},
|
||||||
|
password_field={"role": "champ", "text": "Mot de passe"},
|
||||||
|
submit_button={"role": "bouton", "text": "Connexion"},
|
||||||
|
)
|
||||||
|
ocr = mock_ocr_detailed_client_factory([
|
||||||
|
OcrTokenInfo(text="Login", bbox=(100, 50, 250, 90)),
|
||||||
|
OcrTokenInfo(text="Mot de passe", bbox=(100, 100, 250, 140)),
|
||||||
|
OcrTokenInfo(text="Connexion", bbox=(100, 150, 250, 190)),
|
||||||
|
])
|
||||||
|
vlm = mock_vlm_client_factory({})
|
||||||
|
result = resolve_login_form("/tmp/login.png", config, ocr, vlm)
|
||||||
|
assert result.all_resolved == True
|
||||||
|
assert result.login_field is not None
|
||||||
|
assert result.login_field.method == "ocr_anchor"
|
||||||
|
assert result.password_field is not None
|
||||||
|
assert result.submit_button is not None
|
||||||
|
assert result.method == "ocr_anchor"
|
||||||
|
|
||||||
|
def test_partial_ocr_vlm_fallback(self):
|
||||||
|
"""Login + password by OCR, button by VLM → mixed method."""
|
||||||
|
config = LoginFormConfig(
|
||||||
|
login_field={"role": "champ", "text": "Login"},
|
||||||
|
password_field={"role": "champ", "text": "Password"},
|
||||||
|
submit_button={"role": "bouton", "text": "Connexion"},
|
||||||
|
)
|
||||||
|
ocr = mock_ocr_detailed_client_factory([
|
||||||
|
OcrTokenInfo(text="Login", bbox=(100, 50, 250, 90)),
|
||||||
|
OcrTokenInfo(text="Password", bbox=(100, 100, 250, 140)),
|
||||||
|
# Connexion not in OCR → VLM fallback
|
||||||
|
])
|
||||||
|
vlm = mock_vlm_client_factory({
|
||||||
|
"found": True,
|
||||||
|
"bbox": [0.2, 0.4, 0.4, 0.5],
|
||||||
|
"confidence": 0.85,
|
||||||
|
})
|
||||||
|
result = resolve_login_form("/tmp/login.png", config, ocr, vlm)
|
||||||
|
assert result.all_resolved == True
|
||||||
|
assert result.login_field.method == "ocr_anchor"
|
||||||
|
assert result.submit_button.method == "vlm_grounder"
|
||||||
|
assert result.method == "mixed"
|
||||||
|
|
||||||
|
def test_incomplete_resolution(self):
|
||||||
|
"""Button not found by OCR or VLM → incomplete."""
|
||||||
|
config = LoginFormConfig(
|
||||||
|
login_field={"role": "champ", "text": "Login"},
|
||||||
|
password_field={"role": "champ", "text": "Password"},
|
||||||
|
submit_button={"role": "bouton", "text": "Connexion"},
|
||||||
|
)
|
||||||
|
ocr = mock_ocr_detailed_client_factory([
|
||||||
|
OcrTokenInfo(text="Login", bbox=(100, 50, 250, 90)),
|
||||||
|
OcrTokenInfo(text="Password", bbox=(100, 100, 250, 140)),
|
||||||
|
])
|
||||||
|
vlm = mock_vlm_client_factory({"found": False, "bbox": [], "confidence": 0.0})
|
||||||
|
result = resolve_login_form("/tmp/login.png", config, ocr, vlm)
|
||||||
|
assert result.all_resolved == False
|
||||||
|
assert result.submit_button is None
|
||||||
|
|
||||||
|
def test_cache_hit(self):
|
||||||
|
"""All fields cached → returned directly."""
|
||||||
|
cache = CoordsCache()
|
||||||
|
cache.put("champ:login", (100, 50, 250, 90), (175, 70), "ocr_anchor")
|
||||||
|
cache.put("champ:mot de passe", (100, 100, 250, 140), (175, 120), "ocr_anchor")
|
||||||
|
cache.put("bouton:connexion", (100, 150, 250, 190), (175, 170), "ocr_anchor")
|
||||||
|
|
||||||
|
config = LoginFormConfig(
|
||||||
|
login_field={"role": "champ", "text": "Login"},
|
||||||
|
password_field={"role": "champ", "text": "Mot de passe"},
|
||||||
|
submit_button={"role": "bouton", "text": "Connexion"},
|
||||||
|
)
|
||||||
|
ocr = mock_ocr_detailed_client_factory([])
|
||||||
|
vlm = mock_vlm_client_factory({})
|
||||||
|
result = resolve_login_form(
|
||||||
|
"/tmp/login.png", config, ocr, vlm, coords_cache=cache,
|
||||||
|
)
|
||||||
|
assert result.all_resolved == True
|
||||||
|
assert result.method == "cache"
|
||||||
|
assert result.login_field.center == (175, 70)
|
||||||
|
|
||||||
|
def test_with_dpi_default_config(self):
|
||||||
|
"""Full flow with dpi_urgences_login_config."""
|
||||||
|
config = dpi_urgences_login_config()
|
||||||
|
ocr = mock_ocr_detailed_client_factory([
|
||||||
|
OcrTokenInfo(text="Login", bbox=(100, 50, 250, 90)),
|
||||||
|
OcrTokenInfo(text="Mot de passe", bbox=(100, 100, 250, 140)),
|
||||||
|
OcrTokenInfo(text="Connexion", bbox=(100, 150, 250, 190)),
|
||||||
|
])
|
||||||
|
vlm = mock_vlm_client_factory({})
|
||||||
|
result = resolve_login_form("/tmp/login.png", config, ocr, vlm)
|
||||||
|
assert result.all_resolved == True
|
||||||
|
|
||||||
|
|
||||||
|
# ── LoginResolution describe tests ────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestLoginResolutionDescribe:
|
||||||
|
def test_all_resolved(self):
|
||||||
|
resolution = LoginResolution(
|
||||||
|
login_field=GroundedElement(
|
||||||
|
role="champ", text="Login",
|
||||||
|
bbox=(100, 50, 250, 90), center=(175, 70),
|
||||||
|
confidence=0.9, method="ocr_anchor",
|
||||||
|
),
|
||||||
|
password_field=GroundedElement(
|
||||||
|
role="champ", text="Mot de passe",
|
||||||
|
bbox=(100, 100, 250, 140), center=(175, 120),
|
||||||
|
confidence=0.9, method="ocr_anchor",
|
||||||
|
),
|
||||||
|
submit_button=GroundedElement(
|
||||||
|
role="bouton", text="Connexion",
|
||||||
|
bbox=(100, 150, 250, 190), center=(175, 170),
|
||||||
|
confidence=0.9, method="ocr_anchor",
|
||||||
|
),
|
||||||
|
all_resolved=True,
|
||||||
|
method="ocr_anchor",
|
||||||
|
)
|
||||||
|
desc = resolution.describe()
|
||||||
|
assert "OK" in desc
|
||||||
|
assert "login@" in desc
|
||||||
|
assert "button@" in desc
|
||||||
|
|
||||||
|
def test_incomplete(self):
|
||||||
|
resolution = LoginResolution(
|
||||||
|
login_field=None,
|
||||||
|
password_field=None,
|
||||||
|
submit_button=None,
|
||||||
|
all_resolved=False,
|
||||||
|
method="",
|
||||||
|
)
|
||||||
|
desc = resolution.describe()
|
||||||
|
assert "INCOMPLETE" in desc
|
||||||
|
assert "NOT FOUND" in desc
|
||||||
490
tests/unit/test_visual_verifier.py
Normal file
490
tests/unit/test_visual_verifier.py
Normal file
@@ -0,0 +1,490 @@
|
|||||||
|
"""Tests for core/navigation/visual_verifier.py — OCR-anchored architecture.
|
||||||
|
|
||||||
|
Tests pure functions (normalize_text, fuzzy_match, ocr_presence_check,
|
||||||
|
build_role_confirm_prompt, parse_role_confirm_response) offline,
|
||||||
|
then verifies verify_screen_match with mock OcrClient + VlmClient.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import pytest
|
||||||
|
from core.navigation.visual_verifier import (
|
||||||
|
normalize_text,
|
||||||
|
fuzzy_match,
|
||||||
|
ocr_presence_check,
|
||||||
|
build_role_confirm_prompt,
|
||||||
|
parse_role_confirm_response,
|
||||||
|
verify_screen_match,
|
||||||
|
verify_before,
|
||||||
|
verify_after,
|
||||||
|
ScreenMatchResult,
|
||||||
|
OcrPresenceResult,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ── Mock factories ─────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def mock_ocr_client_factory(tokens: list):
|
||||||
|
"""Factory that creates a mock OcrClient returning the given tokens."""
|
||||||
|
def client(image_path: str) -> list:
|
||||||
|
return tokens
|
||||||
|
return client
|
||||||
|
|
||||||
|
|
||||||
|
def mock_vlm_client_factory(response_json: dict):
|
||||||
|
"""Factory that creates a mock VlmClient returning the given JSON."""
|
||||||
|
def client(image_path: str, prompt: str) -> str:
|
||||||
|
return json.dumps(response_json)
|
||||||
|
return client
|
||||||
|
|
||||||
|
|
||||||
|
# ── normalize_text tests ──────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestNormalizeText:
|
||||||
|
def test_lowercase(self):
|
||||||
|
assert normalize_text("RECHERCHER") == "rechercher"
|
||||||
|
|
||||||
|
def test_strip_accents(self):
|
||||||
|
assert normalize_text("Recherché") == "recherche"
|
||||||
|
|
||||||
|
def test_collapse_whitespace(self):
|
||||||
|
assert normalize_text(" hello world ") == "hello world"
|
||||||
|
|
||||||
|
def test_combined(self):
|
||||||
|
assert normalize_text(" Nom Prénom ") == "nom prenom"
|
||||||
|
|
||||||
|
def test_empty(self):
|
||||||
|
assert normalize_text("") == ""
|
||||||
|
|
||||||
|
def test_numbers_preserved(self):
|
||||||
|
assert normalize_text("IPP 12345") == "ipp 12345"
|
||||||
|
|
||||||
|
|
||||||
|
# ── fuzzy_match tests ─────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestFuzzyMatch:
|
||||||
|
def test_exact_match(self):
|
||||||
|
assert fuzzy_match("Rechercher", "Rechercher") == True
|
||||||
|
|
||||||
|
def test_case_insensitive(self):
|
||||||
|
assert fuzzy_match("rechercher", "RECHERCHER") == True
|
||||||
|
|
||||||
|
def test_accent_match(self):
|
||||||
|
assert fuzzy_match("Recherché", "Recherche") == True
|
||||||
|
|
||||||
|
def test_substring_containment(self):
|
||||||
|
# Short text contained in longer OCR token
|
||||||
|
assert fuzzy_match("Rechercher", "Bouton Rechercher") == True
|
||||||
|
|
||||||
|
def test_reverse_containment(self):
|
||||||
|
# OCR token contained in expected text
|
||||||
|
assert fuzzy_match("Nom Prénom Patient", "Nom") == True
|
||||||
|
|
||||||
|
def test_fuzzy_ratio(self):
|
||||||
|
# Similar but not exact/substring — ratio ~0.90
|
||||||
|
assert fuzzy_match("Connexion", "Connection", threshold=0.8) == True
|
||||||
|
|
||||||
|
def test_no_match(self):
|
||||||
|
assert fuzzy_match("Dashboard", "Login", threshold=0.8) == False
|
||||||
|
|
||||||
|
def test_custom_threshold(self):
|
||||||
|
# "Connection" vs "Connexion" ratio ~0.90, passes at 0.8 but fails at 0.95
|
||||||
|
assert fuzzy_match("Connexion", "Connection", threshold=0.95) == False
|
||||||
|
|
||||||
|
|
||||||
|
# ── ocr_presence_check tests ──────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestOcrPresenceCheck:
|
||||||
|
def test_all_found(self):
|
||||||
|
tokens = ["Rechercher", "Connexion", "Nom Patient"]
|
||||||
|
elements = [
|
||||||
|
{"role": "bouton", "text": "Rechercher"},
|
||||||
|
{"role": "bouton", "text": "Connexion"},
|
||||||
|
]
|
||||||
|
result = ocr_presence_check(tokens, elements)
|
||||||
|
assert result.all_found == True
|
||||||
|
assert result.presence_ratio == 1.0
|
||||||
|
assert len(result.missing) == 0
|
||||||
|
assert result.found_texts["Rechercher"] == "Rechercher"
|
||||||
|
|
||||||
|
def test_partial_found(self):
|
||||||
|
tokens = ["Rechercher"]
|
||||||
|
elements = [
|
||||||
|
{"role": "bouton", "text": "Rechercher"},
|
||||||
|
{"role": "bouton", "text": "Connexion"},
|
||||||
|
]
|
||||||
|
result = ocr_presence_check(tokens, elements)
|
||||||
|
assert result.all_found == False
|
||||||
|
assert result.presence_ratio == 0.5
|
||||||
|
assert "bouton: Connexion" in result.missing
|
||||||
|
|
||||||
|
def test_none_found(self):
|
||||||
|
tokens = ["Accueil", "Paramètres"]
|
||||||
|
elements = [
|
||||||
|
{"role": "bouton", "text": "Rechercher"},
|
||||||
|
]
|
||||||
|
result = ocr_presence_check(tokens, elements)
|
||||||
|
assert result.all_found == False
|
||||||
|
assert result.presence_ratio == 0.0
|
||||||
|
assert "bouton: Rechercher" in result.missing
|
||||||
|
|
||||||
|
def test_fuzzy_match_in_presence(self):
|
||||||
|
tokens = ["Rechércher"] # OCR with accent variation
|
||||||
|
elements = [{"role": "bouton", "text": "Rechercher"}]
|
||||||
|
result = ocr_presence_check(tokens, elements)
|
||||||
|
assert result.all_found == True
|
||||||
|
|
||||||
|
def test_empty_tokens(self):
|
||||||
|
result = ocr_presence_check([], [{"role": "bouton", "text": "Login"}])
|
||||||
|
assert result.all_found == False
|
||||||
|
assert result.presence_ratio == 0.0
|
||||||
|
|
||||||
|
def test_empty_elements(self):
|
||||||
|
result = ocr_presence_check(["Login", "Password"], [])
|
||||||
|
assert result.all_found == True
|
||||||
|
assert result.presence_ratio == 1.0
|
||||||
|
|
||||||
|
def test_no_text_key(self):
|
||||||
|
elements = [{"role": "page"}] # no text key
|
||||||
|
result = ocr_presence_check(["Dashboard"], elements)
|
||||||
|
assert result.all_found == True # no text to check → trivially found
|
||||||
|
|
||||||
|
def test_multiple_elements_same_text(self):
|
||||||
|
tokens = ["Connexion"]
|
||||||
|
elements = [
|
||||||
|
{"role": "bouton", "text": "Connexion"},
|
||||||
|
{"role": "label", "text": "Connexion"},
|
||||||
|
]
|
||||||
|
result = ocr_presence_check(tokens, elements)
|
||||||
|
assert result.all_found == True
|
||||||
|
|
||||||
|
|
||||||
|
# ── build_role_confirm_prompt tests ───────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestBuildRoleConfirmPrompt:
|
||||||
|
def test_basic_prompt(self):
|
||||||
|
found = [
|
||||||
|
{"text": "Rechercher", "expected_role": "bouton", "matched_ocr": "Rechercher"},
|
||||||
|
]
|
||||||
|
expected = [{"role": "bouton", "text": "Rechercher"}]
|
||||||
|
prompt = build_role_confirm_prompt(found, expected)
|
||||||
|
assert "Text \"Rechercher\"" in prompt
|
||||||
|
assert "expected role: bouton" in prompt
|
||||||
|
assert "role_confirmed" in prompt
|
||||||
|
|
||||||
|
def test_with_context(self):
|
||||||
|
found = [
|
||||||
|
{"text": "Connexion", "expected_role": "bouton", "matched_ocr": "Connexion"},
|
||||||
|
]
|
||||||
|
expected = [{"role": "bouton", "text": "Connexion"}]
|
||||||
|
prompt = build_role_confirm_prompt(found, expected, context="page login DPI")
|
||||||
|
assert "Context: page login DPI" in prompt
|
||||||
|
|
||||||
|
def test_multiple_elements(self):
|
||||||
|
found = [
|
||||||
|
{"text": "Login", "expected_role": "champ", "matched_ocr": "Login"},
|
||||||
|
{"text": "Password", "expected_role": "champ", "matched_ocr": "Password"},
|
||||||
|
{"text": "Connexion", "expected_role": "bouton", "matched_ocr": "Connexion"},
|
||||||
|
]
|
||||||
|
expected = [
|
||||||
|
{"role": "champ", "text": "Login"},
|
||||||
|
{"role": "champ", "text": "Password"},
|
||||||
|
{"role": "bouton", "text": "Connexion"},
|
||||||
|
]
|
||||||
|
prompt = build_role_confirm_prompt(found, expected)
|
||||||
|
assert "1." in prompt
|
||||||
|
assert "2." in prompt
|
||||||
|
assert "3." in prompt
|
||||||
|
|
||||||
|
def test_no_self_declaration(self):
|
||||||
|
"""Prompt must NOT ask VLM to declare presence — only role."""
|
||||||
|
found = [
|
||||||
|
{"text": "Login", "expected_role": "champ", "matched_ocr": "Login"},
|
||||||
|
]
|
||||||
|
expected = [{"role": "champ", "text": "Login"}]
|
||||||
|
prompt = build_role_confirm_prompt(found, expected)
|
||||||
|
assert "present" not in prompt.lower() or "confirmed" in prompt.lower()
|
||||||
|
|
||||||
|
|
||||||
|
# ── parse_role_confirm_response tests ─────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestParseRoleConfirmResponse:
|
||||||
|
def test_valid_json(self):
|
||||||
|
data = json.dumps({
|
||||||
|
"confirmed": [
|
||||||
|
{"index": 1, "role_confirmed": True, "actual_role": "bouton", "confidence": 0.92},
|
||||||
|
],
|
||||||
|
"overall_confidence": 0.92,
|
||||||
|
})
|
||||||
|
result = parse_role_confirm_response(data)
|
||||||
|
assert len(result["confirmed"]) == 1
|
||||||
|
assert result["overall_confidence"] == 0.92
|
||||||
|
|
||||||
|
def test_json_in_markdown(self):
|
||||||
|
vlm_text = "```json\n{\"confirmed\": [], \"overall_confidence\": 0.0}\n```"
|
||||||
|
result = parse_role_confirm_response(vlm_text)
|
||||||
|
assert result["overall_confidence"] == 0.0
|
||||||
|
|
||||||
|
def test_garbled_response(self):
|
||||||
|
result = parse_role_confirm_response("I cannot determine the roles")
|
||||||
|
assert result["overall_confidence"] == 0.0
|
||||||
|
assert len(result["confirmed"]) == 0
|
||||||
|
|
||||||
|
def test_confidence_as_string(self):
|
||||||
|
data = json.dumps({"confirmed": [], "overall_confidence": "0.85"})
|
||||||
|
result = parse_role_confirm_response(data)
|
||||||
|
assert result["overall_confidence"] == 0.85
|
||||||
|
|
||||||
|
|
||||||
|
# ── verify_screen_match (OCR-anchored) tests ─────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestVerifyScreenMatchOcrAnchored:
|
||||||
|
def test_full_match(self):
|
||||||
|
ocr = mock_ocr_client_factory(["Rechercher", "Connexion", "Dashboard"])
|
||||||
|
vlm = mock_vlm_client_factory({
|
||||||
|
"confirmed": [
|
||||||
|
{"index": 1, "role_confirmed": True, "actual_role": "bouton", "confidence": 0.92},
|
||||||
|
],
|
||||||
|
"overall_confidence": 0.92,
|
||||||
|
})
|
||||||
|
result = verify_screen_match(
|
||||||
|
"/tmp/test.png",
|
||||||
|
[{"role": "bouton", "text": "Rechercher"}],
|
||||||
|
ocr_client=ocr,
|
||||||
|
vlm_client=vlm,
|
||||||
|
)
|
||||||
|
assert result.match == True
|
||||||
|
assert result.confidence >= 0.7
|
||||||
|
|
||||||
|
def test_ocr_presence_fail(self):
|
||||||
|
"""OCR doesn't find expected text → mismatch (deterministic, no VLM needed)."""
|
||||||
|
ocr = mock_ocr_client_factory(["Accueil", "Paramètres"])
|
||||||
|
vlm = mock_vlm_client_factory({})
|
||||||
|
result = verify_screen_match(
|
||||||
|
"/tmp/test.png",
|
||||||
|
[{"role": "bouton", "text": "Rechercher"}],
|
||||||
|
ocr_client=ocr,
|
||||||
|
vlm_client=vlm,
|
||||||
|
)
|
||||||
|
assert result.match == False
|
||||||
|
assert "OCR presence" in result.reason
|
||||||
|
assert len(result.mismatches) > 0
|
||||||
|
|
||||||
|
def test_role_not_confirmed(self):
|
||||||
|
"""OCR finds text, VLM says it's a label not a button → mismatch."""
|
||||||
|
ocr = mock_ocr_client_factory(["Rechercher"])
|
||||||
|
vlm = mock_vlm_client_factory({
|
||||||
|
"confirmed": [
|
||||||
|
{"index": 1, "role_confirmed": False, "actual_role": "label", "confidence": 0.6},
|
||||||
|
],
|
||||||
|
"overall_confidence": 0.6,
|
||||||
|
})
|
||||||
|
result = verify_screen_match(
|
||||||
|
"/tmp/test.png",
|
||||||
|
[{"role": "bouton", "text": "Rechercher"}],
|
||||||
|
ocr_client=ocr,
|
||||||
|
vlm_client=vlm,
|
||||||
|
)
|
||||||
|
assert result.match == False
|
||||||
|
|
||||||
|
def test_ocr_error(self):
|
||||||
|
"""OCR engine fails → fail-safe mismatch."""
|
||||||
|
def failing_ocr(image_path):
|
||||||
|
raise RuntimeError("OCR engine down")
|
||||||
|
vlm = mock_vlm_client_factory({})
|
||||||
|
result = verify_screen_match(
|
||||||
|
"/tmp/test.png",
|
||||||
|
[{"role": "bouton", "text": "Rechercher"}],
|
||||||
|
ocr_client=failing_ocr,
|
||||||
|
vlm_client=vlm,
|
||||||
|
)
|
||||||
|
assert result.match == False
|
||||||
|
assert "OCR error" in result.reason
|
||||||
|
|
||||||
|
def test_vlm_error_partial_match(self):
|
||||||
|
"""OCR finds texts, VLM fails → partial match (presence OK, role unknown)."""
|
||||||
|
ocr = mock_ocr_client_factory(["Rechercher"])
|
||||||
|
def failing_vlm(image_path, prompt):
|
||||||
|
raise RuntimeError("VLM service down")
|
||||||
|
result = verify_screen_match(
|
||||||
|
"/tmp/test.png",
|
||||||
|
[{"role": "bouton", "text": "Rechercher"}],
|
||||||
|
ocr_client=ocr,
|
||||||
|
vlm_client=failing_vlm,
|
||||||
|
)
|
||||||
|
# Presence confirmed by OCR → partial match, confidence=0.5
|
||||||
|
assert result.match == True
|
||||||
|
assert result.confidence == 0.5
|
||||||
|
assert "VLM role confirm failed" in result.reason
|
||||||
|
|
||||||
|
def test_no_expected_elements(self):
|
||||||
|
ocr = mock_ocr_client_factory(["Login"])
|
||||||
|
vlm = mock_vlm_client_factory({})
|
||||||
|
result = verify_screen_match("/tmp/test.png", [], ocr_client=ocr, vlm_client=vlm)
|
||||||
|
assert result.match == True
|
||||||
|
assert result.confidence == 1.0
|
||||||
|
|
||||||
|
def test_describe_match(self):
|
||||||
|
result = ScreenMatchResult(match=True, confidence=0.92)
|
||||||
|
assert "OK" in result.describe()
|
||||||
|
|
||||||
|
def test_describe_mismatch(self):
|
||||||
|
result = ScreenMatchResult(
|
||||||
|
match=False, confidence=0.3,
|
||||||
|
mismatches=["bouton: Rechercher"],
|
||||||
|
)
|
||||||
|
assert "mismatch" in result.describe()
|
||||||
|
|
||||||
|
def test_multiple_elements_mixed(self):
|
||||||
|
"""2 elements: 1 found+role OK, 1 not found in OCR → mismatch."""
|
||||||
|
ocr = mock_ocr_client_factory(["Connexion"])
|
||||||
|
vlm = mock_vlm_client_factory({
|
||||||
|
"confirmed": [
|
||||||
|
{"index": 1, "role_confirmed": True, "actual_role": "bouton", "confidence": 0.9},
|
||||||
|
],
|
||||||
|
"overall_confidence": 0.9,
|
||||||
|
})
|
||||||
|
result = verify_screen_match(
|
||||||
|
"/tmp/test.png",
|
||||||
|
[
|
||||||
|
{"role": "bouton", "text": "Connexion"},
|
||||||
|
{"role": "champ", "text": "Nom Patient"},
|
||||||
|
],
|
||||||
|
ocr_client=ocr,
|
||||||
|
vlm_client=vlm,
|
||||||
|
)
|
||||||
|
assert result.match == False # "Nom Patient" not found by OCR
|
||||||
|
|
||||||
|
def test_fuzzy_ocr_match(self):
|
||||||
|
"""OCR reads 'Rechércher' (accent), expected 'Rechercher' → still found."""
|
||||||
|
ocr = mock_ocr_client_factory(["Rechércher"])
|
||||||
|
vlm = mock_vlm_client_factory({
|
||||||
|
"confirmed": [
|
||||||
|
{"index": 1, "role_confirmed": True, "actual_role": "bouton", "confidence": 0.9},
|
||||||
|
],
|
||||||
|
"overall_confidence": 0.9,
|
||||||
|
})
|
||||||
|
result = verify_screen_match(
|
||||||
|
"/tmp/test.png",
|
||||||
|
[{"role": "bouton", "text": "Rechercher"}],
|
||||||
|
ocr_client=ocr,
|
||||||
|
vlm_client=vlm,
|
||||||
|
)
|
||||||
|
assert result.match == True
|
||||||
|
|
||||||
|
def test_no_text_elements_trivially_match(self):
|
||||||
|
"""Elements without text key → no presence check needed → trivially OK."""
|
||||||
|
ocr = mock_ocr_client_factory(["Dashboard"])
|
||||||
|
vlm = mock_vlm_client_factory({})
|
||||||
|
result = verify_screen_match(
|
||||||
|
"/tmp/test.png",
|
||||||
|
[{"role": "page"}],
|
||||||
|
ocr_client=ocr,
|
||||||
|
vlm_client=vlm,
|
||||||
|
)
|
||||||
|
assert result.match == True
|
||||||
|
|
||||||
|
|
||||||
|
# ── verify_before / verify_after tests ────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestVerifyBeforeAfter:
|
||||||
|
def test_verify_before_match(self):
|
||||||
|
ocr = mock_ocr_client_factory(["Login", "Password", "Connexion"])
|
||||||
|
vlm = mock_vlm_client_factory({
|
||||||
|
"confirmed": [
|
||||||
|
{"index": 1, "role_confirmed": True, "actual_role": "champ", "confidence": 0.85},
|
||||||
|
],
|
||||||
|
"overall_confidence": 0.85,
|
||||||
|
})
|
||||||
|
result = verify_before(
|
||||||
|
"/tmp/login.png",
|
||||||
|
[{"role": "champ", "text": "Login"}],
|
||||||
|
ocr_client=ocr,
|
||||||
|
vlm_client=vlm,
|
||||||
|
context="page login",
|
||||||
|
)
|
||||||
|
assert result.match == True
|
||||||
|
|
||||||
|
def test_verify_after_higher_threshold(self):
|
||||||
|
"""verify_after uses min_confidence=0.8. VLM returns 0.75 → mismatch."""
|
||||||
|
ocr = mock_ocr_client_factory(["Dashboard"])
|
||||||
|
vlm = mock_vlm_client_factory({
|
||||||
|
"confirmed": [
|
||||||
|
{"index": 1, "role_confirmed": True, "actual_role": "page", "confidence": 0.75},
|
||||||
|
],
|
||||||
|
"overall_confidence": 0.75,
|
||||||
|
})
|
||||||
|
result = verify_after(
|
||||||
|
"/tmp/dashboard.png",
|
||||||
|
[{"role": "page", "text": "Dashboard"}],
|
||||||
|
ocr_client=ocr,
|
||||||
|
vlm_client=vlm,
|
||||||
|
)
|
||||||
|
# 0.75 < 0.8 threshold → role mismatch
|
||||||
|
assert result.match == False
|
||||||
|
|
||||||
|
def test_verify_after_passes_at_0_8(self):
|
||||||
|
ocr = mock_ocr_client_factory(["Dashboard"])
|
||||||
|
vlm = mock_vlm_client_factory({
|
||||||
|
"confirmed": [
|
||||||
|
{"index": 1, "role_confirmed": True, "actual_role": "page", "confidence": 0.85},
|
||||||
|
],
|
||||||
|
"overall_confidence": 0.85,
|
||||||
|
})
|
||||||
|
result = verify_after(
|
||||||
|
"/tmp/dashboard.png",
|
||||||
|
[{"role": "page", "text": "Dashboard"}],
|
||||||
|
ocr_client=ocr,
|
||||||
|
vlm_client=vlm,
|
||||||
|
)
|
||||||
|
assert result.match == True
|
||||||
|
|
||||||
|
def test_verify_before_ocr_missing(self):
|
||||||
|
"""Pre-action: expected text not on screen → mismatch (can't proceed)."""
|
||||||
|
ocr = mock_ocr_client_factory(["Accueil"])
|
||||||
|
vlm = mock_vlm_client_factory({})
|
||||||
|
result = verify_before(
|
||||||
|
"/tmp/page.png",
|
||||||
|
[{"role": "bouton", "text": "Connexion"}],
|
||||||
|
ocr_client=ocr,
|
||||||
|
vlm_client=vlm,
|
||||||
|
context="pre-login",
|
||||||
|
)
|
||||||
|
assert result.match == False
|
||||||
|
assert "OCR presence" in result.reason
|
||||||
|
|
||||||
|
|
||||||
|
# ── OcrPresenceResult dataclass tests ─────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestOcrPresenceResult:
|
||||||
|
def test_presence_ratio_all_found(self):
|
||||||
|
result = OcrPresenceResult(
|
||||||
|
found_texts={"Login": "Login", "Password": "Password"},
|
||||||
|
missing=[],
|
||||||
|
all_found=True,
|
||||||
|
)
|
||||||
|
assert result.presence_ratio == 1.0
|
||||||
|
|
||||||
|
def test_presence_ratio_half_found(self):
|
||||||
|
result = OcrPresenceResult(
|
||||||
|
found_texts={"Login": "Login", "Password": ""},
|
||||||
|
missing=["champ: Password"],
|
||||||
|
all_found=False,
|
||||||
|
)
|
||||||
|
assert result.presence_ratio == 0.5
|
||||||
|
|
||||||
|
def test_presence_ratio_empty(self):
|
||||||
|
result = OcrPresenceResult(
|
||||||
|
found_texts={},
|
||||||
|
missing=[],
|
||||||
|
all_found=True,
|
||||||
|
)
|
||||||
|
assert result.presence_ratio == 1.0
|
||||||
Reference in New Issue
Block a user