fix(p0): secure agent revocation and R6 worker queue
This commit is contained in:
@@ -173,6 +173,9 @@ class AgentRegistry:
|
||||
# Deja enrolle et actif -> conflit explicit
|
||||
raise AgentAlreadyEnrolledError(dict(existing))
|
||||
|
||||
if existing["uninstall_reason"] == "admin_revoke":
|
||||
raise AgentRevokedError(dict(existing))
|
||||
|
||||
# Agent desinstalle : reactivation si autorise (defaut)
|
||||
if not allow_reactivate:
|
||||
raise AgentAlreadyEnrolledError(dict(existing))
|
||||
@@ -273,13 +276,15 @@ class AgentRegistry:
|
||||
"""Met a jour last_seen_at (appel depuis le stream / heartbeat).
|
||||
|
||||
Silencieux si l'agent est inconnu (evite les erreurs sur vieux clients).
|
||||
Ne reactive jamais un agent desinstalle/revoque.
|
||||
"""
|
||||
if not machine_id:
|
||||
return
|
||||
now = _utc_now_iso()
|
||||
with _DB_LOCK, self._connect() as conn:
|
||||
conn.execute(
|
||||
"UPDATE enrolled_agents SET last_seen_at = ? WHERE machine_id = ?",
|
||||
"UPDATE enrolled_agents SET last_seen_at = ? "
|
||||
"WHERE machine_id = ? AND status = 'active'",
|
||||
(now, machine_id),
|
||||
)
|
||||
conn.commit()
|
||||
@@ -294,3 +299,14 @@ class AgentAlreadyEnrolledError(Exception):
|
||||
f"machine_id={existing_row.get('machine_id')} deja enrole "
|
||||
f"(status={existing_row.get('status')})"
|
||||
)
|
||||
|
||||
|
||||
class AgentRevokedError(Exception):
|
||||
"""Levee si un administrateur a revoque ce machine_id."""
|
||||
|
||||
def __init__(self, existing_row: Dict[str, Any]):
|
||||
self.existing = existing_row
|
||||
super().__init__(
|
||||
f"machine_id={existing_row.get('machine_id')} revoque "
|
||||
f"(reason={existing_row.get('uninstall_reason')})"
|
||||
)
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -25,6 +25,7 @@ Le worker :
|
||||
5. Se suspend quand un replay est actif (libère le GPU)
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import signal
|
||||
@@ -67,6 +68,7 @@ class VLMWorker:
|
||||
self._running = False
|
||||
self._processor = None # Initialisé au premier besoin (lazy loading GPU)
|
||||
self._current_session: Optional[str] = None
|
||||
self._started_at: str = datetime.now().isoformat()
|
||||
|
||||
# Stats
|
||||
self._stats: Dict[str, int] = {
|
||||
@@ -83,7 +85,10 @@ class VLMWorker:
|
||||
if self._processor is None:
|
||||
logger.info("Initialisation du StreamProcessor (chargement GPU)...")
|
||||
from .stream_processor import StreamProcessor
|
||||
self._processor = StreamProcessor(data_dir=str(LIVE_SESSIONS_DIR))
|
||||
self._processor = StreamProcessor(
|
||||
data_dir=str(DATA_DIR),
|
||||
enable_vlm=True,
|
||||
)
|
||||
logger.info("StreamProcessor initialisé.")
|
||||
return self._processor
|
||||
|
||||
@@ -98,6 +103,11 @@ class VLMWorker:
|
||||
logger.info(" Sessions dir : %s", LIVE_SESSIONS_DIR)
|
||||
logger.info(" Poll interval : %ds", POLL_INTERVAL)
|
||||
|
||||
# N2 + N3 : santé initiale + signal READY systemd dès le démarrage
|
||||
# (avant tout chargement GPU, pour ne pas dépasser le timeout de start).
|
||||
self._write_health("healthy")
|
||||
self._sd_notify("READY=1")
|
||||
|
||||
while self._running:
|
||||
try:
|
||||
# Vérifier si un replay est actif
|
||||
@@ -110,6 +120,7 @@ class VLMWorker:
|
||||
if session_id:
|
||||
self._process_session(session_id)
|
||||
else:
|
||||
self._write_health("healthy") # N2 : cycle idle
|
||||
time.sleep(POLL_INTERVAL)
|
||||
|
||||
except KeyboardInterrupt:
|
||||
@@ -119,6 +130,7 @@ class VLMWorker:
|
||||
logger.error("Erreur dans la boucle principale : %s", e, exc_info=True)
|
||||
time.sleep(5) # Éviter une boucle d'erreurs rapide
|
||||
|
||||
self._write_health("stopped") # N2 : santé finale
|
||||
logger.info("VLM Worker arrêté.")
|
||||
|
||||
def stop(self):
|
||||
@@ -126,6 +138,103 @@ class VLMWorker:
|
||||
self._running = False
|
||||
logger.info("Arrêt demandé.")
|
||||
|
||||
# =========================================================================
|
||||
# N2 — Health file (_worker_health.json)
|
||||
# =========================================================================
|
||||
#
|
||||
# Garde-fou anti-blocage silencieux : expose l'état de santé du worker sur
|
||||
# disque pour qu'un superviseur (humain, dashboard, watchdog) détecte un
|
||||
# worker dégradé sans avoir à fouiller les logs. Écriture atomique.
|
||||
#
|
||||
# CONFIDENTIALITÉ (HDS) : n'écrit AUCUNE donnée patient — uniquement des
|
||||
# identifiants techniques (session_id), des compteurs et des booléens de
|
||||
# composants. Jamais d'OCR, de noms de fichiers screenshots, ni de contenu
|
||||
# de session.
|
||||
|
||||
def _sd_notify(self, state: str) -> bool:
|
||||
"""Notifie systemd via $NOTIFY_SOCKET, sans dépendance `systemd.daemon`.
|
||||
|
||||
Implémentation pure socket (AF_UNIX SOCK_DGRAM) : fonctionne sous systemd
|
||||
`Type=notify` pour `READY=1` et le heartbeat `WATCHDOG=1`. No-op silencieux
|
||||
hors systemd (variable absente) ou en cas d'erreur — jamais bloquant.
|
||||
Retourne True si le message a été émis.
|
||||
"""
|
||||
addr = os.environ.get("NOTIFY_SOCKET")
|
||||
if not addr:
|
||||
return False
|
||||
try:
|
||||
import socket
|
||||
|
||||
# Namespace abstrait systemd : '@' → octet nul de préfixe
|
||||
connect_addr = "\0" + addr[1:] if addr.startswith("@") else addr
|
||||
with socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) as sock:
|
||||
sock.connect(connect_addr)
|
||||
sock.sendall(state.encode("utf-8"))
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.debug("sd_notify(%s) échoué : %s", state, e)
|
||||
return False
|
||||
|
||||
def _health_components(self) -> Dict[str, bool]:
|
||||
"""Statut booléen de chaque composant lourd, dérivé du processor."""
|
||||
proc = self._processor
|
||||
return {
|
||||
"screen_analyzer": proc is not None and getattr(proc, "_screen_analyzer", None) is not None,
|
||||
"clip_embedder": proc is not None and getattr(proc, "_clip_embedder", None) is not None,
|
||||
"faiss_manager": proc is not None and getattr(proc, "_faiss_manager", None) is not None,
|
||||
"state_embedding_builder": proc is not None and getattr(proc, "_state_embedding_builder", None) is not None,
|
||||
}
|
||||
|
||||
def _write_health(self, status: str) -> None:
|
||||
"""Écrit data/training/_worker_health.json de façon atomique.
|
||||
|
||||
`status` attendu : healthy | busy | degraded | stopped. Si le worker
|
||||
tourne en mode VLM mais que ScreenAnalyzer est absent, le statut est
|
||||
forcé à 'degraded' quelle que soit la valeur demandée.
|
||||
"""
|
||||
try:
|
||||
components = self._health_components()
|
||||
|
||||
proc = self._processor
|
||||
vlm_mode = proc is not None and getattr(proc, "_enable_vlm", False)
|
||||
if vlm_mode and not components["screen_analyzer"]:
|
||||
status = "degraded"
|
||||
|
||||
queue_path = DATA_DIR / "_worker_queue.txt"
|
||||
try:
|
||||
queue_length = len(
|
||||
[ln for ln in queue_path.read_text(encoding="utf-8").splitlines() if ln.strip()]
|
||||
) if queue_path.exists() else 0
|
||||
except Exception:
|
||||
queue_length = 0
|
||||
|
||||
payload = {
|
||||
"pid": os.getpid(),
|
||||
"started_at": self._started_at,
|
||||
"last_cycle": datetime.now().isoformat(),
|
||||
"current_session": self._current_session,
|
||||
"queue_length": queue_length,
|
||||
"components": components,
|
||||
"stats": dict(self._stats),
|
||||
"status": status,
|
||||
}
|
||||
|
||||
health_path = DATA_DIR / "_worker_health.json"
|
||||
tmp_path = health_path.with_suffix(".json.tmp")
|
||||
tmp_path.write_text(
|
||||
json.dumps(payload, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
tmp_path.rename(health_path)
|
||||
except Exception as e:
|
||||
# Le health file est un garde-fou, jamais un point de défaillance.
|
||||
logger.warning("Écriture health file échouée : %s", e)
|
||||
|
||||
# N3 : chaque écriture santé sert aussi de heartbeat watchdog systemd
|
||||
# (sauf à l'arrêt). No-op hors systemd.
|
||||
if status != "stopped":
|
||||
self._sd_notify("WATCHDOG=1")
|
||||
|
||||
# =========================================================================
|
||||
# Queue management (fichier _worker_queue.txt)
|
||||
# =========================================================================
|
||||
@@ -206,6 +315,9 @@ class VLMWorker:
|
||||
REPLAY_WAIT_TIMEOUT,
|
||||
)
|
||||
break
|
||||
# N3 : heartbeat pendant la pause replay (peut durer jusqu'à 120s,
|
||||
# sinon le watchdog tuerait un worker pourtant sain et en attente).
|
||||
self._sd_notify("WATCHDOG=1")
|
||||
time.sleep(REPLAY_CHECK_INTERVAL)
|
||||
|
||||
elapsed = time.time() - start
|
||||
@@ -220,6 +332,7 @@ class VLMWorker:
|
||||
"""Traite une session complète (analyse VLM + construction workflow)."""
|
||||
self._current_session = session_id
|
||||
logger.info("=== Début traitement session %s ===", session_id)
|
||||
self._write_health("busy") # N2 : début de session
|
||||
start_time = time.time()
|
||||
|
||||
try:
|
||||
@@ -331,6 +444,7 @@ class VLMWorker:
|
||||
|
||||
finally:
|
||||
self._current_session = None
|
||||
self._write_health("healthy") # N2 : fin de session (ou degraded auto)
|
||||
|
||||
logger.info("=== Fin traitement session %s ===", session_id)
|
||||
|
||||
@@ -347,6 +461,8 @@ class VLMWorker:
|
||||
f" ({shot_id})" if shot_id else "",
|
||||
)
|
||||
|
||||
self._write_health("busy") # N2 : heartbeat à chaque screenshot
|
||||
|
||||
# Vérifier si un replay est devenu actif pendant le traitement
|
||||
if self._is_replay_active():
|
||||
logger.info(
|
||||
|
||||
@@ -20,6 +20,15 @@ from typing import Any, Dict, List, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
try:
|
||||
from agent_v0.agent_v1.ui.message_contract import (
|
||||
coerce_supervised_pause_message,
|
||||
warn_visible_message,
|
||||
)
|
||||
except Exception: # pragma: no cover - fallback for partial server deployments
|
||||
coerce_supervised_pause_message = None
|
||||
warn_visible_message = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class PausePayload:
|
||||
@@ -50,8 +59,25 @@ def build_pause_payload(
|
||||
last_screenshot: Optional[str],
|
||||
) -> PausePayload:
|
||||
"""Construit le payload de pause enrichi pour une action pause_for_human."""
|
||||
params = action.get("parameters") or {}
|
||||
message = params.get("message", "Validation requise")
|
||||
params = dict(action.get("parameters") or {})
|
||||
for key in ("message", "safety_level", "safety_checks", "pause_reason"):
|
||||
if key not in params or params.get(key) in (None, "", []):
|
||||
if action.get(key) not in (None, "", []):
|
||||
params[key] = action.get(key)
|
||||
|
||||
raw_message = (
|
||||
params.get("message")
|
||||
or action.get("message")
|
||||
or action.get("intention")
|
||||
or ""
|
||||
)
|
||||
message = _coerce_pause_message(
|
||||
raw_message,
|
||||
intention=params.get("intention") or action.get("intention") or action.get("description"),
|
||||
attendu=params.get("attendu") or params.get("expected") or action.get("expected"),
|
||||
vu=params.get("vu") or params.get("observed") or action.get("observed"),
|
||||
demande=params.get("demande") or params.get("request"),
|
||||
)
|
||||
safety_level = params.get("safety_level")
|
||||
declarative = params.get("safety_checks") or []
|
||||
|
||||
@@ -90,11 +116,60 @@ def build_pause_payload(
|
||||
|
||||
return PausePayload(
|
||||
checks=checks,
|
||||
pause_reason="",
|
||||
pause_reason=params.get("pause_reason", ""),
|
||||
message=message,
|
||||
)
|
||||
|
||||
|
||||
def _coerce_pause_message(
|
||||
message: Any = "",
|
||||
*,
|
||||
intention: Any = "",
|
||||
attendu: Any = "",
|
||||
vu: Any = "",
|
||||
demande: Any = "",
|
||||
) -> str:
|
||||
if warn_visible_message is not None:
|
||||
warn_visible_message(
|
||||
message,
|
||||
source="safety_checks_provider._coerce_pause_message.raw",
|
||||
supervised_pause=False,
|
||||
)
|
||||
|
||||
if coerce_supervised_pause_message is not None:
|
||||
result = coerce_supervised_pause_message(
|
||||
message,
|
||||
intention=intention,
|
||||
attendu=attendu,
|
||||
vu=vu,
|
||||
demande=demande,
|
||||
)
|
||||
if warn_visible_message is not None:
|
||||
warn_visible_message(
|
||||
result,
|
||||
source="safety_checks_provider._coerce_pause_message.final",
|
||||
supervised_pause=True,
|
||||
)
|
||||
return result
|
||||
|
||||
fallback_request = "indiquer si je peux continuer ou corriger l'action attendue"
|
||||
result = "\n".join(
|
||||
(
|
||||
f"J'essaie de : {intention or 'continuer une etape supervisee'}",
|
||||
f"J'attendais : {attendu or 'un accord humain clair avant de continuer'}",
|
||||
f"Je vois : {vu or 'je suis sur une etape qui demande une verification humaine'}",
|
||||
f"Peux-tu : {demande or message or fallback_request}",
|
||||
)
|
||||
)
|
||||
if warn_visible_message is not None:
|
||||
warn_visible_message(
|
||||
result,
|
||||
source="safety_checks_provider._coerce_pause_message.final_fallback",
|
||||
supervised_pause=True,
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def _call_llm_for_contextual_checks(
|
||||
action: Dict[str, Any],
|
||||
replay_state: Dict[str, Any],
|
||||
|
||||
@@ -37,6 +37,11 @@ _MODIFIER_ONLY_KEYS = {
|
||||
"meta", "meta_l", "meta_r", "super", "super_l", "super_r",
|
||||
}
|
||||
|
||||
_STANDALONE_SYSTEM_KEYS = {
|
||||
"win", "win_l", "win_r", "cmd", "cmd_l", "cmd_r",
|
||||
"windows", "meta", "meta_l", "meta_r", "super", "super_l", "super_r",
|
||||
}
|
||||
|
||||
# Mapping numpad vk codes → caractères (layout-indépendant)
|
||||
_NUMPAD_VK_MAP = {
|
||||
96: '0', 97: '1', 98: '2', 99: '3', 100: '4',
|
||||
@@ -69,6 +74,18 @@ def _is_modifier_only(keys: list) -> bool:
|
||||
return all(k.lower() in _MODIFIER_ONLY_KEYS for k in keys)
|
||||
|
||||
|
||||
def _is_standalone_system_key(keys: list) -> bool:
|
||||
"""True pour les touches système seules qui sont des gestes utiles."""
|
||||
if len(keys) != 1:
|
||||
return False
|
||||
return str(keys[0]).lower() in _STANDALONE_SYSTEM_KEYS
|
||||
|
||||
|
||||
def _is_ignorable_modifier_only(keys: list) -> bool:
|
||||
"""True pour les modificateurs seuls qui ne doivent pas devenir replay."""
|
||||
return _is_modifier_only(keys) and not _is_standalone_system_key(keys)
|
||||
|
||||
|
||||
def _sanitize_keys(keys: list) -> list:
|
||||
"""Nettoyer une liste de touches : convertir les caractères de contrôle."""
|
||||
cleaned = []
|
||||
@@ -94,7 +111,7 @@ def _is_parasitic_event(event_data: Dict[str, Any]) -> bool:
|
||||
|
||||
if event_type in ("key_press", "key_combo"):
|
||||
keys = event_data.get("keys", event_data.get("data", {}).get("keys", []))
|
||||
if not keys or _is_modifier_only(keys):
|
||||
if not keys or _is_ignorable_modifier_only(keys):
|
||||
return True
|
||||
|
||||
elif event_type == "text_input":
|
||||
@@ -203,7 +220,7 @@ def _filter_parasitic_steps(steps: list) -> list:
|
||||
s for s in steps
|
||||
if not (
|
||||
s.get("type") in ("key_combo", "key_press")
|
||||
and _is_modifier_only(s.get("keys", []))
|
||||
and _is_ignorable_modifier_only(s.get("keys", []))
|
||||
)
|
||||
]
|
||||
|
||||
@@ -266,7 +283,7 @@ def clean_enriched_actions(actions: list) -> list:
|
||||
# key_combo : sanitiser les touches, puis filtrer les modificateurs seuls
|
||||
if atype == 'key_combo':
|
||||
keys = _sanitize_keys(a.get('keys', []))
|
||||
if _is_modifier_only(keys):
|
||||
if _is_ignorable_modifier_only(keys):
|
||||
continue
|
||||
if not keys:
|
||||
continue
|
||||
@@ -328,6 +345,12 @@ _IGNORED_EVENT_TYPES = frozenset({
|
||||
_POST_COMBO_WAITS = {
|
||||
# (tuple de touches normalisées, triées en minuscule) -> wait_ms
|
||||
# NB : les tuples sont sorted() alphabétiquement
|
||||
('win',): 2000, # Win seul → menu/recherche Windows
|
||||
('cmd',): 2000,
|
||||
('s', 'win'): 2000, # Win+S → Recherche Windows
|
||||
('cmd', 's'): 2000,
|
||||
('escape',): 800, # Escape → fermeture menu/dialogue
|
||||
('esc',): 800,
|
||||
('r', 'win'): 3000, # Win+R → Exécuter
|
||||
('r', 'super'): 3000,
|
||||
('meta', 'r'): 3000,
|
||||
@@ -956,8 +979,32 @@ def enrich_click_from_screenshot(
|
||||
vlm_description = ", ".join(vlm_parts) if vlm_parts else ""
|
||||
|
||||
# ── 4. SomEngine : identifier l'élément cliqué ──
|
||||
# C2d-bis (2026-05-25) court-circuits :
|
||||
# Niveau A : si vision_info.text déjà présent, le code priorise vision_info.text
|
||||
# ligne 974-981 de toute façon → SomEngine redondant (économie ~1.2s/clic CPU).
|
||||
# Niveau B : flag RPA_SKIP_BUILD_VISION=true (alias RPA_SKIP_BUILD_VLM)
|
||||
# skip total SomEngine + gemma4 (économie ~4s/clic). Défaut OFF
|
||||
# pour préserver comportement historique.
|
||||
has_vision_text = bool(isinstance(vision_info, dict) and vision_info.get("text"))
|
||||
_skip_flag_raw = (
|
||||
os.environ.get("RPA_SKIP_BUILD_VISION")
|
||||
or os.environ.get("RPA_SKIP_BUILD_VLM")
|
||||
or "0"
|
||||
)
|
||||
skip_build_vision = _skip_flag_raw.strip().lower() in ("1", "true", "yes")
|
||||
|
||||
som_elem = None
|
||||
if session_dir and screenshot_id:
|
||||
if skip_build_vision:
|
||||
logger.debug(
|
||||
"[PERF] vision.skip_som reason=RPA_SKIP_BUILD_VISION click=(%d,%d)",
|
||||
click_x, click_y,
|
||||
)
|
||||
elif has_vision_text:
|
||||
logger.debug(
|
||||
"[PERF] vision.skip_som reason=vision_info.text click=(%d,%d) text=%r",
|
||||
click_x, click_y, vision_info.get("text", "")[:40],
|
||||
)
|
||||
elif session_dir and screenshot_id:
|
||||
# Appeler _som_identify_clicked_element via un event_data minimal
|
||||
fake_event = {
|
||||
"screenshot_id": screenshot_id,
|
||||
@@ -981,10 +1028,15 @@ def enrich_click_from_screenshot(
|
||||
text_source = "ocr"
|
||||
|
||||
# ── 5b. Gemma4 : identifier l'élément cliqué via le screenshot fenêtre ──
|
||||
# Quand l'OCR et SomEngine ne trouvent pas de texte, gemma4 (port 11435)
|
||||
# reçoit le screenshot fenêtre + la position du clic et décrit l'élément.
|
||||
# Un seul appel, une seule fois, pendant l'enregistrement.
|
||||
if not element_text:
|
||||
# Quand l'OCR et SomEngine ne trouvent pas de texte, gemma4 reçoit le
|
||||
# screenshot fenêtre + la position du clic et décrit l'élément.
|
||||
# Skippé si RPA_SKIP_BUILD_VISION actif (Niveau B C2d-bis).
|
||||
if not element_text and skip_build_vision:
|
||||
logger.debug(
|
||||
"[PERF] vision.skip_gemma4 reason=RPA_SKIP_BUILD_VISION click=(%d,%d)",
|
||||
click_x, click_y,
|
||||
)
|
||||
elif not element_text:
|
||||
# Essayer avec le screenshot fenêtre (contexte complet)
|
||||
win_screenshot = None
|
||||
if session_dir and screenshot_id:
|
||||
@@ -1320,6 +1372,157 @@ def _infer_close_tab_target(
|
||||
return None
|
||||
|
||||
|
||||
def _is_notepad_title(title: str) -> bool:
|
||||
"""Retourne True pour les fenêtres Bloc-notes modernes/françaises."""
|
||||
lowered = str(title or "").casefold()
|
||||
return "bloc-notes" in lowered or "notepad" in lowered
|
||||
|
||||
|
||||
def _infer_save_dialog_primary_button_target(
|
||||
raw_events: list,
|
||||
click_event: Dict[str, Any],
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Détecter le bouton primaire du dialogue Windows ``Enregistrer sous``.
|
||||
|
||||
Pattern réel ``sess_20260520T102916_066851`` :
|
||||
- clic dans la fenêtre ``Enregistrer sous`` en bas de la boîte ;
|
||||
- focus immédiat de retour vers ``... – Bloc-notes``.
|
||||
|
||||
Quand OCR/SomEngine sont skippés au build, ce clic restait seulement
|
||||
décrit par position + crop. Le template matching pouvait alors dériver.
|
||||
On encode donc l'intention UI stable : bouton ``Enregistrer``.
|
||||
"""
|
||||
if click_event.get("type") != "mouse_click":
|
||||
return None
|
||||
|
||||
window = click_event.get("window", {})
|
||||
if not isinstance(window, dict):
|
||||
return None
|
||||
|
||||
from_title = str(window.get("title", "") or "").strip()
|
||||
app_name = str(window.get("app_name", "") or "").strip().lower()
|
||||
if from_title.casefold() != "enregistrer sous":
|
||||
return None
|
||||
if app_name and "notepad" not in app_name:
|
||||
return None
|
||||
|
||||
window_capture = click_event.get("window_capture", {})
|
||||
if not isinstance(window_capture, dict):
|
||||
return None
|
||||
click_relative = window_capture.get("click_relative")
|
||||
window_size = window_capture.get("window_size")
|
||||
if not (
|
||||
isinstance(click_relative, list)
|
||||
and len(click_relative) == 2
|
||||
and isinstance(window_size, list)
|
||||
and len(window_size) == 2
|
||||
):
|
||||
return None
|
||||
try:
|
||||
rel_y = float(click_relative[1])
|
||||
win_h = float(window_size[1])
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
if win_h <= 0 or rel_y / win_h < 0.78:
|
||||
# Boutons Enregistrer/Annuler en bas de dialogue.
|
||||
return None
|
||||
|
||||
click_ts = click_event.get("timestamp")
|
||||
click_pos = click_event.get("pos") or []
|
||||
match_idx = None
|
||||
for idx, raw_evt in enumerate(raw_events):
|
||||
event_data = raw_evt.get("event", raw_evt)
|
||||
if event_data.get("type") != "mouse_click":
|
||||
continue
|
||||
if event_data.get("timestamp") != click_ts:
|
||||
continue
|
||||
if (event_data.get("pos") or []) != click_pos:
|
||||
continue
|
||||
match_idx = idx
|
||||
break
|
||||
|
||||
if match_idx is None:
|
||||
return None
|
||||
|
||||
for follow_evt in raw_events[match_idx + 1: match_idx + 6]:
|
||||
follow_data = follow_evt.get("event", follow_evt)
|
||||
follow_type = follow_data.get("type", "")
|
||||
if follow_type in {"mouse_click", "text_input", "key_press", "key_combo"}:
|
||||
return None
|
||||
if follow_type != "window_focus_change":
|
||||
continue
|
||||
|
||||
to_info = follow_data.get("to", {})
|
||||
if not isinstance(to_info, dict):
|
||||
continue
|
||||
to_title = str(to_info.get("title", "") or "").strip()
|
||||
to_app = str(to_info.get("app_name", "") or "").strip().lower()
|
||||
if "notepad" not in to_app or not _is_notepad_title(to_title):
|
||||
continue
|
||||
|
||||
follow_ts = follow_data.get("timestamp")
|
||||
if (
|
||||
isinstance(click_ts, (int, float))
|
||||
and isinstance(follow_ts, (int, float))
|
||||
and follow_ts - click_ts > 3.0
|
||||
):
|
||||
break
|
||||
|
||||
return {
|
||||
"by_text": "Enregistrer",
|
||||
"by_role": "button",
|
||||
"window_title": "Enregistrer sous",
|
||||
"context_hints": {
|
||||
"window_title": "Enregistrer sous",
|
||||
"interaction": "save_dialog_primary_button",
|
||||
"expected_after_window": to_title,
|
||||
},
|
||||
"vlm_description": (
|
||||
"Dans la fenêtre 'Enregistrer sous', le bouton principal "
|
||||
"'Enregistrer' en bas de la boîte de dialogue"
|
||||
),
|
||||
}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _is_post_save_out_of_window_click(event_data: dict) -> bool:
|
||||
"""Vrai pour un clic parasite hors fenêtre juste après sauvegarde Notepad."""
|
||||
if event_data.get("type") != "mouse_click":
|
||||
return False
|
||||
window = event_data.get("window", {})
|
||||
if not isinstance(window, dict):
|
||||
return False
|
||||
if not _is_notepad_title(str(window.get("title", "") or "")):
|
||||
return False
|
||||
|
||||
window_capture = event_data.get("window_capture", {})
|
||||
if not isinstance(window_capture, dict):
|
||||
return False
|
||||
if window_capture.get("click_inside_window") is False:
|
||||
return True
|
||||
|
||||
click_relative = window_capture.get("click_relative")
|
||||
window_size = window_capture.get("window_size")
|
||||
if not (
|
||||
isinstance(click_relative, list)
|
||||
and len(click_relative) == 2
|
||||
and isinstance(window_size, list)
|
||||
and len(window_size) == 2
|
||||
):
|
||||
return False
|
||||
try:
|
||||
rel_x = float(click_relative[0])
|
||||
rel_y = float(click_relative[1])
|
||||
win_w = float(window_size[0])
|
||||
win_h = float(window_size[1])
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
return win_w > 0 and win_h > 0 and (
|
||||
rel_x < 0 or rel_y < 0 or rel_x > win_w or rel_y > win_h
|
||||
)
|
||||
|
||||
|
||||
def _attach_expected_window_before(actions: list, raw_events: list) -> None:
|
||||
"""Attacher la fenêtre attendue AVANT chaque clic en rejouant les
|
||||
raw events et en conservant le dernier ``window_focus_change.to.title``.
|
||||
@@ -1463,6 +1666,17 @@ def _enrich_actions_with_intentions(
|
||||
"""
|
||||
import requests as _requests
|
||||
|
||||
skip_flag = (
|
||||
os.environ.get("RPA_SKIP_INTENTION_ENRICHMENT")
|
||||
or os.environ.get("RPA_SKIP_ENRICHMENT")
|
||||
or ""
|
||||
)
|
||||
if skip_flag.strip().lower() in {"1", "true", "yes", "on"}:
|
||||
logger.info(
|
||||
"Enrichissement intentions désactivé par RPA_SKIP_INTENTION_ENRICHMENT"
|
||||
)
|
||||
return
|
||||
|
||||
gemma4_port = os.environ.get("GEMMA4_PORT", _GEMMA4_PORT)
|
||||
gemma4_url = f"http://localhost:{gemma4_port}/api/chat"
|
||||
|
||||
@@ -1659,6 +1873,21 @@ def build_replay_from_raw_events(
|
||||
if not events:
|
||||
return []
|
||||
|
||||
# C2b 2026-05-25 : instrumentation [PERF] des étapes de build_replay
|
||||
# (décomposition des ~22s restantes après skip enrichissement gemma4).
|
||||
# Préfixe [PERF] cohérent avec arbitrage Codex D3 10:19. Pas de flag :
|
||||
# spans build hors boucle chaude, info permanente OK.
|
||||
import time as _time_perf
|
||||
_perf_t_step = _time_perf.perf_counter()
|
||||
_perf_t_total = _perf_t_step
|
||||
|
||||
def _perf_log(step: str) -> None:
|
||||
nonlocal _perf_t_step
|
||||
now = _time_perf.perf_counter()
|
||||
elapsed_ms = (now - _perf_t_step) * 1000
|
||||
logger.info("[PERF] build.%s session=%s elapsed_ms=%.0f", step, session_id, elapsed_ms)
|
||||
_perf_t_step = now
|
||||
|
||||
# Résoudre le répertoire de session pour les crops visuels
|
||||
session_dir_path = Path(session_dir) if session_dir else None
|
||||
if session_dir_path and not session_dir_path.is_dir():
|
||||
@@ -1675,6 +1904,8 @@ def build_replay_from_raw_events(
|
||||
bool(session_dir_path),
|
||||
)
|
||||
|
||||
_perf_log("step1_extract_resolution")
|
||||
|
||||
# ── 2. Filtrer et normaliser les événements ──
|
||||
actionable_events = []
|
||||
saw_save_combo = False # Tracker Ctrl+S / Ctrl+Shift+S pour la coupure systray
|
||||
@@ -1714,8 +1945,19 @@ def build_replay_from_raw_events(
|
||||
)
|
||||
break
|
||||
|
||||
if _is_post_save_out_of_window_click(event_data):
|
||||
logger.debug(
|
||||
"Coupure du replay : clic post-save hors fenêtre applicative "
|
||||
"(window=%s, click_relative=%s)",
|
||||
(event_data.get("window") or {}).get("title", ""),
|
||||
(event_data.get("window_capture") or {}).get("click_relative"),
|
||||
)
|
||||
break
|
||||
|
||||
actionable_events.append(event_data)
|
||||
|
||||
_perf_log("step2_filter_normalize")
|
||||
|
||||
# ── 3. Fusionner les text_input consécutifs ──
|
||||
# Tous les text_input consécutifs sont fusionnés en un seul, indépendamment
|
||||
# du gap temporel. L'utilisateur tape lettre par lettre mais on veut un
|
||||
@@ -1854,6 +2096,8 @@ def build_replay_from_raw_events(
|
||||
original[:50],
|
||||
)
|
||||
|
||||
_perf_log("step3_merge_text_input")
|
||||
|
||||
# ── 4. Convertir en actions replay normalisées ──
|
||||
actions = []
|
||||
last_ts = 0.0
|
||||
@@ -1977,6 +2221,22 @@ def build_replay_from_raw_events(
|
||||
target_spec["context_hints"] = context_hints
|
||||
action["visual_mode"] = True
|
||||
|
||||
save_dialog_target = _infer_save_dialog_primary_button_target(events, evt)
|
||||
if save_dialog_target:
|
||||
target_spec = action.setdefault("target_spec", {})
|
||||
target_spec["by_text"] = save_dialog_target["by_text"]
|
||||
target_spec["by_text_source"] = "heuristic"
|
||||
target_spec["by_role"] = save_dialog_target["by_role"]
|
||||
target_spec["window_title"] = save_dialog_target["window_title"]
|
||||
target_spec["vlm_description"] = save_dialog_target["vlm_description"]
|
||||
context_hints = dict(target_spec.get("context_hints") or {})
|
||||
context_hints.update(save_dialog_target["context_hints"])
|
||||
target_spec["context_hints"] = context_hints
|
||||
expected_after_window = context_hints.get("expected_after_window")
|
||||
if expected_after_window:
|
||||
action["expected_window_title"] = expected_after_window
|
||||
action["visual_mode"] = True
|
||||
|
||||
elif evt_type == "text_input":
|
||||
text = evt.get("text", "")
|
||||
if not text:
|
||||
@@ -2027,8 +2287,11 @@ def build_replay_from_raw_events(
|
||||
|
||||
actions.append(action)
|
||||
|
||||
_perf_log("step4_convert_actions_and_crops")
|
||||
|
||||
# ── 5. Nettoyage global (dédup combos, sanitize, merge texte, waits) ──
|
||||
actions = clean_enriched_actions(actions)
|
||||
_perf_log("step5_clean_enriched_actions")
|
||||
|
||||
# ── 6. Insérer des waits contextuels après raccourcis critiques ──
|
||||
final_actions = []
|
||||
@@ -2043,6 +2306,8 @@ def build_replay_from_raw_events(
|
||||
"duration_ms": post_wait,
|
||||
})
|
||||
|
||||
_perf_log("step6_insert_contextual_waits")
|
||||
|
||||
# ── 7. Dernier nettoyage des waits consécutifs ──
|
||||
result = []
|
||||
for a in final_actions:
|
||||
@@ -2055,12 +2320,16 @@ def build_replay_from_raw_events(
|
||||
continue
|
||||
result.append(a)
|
||||
|
||||
_perf_log("step7_cleanup_consecutive_waits")
|
||||
|
||||
# ── 8. Attacher les screenshots de référence (état attendu après action) ──
|
||||
# Les screenshots res_shot_XXXX.png capturés 1s après chaque action pendant
|
||||
# l'enregistrement servent de référence pour le contrôle visuel.
|
||||
if session_dir_path:
|
||||
_attach_expected_screenshots(result, events, session_dir_path)
|
||||
|
||||
_perf_log("step8_attach_screenshots")
|
||||
|
||||
# ── 9. Enrichir avec expected_window_title (titre fenêtre attendu après le clic) ──
|
||||
# Pour la vérification post-action : le titre de la fenêtre APRÈS le clic
|
||||
# est le window_title du PROCHAIN clic dans la séquence.
|
||||
@@ -2087,6 +2356,8 @@ def build_replay_from_raw_events(
|
||||
# il prime sur target_spec.window_title obsolète.
|
||||
_attach_expected_window_before(result, events)
|
||||
|
||||
_perf_log("step9_expected_window_title")
|
||||
|
||||
# ── 10. Enrichir avec intention + expected_result via gemma4 (Critic) ──
|
||||
# gemma4 analyse chaque action dans son contexte pour produire :
|
||||
# - intention : ce que l'utilisateur veut accomplir
|
||||
@@ -2099,6 +2370,8 @@ def build_replay_from_raw_events(
|
||||
if session_dir_path:
|
||||
_enrich_actions_with_intentions(result, session_dir_path)
|
||||
|
||||
_perf_log("step10_enrich_intentions_gemma4")
|
||||
|
||||
# ── 11. Consolider avec les apprentissages passés ──
|
||||
# Les replays précédents ont enregistré quelles méthodes marchent
|
||||
# pour quels éléments. On réinjecte ces connaissances dans le workflow.
|
||||
@@ -2115,6 +2388,10 @@ def build_replay_from_raw_events(
|
||||
except Exception as e:
|
||||
logger.debug("Consolidation apprentissage échouée : %s", e)
|
||||
|
||||
_perf_log("step11_replay_learner_consolidation")
|
||||
_total_ms = (_time_perf.perf_counter() - _perf_t_total) * 1000
|
||||
logger.info("[PERF] build.TOTAL session=%s total_ms=%.0f", session_id, _total_ms)
|
||||
|
||||
# Stats visual replay
|
||||
visual_clicks = sum(
|
||||
1 for a in result
|
||||
@@ -2148,8 +2425,9 @@ class StreamProcessor:
|
||||
4. finalize_session() — construit le Workflow via GraphBuilder (DBSCAN)
|
||||
"""
|
||||
|
||||
def __init__(self, data_dir: str = "data/training"):
|
||||
def __init__(self, data_dir: str = "data/training", enable_vlm: bool = False):
|
||||
self.data_dir = Path(data_dir)
|
||||
self._enable_vlm = enable_vlm
|
||||
persist_dir = str(self.data_dir / "streaming_sessions")
|
||||
live_sessions_dir = str(self.data_dir / "live_sessions")
|
||||
self.session_manager = LiveSessionManager(
|
||||
@@ -2290,10 +2568,12 @@ class StreamProcessor:
|
||||
"""
|
||||
if self._initialized:
|
||||
return
|
||||
# Marquer comme initialisé SANS charger les composants GPU
|
||||
self._initialized = True
|
||||
logger.info("StreamProcessor initialisé en mode LÉGER (pas de GPU, pas de VLM)")
|
||||
return
|
||||
if not self._enable_vlm:
|
||||
# Marquer comme initialisé SANS charger les composants GPU. Le serveur
|
||||
# HTTP reste en mode léger ; le worker dédié active enable_vlm=True.
|
||||
self._initialized = True
|
||||
logger.info("StreamProcessor initialisé en mode LÉGER (pas de GPU, pas de VLM)")
|
||||
return
|
||||
|
||||
with self._lock:
|
||||
if self._initialized:
|
||||
@@ -2357,6 +2637,20 @@ class StreamProcessor:
|
||||
logger.error(f" Erreur init FAISSManager: {e}")
|
||||
self._faiss_manager = None
|
||||
|
||||
# N1 anti-poison : en mode VLM, un ScreenAnalyzer absent rend le worker
|
||||
# incapable d'enrichir le moindre screenshot. Ne PAS figer
|
||||
# _initialized=True dans ce cas, sinon l'échec (souvent transitoire :
|
||||
# contention GPU au boot, OOM passager) est mis en cache pour toute la
|
||||
# vie du process — c'est précisément ce qui a provoqué le blocage R6 de
|
||||
# 5 jours (worker vivant mais 0 enrichissement, sans alarme). On laisse
|
||||
# _initialized à False pour réessayer au screenshot / cycle suivant.
|
||||
if self._screen_analyzer is None:
|
||||
logger.critical(
|
||||
"Worker VLM DÉGRADÉ : ScreenAnalyzer indisponible après init "
|
||||
"(_initialized laissé à False, retry au prochain cycle)."
|
||||
)
|
||||
return
|
||||
|
||||
self._initialized = True
|
||||
logger.info("Composants core initialisés.")
|
||||
|
||||
@@ -3115,7 +3409,7 @@ class StreamProcessor:
|
||||
# pour que ScreenAnalyzer crée des ScreenStates avec les bons titres de fenêtre
|
||||
self._restore_window_events(session_id, session_dir)
|
||||
|
||||
# Restaurer les événements utilisateur (mouse_click, text_input, key_press)
|
||||
# Restaurer les événements utilisateur (mouse_click, text_input, key_press, key_combo)
|
||||
# depuis live_events.jsonl → session.events, pour que to_raw_session()
|
||||
# puisse les passer au GraphBuilder (construction des edges/actions)
|
||||
self._restore_user_events(session_id, session_dir)
|
||||
@@ -3377,7 +3671,7 @@ class StreamProcessor:
|
||||
def _restore_user_events(self, session_id: str, session_dir: Path):
|
||||
"""Restaurer les événements utilisateur depuis live_events.jsonl.
|
||||
|
||||
Charge les événements d'action (mouse_click, text_input, key_press)
|
||||
Charge les événements d'action (mouse_click, text_input, key_press, key_combo)
|
||||
dans session.events via session_manager.add_event().
|
||||
Sans cela, to_raw_session() retourne une liste d'events vide,
|
||||
et le GraphBuilder ne peut pas construire les actions des edges.
|
||||
@@ -3423,7 +3717,7 @@ class StreamProcessor:
|
||||
evt_type = event_data.get("type", "")
|
||||
ts = float(event_data.get("timestamp", raw.get("timestamp", 0)))
|
||||
|
||||
if evt_type not in ("mouse_click", "text_input", "key_press"):
|
||||
if evt_type not in ("mouse_click", "text_input", "key_press", "key_combo"):
|
||||
continue
|
||||
|
||||
# Construire le dict d'événement pour add_event()
|
||||
@@ -3438,8 +3732,11 @@ class StreamProcessor:
|
||||
evt_dict["button"] = event_data.get("button", "left")
|
||||
elif evt_type == "text_input":
|
||||
evt_dict["text"] = event_data.get("text", "")
|
||||
elif evt_type == "key_press":
|
||||
elif evt_type in ("key_press", "key_combo"):
|
||||
evt_dict["keys"] = event_data.get("keys", [])
|
||||
raw_keys = event_data.get("raw_keys")
|
||||
if raw_keys:
|
||||
evt_dict["raw_keys"] = raw_keys
|
||||
|
||||
# Copier window info si disponible
|
||||
window = event_data.get("window")
|
||||
|
||||
@@ -34,8 +34,16 @@ class StreamWorker:
|
||||
self.running = False
|
||||
self.processed_files: Set[str] = set()
|
||||
|
||||
# StreamProcessor partagé (créé si non fourni)
|
||||
self.processor = processor or StreamProcessor(data_dir=str(self.live_dir))
|
||||
# StreamProcessor partagé (créé si non fourni). En mode standalone,
|
||||
# live_dir pointe normalement vers data/training/live_sessions ; le
|
||||
# processor doit garder data/training comme racine pour workflows/.
|
||||
processor_data_dir = (
|
||||
self.live_dir.parent if self.live_dir.name == "live_sessions" else self.live_dir
|
||||
)
|
||||
self.processor = processor or StreamProcessor(
|
||||
data_dir=str(processor_data_dir),
|
||||
enable_vlm=True,
|
||||
)
|
||||
|
||||
self._thread: threading.Thread = None
|
||||
|
||||
|
||||
Reference in New Issue
Block a user