snapshot: WIP 5j replay reliability (B1 watchdog + dialog handlers + grounding drift)

Snapshot avant correction du blocage relance Léa (3 incidents 24h: SSH refusé,
polls morts ×2). Point de rollback stable.

Contenu:
- agent_v1/core/executor.py: 5 patchs dialog handling (saveas drift, close_tab
  hotkey fallback, confirm_save Unicode apostrophe, foreground dialog
  recontextualization, runtime_dialog in-loop) + helpers normalize_window_hint,
  requires_post_verify_window_transition
- agent_v1/core/grounding.py: garde drift template fix (fallback_x/y plumbed)
- server_v1/replay_watchdog.py (NEW): orphan watchdog B1, scan 10s timeout 30s
- server_v1/api_stream.py: dispatched_action plumbing, watchdog lifespan,
  metrics endpoint
- server_v1/replay_engine.py: _schedule_retry préserve original_action +
  dispatched_action
- stream_processor.py: gardes _infer_tab_switch_target (no false switch_tab
  on save_as dialog open) + _attach_expected_window_before
- tests/integration: test_replay_watchdog.py (8 cas), test_stream_processor.py
- tests/unit: test_executor_verify_window_guard.py (start_button, close_tab,
  runtime_dialog, post_verify, transition fallbacks)

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Dom
2026-05-24 16:48:37 +02:00
parent 5ea4960e65
commit 7df51d2c79
47 changed files with 9811 additions and 451 deletions

File diff suppressed because it is too large Load Diff

View File

@@ -74,6 +74,142 @@ class GroundingEngine:
"""
self._executor = executor
@staticmethod
def _should_scope_to_active_window(target_spec: Dict[str, Any]) -> bool:
"""Déterminer si le grounding doit être limité à la fenêtre active."""
if str(target_spec.get("screen_scope", "")).strip().lower() == "full_screen":
return False
by_role = str(target_spec.get("by_role", "")).strip().lower()
if by_role in {"start_button"}:
return False
return True
@staticmethod
def _targets_lea_window(target_spec: Dict[str, Any]) -> bool:
"""Déterminer si la cible pointe explicitement vers l'UI de Léa."""
try:
from ..ui.messages import est_fenetre_lea
except Exception:
return False
context_hints = target_spec.get("context_hints") or {}
hints = [
target_spec.get("window_title", ""),
context_hints.get("window_title", ""),
target_spec.get("vlm_description", ""),
target_spec.get("by_text", ""),
]
return any(est_fenetre_lea(str(hint)) for hint in hints if hint)
@staticmethod
def _is_plausible_window_rect(
rect: Optional[List[int]],
title: str,
screen_width: int,
screen_height: int,
) -> bool:
"""Valider qu'un rect actif ressemble à une vraie fenêtre utilisable.
Rejette explicitement les zones système "bar-like" (taskbar, systray)
et les titres inconnus/bruités. Le grounding ne doit jamais se
contraindre à une zone non validée.
"""
if not rect or len(rect) != 4:
return False
try:
from ..ui.messages import est_fenetre_bruit
except Exception:
def est_fenetre_bruit(_title: str) -> bool:
return not _title or _title.strip().lower() == "unknown_window"
w = rect[2] - rect[0]
h = rect[3] - rect[1]
title_clean = str(title or "").strip()
if w <= 50 or h <= 50:
return False
title_lower = title_clean.lower()
is_unknown_title = not title_clean or title_lower == "unknown_window"
if not is_unknown_title and est_fenetre_bruit(title_clean):
return False
# Une zone très plate, surtout en bas d'écran et très large, est
# typiquement une barre des tâches / systray, pas une vraie fenêtre.
# On réduit le seuil de hauteur à 120px pour ne pas rejeter les petits modaux.
is_bar_like = (
h < 120
or (w > 0.9 * screen_width and h < 0.15 * screen_height)
)
# Exception : si le titre contient un mot-clé de dialogue connu,
# on considère que c'est plausible même si c'est petit.
keywords = ["enregistrer sous", "save as", "voulez-vous", "confirm", "attention", "error", "erreur"]
if any(k in title_lower for k in keywords):
return h >= 80 # Un dialogue fait au moins 80px (titre + bouton)
return not is_bar_like
@staticmethod
def _visual_scope_hints(target_spec: Dict[str, Any]) -> List[str]:
"""Construire des indices textuels à chercher dans le crop fenêtre."""
hints: List[str] = []
raw_hints = [
target_spec.get("window_title", ""),
(target_spec.get("context_hints") or {}).get("window_title", ""),
target_spec.get("by_text", ""),
]
for raw in raw_hints:
text = str(raw or "").strip()
if not text:
continue
text = text.lstrip("*").strip()
variants = [text]
for sep in (" ", " - ", ""):
if sep in text:
variants.extend(part.strip().lstrip("*") for part in text.split(sep))
for variant in variants:
if variant and len(variant) >= 3 and variant not in hints:
hints.append(variant)
return hints
def _window_crop_matches_target_visually(
self,
screenshot_b64: str,
target_spec: Dict[str, Any],
) -> bool:
"""Vérifier visuellement qu'un crop contraint contient la bonne cible.
Principe: ne jamais faire confiance au rect système seul. Si aucun
indice textuel n'est disponible, on laisse passer le crop plausible
pour ne pas sur-bloquer les cibles purement iconiques.
"""
hints = self._visual_scope_hints(target_spec)
if not hints:
return True
finder = getattr(self._executor, "_find_text_on_screen", None)
if not callable(finder):
return True
for hint in hints:
try:
if finder(screenshot_b64, hint):
logger.info(
"Grounding fenêtre validé visuellement via '%s'",
hint,
)
return True
except Exception as e:
logger.debug("Validation visuelle du crop échouée pour '%s': %s", hint, e)
logger.info(
"Grounding plein écran : crop fenêtre rejeté par validation visuelle "
"(hints=%s)",
hints,
)
return False
def locate(
self,
server_url: str,
@@ -128,35 +264,63 @@ class GroundingEngine:
t_start = time.time()
# ── Capture contrainte à la fenêtre active ──
# Le grounding ne voit QUE la fenêtre attendue — pas la taskbar,
# pas le systray, pas les autres apps. Comme un humain qui regarde
# l'application sur laquelle il travaille.
window_rect = None
try:
from ..window_info_crossplatform import get_active_window_rect
win_info = get_active_window_rect()
if win_info and win_info.get("rect"):
r = win_info["rect"] # [left, top, right, bottom]
# Validation : fenêtre visible et pas minuscule
w = r[2] - r[0]
h = r[3] - r[1]
if w > 50 and h > 50:
window_rect = {
"left": max(0, r[0]),
"top": max(0, r[1]),
"width": min(w, screen_width),
"height": min(h, screen_height),
}
logger.info(
f"Grounding contraint à la fenêtre : "
f"{window_rect['width']}x{window_rect['height']} "
f"à ({window_rect['left']}, {window_rect['top']})"
)
except Exception as e:
logger.debug(f"Pas de window rect disponible : {e}")
active_title = ""
if self._should_scope_to_active_window(target_spec):
# ── Capture contrainte à la fenêtre active ──
# Le grounding ne voit QUE la fenêtre attendue — pas la taskbar,
# pas le systray, pas les autres apps. Comme un humain qui regarde
# l'application sur laquelle il travaille.
try:
from ..window_info_crossplatform import get_active_window_rect
from ..ui.messages import est_fenetre_lea
win_info = get_active_window_rect()
if win_info and win_info.get("rect"):
active_title = str(win_info.get("title", "") or "")
if est_fenetre_lea(active_title) and not self._targets_lea_window(target_spec):
logger.info(
"Grounding plein écran : fenêtre active Léa ignorée pour "
"cible externe (%s)",
target_spec.get("by_text", "") or target_spec.get("by_role", ""),
)
win_info = None
if win_info and win_info.get("rect"):
r = win_info["rect"] # [left, top, right, bottom]
if self._is_plausible_window_rect(r, active_title, screen_width, screen_height):
w = r[2] - r[0]
h = r[3] - r[1]
window_rect = {
"left": max(0, r[0]),
"top": max(0, r[1]),
"width": min(w, screen_width),
"height": min(h, screen_height),
}
logger.info(
f"Grounding contraint à la fenêtre : "
f"{window_rect['width']}x{window_rect['height']} "
f"à ({window_rect['left']}, {window_rect['top']})"
)
else:
logger.info(
"Grounding plein écran : rect actif rejeté "
"(title='%s', rect=%s)",
active_title,
r,
)
except Exception as e:
logger.debug(f"Pas de window rect disponible : {e}")
else:
logger.info(
"Grounding plein écran pour by_role='%s'",
target_spec.get("by_role", ""),
)
screenshot_b64 = self._capture_window_or_screen(window_rect)
if window_rect and screenshot_b64:
if not self._window_crop_matches_target_visually(screenshot_b64, target_spec):
window_rect = None
screenshot_b64 = self._capture_window_or_screen(None)
if not screenshot_b64:
return GroundingResult(
found=False, detail="Capture screenshot échouée",
@@ -186,6 +350,18 @@ class GroundingEngine:
result.elapsed_ms = (time.time() - t_start) * 1000
return result
if target_spec.get("allow_position_fallback"):
if 0.0 <= fallback_x <= 1.0 and 0.0 <= fallback_y <= 1.0:
return GroundingResult(
found=True,
x_pct=fallback_x,
y_pct=fallback_y,
method="position_fallback",
score=0.2,
detail="fallback positionnel explicite",
elapsed_ms=(time.time() - t_start) * 1000,
)
return GroundingResult(
found=False,
detail=f"Toutes les stratégies ont échoué ({', '.join(strategies)})",
@@ -258,7 +434,12 @@ class GroundingEngine:
anchor_b64 = target_spec.get("anchor_image_base64", "")
if anchor_b64:
raw = self._executor._template_match_anchor(
screenshot_b64, anchor_b64, screen_width, screen_height,
screenshot_b64,
anchor_b64,
screen_width,
screen_height,
fallback_x_pct=fallback_x,
fallback_y_pct=fallback_y,
)
if raw and raw.get("resolved"):
return GroundingResult(

View File

@@ -0,0 +1,39 @@
"""Dispatch léger du contrat enrichi de /finalize côté agent."""
from __future__ import annotations
import logging
from typing import Any, Dict
logger = logging.getLogger(__name__)
def dispatch_finalize_result(ui: Any, payload: Dict[str, Any], replay_name: str) -> None:
"""Router le résultat de /finalize vers la bonne surface UI agent."""
if not isinstance(payload, dict):
return
replay_request = payload.get("replay_request") or {}
replay_launch = payload.get("replay_launch") or {}
if replay_launch.get("status") == "started":
logger.info("Replay direct déjà lancé par le serveur après finalize")
return
if not payload.get("replay_ready") or not replay_request:
return
if replay_launch.get("status") == "failed":
logger.warning(
"Auto-replay serveur échoué après finalize, proposition manuelle"
)
if ui is None or not hasattr(ui, "offer_finalize_replay"):
logger.info("UI indisponible pour proposer un test immédiat")
return
ui.offer_finalize_replay(
replay_request,
replay_name or "la tâche que vous venez d'enregistrer",
)

View File

@@ -28,6 +28,7 @@ from .ui.chat_window import ChatWindow
from .ui.capture_server import CaptureServer
from .session.storage import SessionStorage
from .vision.capturer import VisionCapturer
from .finalize_contract import dispatch_finalize_result
# Import optionnel du client serveur (pour le chat et les workflows)
# Deux chemins : relatif (depuis agent_v0.agent_v1) ou absolu (depuis C:\rpa_vision\agent_v1)
@@ -80,6 +81,7 @@ class AgentV1:
self._executor = None
# Flag pour indiquer qu'un replay est en cours (eviter les conflits)
self._replay_active = False
self._last_recording_name = ""
# Etat partage entre systray et chat (source de verite unique)
self._state = AgentState()
@@ -210,12 +212,14 @@ class AgentV1:
time.sleep(30) # Vérifier toutes les 30s
def start_session(self, workflow_name):
self._last_recording_name = workflow_name
self.session_id = f"sess_{time.strftime('%Y%m%dT%H%M%S')}_{uuid.uuid4().hex[:6]}"
self.session_dir = self.storage.get_session_dir(self.session_id)
self.vision = VisionCapturer(str(self.session_dir))
self.streamer = TraceStreamer(self.session_id, machine_id=self.machine_id)
self.streamer.set_on_finalize_result(self._on_finalize_result)
self.captor = EventCaptorV1(self._on_event_bridge)
# Initialiser l'executeur partage
@@ -325,6 +329,15 @@ class AgentV1:
# pour enchainer les actions du workflow
time.sleep(0.2)
else:
if getattr(self._executor, "_replay_paused", False):
if not self._replay_active:
self._replay_active = True
self.ui.set_replay_active(True)
self._state.set_replay_active(True)
poll_delay = getattr(self._executor, '_poll_backoff', REPLAY_POLL_INTERVAL)
time.sleep(max(poll_delay, REPLAY_POLL_INTERVAL))
continue
# Pas d'action en attente — utiliser le backoff de l'executor
# (augmente si le serveur est indisponible, reset a 1s sinon)
if self._replay_active:
@@ -429,6 +442,11 @@ class AgentV1:
f"agent_{self.user_id}"
)
def _on_finalize_result(self, payload: dict) -> None:
"""Réagir au contrat enrichi de /finalize côté agent."""
replay_name = self._last_recording_name or "la tâche que vous venez d'enregistrer"
dispatch_finalize_result(self.ui, payload, replay_name)
_last_heartbeat_hash: str = ""
def _heartbeat_loop(self):

View File

@@ -30,6 +30,7 @@ import os
import queue
import threading
import time
from typing import Callable, Optional
import requests
from PIL import Image
@@ -95,6 +96,11 @@ class TraceStreamer:
# Initialisé paresseusement pour ne pas payer le coût SQLite en dehors
# d'un streaming actif.
self._buffer: PersistentBuffer | None = None
self._on_finalize_result: Optional[Callable[[dict], None]] = None
def set_on_finalize_result(self, callback: Optional[Callable[[dict], None]]) -> None:
"""Définir un callback appelé avec le payload JSON de /finalize."""
self._on_finalize_result = callback
def _get_buffer(self) -> PersistentBuffer:
"""Retourne le buffer persistant, en l'initialisant au besoin."""
@@ -621,6 +627,14 @@ class TraceStreamer:
if resp.ok:
result = resp.json()
logger.info(f"Session finalisée: {result}")
if self._on_finalize_result is not None:
try:
self._on_finalize_result(result)
except Exception as cb_error:
logger.warning(
"Callback finalize ignoré après erreur: %s",
cb_error,
)
else:
logger.warning(f"Finalisation échouée: {resp.status_code}")
except Exception as e:

View File

@@ -158,14 +158,25 @@ class CaptureHandler(BaseHTTPRequestHandler):
"""Capture l'ecran principal et le renvoie en base64 JPEG."""
t0 = time.perf_counter()
try:
import mss
from PIL import Image
from ..vision.capturer import (
capture_foreground_window_image,
capture_screen_image,
)
with mss.mss() as sct:
monitor = sct.monitors[1] # ecran principal
raw = sct.grab(monitor)
img = Image.frombytes("RGB", raw.size, raw.bgra, "raw", "BGRX")
_monitor, img, meta = capture_screen_image()
if img is None:
img, win_meta = capture_foreground_window_image()
meta.update(win_meta)
if img is None:
elapsed_ms = (time.perf_counter() - t0) * 1000
logger.error("Erreur capture : aucun backend exploitable (%s)", meta)
self._send_json(503, {
"error": "capture_unavailable",
"source": meta.get("backend", "unknown"),
"capture_ms": round(elapsed_ms),
"diagnostics": meta,
})
return
# Floutage des données sensibles (conformité AI Act)
if BLUR_SENSITIVE:
@@ -180,15 +191,22 @@ class CaptureHandler(BaseHTTPRequestHandler):
img_b64 = base64.b64encode(buf.getvalue()).decode()
elapsed_ms = (time.perf_counter() - t0) * 1000
logger.info(f"Capture {img.width}x{img.height} en {elapsed_ms:.0f}ms")
logger.info(
"Capture %sx%s via %s en %.0fms",
img.width,
img.height,
meta.get("backend", "unknown"),
elapsed_ms,
)
self._send_json(200, {
"image": img_b64,
"width": img.width,
"height": img.height,
"format": "jpeg",
"source": "windows_live",
"source": meta.get("backend", "windows_live"),
"capture_ms": round(elapsed_ms),
"diagnostics": meta,
})
except Exception as e:

View File

@@ -894,6 +894,34 @@ class ChatWindow:
except Exception:
logger.debug("clear chat history silenced", exc_info=True)
@staticmethod
def _compute_paused_bubble_height(reason_str: str) -> tuple:
"""Calcule la hauteur du Text (en lignes) + si une scrollbar est
nécessaire pour le message d'une bulle paused.
Patch 22 mai 2026 — fix troncature : on prend en compte les \\n
explicites (les `reason` serveur peuvent lister plusieurs
candidats avec un saut de ligne par item) en plus de la longueur
en caractères, et on active la scrollbar dès que le cap est
atteint pour éviter que du contenu disparaisse silencieusement.
Retourne ``(height_lines, needs_scrollbar)``.
"""
if not reason_str:
return 2, False
text = str(reason_str)
# Estimation : ~60 chars/ligne effectifs avec wraplength.
wrapped_lines = (len(text) // 60) + 1
explicit_lines = text.count("\n") + 1
estimated = max(wrapped_lines, explicit_lines)
cap = 12
height = max(2, min(cap, estimated))
# Scrollbar dès que le cap est atteint OU contenu long (filet
# textuel : ≥ 200 chars implique souvent un débordement visuel
# même quand les lignes brutes sont peu nombreuses).
needs_scroll = (estimated >= cap) or (len(text) > 200)
return height, needs_scroll
def _render_paused_bubble(self, payload: Dict[str, Any]) -> None:
tk = self._tk
if getattr(self, "_msg_frame", None) is None:
@@ -923,22 +951,23 @@ class ChatWindow:
# Message scrollable pour les longs reasons (ex: 200+ chars depuis le serveur).
# On utilise un Text en mode read-only avec hauteur calculée selon la longueur.
# Au-delà de 280 chars, scrollbar interne ; sinon Text auto-fitté.
# Patch 22 mai 2026 : prendre en compte les \n explicites (titres
# fenêtre / patterns) et activer la scrollbar dès que le cap de
# hauteur est atteint — sinon les bulles de pause étaient
# tronquées visuellement sans aucun ascenseur visible.
reason_str = str(reason)
# Estimation simple : ~70 chars/ligne avec wraplength
approx_lines = max(2, min(8, (len(reason_str) // 60) + 1))
height_lines, needs_scroll = self._compute_paused_bubble_height(reason_str)
msg_frame = tk.Frame(inner, bg=PAUSED_BG)
msg_frame.pack(fill=tk.X, anchor=tk.W, pady=(6, 0))
reason_text = tk.Text(
msg_frame, bg=PAUSED_BG, fg=PAUSED_FG,
font=FONT_MSG, wrap=tk.WORD, bd=0, height=approx_lines,
font=FONT_MSG, wrap=tk.WORD, bd=0, height=height_lines,
highlightthickness=0, relief=tk.FLAT, cursor="arrow",
)
reason_text.insert("1.0", reason_str)
reason_text.configure(state="disabled")
reason_text.pack(side=tk.LEFT, fill=tk.X, expand=True)
# Scrollbar interne uniquement si le contenu déborde (long messages)
if len(reason_str) > 280:
if needs_scroll:
reason_scroll = tk.Scrollbar(
msg_frame, orient=tk.VERTICAL,
command=reason_text.yview, width=8,
@@ -1019,27 +1048,40 @@ class ChatWindow:
UX fix 8 mai 2026 : on désactive les 2 boutons et on affiche un message
de feedback dès le clic, sans attendre l'ack serveur. Le bus émet en
arrière-plan ; si la connexion est tombée, on log un warning visible.
Fallback HTTP 22 mai 2026 : si le bus SocketIO est déconnecté, on
retombe sur un POST direct ``/replay/{id}/resume`` via
``server_client``. Si les deux échouent, on ré-active les boutons
et on saute l'auto-hide pour permettre à l'utilisateur de
réessayer manuellement (sinon le replay reste figé côté serveur).
"""
if not replay_id:
self._update_paused_feedback("⚠ replay_id manquant — impossible de relancer")
return
emitted = False
if self._bus is not None and self._bus.connected:
emitted = self._bus.resume_replay(replay_id)
# Feedback immédiat : disable boutons + message
emitted, channel = self._dispatch_paused_action(
replay_id,
bus_method="resume_replay",
client_method="resume_replay",
)
self._disable_paused_buttons()
if emitted:
self._update_paused_feedback("→ Reprise demandée…")
logger.info("paused_bubble: lea:replay_resume émis pour %s", replay_id)
else:
self._update_paused_feedback("⚠ Bus indisponible — réessayez dans 5s")
logger.warning("paused_bubble: bus déconnecté, resume non émis")
# UX fix mai 2026 : minimiser la fenêtre vers le systray après 500ms
# (laisse à l'utilisateur le temps de voir "Reprise demandée…").
try:
self._root.after(500, self._do_hide)
except Exception:
logger.debug("auto-hide on resume silenced", exc_info=True)
logger.info(
"paused_bubble: replay_resume émis pour %s via %s",
replay_id, channel,
)
try:
self._root.after(500, self._do_hide)
except Exception:
logger.debug("auto-hide on resume silenced", exc_info=True)
return
# Échec sur les deux canaux : laisser l'utilisateur réessayer.
self._update_paused_feedback("⚠ Serveur injoignable — réessayez")
self._enable_paused_buttons()
logger.warning(
"paused_bubble: bus et HTTP indisponibles, resume non émis "
"pour %s", replay_id,
)
def _on_paused_abort(self, replay_id: str) -> None:
"""Bouton Annuler : émettre lea:replay_abort + fermeture locale immédiate.
@@ -1048,17 +1090,30 @@ class ChatWindow:
n'envoie pas de lea:resumed pour un abort, donc sans cette fermeture
locale la bulle restait coincée — c'était la cause de "Annuler ne
fonctionne pas" rapportée par Dom).
Fallback HTTP 22 mai 2026 : symétrique de ``_on_paused_resume`` —
si le bus est déconnecté, POST direct ``/replay/{id}/cancel``.
L'abort ferme la bulle localement quelle que soit l'issue (l'état
serveur sera réconcilié au prochain poll /replay/next).
"""
emitted = False
if self._bus is not None and self._bus.connected:
emitted = self._bus.abort_replay(replay_id)
emitted, channel = self._dispatch_paused_action(
replay_id,
bus_method="abort_replay",
client_method="abort_replay",
)
self._disable_paused_buttons()
if emitted:
self._update_paused_feedback("✗ Annulé")
logger.info("paused_bubble: lea:replay_abort émis pour %s", replay_id)
logger.info(
"paused_bubble: replay_abort émis pour %s via %s",
replay_id, channel,
)
else:
self._update_paused_feedback("✗ Annulé (bus indisponible)")
logger.warning("paused_bubble: bus déconnecté, abort non émis")
self._update_paused_feedback("✗ Annulé (serveur injoignable)")
logger.warning(
"paused_bubble: bus et HTTP indisponibles, abort non émis "
"pour %s", replay_id,
)
# Fermer la bulle en local (l'abort n'a pas de lea:resumed associé)
self._close_active_paused_bubble(reason="abort_local")
# UX fix mai 2026 : minimiser la fenêtre après 500ms (cohérence
@@ -1068,6 +1123,34 @@ class ChatWindow:
except Exception:
logger.debug("auto-hide on abort silenced", exc_info=True)
def _dispatch_paused_action(
self,
replay_id: str,
bus_method: str,
client_method: str,
) -> tuple:
"""Envoyer une action de bulle paused via bus puis fallback HTTP.
Retourne ``(emitted, channel)`` où ``channel`` vaut ``"bus"``,
``"http"`` ou ``""`` (aucun chemin n'a abouti).
"""
if self._bus is not None and getattr(self._bus, "connected", False):
try:
if getattr(self._bus, bus_method)(replay_id):
return True, "bus"
except Exception:
logger.debug("paused_bubble: bus %s silenced", bus_method, exc_info=True)
if self._server_client is not None and hasattr(self._server_client, client_method):
try:
if getattr(self._server_client, client_method)(replay_id):
return True, "http"
except Exception:
logger.debug(
"paused_bubble: server_client %s silenced",
client_method, exc_info=True,
)
return False, ""
def _disable_paused_buttons(self) -> None:
if not self._active_paused_bubble:
return
@@ -1077,6 +1160,19 @@ class ChatWindow:
except Exception:
logger.debug("disable paused buttons silenced", exc_info=True)
def _enable_paused_buttons(self) -> None:
"""Ré-activer les boutons Continuer/Annuler de la bulle paused
active. Appelé quand l'envoi a échoué sur tous les canaux —
l'utilisateur doit pouvoir réessayer manuellement.
"""
if not self._active_paused_bubble:
return
try:
self._active_paused_bubble["btn_resume"].config(state="normal")
self._active_paused_bubble["btn_abort"].config(state="normal")
except Exception:
logger.debug("enable paused buttons silenced", exc_info=True)
def _update_paused_feedback(self, text: str) -> None:
if not self._active_paused_bubble:
return

View File

@@ -504,6 +504,100 @@ class SmartTrayV1:
threading.Thread(target=_replay, daemon=True).start()
def _launch_replay_request(
self,
replay_request: Dict[str, Any],
replay_name: str,
) -> None:
"""Lance un replay direct depuis un payload `replay_request` serveur."""
endpoint = (replay_request or {}).get("endpoint", "")
session_id = (replay_request or {}).get("session_id", "")
machine_id = (replay_request or {}).get("machine_id") or self.machine_id
if endpoint != "/api/v1/traces/stream/replay-session" or not session_id:
logger.warning("Replay request non supporté: %s", replay_request)
self._notifier.notify(
"Léa",
"Je ne peux pas lancer ce test automatique pour le moment.",
)
return
def _replay():
if self.server_client is None:
return
with self._state_lock:
self._replay_active = True
self._update_icon()
self._notifier.notify(
"Léa",
f"Le système d'intelligence artificielle exécute la "
f"tâche '{replay_name}' sur votre écran.",
)
try:
import requests
auth_headers = {}
if self.server_client is not None:
auth_headers = self.server_client._auth_headers()
resp = requests.post(
f"{self.server_client._stream_base}{endpoint}",
params={
"session_id": session_id,
"machine_id": machine_id,
},
headers=auth_headers,
timeout=30,
allow_redirects=False,
)
if resp.ok:
logger.info(
"Replay direct démarré pour session %s (machine=%s)",
session_id,
machine_id,
)
else:
self._notifier.notify(
"Léa",
"Hmm, le serveur a refusé le test immédiat.",
)
except Exception as e:
logger.error("Erreur lancement replay direct : %s", e)
self._notifier.notify(
"Léa",
f"Oups, un problème : {e}",
)
finally:
with self._state_lock:
self._replay_active = False
self._update_icon()
threading.Thread(target=_replay, daemon=True).start()
def offer_finalize_replay(
self,
replay_request: Dict[str, Any],
replay_name: str,
) -> None:
"""Proposer à l'utilisateur de tester immédiatement la tâche apprise."""
if not replay_request or not replay_request.get("session_id"):
return
def _offer():
self._notifier.notify(
"Léa",
f"J'ai compris la tâche '{replay_name}'. Voulez-vous la tester ?",
)
if not _ask_consent(
"Léa — Test immédiat",
f"J'ai compris la tâche '{replay_name}'. "
"Voulez-vous la tester maintenant ?",
):
return
self._launch_replay_request(replay_request, replay_name)
threading.Thread(target=_offer, daemon=True).start()
def _on_emergency_stop(self, _icon=None, _item=None) -> None:
"""Arret d'urgence — stoppe TOUTES les activites de l'agent immediatement.

View File

@@ -15,7 +15,7 @@ import time
import logging
import hashlib
import platform
from typing import Any, Dict, List, Optional
from typing import Any, Dict, List, Optional, Tuple
from PIL import Image, ImageFilter, ImageStat
import mss
from ..config import TARGETED_CROP_SIZE, SCREENSHOT_QUALITY, BLUR_SENSITIVE
@@ -86,6 +86,337 @@ def _enrich_with_monitor_info(payload: dict) -> dict:
payload["monitors_geometry"] = _get_monitors_geometry()
return payload
# Garde dimensions monitor (démo GHT 19 mai 2026) : mss.monitors[1] peut
# retourner intermittemment des dims tronquées (cas observé 2560×60). Utiliser
# ces dims pour normaliser des coords empoisonne la mémoire (TargetMemoryStore).
MIN_MONITOR_WIDTH = 200
MIN_MONITOR_HEIGHT = 200
MONITOR_MAX_ATTEMPTS = 2
MONITOR_RETRY_DELAY_S = 0.05
BLACK_FRAME_MEAN_MAX = 1.0
BLACK_FRAME_STDDEV_MAX = 1.0
BLACK_FRAME_MAX_LUMA = 3
def _is_monitor_sane(monitor) -> bool:
"""True si les dims du monitor sont au-dessus du seuil de plausibilité."""
if not isinstance(monitor, dict):
return False
w = monitor.get("width", 0) or 0
h = monitor.get("height", 0) or 0
return w >= MIN_MONITOR_WIDTH and h >= MIN_MONITOR_HEIGHT
def _dim_str(monitor) -> str:
"""Représentation courte WxH pour les logs (gère monitor=None)."""
if not isinstance(monitor, dict):
return "?x?"
return f"{monitor.get('width', '?')}x{monitor.get('height', '?')}"
def _acquire_safe_grab(max_attempts: int = MONITOR_MAX_ATTEMPTS,
retry_delay_s: float = MONITOR_RETRY_DELAY_S,
allow_secondary_fallback: bool = True):
"""Ouvre mss et capture un monitor avec dimensions plausibles.
Stratégie en cascade :
1. À chaque tentative, ouvrir un nouveau `mss.mss()` (peut rafraîchir le
cache interne) et examiner monitors[1..n].
2. Préférer monitors[1] (écran principal physique). Si aberrant ET
`allow_secondary_fallback=True`, prendre le premier monitors[2..n]
sain avec un WARNING explicite.
3. Si `allow_secondary_fallback=False`, on n'accepte QUE monitors[1].
Utile pour les méthodes qui reçoivent des coordonnées (x, y) en
système écran composite : capturer un monitor secondaire produirait
une image saine mais décalée par rapport à ces coords.
4. Si aucune dim plausible : attendre `retry_delay_s` et retenter.
5. Après `max_attempts` infructueuses : log ERROR et retourner
(None, None) pour que l'appelant tombe en sortie d'erreur explicite.
Args:
max_attempts: nombre de tentatives mss avant abandon.
retry_delay_s: délai entre tentatives.
allow_secondary_fallback: si False, refuser monitors[2..n] (fail-closed
pour les méthodes coord-bearing).
Returns:
Tuple (monitor_dict, PIL.Image) si capture saine réussie,
(None, None) sinon.
"""
last_aberrant = None
secondary_seen = False # un monitor secondaire sain a été vu mais refusé
for attempt in range(max_attempts):
with mss.mss() as sct:
monitors = list(sct.monitors) if sct.monitors else []
chosen = None
chosen_idx = None
for idx in range(1, len(monitors)):
candidate = monitors[idx]
if not _is_monitor_sane(candidate):
last_aberrant = candidate
logger.warning(
"Monitor[%d] dims aberrantes (%s, seuil %dx%d) "
"— attempt %d/%d",
idx, _dim_str(candidate),
MIN_MONITOR_WIDTH, MIN_MONITOR_HEIGHT,
attempt + 1, max_attempts,
)
continue
# Monitor sain trouvé
if idx == 1 or allow_secondary_fallback:
chosen = candidate
chosen_idx = idx
break
# Sinon : sain mais secondaire interdit pour cet appelant
secondary_seen = True
logger.warning(
"Monitor[%d] sain (%s) mais fallback secondaire refusé "
"(allow_secondary_fallback=False) — capture cohérente "
"des coords impossible",
idx, _dim_str(candidate),
)
if chosen is not None:
if chosen_idx != 1 or attempt > 0:
logger.warning(
"Capture fallback : monitor[%d] dim=%s, attempt=%d",
chosen_idx, _dim_str(chosen), attempt + 1,
)
sct_img = sct.grab(chosen)
img = Image.frombytes(
"RGB", sct_img.size, sct_img.bgra, "raw", "BGRX",
)
return chosen, img
if attempt < max_attempts - 1:
time.sleep(retry_delay_s)
if secondary_seen and not allow_secondary_fallback:
logger.error(
"Capture abandonnée : monitor[1] aberrant après %d tentatives "
"(dernier vu %s) et fallback secondaire désactivé "
"pour préserver la cohérence des coordonnées",
max_attempts, _dim_str(last_aberrant),
)
else:
logger.error(
"Aucun monitor avec dims plausibles trouvé après %d tentatives "
"(dernier vu : %s, seuil %dx%d) — capture abandonnée",
max_attempts, _dim_str(last_aberrant),
MIN_MONITOR_WIDTH, MIN_MONITOR_HEIGHT,
)
return None, None
def _compute_luma_stats(img: Image.Image) -> Dict[str, float | int]:
"""Retourne des stats simples de luminance pour diagnostiquer un frame noir."""
gray = img.convert("L")
stat = ImageStat.Stat(gray)
min_luma, max_luma = gray.getextrema()
return {
"mean": round(float(stat.mean[0]) if stat.mean else 0.0, 2),
"stddev": round(float(stat.stddev[0]) if stat.stddev else 0.0, 2),
"min": int(min_luma),
"max": int(max_luma),
}
def _is_effectively_black(img: Image.Image) -> bool:
"""Heuristique fail-closed pour refuser un screenshot pratiquement noir."""
stats = _compute_luma_stats(img)
return (
stats["max"] <= BLACK_FRAME_MAX_LUMA
and stats["mean"] <= BLACK_FRAME_MEAN_MAX
and stats["stddev"] <= BLACK_FRAME_STDDEV_MAX
)
def _capture_via_imagegrab() -> Tuple[Optional[Dict[str, int]], Optional[Image.Image], Dict[str, Any]]:
"""Fallback Windows via Pillow/ImageGrab.
Utile quand `mss` retourne un frame noir alors que la session graphique
utilisateur reste visible.
"""
if _SYSTEM != "Windows":
return None, None, {"backend": "imagegrab", "error": "unsupported_platform"}
try:
from PIL import ImageGrab
except ImportError as exc:
return None, None, {"backend": "imagegrab", "error": str(exc)}
try:
img = ImageGrab.grab(all_screens=True)
except Exception as exc:
logger.warning("ImageGrab indisponible pour le fallback capture : %s", exc)
return None, None, {"backend": "imagegrab", "error": str(exc)}
monitor = {"left": 0, "top": 0, "width": img.width, "height": img.height}
return monitor, img, {
"backend": "imagegrab",
"luma": _compute_luma_stats(img),
}
def capture_screen_image(
allow_secondary_fallback: bool = True,
) -> Tuple[Optional[Dict[str, int]], Optional[Image.Image], Dict[str, Any]]:
"""Capture plein écran avec diagnostic noir + fallback Windows.
Returns:
(monitor, image, meta) où image peut être None si aucun backend plein
écran n'a produit une image exploitable.
"""
monitor, img = _acquire_safe_grab(
allow_secondary_fallback=allow_secondary_fallback
)
meta: Dict[str, Any] = {"backend": "mss"}
if img is not None:
meta["luma"] = _compute_luma_stats(img)
if not _is_effectively_black(img):
return monitor, img, meta
logger.warning(
"Capture mss quasi noire (%s) — tentative de fallback",
meta["luma"],
)
meta["mss_black_frame"] = True
else:
meta["mss_unavailable"] = True
fallback_monitor, fallback_img, fallback_meta = _capture_via_imagegrab()
if fallback_img is not None:
if not _is_effectively_black(fallback_img):
logger.warning(
"Capture fallback via ImageGrab (%sx%s)",
fallback_img.width,
fallback_img.height,
)
return fallback_monitor, fallback_img, fallback_meta
logger.warning(
"Capture ImageGrab quasi noire (%s)",
fallback_meta.get("luma"),
)
meta["imagegrab_black_frame"] = True
meta["imagegrab_error"] = fallback_meta.get("error")
return None, None, meta
def _capture_window_image_windows(
hwnd: int,
width: int,
height: int,
) -> Tuple[Optional[Image.Image], Dict[str, Any]]:
"""Capture une fenêtre Windows via PrintWindow.
Fallback utile quand la capture plein écran est noire mais que la fenêtre
active reste imprimable par l'API Win32.
"""
if _SYSTEM != "Windows":
return None, {"backend": "printwindow", "error": "unsupported_platform"}
try:
import ctypes
import win32gui
import win32ui
except ImportError as exc:
return None, {"backend": "printwindow", "error": str(exc)}
last_error = None
for flag in (3, 2, 0):
wnd_dc = None
src_dc = None
mem_dc = None
bmp = None
try:
wnd_dc = win32gui.GetWindowDC(hwnd)
if not wnd_dc:
raise RuntimeError("GetWindowDC a retourné 0")
src_dc = win32ui.CreateDCFromHandle(wnd_dc)
mem_dc = src_dc.CreateCompatibleDC()
bmp = win32ui.CreateBitmap()
bmp.CreateCompatibleBitmap(src_dc, width, height)
mem_dc.SelectObject(bmp)
result = ctypes.windll.user32.PrintWindow(
hwnd, mem_dc.GetSafeHdc(), flag
)
bits = bmp.GetBitmapBits(True)
img = Image.frombuffer(
"RGB", (width, height), bits, "raw", "BGRX", 0, 1
)
luma = _compute_luma_stats(img)
if result or not _is_effectively_black(img):
return img, {
"backend": f"printwindow:{flag}",
"printwindow_result": int(result),
"luma": luma,
}
except Exception as exc:
last_error = str(exc)
finally:
try:
if bmp is not None:
win32gui.DeleteObject(bmp.GetHandle())
except Exception:
pass
try:
if mem_dc is not None:
mem_dc.DeleteDC()
except Exception:
pass
try:
if src_dc is not None:
src_dc.DeleteDC()
except Exception:
pass
try:
if wnd_dc is not None:
win32gui.ReleaseDC(hwnd, wnd_dc)
except Exception:
pass
return None, {
"backend": "printwindow",
"error": last_error or "no_usable_frame",
}
def capture_foreground_window_image() -> Tuple[Optional[Image.Image], Dict[str, Any]]:
"""Capture la fenêtre au focus via API native si disponible."""
try:
from ..window_info_crossplatform import get_active_window_rect
rect_info = get_active_window_rect()
except Exception as exc:
return None, {"backend": "printwindow", "error": str(exc)}
if not rect_info:
return None, {"backend": "printwindow", "error": "active_window_unavailable"}
win_w, win_h = rect_info.get("size", [0, 0])
hwnd = rect_info.get("hwnd")
if not hwnd or win_w <= 0 or win_h <= 0:
return None, {
"backend": "printwindow",
"error": "active_window_handle_unavailable",
"title": rect_info.get("title", "unknown_window"),
}
img, meta = _capture_window_image_windows(hwnd, win_w, win_h)
if img is None:
return None, meta
meta.update(
{
"title": rect_info.get("title", "unknown_window"),
"app_name": rect_info.get("app_name", "unknown_app"),
"rect": rect_info.get("rect"),
"window_size": rect_info.get("size"),
"hwnd": hwnd,
}
)
return img, meta
class VisionCapturer:
def __init__(self, session_dir: str):
self.session_dir = session_dir
@@ -103,25 +434,35 @@ class VisionCapturer:
(utile pour le contextualisation des heartbeats côté serveur).
"""
try:
with mss.mss() as sct:
monitor = sct.monitors[1]
sct_img = sct.grab(monitor)
img = Image.frombytes("RGB", sct_img.size, sct_img.bgra, "raw", "BGRX")
_monitor, img, meta = capture_screen_image()
if img is None:
img, win_meta = capture_foreground_window_image()
if img is None:
logger.error(
"Capture plein contexte indisponible (meta=%s, window=%s)",
meta,
win_meta,
)
return ""
logger.warning(
"Capture plein contexte dégradée via fenêtre active (%s)",
win_meta.get("backend"),
)
# Détection de changement (pour Heartbeat)
if not force:
current_hash = self._compute_quick_hash(img)
if current_hash == self.last_img_hash:
return "" # Pas de changement, on économise la fibre
self.last_img_hash = current_hash
# Détection de changement (pour Heartbeat)
if not force:
current_hash = self._compute_quick_hash(img)
if current_hash == self.last_img_hash:
return "" # Pas de changement, on économise la fibre
self.last_img_hash = current_hash
# Floutage des données sensibles (conformité AI Act)
if BLUR_SENSITIVE:
blur_sensitive_regions(img)
# Floutage des données sensibles (conformité AI Act)
if BLUR_SENSITIVE:
blur_sensitive_regions(img)
path = os.path.join(self.shots_dir, f"context_{int(time.time())}_{name_suffix}.png")
img.save(path, "PNG", quality=SCREENSHOT_QUALITY)
return path
path = os.path.join(self.shots_dir, f"context_{int(time.time())}_{name_suffix}.png")
img.save(path, "PNG", quality=SCREENSHOT_QUALITY)
return path
except Exception as e:
logger.error(f"Erreur Context Capture: {e}")
return ""
@@ -145,46 +486,62 @@ class VisionCapturer:
sont toujours retournés (fallback gracieux).
"""
try:
with mss.mss() as sct:
full_path = os.path.join(self.shots_dir, f"{screenshot_id}_full.png")
monitor = sct.monitors[1]
sct_img = sct.grab(monitor)
img = Image.frombytes("RGB", sct_img.size, sct_img.bgra, "raw", "BGRX")
# Capture du Crop (Cœur de l'apprentissage qwen3-vl)
crop_path = os.path.join(self.shots_dir, f"{screenshot_id}_crop.png")
w, h = TARGETED_CROP_SIZE
left = max(0, x - w // 2)
top = max(0, y - h // 2)
crop_img = img.crop((left, top, left + w, top + h))
if anonymize:
crop_img = crop_img.filter(ImageFilter.GaussianBlur(radius=4))
# Floutage des données sensibles (conformité AI Act)
if BLUR_SENSITIVE:
blur_sensitive_regions(img)
blur_sensitive_regions(crop_img)
img.save(full_path, "PNG", quality=SCREENSHOT_QUALITY)
crop_img.save(crop_path, "PNG", quality=SCREENSHOT_QUALITY)
# Mise à jour du hash pour le prochain heartbeat
self.last_img_hash = self._compute_quick_hash(img)
result = {"full": full_path, "crop": crop_path}
# --- Capture de la fenêtre active ---
# Ajout non-bloquant : enrichit le résultat avec l'image
# de la fenêtre seule + métadonnées (titre, rect, clic relatif)
window_info = self.capture_active_window(x, y, screenshot_id, full_img=img)
# Coords (x, y) sont en système écran composite ; cropper depuis
# un monitor secondaire (offset ≠ 0) produirait une image saine
# mais décalée → fail-closed sur fallback secondaire.
_monitor, img, meta = capture_screen_image(
allow_secondary_fallback=False
)
if img is None:
window_info = self.capture_active_window(
x, y, screenshot_id, full_img=None
)
if window_info:
result["window_capture"] = window_info
result = {"window_capture": window_info}
_enrich_with_monitor_info(result)
logger.warning(
"capture_dual dégradée: fenêtre active seule (%s)",
meta,
)
return result
return {}
# QW1 — enrichissement multi-écrans (additif, fallback gracieux)
_enrich_with_monitor_info(result)
full_path = os.path.join(self.shots_dir, f"{screenshot_id}_full.png")
return result
# Capture du Crop (Cœur de l'apprentissage qwen3-vl)
crop_path = os.path.join(self.shots_dir, f"{screenshot_id}_crop.png")
w, h = TARGETED_CROP_SIZE
left = max(0, x - w // 2)
top = max(0, y - h // 2)
crop_img = img.crop((left, top, left + w, top + h))
if anonymize:
crop_img = crop_img.filter(ImageFilter.GaussianBlur(radius=4))
# Floutage des données sensibles (conformité AI Act)
if BLUR_SENSITIVE:
blur_sensitive_regions(img)
blur_sensitive_regions(crop_img)
img.save(full_path, "PNG", quality=SCREENSHOT_QUALITY)
crop_img.save(crop_path, "PNG", quality=SCREENSHOT_QUALITY)
# Mise à jour du hash pour le prochain heartbeat
self.last_img_hash = self._compute_quick_hash(img)
result = {"full": full_path, "crop": crop_path}
# --- Capture de la fenêtre active ---
# Ajout non-bloquant : enrichit le résultat avec l'image
# de la fenêtre seule + métadonnées (titre, rect, clic relatif)
window_info = self.capture_active_window(x, y, screenshot_id, full_img=img)
if window_info:
result["window_capture"] = window_info
# QW1 — enrichissement multi-écrans (additif, fallback gracieux)
_enrich_with_monitor_info(result)
return result
except Exception as e:
logger.error(f"Erreur Dual Capture: {e}")
return {}
@@ -239,33 +596,54 @@ class VisionCapturer:
# Si le clic est en dehors de la fenêtre, on le signale mais on continue
click_inside = (0 <= click_rel_x <= win_w and 0 <= click_rel_y <= win_h)
window_img = None
# --- Crop de la fenêtre depuis le plein écran ---
if full_img is None:
# Pas de screenshot fourni — en capturer un (cas standalone)
# Pas de screenshot fourni — en capturer un (cas standalone).
# win_rect est en coords globales ; cropper depuis un monitor
# secondaire produirait une image décalée → fail-closed sur
# fallback secondaire.
try:
with mss.mss() as sct:
monitor = sct.monitors[1]
sct_img = sct.grab(monitor)
full_img = Image.frombytes(
"RGB", sct_img.size, sct_img.bgra, "raw", "BGRX"
)
_monitor, full_img, _meta = capture_screen_image(
allow_secondary_fallback=False
)
except Exception as e:
logger.error(f"Erreur capture plein écran pour fenêtre : {e}")
return None
full_img = None
# Borner le crop aux limites de l'image plein écran
img_w, img_h = full_img.size
crop_left = max(0, win_left)
crop_top = max(0, win_top)
crop_right = min(img_w, win_right)
crop_bottom = min(img_h, win_bottom)
if full_img is not None and not _is_effectively_black(full_img):
img_w, img_h = full_img.size
crop_left = max(0, win_left)
crop_top = max(0, win_top)
crop_right = min(img_w, win_right)
crop_bottom = min(img_h, win_bottom)
if crop_right <= crop_left or crop_bottom <= crop_top:
logger.debug("Fenêtre hors écran — skip capture fenêtre")
if crop_right > crop_left and crop_bottom > crop_top:
window_img = full_img.crop(
(crop_left, crop_top, crop_right, crop_bottom)
)
else:
logger.debug("Fenêtre hors écran — fallback natif si possible")
elif full_img is not None:
logger.warning(
"capture_active_window: screenshot plein écran noir, fallback natif"
)
if window_img is None and rect_info.get("hwnd"):
window_img, native_meta = _capture_window_image_windows(
rect_info["hwnd"], win_w, win_h
)
if window_img is not None:
logger.warning(
"capture_active_window via fallback natif (%s)",
native_meta.get("backend"),
)
if window_img is None:
logger.debug("Fenêtre hors écran ou capture native indisponible")
return None
window_img = full_img.crop((crop_left, crop_top, crop_right, crop_bottom))
# Floutage conformité AI Act
if BLUR_SENSITIVE:
blur_sensitive_regions(window_img)