snapshot: WIP 5j replay reliability (B1 watchdog + dialog handlers + grounding drift)
Snapshot avant correction du blocage relance Léa (3 incidents 24h: SSH refusé, polls morts ×2). Point de rollback stable. Contenu: - agent_v1/core/executor.py: 5 patchs dialog handling (saveas drift, close_tab hotkey fallback, confirm_save Unicode apostrophe, foreground dialog recontextualization, runtime_dialog in-loop) + helpers normalize_window_hint, requires_post_verify_window_transition - agent_v1/core/grounding.py: garde drift template fix (fallback_x/y plumbed) - server_v1/replay_watchdog.py (NEW): orphan watchdog B1, scan 10s timeout 30s - server_v1/api_stream.py: dispatched_action plumbing, watchdog lifespan, metrics endpoint - server_v1/replay_engine.py: _schedule_retry préserve original_action + dispatched_action - stream_processor.py: gardes _infer_tab_switch_target (no false switch_tab on save_as dialog open) + _attach_expected_window_before - tests/integration: test_replay_watchdog.py (8 cas), test_stream_processor.py - tests/unit: test_executor_verify_window_guard.py (start_button, close_tab, runtime_dialog, post_verify, transition fallbacks) Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
File diff suppressed because it is too large
Load Diff
@@ -74,6 +74,142 @@ class GroundingEngine:
|
||||
"""
|
||||
self._executor = executor
|
||||
|
||||
@staticmethod
|
||||
def _should_scope_to_active_window(target_spec: Dict[str, Any]) -> bool:
|
||||
"""Déterminer si le grounding doit être limité à la fenêtre active."""
|
||||
if str(target_spec.get("screen_scope", "")).strip().lower() == "full_screen":
|
||||
return False
|
||||
|
||||
by_role = str(target_spec.get("by_role", "")).strip().lower()
|
||||
if by_role in {"start_button"}:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _targets_lea_window(target_spec: Dict[str, Any]) -> bool:
|
||||
"""Déterminer si la cible pointe explicitement vers l'UI de Léa."""
|
||||
try:
|
||||
from ..ui.messages import est_fenetre_lea
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
context_hints = target_spec.get("context_hints") or {}
|
||||
hints = [
|
||||
target_spec.get("window_title", ""),
|
||||
context_hints.get("window_title", ""),
|
||||
target_spec.get("vlm_description", ""),
|
||||
target_spec.get("by_text", ""),
|
||||
]
|
||||
return any(est_fenetre_lea(str(hint)) for hint in hints if hint)
|
||||
|
||||
@staticmethod
|
||||
def _is_plausible_window_rect(
|
||||
rect: Optional[List[int]],
|
||||
title: str,
|
||||
screen_width: int,
|
||||
screen_height: int,
|
||||
) -> bool:
|
||||
"""Valider qu'un rect actif ressemble à une vraie fenêtre utilisable.
|
||||
|
||||
Rejette explicitement les zones système "bar-like" (taskbar, systray)
|
||||
et les titres inconnus/bruités. Le grounding ne doit jamais se
|
||||
contraindre à une zone non validée.
|
||||
"""
|
||||
if not rect or len(rect) != 4:
|
||||
return False
|
||||
|
||||
try:
|
||||
from ..ui.messages import est_fenetre_bruit
|
||||
except Exception:
|
||||
def est_fenetre_bruit(_title: str) -> bool:
|
||||
return not _title or _title.strip().lower() == "unknown_window"
|
||||
|
||||
w = rect[2] - rect[0]
|
||||
h = rect[3] - rect[1]
|
||||
title_clean = str(title or "").strip()
|
||||
if w <= 50 or h <= 50:
|
||||
return False
|
||||
title_lower = title_clean.lower()
|
||||
is_unknown_title = not title_clean or title_lower == "unknown_window"
|
||||
if not is_unknown_title and est_fenetre_bruit(title_clean):
|
||||
return False
|
||||
|
||||
# Une zone très plate, surtout en bas d'écran et très large, est
|
||||
# typiquement une barre des tâches / systray, pas une vraie fenêtre.
|
||||
# On réduit le seuil de hauteur à 120px pour ne pas rejeter les petits modaux.
|
||||
is_bar_like = (
|
||||
h < 120
|
||||
or (w > 0.9 * screen_width and h < 0.15 * screen_height)
|
||||
)
|
||||
|
||||
# Exception : si le titre contient un mot-clé de dialogue connu,
|
||||
# on considère que c'est plausible même si c'est petit.
|
||||
keywords = ["enregistrer sous", "save as", "voulez-vous", "confirm", "attention", "error", "erreur"]
|
||||
if any(k in title_lower for k in keywords):
|
||||
return h >= 80 # Un dialogue fait au moins 80px (titre + bouton)
|
||||
|
||||
return not is_bar_like
|
||||
|
||||
@staticmethod
|
||||
def _visual_scope_hints(target_spec: Dict[str, Any]) -> List[str]:
|
||||
"""Construire des indices textuels à chercher dans le crop fenêtre."""
|
||||
hints: List[str] = []
|
||||
raw_hints = [
|
||||
target_spec.get("window_title", ""),
|
||||
(target_spec.get("context_hints") or {}).get("window_title", ""),
|
||||
target_spec.get("by_text", ""),
|
||||
]
|
||||
for raw in raw_hints:
|
||||
text = str(raw or "").strip()
|
||||
if not text:
|
||||
continue
|
||||
text = text.lstrip("*").strip()
|
||||
variants = [text]
|
||||
for sep in (" – ", " - ", " — "):
|
||||
if sep in text:
|
||||
variants.extend(part.strip().lstrip("*") for part in text.split(sep))
|
||||
for variant in variants:
|
||||
if variant and len(variant) >= 3 and variant not in hints:
|
||||
hints.append(variant)
|
||||
return hints
|
||||
|
||||
def _window_crop_matches_target_visually(
|
||||
self,
|
||||
screenshot_b64: str,
|
||||
target_spec: Dict[str, Any],
|
||||
) -> bool:
|
||||
"""Vérifier visuellement qu'un crop contraint contient la bonne cible.
|
||||
|
||||
Principe: ne jamais faire confiance au rect système seul. Si aucun
|
||||
indice textuel n'est disponible, on laisse passer le crop plausible
|
||||
pour ne pas sur-bloquer les cibles purement iconiques.
|
||||
"""
|
||||
hints = self._visual_scope_hints(target_spec)
|
||||
if not hints:
|
||||
return True
|
||||
|
||||
finder = getattr(self._executor, "_find_text_on_screen", None)
|
||||
if not callable(finder):
|
||||
return True
|
||||
|
||||
for hint in hints:
|
||||
try:
|
||||
if finder(screenshot_b64, hint):
|
||||
logger.info(
|
||||
"Grounding fenêtre validé visuellement via '%s'",
|
||||
hint,
|
||||
)
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.debug("Validation visuelle du crop échouée pour '%s': %s", hint, e)
|
||||
logger.info(
|
||||
"Grounding plein écran : crop fenêtre rejeté par validation visuelle "
|
||||
"(hints=%s)",
|
||||
hints,
|
||||
)
|
||||
return False
|
||||
|
||||
def locate(
|
||||
self,
|
||||
server_url: str,
|
||||
@@ -128,35 +264,63 @@ class GroundingEngine:
|
||||
|
||||
t_start = time.time()
|
||||
|
||||
# ── Capture contrainte à la fenêtre active ──
|
||||
# Le grounding ne voit QUE la fenêtre attendue — pas la taskbar,
|
||||
# pas le systray, pas les autres apps. Comme un humain qui regarde
|
||||
# l'application sur laquelle il travaille.
|
||||
window_rect = None
|
||||
try:
|
||||
from ..window_info_crossplatform import get_active_window_rect
|
||||
win_info = get_active_window_rect()
|
||||
if win_info and win_info.get("rect"):
|
||||
r = win_info["rect"] # [left, top, right, bottom]
|
||||
# Validation : fenêtre visible et pas minuscule
|
||||
w = r[2] - r[0]
|
||||
h = r[3] - r[1]
|
||||
if w > 50 and h > 50:
|
||||
window_rect = {
|
||||
"left": max(0, r[0]),
|
||||
"top": max(0, r[1]),
|
||||
"width": min(w, screen_width),
|
||||
"height": min(h, screen_height),
|
||||
}
|
||||
logger.info(
|
||||
f"Grounding contraint à la fenêtre : "
|
||||
f"{window_rect['width']}x{window_rect['height']} "
|
||||
f"à ({window_rect['left']}, {window_rect['top']})"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug(f"Pas de window rect disponible : {e}")
|
||||
active_title = ""
|
||||
if self._should_scope_to_active_window(target_spec):
|
||||
# ── Capture contrainte à la fenêtre active ──
|
||||
# Le grounding ne voit QUE la fenêtre attendue — pas la taskbar,
|
||||
# pas le systray, pas les autres apps. Comme un humain qui regarde
|
||||
# l'application sur laquelle il travaille.
|
||||
try:
|
||||
from ..window_info_crossplatform import get_active_window_rect
|
||||
from ..ui.messages import est_fenetre_lea
|
||||
|
||||
win_info = get_active_window_rect()
|
||||
if win_info and win_info.get("rect"):
|
||||
active_title = str(win_info.get("title", "") or "")
|
||||
if est_fenetre_lea(active_title) and not self._targets_lea_window(target_spec):
|
||||
logger.info(
|
||||
"Grounding plein écran : fenêtre active Léa ignorée pour "
|
||||
"cible externe (%s)",
|
||||
target_spec.get("by_text", "") or target_spec.get("by_role", ""),
|
||||
)
|
||||
win_info = None
|
||||
if win_info and win_info.get("rect"):
|
||||
r = win_info["rect"] # [left, top, right, bottom]
|
||||
if self._is_plausible_window_rect(r, active_title, screen_width, screen_height):
|
||||
w = r[2] - r[0]
|
||||
h = r[3] - r[1]
|
||||
window_rect = {
|
||||
"left": max(0, r[0]),
|
||||
"top": max(0, r[1]),
|
||||
"width": min(w, screen_width),
|
||||
"height": min(h, screen_height),
|
||||
}
|
||||
logger.info(
|
||||
f"Grounding contraint à la fenêtre : "
|
||||
f"{window_rect['width']}x{window_rect['height']} "
|
||||
f"à ({window_rect['left']}, {window_rect['top']})"
|
||||
)
|
||||
else:
|
||||
logger.info(
|
||||
"Grounding plein écran : rect actif rejeté "
|
||||
"(title='%s', rect=%s)",
|
||||
active_title,
|
||||
r,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug(f"Pas de window rect disponible : {e}")
|
||||
else:
|
||||
logger.info(
|
||||
"Grounding plein écran pour by_role='%s'",
|
||||
target_spec.get("by_role", ""),
|
||||
)
|
||||
|
||||
screenshot_b64 = self._capture_window_or_screen(window_rect)
|
||||
if window_rect and screenshot_b64:
|
||||
if not self._window_crop_matches_target_visually(screenshot_b64, target_spec):
|
||||
window_rect = None
|
||||
screenshot_b64 = self._capture_window_or_screen(None)
|
||||
if not screenshot_b64:
|
||||
return GroundingResult(
|
||||
found=False, detail="Capture screenshot échouée",
|
||||
@@ -186,6 +350,18 @@ class GroundingEngine:
|
||||
result.elapsed_ms = (time.time() - t_start) * 1000
|
||||
return result
|
||||
|
||||
if target_spec.get("allow_position_fallback"):
|
||||
if 0.0 <= fallback_x <= 1.0 and 0.0 <= fallback_y <= 1.0:
|
||||
return GroundingResult(
|
||||
found=True,
|
||||
x_pct=fallback_x,
|
||||
y_pct=fallback_y,
|
||||
method="position_fallback",
|
||||
score=0.2,
|
||||
detail="fallback positionnel explicite",
|
||||
elapsed_ms=(time.time() - t_start) * 1000,
|
||||
)
|
||||
|
||||
return GroundingResult(
|
||||
found=False,
|
||||
detail=f"Toutes les stratégies ont échoué ({', '.join(strategies)})",
|
||||
@@ -258,7 +434,12 @@ class GroundingEngine:
|
||||
anchor_b64 = target_spec.get("anchor_image_base64", "")
|
||||
if anchor_b64:
|
||||
raw = self._executor._template_match_anchor(
|
||||
screenshot_b64, anchor_b64, screen_width, screen_height,
|
||||
screenshot_b64,
|
||||
anchor_b64,
|
||||
screen_width,
|
||||
screen_height,
|
||||
fallback_x_pct=fallback_x,
|
||||
fallback_y_pct=fallback_y,
|
||||
)
|
||||
if raw and raw.get("resolved"):
|
||||
return GroundingResult(
|
||||
|
||||
39
agent_v0/agent_v1/finalize_contract.py
Normal file
39
agent_v0/agent_v1/finalize_contract.py
Normal file
@@ -0,0 +1,39 @@
|
||||
"""Dispatch léger du contrat enrichi de /finalize côté agent."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any, Dict
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def dispatch_finalize_result(ui: Any, payload: Dict[str, Any], replay_name: str) -> None:
|
||||
"""Router le résultat de /finalize vers la bonne surface UI agent."""
|
||||
if not isinstance(payload, dict):
|
||||
return
|
||||
|
||||
replay_request = payload.get("replay_request") or {}
|
||||
replay_launch = payload.get("replay_launch") or {}
|
||||
|
||||
if replay_launch.get("status") == "started":
|
||||
logger.info("Replay direct déjà lancé par le serveur après finalize")
|
||||
return
|
||||
|
||||
if not payload.get("replay_ready") or not replay_request:
|
||||
return
|
||||
|
||||
if replay_launch.get("status") == "failed":
|
||||
logger.warning(
|
||||
"Auto-replay serveur échoué après finalize, proposition manuelle"
|
||||
)
|
||||
|
||||
if ui is None or not hasattr(ui, "offer_finalize_replay"):
|
||||
logger.info("UI indisponible pour proposer un test immédiat")
|
||||
return
|
||||
|
||||
ui.offer_finalize_replay(
|
||||
replay_request,
|
||||
replay_name or "la tâche que vous venez d'enregistrer",
|
||||
)
|
||||
@@ -28,6 +28,7 @@ from .ui.chat_window import ChatWindow
|
||||
from .ui.capture_server import CaptureServer
|
||||
from .session.storage import SessionStorage
|
||||
from .vision.capturer import VisionCapturer
|
||||
from .finalize_contract import dispatch_finalize_result
|
||||
|
||||
# Import optionnel du client serveur (pour le chat et les workflows)
|
||||
# Deux chemins : relatif (depuis agent_v0.agent_v1) ou absolu (depuis C:\rpa_vision\agent_v1)
|
||||
@@ -80,6 +81,7 @@ class AgentV1:
|
||||
self._executor = None
|
||||
# Flag pour indiquer qu'un replay est en cours (eviter les conflits)
|
||||
self._replay_active = False
|
||||
self._last_recording_name = ""
|
||||
|
||||
# Etat partage entre systray et chat (source de verite unique)
|
||||
self._state = AgentState()
|
||||
@@ -210,12 +212,14 @@ class AgentV1:
|
||||
time.sleep(30) # Vérifier toutes les 30s
|
||||
|
||||
def start_session(self, workflow_name):
|
||||
self._last_recording_name = workflow_name
|
||||
self.session_id = f"sess_{time.strftime('%Y%m%dT%H%M%S')}_{uuid.uuid4().hex[:6]}"
|
||||
self.session_dir = self.storage.get_session_dir(self.session_id)
|
||||
|
||||
self.vision = VisionCapturer(str(self.session_dir))
|
||||
|
||||
self.streamer = TraceStreamer(self.session_id, machine_id=self.machine_id)
|
||||
self.streamer.set_on_finalize_result(self._on_finalize_result)
|
||||
self.captor = EventCaptorV1(self._on_event_bridge)
|
||||
|
||||
# Initialiser l'executeur partage
|
||||
@@ -325,6 +329,15 @@ class AgentV1:
|
||||
# pour enchainer les actions du workflow
|
||||
time.sleep(0.2)
|
||||
else:
|
||||
if getattr(self._executor, "_replay_paused", False):
|
||||
if not self._replay_active:
|
||||
self._replay_active = True
|
||||
self.ui.set_replay_active(True)
|
||||
self._state.set_replay_active(True)
|
||||
poll_delay = getattr(self._executor, '_poll_backoff', REPLAY_POLL_INTERVAL)
|
||||
time.sleep(max(poll_delay, REPLAY_POLL_INTERVAL))
|
||||
continue
|
||||
|
||||
# Pas d'action en attente — utiliser le backoff de l'executor
|
||||
# (augmente si le serveur est indisponible, reset a 1s sinon)
|
||||
if self._replay_active:
|
||||
@@ -429,6 +442,11 @@ class AgentV1:
|
||||
f"agent_{self.user_id}"
|
||||
)
|
||||
|
||||
def _on_finalize_result(self, payload: dict) -> None:
|
||||
"""Réagir au contrat enrichi de /finalize côté agent."""
|
||||
replay_name = self._last_recording_name or "la tâche que vous venez d'enregistrer"
|
||||
dispatch_finalize_result(self.ui, payload, replay_name)
|
||||
|
||||
_last_heartbeat_hash: str = ""
|
||||
|
||||
def _heartbeat_loop(self):
|
||||
|
||||
@@ -30,6 +30,7 @@ import os
|
||||
import queue
|
||||
import threading
|
||||
import time
|
||||
from typing import Callable, Optional
|
||||
|
||||
import requests
|
||||
from PIL import Image
|
||||
@@ -95,6 +96,11 @@ class TraceStreamer:
|
||||
# Initialisé paresseusement pour ne pas payer le coût SQLite en dehors
|
||||
# d'un streaming actif.
|
||||
self._buffer: PersistentBuffer | None = None
|
||||
self._on_finalize_result: Optional[Callable[[dict], None]] = None
|
||||
|
||||
def set_on_finalize_result(self, callback: Optional[Callable[[dict], None]]) -> None:
|
||||
"""Définir un callback appelé avec le payload JSON de /finalize."""
|
||||
self._on_finalize_result = callback
|
||||
|
||||
def _get_buffer(self) -> PersistentBuffer:
|
||||
"""Retourne le buffer persistant, en l'initialisant au besoin."""
|
||||
@@ -621,6 +627,14 @@ class TraceStreamer:
|
||||
if resp.ok:
|
||||
result = resp.json()
|
||||
logger.info(f"Session finalisée: {result}")
|
||||
if self._on_finalize_result is not None:
|
||||
try:
|
||||
self._on_finalize_result(result)
|
||||
except Exception as cb_error:
|
||||
logger.warning(
|
||||
"Callback finalize ignoré après erreur: %s",
|
||||
cb_error,
|
||||
)
|
||||
else:
|
||||
logger.warning(f"Finalisation échouée: {resp.status_code}")
|
||||
except Exception as e:
|
||||
|
||||
@@ -158,14 +158,25 @@ class CaptureHandler(BaseHTTPRequestHandler):
|
||||
"""Capture l'ecran principal et le renvoie en base64 JPEG."""
|
||||
t0 = time.perf_counter()
|
||||
try:
|
||||
import mss
|
||||
from PIL import Image
|
||||
from ..vision.capturer import (
|
||||
capture_foreground_window_image,
|
||||
capture_screen_image,
|
||||
)
|
||||
|
||||
with mss.mss() as sct:
|
||||
monitor = sct.monitors[1] # ecran principal
|
||||
raw = sct.grab(monitor)
|
||||
|
||||
img = Image.frombytes("RGB", raw.size, raw.bgra, "raw", "BGRX")
|
||||
_monitor, img, meta = capture_screen_image()
|
||||
if img is None:
|
||||
img, win_meta = capture_foreground_window_image()
|
||||
meta.update(win_meta)
|
||||
if img is None:
|
||||
elapsed_ms = (time.perf_counter() - t0) * 1000
|
||||
logger.error("Erreur capture : aucun backend exploitable (%s)", meta)
|
||||
self._send_json(503, {
|
||||
"error": "capture_unavailable",
|
||||
"source": meta.get("backend", "unknown"),
|
||||
"capture_ms": round(elapsed_ms),
|
||||
"diagnostics": meta,
|
||||
})
|
||||
return
|
||||
|
||||
# Floutage des données sensibles (conformité AI Act)
|
||||
if BLUR_SENSITIVE:
|
||||
@@ -180,15 +191,22 @@ class CaptureHandler(BaseHTTPRequestHandler):
|
||||
img_b64 = base64.b64encode(buf.getvalue()).decode()
|
||||
|
||||
elapsed_ms = (time.perf_counter() - t0) * 1000
|
||||
logger.info(f"Capture {img.width}x{img.height} en {elapsed_ms:.0f}ms")
|
||||
logger.info(
|
||||
"Capture %sx%s via %s en %.0fms",
|
||||
img.width,
|
||||
img.height,
|
||||
meta.get("backend", "unknown"),
|
||||
elapsed_ms,
|
||||
)
|
||||
|
||||
self._send_json(200, {
|
||||
"image": img_b64,
|
||||
"width": img.width,
|
||||
"height": img.height,
|
||||
"format": "jpeg",
|
||||
"source": "windows_live",
|
||||
"source": meta.get("backend", "windows_live"),
|
||||
"capture_ms": round(elapsed_ms),
|
||||
"diagnostics": meta,
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
|
||||
@@ -894,6 +894,34 @@ class ChatWindow:
|
||||
except Exception:
|
||||
logger.debug("clear chat history silenced", exc_info=True)
|
||||
|
||||
@staticmethod
|
||||
def _compute_paused_bubble_height(reason_str: str) -> tuple:
|
||||
"""Calcule la hauteur du Text (en lignes) + si une scrollbar est
|
||||
nécessaire pour le message d'une bulle paused.
|
||||
|
||||
Patch 22 mai 2026 — fix troncature : on prend en compte les \\n
|
||||
explicites (les `reason` serveur peuvent lister plusieurs
|
||||
candidats avec un saut de ligne par item) en plus de la longueur
|
||||
en caractères, et on active la scrollbar dès que le cap est
|
||||
atteint pour éviter que du contenu disparaisse silencieusement.
|
||||
|
||||
Retourne ``(height_lines, needs_scrollbar)``.
|
||||
"""
|
||||
if not reason_str:
|
||||
return 2, False
|
||||
text = str(reason_str)
|
||||
# Estimation : ~60 chars/ligne effectifs avec wraplength.
|
||||
wrapped_lines = (len(text) // 60) + 1
|
||||
explicit_lines = text.count("\n") + 1
|
||||
estimated = max(wrapped_lines, explicit_lines)
|
||||
cap = 12
|
||||
height = max(2, min(cap, estimated))
|
||||
# Scrollbar dès que le cap est atteint OU contenu long (filet
|
||||
# textuel : ≥ 200 chars implique souvent un débordement visuel
|
||||
# même quand les lignes brutes sont peu nombreuses).
|
||||
needs_scroll = (estimated >= cap) or (len(text) > 200)
|
||||
return height, needs_scroll
|
||||
|
||||
def _render_paused_bubble(self, payload: Dict[str, Any]) -> None:
|
||||
tk = self._tk
|
||||
if getattr(self, "_msg_frame", None) is None:
|
||||
@@ -923,22 +951,23 @@ class ChatWindow:
|
||||
|
||||
# Message scrollable pour les longs reasons (ex: 200+ chars depuis le serveur).
|
||||
# On utilise un Text en mode read-only avec hauteur calculée selon la longueur.
|
||||
# Au-delà de 280 chars, scrollbar interne ; sinon Text auto-fitté.
|
||||
# Patch 22 mai 2026 : prendre en compte les \n explicites (titres
|
||||
# fenêtre / patterns) et activer la scrollbar dès que le cap de
|
||||
# hauteur est atteint — sinon les bulles de pause étaient
|
||||
# tronquées visuellement sans aucun ascenseur visible.
|
||||
reason_str = str(reason)
|
||||
# Estimation simple : ~70 chars/ligne avec wraplength
|
||||
approx_lines = max(2, min(8, (len(reason_str) // 60) + 1))
|
||||
height_lines, needs_scroll = self._compute_paused_bubble_height(reason_str)
|
||||
msg_frame = tk.Frame(inner, bg=PAUSED_BG)
|
||||
msg_frame.pack(fill=tk.X, anchor=tk.W, pady=(6, 0))
|
||||
reason_text = tk.Text(
|
||||
msg_frame, bg=PAUSED_BG, fg=PAUSED_FG,
|
||||
font=FONT_MSG, wrap=tk.WORD, bd=0, height=approx_lines,
|
||||
font=FONT_MSG, wrap=tk.WORD, bd=0, height=height_lines,
|
||||
highlightthickness=0, relief=tk.FLAT, cursor="arrow",
|
||||
)
|
||||
reason_text.insert("1.0", reason_str)
|
||||
reason_text.configure(state="disabled")
|
||||
reason_text.pack(side=tk.LEFT, fill=tk.X, expand=True)
|
||||
# Scrollbar interne uniquement si le contenu déborde (long messages)
|
||||
if len(reason_str) > 280:
|
||||
if needs_scroll:
|
||||
reason_scroll = tk.Scrollbar(
|
||||
msg_frame, orient=tk.VERTICAL,
|
||||
command=reason_text.yview, width=8,
|
||||
@@ -1019,27 +1048,40 @@ class ChatWindow:
|
||||
UX fix 8 mai 2026 : on désactive les 2 boutons et on affiche un message
|
||||
de feedback dès le clic, sans attendre l'ack serveur. Le bus émet en
|
||||
arrière-plan ; si la connexion est tombée, on log un warning visible.
|
||||
|
||||
Fallback HTTP 22 mai 2026 : si le bus SocketIO est déconnecté, on
|
||||
retombe sur un POST direct ``/replay/{id}/resume`` via
|
||||
``server_client``. Si les deux échouent, on ré-active les boutons
|
||||
et on saute l'auto-hide pour permettre à l'utilisateur de
|
||||
réessayer manuellement (sinon le replay reste figé côté serveur).
|
||||
"""
|
||||
if not replay_id:
|
||||
self._update_paused_feedback("⚠ replay_id manquant — impossible de relancer")
|
||||
return
|
||||
emitted = False
|
||||
if self._bus is not None and self._bus.connected:
|
||||
emitted = self._bus.resume_replay(replay_id)
|
||||
# Feedback immédiat : disable boutons + message
|
||||
emitted, channel = self._dispatch_paused_action(
|
||||
replay_id,
|
||||
bus_method="resume_replay",
|
||||
client_method="resume_replay",
|
||||
)
|
||||
self._disable_paused_buttons()
|
||||
if emitted:
|
||||
self._update_paused_feedback("→ Reprise demandée…")
|
||||
logger.info("paused_bubble: lea:replay_resume émis pour %s", replay_id)
|
||||
else:
|
||||
self._update_paused_feedback("⚠ Bus indisponible — réessayez dans 5s")
|
||||
logger.warning("paused_bubble: bus déconnecté, resume non émis")
|
||||
# UX fix mai 2026 : minimiser la fenêtre vers le systray après 500ms
|
||||
# (laisse à l'utilisateur le temps de voir "Reprise demandée…").
|
||||
try:
|
||||
self._root.after(500, self._do_hide)
|
||||
except Exception:
|
||||
logger.debug("auto-hide on resume silenced", exc_info=True)
|
||||
logger.info(
|
||||
"paused_bubble: replay_resume émis pour %s via %s",
|
||||
replay_id, channel,
|
||||
)
|
||||
try:
|
||||
self._root.after(500, self._do_hide)
|
||||
except Exception:
|
||||
logger.debug("auto-hide on resume silenced", exc_info=True)
|
||||
return
|
||||
# Échec sur les deux canaux : laisser l'utilisateur réessayer.
|
||||
self._update_paused_feedback("⚠ Serveur injoignable — réessayez")
|
||||
self._enable_paused_buttons()
|
||||
logger.warning(
|
||||
"paused_bubble: bus et HTTP indisponibles, resume non émis "
|
||||
"pour %s", replay_id,
|
||||
)
|
||||
|
||||
def _on_paused_abort(self, replay_id: str) -> None:
|
||||
"""Bouton Annuler : émettre lea:replay_abort + fermeture locale immédiate.
|
||||
@@ -1048,17 +1090,30 @@ class ChatWindow:
|
||||
n'envoie pas de lea:resumed pour un abort, donc sans cette fermeture
|
||||
locale la bulle restait coincée — c'était la cause de "Annuler ne
|
||||
fonctionne pas" rapportée par Dom).
|
||||
|
||||
Fallback HTTP 22 mai 2026 : symétrique de ``_on_paused_resume`` —
|
||||
si le bus est déconnecté, POST direct ``/replay/{id}/cancel``.
|
||||
L'abort ferme la bulle localement quelle que soit l'issue (l'état
|
||||
serveur sera réconcilié au prochain poll /replay/next).
|
||||
"""
|
||||
emitted = False
|
||||
if self._bus is not None and self._bus.connected:
|
||||
emitted = self._bus.abort_replay(replay_id)
|
||||
emitted, channel = self._dispatch_paused_action(
|
||||
replay_id,
|
||||
bus_method="abort_replay",
|
||||
client_method="abort_replay",
|
||||
)
|
||||
self._disable_paused_buttons()
|
||||
if emitted:
|
||||
self._update_paused_feedback("✗ Annulé")
|
||||
logger.info("paused_bubble: lea:replay_abort émis pour %s", replay_id)
|
||||
logger.info(
|
||||
"paused_bubble: replay_abort émis pour %s via %s",
|
||||
replay_id, channel,
|
||||
)
|
||||
else:
|
||||
self._update_paused_feedback("✗ Annulé (bus indisponible)")
|
||||
logger.warning("paused_bubble: bus déconnecté, abort non émis")
|
||||
self._update_paused_feedback("✗ Annulé (serveur injoignable)")
|
||||
logger.warning(
|
||||
"paused_bubble: bus et HTTP indisponibles, abort non émis "
|
||||
"pour %s", replay_id,
|
||||
)
|
||||
# Fermer la bulle en local (l'abort n'a pas de lea:resumed associé)
|
||||
self._close_active_paused_bubble(reason="abort_local")
|
||||
# UX fix mai 2026 : minimiser la fenêtre après 500ms (cohérence
|
||||
@@ -1068,6 +1123,34 @@ class ChatWindow:
|
||||
except Exception:
|
||||
logger.debug("auto-hide on abort silenced", exc_info=True)
|
||||
|
||||
def _dispatch_paused_action(
|
||||
self,
|
||||
replay_id: str,
|
||||
bus_method: str,
|
||||
client_method: str,
|
||||
) -> tuple:
|
||||
"""Envoyer une action de bulle paused via bus puis fallback HTTP.
|
||||
|
||||
Retourne ``(emitted, channel)`` où ``channel`` vaut ``"bus"``,
|
||||
``"http"`` ou ``""`` (aucun chemin n'a abouti).
|
||||
"""
|
||||
if self._bus is not None and getattr(self._bus, "connected", False):
|
||||
try:
|
||||
if getattr(self._bus, bus_method)(replay_id):
|
||||
return True, "bus"
|
||||
except Exception:
|
||||
logger.debug("paused_bubble: bus %s silenced", bus_method, exc_info=True)
|
||||
if self._server_client is not None and hasattr(self._server_client, client_method):
|
||||
try:
|
||||
if getattr(self._server_client, client_method)(replay_id):
|
||||
return True, "http"
|
||||
except Exception:
|
||||
logger.debug(
|
||||
"paused_bubble: server_client %s silenced",
|
||||
client_method, exc_info=True,
|
||||
)
|
||||
return False, ""
|
||||
|
||||
def _disable_paused_buttons(self) -> None:
|
||||
if not self._active_paused_bubble:
|
||||
return
|
||||
@@ -1077,6 +1160,19 @@ class ChatWindow:
|
||||
except Exception:
|
||||
logger.debug("disable paused buttons silenced", exc_info=True)
|
||||
|
||||
def _enable_paused_buttons(self) -> None:
|
||||
"""Ré-activer les boutons Continuer/Annuler de la bulle paused
|
||||
active. Appelé quand l'envoi a échoué sur tous les canaux —
|
||||
l'utilisateur doit pouvoir réessayer manuellement.
|
||||
"""
|
||||
if not self._active_paused_bubble:
|
||||
return
|
||||
try:
|
||||
self._active_paused_bubble["btn_resume"].config(state="normal")
|
||||
self._active_paused_bubble["btn_abort"].config(state="normal")
|
||||
except Exception:
|
||||
logger.debug("enable paused buttons silenced", exc_info=True)
|
||||
|
||||
def _update_paused_feedback(self, text: str) -> None:
|
||||
if not self._active_paused_bubble:
|
||||
return
|
||||
|
||||
@@ -504,6 +504,100 @@ class SmartTrayV1:
|
||||
|
||||
threading.Thread(target=_replay, daemon=True).start()
|
||||
|
||||
def _launch_replay_request(
|
||||
self,
|
||||
replay_request: Dict[str, Any],
|
||||
replay_name: str,
|
||||
) -> None:
|
||||
"""Lance un replay direct depuis un payload `replay_request` serveur."""
|
||||
endpoint = (replay_request or {}).get("endpoint", "")
|
||||
session_id = (replay_request or {}).get("session_id", "")
|
||||
machine_id = (replay_request or {}).get("machine_id") or self.machine_id
|
||||
|
||||
if endpoint != "/api/v1/traces/stream/replay-session" or not session_id:
|
||||
logger.warning("Replay request non supporté: %s", replay_request)
|
||||
self._notifier.notify(
|
||||
"Léa",
|
||||
"Je ne peux pas lancer ce test automatique pour le moment.",
|
||||
)
|
||||
return
|
||||
|
||||
def _replay():
|
||||
if self.server_client is None:
|
||||
return
|
||||
|
||||
with self._state_lock:
|
||||
self._replay_active = True
|
||||
self._update_icon()
|
||||
self._notifier.notify(
|
||||
"Léa",
|
||||
f"Le système d'intelligence artificielle exécute la "
|
||||
f"tâche '{replay_name}' sur votre écran.",
|
||||
)
|
||||
|
||||
try:
|
||||
import requests
|
||||
auth_headers = {}
|
||||
if self.server_client is not None:
|
||||
auth_headers = self.server_client._auth_headers()
|
||||
resp = requests.post(
|
||||
f"{self.server_client._stream_base}{endpoint}",
|
||||
params={
|
||||
"session_id": session_id,
|
||||
"machine_id": machine_id,
|
||||
},
|
||||
headers=auth_headers,
|
||||
timeout=30,
|
||||
allow_redirects=False,
|
||||
)
|
||||
if resp.ok:
|
||||
logger.info(
|
||||
"Replay direct démarré pour session %s (machine=%s)",
|
||||
session_id,
|
||||
machine_id,
|
||||
)
|
||||
else:
|
||||
self._notifier.notify(
|
||||
"Léa",
|
||||
"Hmm, le serveur a refusé le test immédiat.",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error("Erreur lancement replay direct : %s", e)
|
||||
self._notifier.notify(
|
||||
"Léa",
|
||||
f"Oups, un problème : {e}",
|
||||
)
|
||||
finally:
|
||||
with self._state_lock:
|
||||
self._replay_active = False
|
||||
self._update_icon()
|
||||
|
||||
threading.Thread(target=_replay, daemon=True).start()
|
||||
|
||||
def offer_finalize_replay(
|
||||
self,
|
||||
replay_request: Dict[str, Any],
|
||||
replay_name: str,
|
||||
) -> None:
|
||||
"""Proposer à l'utilisateur de tester immédiatement la tâche apprise."""
|
||||
if not replay_request or not replay_request.get("session_id"):
|
||||
return
|
||||
|
||||
def _offer():
|
||||
self._notifier.notify(
|
||||
"Léa",
|
||||
f"J'ai compris la tâche '{replay_name}'. Voulez-vous la tester ?",
|
||||
)
|
||||
if not _ask_consent(
|
||||
"Léa — Test immédiat",
|
||||
f"J'ai compris la tâche '{replay_name}'. "
|
||||
"Voulez-vous la tester maintenant ?",
|
||||
):
|
||||
return
|
||||
self._launch_replay_request(replay_request, replay_name)
|
||||
|
||||
threading.Thread(target=_offer, daemon=True).start()
|
||||
|
||||
def _on_emergency_stop(self, _icon=None, _item=None) -> None:
|
||||
"""Arret d'urgence — stoppe TOUTES les activites de l'agent immediatement.
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@ import time
|
||||
import logging
|
||||
import hashlib
|
||||
import platform
|
||||
from typing import Any, Dict, List, Optional
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
from PIL import Image, ImageFilter, ImageStat
|
||||
import mss
|
||||
from ..config import TARGETED_CROP_SIZE, SCREENSHOT_QUALITY, BLUR_SENSITIVE
|
||||
@@ -86,6 +86,337 @@ def _enrich_with_monitor_info(payload: dict) -> dict:
|
||||
payload["monitors_geometry"] = _get_monitors_geometry()
|
||||
return payload
|
||||
|
||||
|
||||
# Garde dimensions monitor (démo GHT 19 mai 2026) : mss.monitors[1] peut
|
||||
# retourner intermittemment des dims tronquées (cas observé 2560×60). Utiliser
|
||||
# ces dims pour normaliser des coords empoisonne la mémoire (TargetMemoryStore).
|
||||
MIN_MONITOR_WIDTH = 200
|
||||
MIN_MONITOR_HEIGHT = 200
|
||||
MONITOR_MAX_ATTEMPTS = 2
|
||||
MONITOR_RETRY_DELAY_S = 0.05
|
||||
BLACK_FRAME_MEAN_MAX = 1.0
|
||||
BLACK_FRAME_STDDEV_MAX = 1.0
|
||||
BLACK_FRAME_MAX_LUMA = 3
|
||||
|
||||
|
||||
def _is_monitor_sane(monitor) -> bool:
|
||||
"""True si les dims du monitor sont au-dessus du seuil de plausibilité."""
|
||||
if not isinstance(monitor, dict):
|
||||
return False
|
||||
w = monitor.get("width", 0) or 0
|
||||
h = monitor.get("height", 0) or 0
|
||||
return w >= MIN_MONITOR_WIDTH and h >= MIN_MONITOR_HEIGHT
|
||||
|
||||
|
||||
def _dim_str(monitor) -> str:
|
||||
"""Représentation courte WxH pour les logs (gère monitor=None)."""
|
||||
if not isinstance(monitor, dict):
|
||||
return "?x?"
|
||||
return f"{monitor.get('width', '?')}x{monitor.get('height', '?')}"
|
||||
|
||||
|
||||
def _acquire_safe_grab(max_attempts: int = MONITOR_MAX_ATTEMPTS,
|
||||
retry_delay_s: float = MONITOR_RETRY_DELAY_S,
|
||||
allow_secondary_fallback: bool = True):
|
||||
"""Ouvre mss et capture un monitor avec dimensions plausibles.
|
||||
|
||||
Stratégie en cascade :
|
||||
1. À chaque tentative, ouvrir un nouveau `mss.mss()` (peut rafraîchir le
|
||||
cache interne) et examiner monitors[1..n].
|
||||
2. Préférer monitors[1] (écran principal physique). Si aberrant ET
|
||||
`allow_secondary_fallback=True`, prendre le premier monitors[2..n]
|
||||
sain avec un WARNING explicite.
|
||||
3. Si `allow_secondary_fallback=False`, on n'accepte QUE monitors[1].
|
||||
Utile pour les méthodes qui reçoivent des coordonnées (x, y) en
|
||||
système écran composite : capturer un monitor secondaire produirait
|
||||
une image saine mais décalée par rapport à ces coords.
|
||||
4. Si aucune dim plausible : attendre `retry_delay_s` et retenter.
|
||||
5. Après `max_attempts` infructueuses : log ERROR et retourner
|
||||
(None, None) pour que l'appelant tombe en sortie d'erreur explicite.
|
||||
|
||||
Args:
|
||||
max_attempts: nombre de tentatives mss avant abandon.
|
||||
retry_delay_s: délai entre tentatives.
|
||||
allow_secondary_fallback: si False, refuser monitors[2..n] (fail-closed
|
||||
pour les méthodes coord-bearing).
|
||||
|
||||
Returns:
|
||||
Tuple (monitor_dict, PIL.Image) si capture saine réussie,
|
||||
(None, None) sinon.
|
||||
"""
|
||||
last_aberrant = None
|
||||
secondary_seen = False # un monitor secondaire sain a été vu mais refusé
|
||||
for attempt in range(max_attempts):
|
||||
with mss.mss() as sct:
|
||||
monitors = list(sct.monitors) if sct.monitors else []
|
||||
chosen = None
|
||||
chosen_idx = None
|
||||
for idx in range(1, len(monitors)):
|
||||
candidate = monitors[idx]
|
||||
if not _is_monitor_sane(candidate):
|
||||
last_aberrant = candidate
|
||||
logger.warning(
|
||||
"Monitor[%d] dims aberrantes (%s, seuil %dx%d) "
|
||||
"— attempt %d/%d",
|
||||
idx, _dim_str(candidate),
|
||||
MIN_MONITOR_WIDTH, MIN_MONITOR_HEIGHT,
|
||||
attempt + 1, max_attempts,
|
||||
)
|
||||
continue
|
||||
# Monitor sain trouvé
|
||||
if idx == 1 or allow_secondary_fallback:
|
||||
chosen = candidate
|
||||
chosen_idx = idx
|
||||
break
|
||||
# Sinon : sain mais secondaire interdit pour cet appelant
|
||||
secondary_seen = True
|
||||
logger.warning(
|
||||
"Monitor[%d] sain (%s) mais fallback secondaire refusé "
|
||||
"(allow_secondary_fallback=False) — capture cohérente "
|
||||
"des coords impossible",
|
||||
idx, _dim_str(candidate),
|
||||
)
|
||||
if chosen is not None:
|
||||
if chosen_idx != 1 or attempt > 0:
|
||||
logger.warning(
|
||||
"Capture fallback : monitor[%d] dim=%s, attempt=%d",
|
||||
chosen_idx, _dim_str(chosen), attempt + 1,
|
||||
)
|
||||
sct_img = sct.grab(chosen)
|
||||
img = Image.frombytes(
|
||||
"RGB", sct_img.size, sct_img.bgra, "raw", "BGRX",
|
||||
)
|
||||
return chosen, img
|
||||
if attempt < max_attempts - 1:
|
||||
time.sleep(retry_delay_s)
|
||||
if secondary_seen and not allow_secondary_fallback:
|
||||
logger.error(
|
||||
"Capture abandonnée : monitor[1] aberrant après %d tentatives "
|
||||
"(dernier vu %s) et fallback secondaire désactivé "
|
||||
"pour préserver la cohérence des coordonnées",
|
||||
max_attempts, _dim_str(last_aberrant),
|
||||
)
|
||||
else:
|
||||
logger.error(
|
||||
"Aucun monitor avec dims plausibles trouvé après %d tentatives "
|
||||
"(dernier vu : %s, seuil %dx%d) — capture abandonnée",
|
||||
max_attempts, _dim_str(last_aberrant),
|
||||
MIN_MONITOR_WIDTH, MIN_MONITOR_HEIGHT,
|
||||
)
|
||||
return None, None
|
||||
|
||||
|
||||
def _compute_luma_stats(img: Image.Image) -> Dict[str, float | int]:
|
||||
"""Retourne des stats simples de luminance pour diagnostiquer un frame noir."""
|
||||
gray = img.convert("L")
|
||||
stat = ImageStat.Stat(gray)
|
||||
min_luma, max_luma = gray.getextrema()
|
||||
return {
|
||||
"mean": round(float(stat.mean[0]) if stat.mean else 0.0, 2),
|
||||
"stddev": round(float(stat.stddev[0]) if stat.stddev else 0.0, 2),
|
||||
"min": int(min_luma),
|
||||
"max": int(max_luma),
|
||||
}
|
||||
|
||||
|
||||
def _is_effectively_black(img: Image.Image) -> bool:
|
||||
"""Heuristique fail-closed pour refuser un screenshot pratiquement noir."""
|
||||
stats = _compute_luma_stats(img)
|
||||
return (
|
||||
stats["max"] <= BLACK_FRAME_MAX_LUMA
|
||||
and stats["mean"] <= BLACK_FRAME_MEAN_MAX
|
||||
and stats["stddev"] <= BLACK_FRAME_STDDEV_MAX
|
||||
)
|
||||
|
||||
|
||||
def _capture_via_imagegrab() -> Tuple[Optional[Dict[str, int]], Optional[Image.Image], Dict[str, Any]]:
|
||||
"""Fallback Windows via Pillow/ImageGrab.
|
||||
|
||||
Utile quand `mss` retourne un frame noir alors que la session graphique
|
||||
utilisateur reste visible.
|
||||
"""
|
||||
if _SYSTEM != "Windows":
|
||||
return None, None, {"backend": "imagegrab", "error": "unsupported_platform"}
|
||||
|
||||
try:
|
||||
from PIL import ImageGrab
|
||||
except ImportError as exc:
|
||||
return None, None, {"backend": "imagegrab", "error": str(exc)}
|
||||
|
||||
try:
|
||||
img = ImageGrab.grab(all_screens=True)
|
||||
except Exception as exc:
|
||||
logger.warning("ImageGrab indisponible pour le fallback capture : %s", exc)
|
||||
return None, None, {"backend": "imagegrab", "error": str(exc)}
|
||||
|
||||
monitor = {"left": 0, "top": 0, "width": img.width, "height": img.height}
|
||||
return monitor, img, {
|
||||
"backend": "imagegrab",
|
||||
"luma": _compute_luma_stats(img),
|
||||
}
|
||||
|
||||
|
||||
def capture_screen_image(
|
||||
allow_secondary_fallback: bool = True,
|
||||
) -> Tuple[Optional[Dict[str, int]], Optional[Image.Image], Dict[str, Any]]:
|
||||
"""Capture plein écran avec diagnostic noir + fallback Windows.
|
||||
|
||||
Returns:
|
||||
(monitor, image, meta) où image peut être None si aucun backend plein
|
||||
écran n'a produit une image exploitable.
|
||||
"""
|
||||
monitor, img = _acquire_safe_grab(
|
||||
allow_secondary_fallback=allow_secondary_fallback
|
||||
)
|
||||
meta: Dict[str, Any] = {"backend": "mss"}
|
||||
|
||||
if img is not None:
|
||||
meta["luma"] = _compute_luma_stats(img)
|
||||
if not _is_effectively_black(img):
|
||||
return monitor, img, meta
|
||||
logger.warning(
|
||||
"Capture mss quasi noire (%s) — tentative de fallback",
|
||||
meta["luma"],
|
||||
)
|
||||
meta["mss_black_frame"] = True
|
||||
else:
|
||||
meta["mss_unavailable"] = True
|
||||
|
||||
fallback_monitor, fallback_img, fallback_meta = _capture_via_imagegrab()
|
||||
if fallback_img is not None:
|
||||
if not _is_effectively_black(fallback_img):
|
||||
logger.warning(
|
||||
"Capture fallback via ImageGrab (%sx%s)",
|
||||
fallback_img.width,
|
||||
fallback_img.height,
|
||||
)
|
||||
return fallback_monitor, fallback_img, fallback_meta
|
||||
logger.warning(
|
||||
"Capture ImageGrab quasi noire (%s)",
|
||||
fallback_meta.get("luma"),
|
||||
)
|
||||
meta["imagegrab_black_frame"] = True
|
||||
|
||||
meta["imagegrab_error"] = fallback_meta.get("error")
|
||||
return None, None, meta
|
||||
|
||||
|
||||
def _capture_window_image_windows(
|
||||
hwnd: int,
|
||||
width: int,
|
||||
height: int,
|
||||
) -> Tuple[Optional[Image.Image], Dict[str, Any]]:
|
||||
"""Capture une fenêtre Windows via PrintWindow.
|
||||
|
||||
Fallback utile quand la capture plein écran est noire mais que la fenêtre
|
||||
active reste imprimable par l'API Win32.
|
||||
"""
|
||||
if _SYSTEM != "Windows":
|
||||
return None, {"backend": "printwindow", "error": "unsupported_platform"}
|
||||
|
||||
try:
|
||||
import ctypes
|
||||
import win32gui
|
||||
import win32ui
|
||||
except ImportError as exc:
|
||||
return None, {"backend": "printwindow", "error": str(exc)}
|
||||
|
||||
last_error = None
|
||||
for flag in (3, 2, 0):
|
||||
wnd_dc = None
|
||||
src_dc = None
|
||||
mem_dc = None
|
||||
bmp = None
|
||||
try:
|
||||
wnd_dc = win32gui.GetWindowDC(hwnd)
|
||||
if not wnd_dc:
|
||||
raise RuntimeError("GetWindowDC a retourné 0")
|
||||
src_dc = win32ui.CreateDCFromHandle(wnd_dc)
|
||||
mem_dc = src_dc.CreateCompatibleDC()
|
||||
bmp = win32ui.CreateBitmap()
|
||||
bmp.CreateCompatibleBitmap(src_dc, width, height)
|
||||
mem_dc.SelectObject(bmp)
|
||||
result = ctypes.windll.user32.PrintWindow(
|
||||
hwnd, mem_dc.GetSafeHdc(), flag
|
||||
)
|
||||
bits = bmp.GetBitmapBits(True)
|
||||
img = Image.frombuffer(
|
||||
"RGB", (width, height), bits, "raw", "BGRX", 0, 1
|
||||
)
|
||||
luma = _compute_luma_stats(img)
|
||||
if result or not _is_effectively_black(img):
|
||||
return img, {
|
||||
"backend": f"printwindow:{flag}",
|
||||
"printwindow_result": int(result),
|
||||
"luma": luma,
|
||||
}
|
||||
except Exception as exc:
|
||||
last_error = str(exc)
|
||||
finally:
|
||||
try:
|
||||
if bmp is not None:
|
||||
win32gui.DeleteObject(bmp.GetHandle())
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
if mem_dc is not None:
|
||||
mem_dc.DeleteDC()
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
if src_dc is not None:
|
||||
src_dc.DeleteDC()
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
if wnd_dc is not None:
|
||||
win32gui.ReleaseDC(hwnd, wnd_dc)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return None, {
|
||||
"backend": "printwindow",
|
||||
"error": last_error or "no_usable_frame",
|
||||
}
|
||||
|
||||
|
||||
def capture_foreground_window_image() -> Tuple[Optional[Image.Image], Dict[str, Any]]:
|
||||
"""Capture la fenêtre au focus via API native si disponible."""
|
||||
try:
|
||||
from ..window_info_crossplatform import get_active_window_rect
|
||||
|
||||
rect_info = get_active_window_rect()
|
||||
except Exception as exc:
|
||||
return None, {"backend": "printwindow", "error": str(exc)}
|
||||
|
||||
if not rect_info:
|
||||
return None, {"backend": "printwindow", "error": "active_window_unavailable"}
|
||||
|
||||
win_w, win_h = rect_info.get("size", [0, 0])
|
||||
hwnd = rect_info.get("hwnd")
|
||||
if not hwnd or win_w <= 0 or win_h <= 0:
|
||||
return None, {
|
||||
"backend": "printwindow",
|
||||
"error": "active_window_handle_unavailable",
|
||||
"title": rect_info.get("title", "unknown_window"),
|
||||
}
|
||||
|
||||
img, meta = _capture_window_image_windows(hwnd, win_w, win_h)
|
||||
if img is None:
|
||||
return None, meta
|
||||
|
||||
meta.update(
|
||||
{
|
||||
"title": rect_info.get("title", "unknown_window"),
|
||||
"app_name": rect_info.get("app_name", "unknown_app"),
|
||||
"rect": rect_info.get("rect"),
|
||||
"window_size": rect_info.get("size"),
|
||||
"hwnd": hwnd,
|
||||
}
|
||||
)
|
||||
return img, meta
|
||||
|
||||
|
||||
class VisionCapturer:
|
||||
def __init__(self, session_dir: str):
|
||||
self.session_dir = session_dir
|
||||
@@ -103,25 +434,35 @@ class VisionCapturer:
|
||||
(utile pour le contextualisation des heartbeats côté serveur).
|
||||
"""
|
||||
try:
|
||||
with mss.mss() as sct:
|
||||
monitor = sct.monitors[1]
|
||||
sct_img = sct.grab(monitor)
|
||||
img = Image.frombytes("RGB", sct_img.size, sct_img.bgra, "raw", "BGRX")
|
||||
_monitor, img, meta = capture_screen_image()
|
||||
if img is None:
|
||||
img, win_meta = capture_foreground_window_image()
|
||||
if img is None:
|
||||
logger.error(
|
||||
"Capture plein contexte indisponible (meta=%s, window=%s)",
|
||||
meta,
|
||||
win_meta,
|
||||
)
|
||||
return ""
|
||||
logger.warning(
|
||||
"Capture plein contexte dégradée via fenêtre active (%s)",
|
||||
win_meta.get("backend"),
|
||||
)
|
||||
|
||||
# Détection de changement (pour Heartbeat)
|
||||
if not force:
|
||||
current_hash = self._compute_quick_hash(img)
|
||||
if current_hash == self.last_img_hash:
|
||||
return "" # Pas de changement, on économise la fibre
|
||||
self.last_img_hash = current_hash
|
||||
# Détection de changement (pour Heartbeat)
|
||||
if not force:
|
||||
current_hash = self._compute_quick_hash(img)
|
||||
if current_hash == self.last_img_hash:
|
||||
return "" # Pas de changement, on économise la fibre
|
||||
self.last_img_hash = current_hash
|
||||
|
||||
# Floutage des données sensibles (conformité AI Act)
|
||||
if BLUR_SENSITIVE:
|
||||
blur_sensitive_regions(img)
|
||||
# Floutage des données sensibles (conformité AI Act)
|
||||
if BLUR_SENSITIVE:
|
||||
blur_sensitive_regions(img)
|
||||
|
||||
path = os.path.join(self.shots_dir, f"context_{int(time.time())}_{name_suffix}.png")
|
||||
img.save(path, "PNG", quality=SCREENSHOT_QUALITY)
|
||||
return path
|
||||
path = os.path.join(self.shots_dir, f"context_{int(time.time())}_{name_suffix}.png")
|
||||
img.save(path, "PNG", quality=SCREENSHOT_QUALITY)
|
||||
return path
|
||||
except Exception as e:
|
||||
logger.error(f"Erreur Context Capture: {e}")
|
||||
return ""
|
||||
@@ -145,46 +486,62 @@ class VisionCapturer:
|
||||
sont toujours retournés (fallback gracieux).
|
||||
"""
|
||||
try:
|
||||
with mss.mss() as sct:
|
||||
full_path = os.path.join(self.shots_dir, f"{screenshot_id}_full.png")
|
||||
monitor = sct.monitors[1]
|
||||
sct_img = sct.grab(monitor)
|
||||
img = Image.frombytes("RGB", sct_img.size, sct_img.bgra, "raw", "BGRX")
|
||||
|
||||
# Capture du Crop (Cœur de l'apprentissage qwen3-vl)
|
||||
crop_path = os.path.join(self.shots_dir, f"{screenshot_id}_crop.png")
|
||||
w, h = TARGETED_CROP_SIZE
|
||||
left = max(0, x - w // 2)
|
||||
top = max(0, y - h // 2)
|
||||
crop_img = img.crop((left, top, left + w, top + h))
|
||||
|
||||
if anonymize:
|
||||
crop_img = crop_img.filter(ImageFilter.GaussianBlur(radius=4))
|
||||
|
||||
# Floutage des données sensibles (conformité AI Act)
|
||||
if BLUR_SENSITIVE:
|
||||
blur_sensitive_regions(img)
|
||||
blur_sensitive_regions(crop_img)
|
||||
|
||||
img.save(full_path, "PNG", quality=SCREENSHOT_QUALITY)
|
||||
crop_img.save(crop_path, "PNG", quality=SCREENSHOT_QUALITY)
|
||||
|
||||
# Mise à jour du hash pour le prochain heartbeat
|
||||
self.last_img_hash = self._compute_quick_hash(img)
|
||||
|
||||
result = {"full": full_path, "crop": crop_path}
|
||||
|
||||
# --- Capture de la fenêtre active ---
|
||||
# Ajout non-bloquant : enrichit le résultat avec l'image
|
||||
# de la fenêtre seule + métadonnées (titre, rect, clic relatif)
|
||||
window_info = self.capture_active_window(x, y, screenshot_id, full_img=img)
|
||||
# Coords (x, y) sont en système écran composite ; cropper depuis
|
||||
# un monitor secondaire (offset ≠ 0) produirait une image saine
|
||||
# mais décalée → fail-closed sur fallback secondaire.
|
||||
_monitor, img, meta = capture_screen_image(
|
||||
allow_secondary_fallback=False
|
||||
)
|
||||
if img is None:
|
||||
window_info = self.capture_active_window(
|
||||
x, y, screenshot_id, full_img=None
|
||||
)
|
||||
if window_info:
|
||||
result["window_capture"] = window_info
|
||||
result = {"window_capture": window_info}
|
||||
_enrich_with_monitor_info(result)
|
||||
logger.warning(
|
||||
"capture_dual dégradée: fenêtre active seule (%s)",
|
||||
meta,
|
||||
)
|
||||
return result
|
||||
return {}
|
||||
|
||||
# QW1 — enrichissement multi-écrans (additif, fallback gracieux)
|
||||
_enrich_with_monitor_info(result)
|
||||
full_path = os.path.join(self.shots_dir, f"{screenshot_id}_full.png")
|
||||
|
||||
return result
|
||||
# Capture du Crop (Cœur de l'apprentissage qwen3-vl)
|
||||
crop_path = os.path.join(self.shots_dir, f"{screenshot_id}_crop.png")
|
||||
w, h = TARGETED_CROP_SIZE
|
||||
left = max(0, x - w // 2)
|
||||
top = max(0, y - h // 2)
|
||||
crop_img = img.crop((left, top, left + w, top + h))
|
||||
|
||||
if anonymize:
|
||||
crop_img = crop_img.filter(ImageFilter.GaussianBlur(radius=4))
|
||||
|
||||
# Floutage des données sensibles (conformité AI Act)
|
||||
if BLUR_SENSITIVE:
|
||||
blur_sensitive_regions(img)
|
||||
blur_sensitive_regions(crop_img)
|
||||
|
||||
img.save(full_path, "PNG", quality=SCREENSHOT_QUALITY)
|
||||
crop_img.save(crop_path, "PNG", quality=SCREENSHOT_QUALITY)
|
||||
|
||||
# Mise à jour du hash pour le prochain heartbeat
|
||||
self.last_img_hash = self._compute_quick_hash(img)
|
||||
|
||||
result = {"full": full_path, "crop": crop_path}
|
||||
|
||||
# --- Capture de la fenêtre active ---
|
||||
# Ajout non-bloquant : enrichit le résultat avec l'image
|
||||
# de la fenêtre seule + métadonnées (titre, rect, clic relatif)
|
||||
window_info = self.capture_active_window(x, y, screenshot_id, full_img=img)
|
||||
if window_info:
|
||||
result["window_capture"] = window_info
|
||||
|
||||
# QW1 — enrichissement multi-écrans (additif, fallback gracieux)
|
||||
_enrich_with_monitor_info(result)
|
||||
|
||||
return result
|
||||
except Exception as e:
|
||||
logger.error(f"Erreur Dual Capture: {e}")
|
||||
return {}
|
||||
@@ -239,33 +596,54 @@ class VisionCapturer:
|
||||
# Si le clic est en dehors de la fenêtre, on le signale mais on continue
|
||||
click_inside = (0 <= click_rel_x <= win_w and 0 <= click_rel_y <= win_h)
|
||||
|
||||
window_img = None
|
||||
|
||||
# --- Crop de la fenêtre depuis le plein écran ---
|
||||
if full_img is None:
|
||||
# Pas de screenshot fourni — en capturer un (cas standalone)
|
||||
# Pas de screenshot fourni — en capturer un (cas standalone).
|
||||
# win_rect est en coords globales ; cropper depuis un monitor
|
||||
# secondaire produirait une image décalée → fail-closed sur
|
||||
# fallback secondaire.
|
||||
try:
|
||||
with mss.mss() as sct:
|
||||
monitor = sct.monitors[1]
|
||||
sct_img = sct.grab(monitor)
|
||||
full_img = Image.frombytes(
|
||||
"RGB", sct_img.size, sct_img.bgra, "raw", "BGRX"
|
||||
)
|
||||
_monitor, full_img, _meta = capture_screen_image(
|
||||
allow_secondary_fallback=False
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Erreur capture plein écran pour fenêtre : {e}")
|
||||
return None
|
||||
full_img = None
|
||||
|
||||
# Borner le crop aux limites de l'image plein écran
|
||||
img_w, img_h = full_img.size
|
||||
crop_left = max(0, win_left)
|
||||
crop_top = max(0, win_top)
|
||||
crop_right = min(img_w, win_right)
|
||||
crop_bottom = min(img_h, win_bottom)
|
||||
if full_img is not None and not _is_effectively_black(full_img):
|
||||
img_w, img_h = full_img.size
|
||||
crop_left = max(0, win_left)
|
||||
crop_top = max(0, win_top)
|
||||
crop_right = min(img_w, win_right)
|
||||
crop_bottom = min(img_h, win_bottom)
|
||||
|
||||
if crop_right <= crop_left or crop_bottom <= crop_top:
|
||||
logger.debug("Fenêtre hors écran — skip capture fenêtre")
|
||||
if crop_right > crop_left and crop_bottom > crop_top:
|
||||
window_img = full_img.crop(
|
||||
(crop_left, crop_top, crop_right, crop_bottom)
|
||||
)
|
||||
else:
|
||||
logger.debug("Fenêtre hors écran — fallback natif si possible")
|
||||
elif full_img is not None:
|
||||
logger.warning(
|
||||
"capture_active_window: screenshot plein écran noir, fallback natif"
|
||||
)
|
||||
|
||||
if window_img is None and rect_info.get("hwnd"):
|
||||
window_img, native_meta = _capture_window_image_windows(
|
||||
rect_info["hwnd"], win_w, win_h
|
||||
)
|
||||
if window_img is not None:
|
||||
logger.warning(
|
||||
"capture_active_window via fallback natif (%s)",
|
||||
native_meta.get("backend"),
|
||||
)
|
||||
|
||||
if window_img is None:
|
||||
logger.debug("Fenêtre hors écran ou capture native indisponible")
|
||||
return None
|
||||
|
||||
window_img = full_img.crop((crop_left, crop_top, crop_right, crop_bottom))
|
||||
|
||||
# Floutage conformité AI Act
|
||||
if BLUR_SENSITIVE:
|
||||
blur_sensitive_regions(window_img)
|
||||
|
||||
Reference in New Issue
Block a user