Dom/rpa_vision_v3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

snapshot: WIP 5j replay reliability (B1 watchdog + dialog handlers + grounding drift)

Browse Source 
Snapshot avant correction du blocage relance Léa (3 incidents 24h: SSH refusé,
polls morts ×2). Point de rollback stable.

Contenu:
- agent_v1/core/executor.py: 5 patchs dialog handling (saveas drift, close_tab
  hotkey fallback, confirm_save Unicode apostrophe, foreground dialog
  recontextualization, runtime_dialog in-loop) + helpers normalize_window_hint,
  requires_post_verify_window_transition
- agent_v1/core/grounding.py: garde drift template fix (fallback_x/y plumbed)
- server_v1/replay_watchdog.py (NEW): orphan watchdog B1, scan 10s timeout 30s
- server_v1/api_stream.py: dispatched_action plumbing, watchdog lifespan,
  metrics endpoint
- server_v1/replay_engine.py: _schedule_retry préserve original_action +
  dispatched_action
- stream_processor.py: gardes _infer_tab_switch_target (no false switch_tab
  on save_as dialog open) + _attach_expected_window_before
- tests/integration: test_replay_watchdog.py (8 cas), test_stream_processor.py
- tests/unit: test_executor_verify_window_guard.py (start_button, close_tab,
  runtime_dialog, post_verify, transition fallbacks)

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Dom
2026-05-24 16:48:37 +02:00
parent 5ea4960e65
commit 7df51d2c79
47 changed files with 9811 additions and 451 deletions
Download Patch File Download Diff File Expand all files Collapse all files

280
agent_v0/server_v1/api_stream.py
View File

@@ -61,7 +61,9 @@ MAX_ACTIONS_PER_REPLAY = 500 # Max actions par requête de replay
MAX_REPLAY_STATES = 1000 # Max entrées dans _replay_states
REPLAY_STATE_TTL_SECONDS = 3600 # Nettoyage auto des replays terminés après 1h
# Actions en cours de retry : action_id -> {"action": ..., "retry_count": N, "replay_id": ...}
# Actions in-flight / retry : action_id -> transport + retry metadata.
# `action` remains the semantic/original action for reporting/retry logic,
# while `dispatched_action` tracks the exact payload last sent to Lea.
_retry_pending: Dict[str, Dict[str, Any]] = {}
# Callbacks d'erreur par replay_id : replay_id -> callback_url
@@ -207,12 +209,14 @@ from .replay_engine import (
_MAX_ACTION_TEXT_LENGTH,
_MAX_KEYS_PER_COMBO,
_KNOWN_KEY_NAMES,
_auto_launch_replay_after_finalize,
_validate_replay_action,
_APP_LAUNCH_COMMANDS,
_APP_VISUAL_SEARCH,
_SETUP_IGNORE_APPS,
_extract_required_apps_from_events,
_extract_required_apps_from_workflow,
_trim_redundant_setup_events,
_resolve_launch_command,
_infer_app_from_window_titles,
_get_visual_search_info,
@@ -475,6 +479,19 @@ def _clear_replay_lock():
logger.error(f"Erreur suppression replay lock : {e}")
def _memory_window_title_for_action(action_meta: Dict[str, Any]) -> str:
"""Résoudre le meilleur window_title disponible pour la mémoire persistante."""
action_meta = action_meta or {}
target_spec = action_meta.get("target_spec") or {}
context_hints = target_spec.get("context_hints") or {}
return (
action_meta.get("expected_window_before", "")
or target_spec.get("window_title", "")
or context_hints.get("window_title", "")
or action_meta.get("window_title", "")
)
def _get_worker_queue_status() -> Dict[str, Any]:
"""Retourne l'état de la queue du worker VLM (pour le monitoring)."""
queue = []
@@ -544,6 +561,34 @@ _machine_replay_target: Dict[str, str] = {}
_replay_states: Dict[str, Dict[str, Any]] = {}
def _remove_queued_action_duplicates(session_id: str, action_id: str) -> int:
"""Retirer d'une queue les copies exactes d'une action déjà acquittée.
Le watchdog peut re-pousser une action orpheline en tête de queue. Si le
report original arrive juste après, cette copie resend doit être jetée,
sinon Léa ré-exécute la même action avec le même `action_id` et peut
toggler l'état UI (ex: touche Windows qui referme Démarrer).
"""
if not session_id or not action_id:
return 0
queue = _replay_queues.get(session_id, [])
if not queue:
return 0
filtered: List[Dict[str, Any]] = []
removed = 0
for queued_action in queue:
queued_id = str((queued_action or {}).get("action_id", "") or "")
if queued_id == action_id:
removed += 1
continue
filtered.append(queued_action)
if removed:
_replay_queues[session_id] = filtered
return removed
class StreamEvent(BaseModel):
session_id: str
timestamp: float
@@ -832,6 +877,16 @@ async def startup():
threading.Thread(target=_preload_easyocr, daemon=True, name="preload_easyocr").start()
from .replay_watchdog import get_or_create_watchdog
app.state.replay_watchdog = get_or_create_watchdog(
retry_pending=_retry_pending,
replay_queues=_replay_queues,
async_lock_factory=_async_replay_lock,
sse_notifier=None,
)
await app.state.replay_watchdog.start()
logger.info(
"API Streaming démarrée — StreamProcessor, Worker et Cleanup prêts. "
"VLM Worker dans un process séparé (run_worker.py)."
@@ -886,6 +941,9 @@ def _load_existing_workflows():
async def shutdown():
global _cleanup_running
_cleanup_running = False
watchdog = getattr(app.state, "replay_watchdog", None)
if watchdog is not None:
await watchdog.stop(timeout_s=3.0)
worker.stop()
# Nettoyer le replay lock au shutdown (sinon le worker VLM resterait bloqué)
_clear_replay_lock()
@@ -1477,17 +1535,24 @@ def _process_screenshot_thread(session_id: str, shot_id: str, path: str):
# =========================================================================
@app.post("/api/v1/traces/stream/finalize")
async def finalize(session_id: str, machine_id: str = "default"):
async def finalize(
session_id: str,
machine_id: str = "default",
launch_replay: bool = False,
):
"""Clôture la session et place le traitement en file d'attente.
Ne bloque plus : marque la session comme finalisée et l'ajoute à la queue
du worker VLM (process séparé) pour analyse + construction workflow.
Le client peut suivre la progression via GET /api/v1/traces/stream/processing/status.
Optionnellement, il peut aussi déclencher immédiatement un replay direct
depuis la session finalisée (chemin Lea-first, sans attendre le workflow VLM).
Args:
session_id: Identifiant de la session à finaliser
machine_id: Identifiant machine (informatif, le machine_id est déjà dans la session)
launch_replay: Si vrai, tente de lancer immédiatement /replay-session
"""
# Vérifier que la session existe
session = processor.session_manager.get_session(session_id)
@@ -1501,6 +1566,10 @@ async def finalize(session_id: str, machine_id: str = "default"):
processor.session_manager.finalize(session_id)
logger.info(f"Session {session_id} finalisée, ajout à la queue du worker VLM")
resolved_machine_id = machine_id
if resolved_machine_id == "default" and getattr(session, "machine_id", ""):
resolved_machine_id = session.machine_id
# Nettoyer les structures d'enrichissement temps réel pour cette session
with _enrichment_lock:
keys_to_remove = [k for k in _pending_click_enrichments if k[0] == session_id]
@@ -1521,17 +1590,70 @@ async def finalize(session_id: str, machine_id: str = "default"):
if shots_dir.exists():
full_shots_count = len(list(shots_dir.glob("shot_*_full.png")))
return {
# Patch 2026-05-23 (brief 0902 deferred-workflow) : par défaut, on
# ne propose plus le replay direct immédiat post-finalize — le chemin
# produit cible est le workflow compilé par le worker VLM. Le client
# attend la disponibilité du workflow nommé pour proposer un test.
# Le replay direct reste accessible (smoke/debug) en activant
# RPA_AUTO_LAUNCH_REPLAY_AFTER_FINALIZE=true côté serveur, OU
# en appelant explicitement POST /api/v1/traces/stream/replay-session
# depuis un outil de test.
_direct_replay_enabled = _auto_launch_replay_after_finalize()
response = {
"status": "queued_for_processing",
"session_id": session_id,
"machine_id": session.machine_id,
"screenshots_to_analyze": full_shots_count,
"replay_ready": _direct_replay_enabled,
"message": (
f"Session finalisée. {full_shots_count} screenshots seront analysés "
"en arrière-plan. Suivez la progression via "
"GET /api/v1/traces/stream/processing/status"
"GET /api/v1/traces/stream/processing/status."
),
}
if _direct_replay_enabled:
response["replay_request"] = {
"endpoint": "/api/v1/traces/stream/replay-session",
"session_id": session_id,
"machine_id": resolved_machine_id,
}
response["message"] += (
" Le replay direct est disponible via "
"POST /api/v1/traces/stream/replay-session"
)
if not launch_replay:
return response
try:
replay_result = await replay_from_session(
session_id=session_id,
machine_id=resolved_machine_id,
)
except HTTPException as exc:
logger.warning(
"Finalize %s : replay direct non lancé (%s)",
session_id,
exc.detail,
)
response["replay_launch"] = {
"status": "failed",
"status_code": exc.status_code,
"detail": exc.detail,
}
response["message"] += (
" Le lancement automatique du replay direct a échoué ; "
"la session reste finalisée et re-jouable manuellement."
)
return response
response["replay_launch"] = {
"status": "started",
"replay": replay_result,
}
response["message"] += " Le replay direct a été lancé immédiatement."
return response
# =========================================================================
@@ -2262,18 +2384,39 @@ async def replay_from_session(
if session_mem and session_mem.events:
_merge_enrichments_into_raw_events(raw_events, session_mem.events)
# ── 3. Construire le replay propre depuis les events bruts ──
# Passer le répertoire de session pour activer le visual replay (crops de référence)
# Répertoire de session utilisé par le visual replay et les anchors setup
session_dir = str(events_file.parent)
# ── 3. Préparer le setup environnement et couper le préambule source ──
setup_actions = []
app_info = _extract_required_apps_from_events(
raw_events,
session_dir=session_dir,
)
replay_raw_events = raw_events
if app_info:
setup_actions = _generate_setup_actions(app_info, setup_id_prefix="setup_sess")
if setup_actions:
replay_raw_events = _trim_redundant_setup_events(raw_events, app_info)
logger.info(
"replay-session %s : %d actions de setup préparées avant le replay "
"(app=%s, cmd=%s, raw_trim=%d→%d)",
session_id, len(setup_actions),
app_info.get("primary_app"), app_info.get("primary_launch_cmd"),
len(raw_events), len(replay_raw_events),
)
# ── 4. Construire le replay propre depuis les events bruts ──
# Passer le répertoire de session pour activer le visual replay (crops de référence)
actions = build_replay_from_raw_events(
raw_events, session_id=session_id, session_dir=session_dir,
replay_raw_events, session_id=session_id, session_dir=session_dir,
)
if not actions:
raise HTTPException(
status_code=400,
detail=f"Session '{session_id}' : aucune action exploitable après nettoyage "
f"({len(raw_events)} événements bruts)"
f"({len(replay_raw_events)} événements bruts)"
)
# Limite de sécurité
@@ -2305,23 +2448,10 @@ async def replay_from_session(
if _gesture_catalog and actions:
actions = _gesture_catalog.optimize_replay_actions(actions)
# ── 3b. Setup environnement — ouvrir les applications nécessaires ──
# Analyser les événements bruts pour détecter quelles applications sont requises
# et injecter des actions de setup en tête de la queue de replay.
setup_actions = []
app_info = _extract_required_apps_from_events(raw_events)
if app_info:
setup_actions = _generate_setup_actions(app_info, setup_id_prefix="setup_sess")
if setup_actions:
actions = setup_actions + actions
logger.info(
"replay-session %s : %d actions de setup injectées avant le replay "
"(app=%s, cmd=%s)",
session_id, len(setup_actions),
app_info.get("primary_app"), app_info.get("primary_launch_cmd"),
)
if setup_actions:
actions = setup_actions + actions
# ── 4. Trouver la session de replay cible (Agent V1 actif) ──
# ── 5. Trouver la session de replay cible (Agent V1 actif) ──
# L'agent actif peut avoir une session différente de la session source
target_session_id = _find_active_agent_session(machine_id=machine_id)
if not target_session_id:
@@ -2335,7 +2465,7 @@ async def replay_from_session(
"Lancez l'Agent V1 sur le PC cible."
)
# ── 5. Injecter dans la queue de replay ──
# ── 6. Injecter dans la queue de replay ──
replay_id = f"replay_sess_{uuid.uuid4().hex[:8]}"
async with _async_replay_lock():
@@ -3265,11 +3395,35 @@ async def get_next_action(session_id: str, machine_id: str = "default"):
# NE PAS écraser si _schedule_retry a déjà mis le bon retry_count
action_id_sent = action.get("action_id", "")
if action_id_sent and action_id_sent not in _retry_pending:
now = time.time()
_retry_pending[action_id_sent] = {
"action": dict(action),
"dispatched_action": dict(action),
"retry_count": 0,
"replay_id": "",
"replay_id": owning_replay.get("replay_id", "") if owning_replay else "",
"session_id": session_id,
"machine_id": machine_id,
"dispatched_at": now,
"first_dispatched_at": now,
"resent_count": 0,
"last_resent_at": 0.0,
}
elif action_id_sent:
existing = _retry_pending.get(action_id_sent)
if existing is not None:
now = time.time()
existing.setdefault("action", dict(action))
existing["dispatched_action"] = dict(action)
existing["replay_id"] = existing.get("replay_id") or (
owning_replay.get("replay_id", "") if owning_replay else ""
)
existing["session_id"] = session_id
existing["machine_id"] = machine_id
existing["dispatched_at"] = now
if not existing.get("first_dispatched_at"):
existing["first_dispatched_at"] = now
existing.setdefault("resent_count", 0)
existing.setdefault("last_resent_at", 0.0)
# [REPLAY] log structuré pour suivre une action à travers toute la chaîne
# Grep facile : journalctl --user -u rpa-streaming -f | grep REPLAY
@@ -3400,6 +3554,15 @@ async def report_action_result(report: ReplayResultReport):
)
return {"status": "no_active_replay", "session_id": session_id}
removed_dupes = _remove_queued_action_duplicates(session_id, action_id)
if removed_dupes:
logger.warning(
"[REPLAY] REPORT cleanup session=%s action_id=%s removed_queue_duplicates=%d",
session_id,
action_id,
removed_dupes,
)
# Récupérer l'info de retry pour cette action (si c'est un retry)
retry_info = _retry_pending.pop(action_id, None)
retry_count = retry_info["retry_count"] if retry_info else 0
@@ -3631,10 +3794,7 @@ async def report_action_result(report: ReplayResultReport):
_current = _actions_meta[_idx] or {}
if _current.get("type") == "click":
_mem_target_spec = _current.get("target_spec") or {}
_mem_window_title = (
_mem_target_spec.get("window_title", "")
or _mem_target_spec.get("expected_window_before", "")
)
_mem_window_title = _memory_window_title_for_action(_current)
if _mem_window_title:
_mem_success = (
@@ -3749,6 +3909,7 @@ async def report_action_result(report: ReplayResultReport):
"target_description": f"Dialogue système : {_sys_category}",
"screenshot_b64": screenshot_after or report.screenshot,
"target_spec": _tspec_sys,
"original_action": dict(original_action or {}),
"reason": "system_dialog",
"system_dialog": _sys_info,
"error_detail": _sys_reason or (report.error or ""),
@@ -3814,6 +3975,7 @@ async def report_action_result(report: ReplayResultReport):
"target_description": _target_desc_ww,
"screenshot_b64": screenshot_after or report.screenshot,
"target_spec": _tspec_ww,
"original_action": dict(original_action or {}),
"reason": "wrong_window",
"error_detail": report.error or "",
}
@@ -3888,6 +4050,7 @@ async def report_action_result(report: ReplayResultReport):
"target_description": _target_desc,
"screenshot_b64": screenshot_after or report.screenshot,
"target_spec": _tspec,
"original_action": dict(original_action or {}),
"reason": "no_screen_change_strict",
"resolution_method": report.resolution_method or "",
"resolution_score": report.resolution_score or 0,
@@ -3947,6 +4110,7 @@ async def report_action_result(report: ReplayResultReport):
"target_description": target_desc,
"screenshot_b64": screenshot_after or report.screenshot,
"target_spec": report.target_spec,
"original_action": dict(original_action or {}),
}
replay_state["pause_message"] = f"Je ne vois pas '{target_desc}' à l'écran"
error_entry = {
@@ -3989,6 +4153,7 @@ async def report_action_result(report: ReplayResultReport):
"target_description": target_desc,
"screenshot_b64": screenshot_after or report.screenshot,
"target_spec": report.target_spec,
"original_action": dict(original_action or {}),
}
replay_state["pause_message"] = f"Je ne vois pas '{target_desc}' à l'écran"
error_entry = {
@@ -4341,8 +4506,14 @@ async def resume_replay(
and failed_action.get("reason") != "user_request"):
# Reconstruire l'action a partir du retry_pending ou de l'original
original_action_id = failed_action["action_id"]
original = failed_action.get("original_action")
if isinstance(original, dict) and original:
original = dict(original)
else:
original = None
# Chercher l'action originale dans les retry_pending
original = _retry_pending.pop(original_action_id, {}).get("action")
if not original:
original = _retry_pending.pop(original_action_id, {}).get("action")
if not original:
# Reconstruire un minimum depuis le failed_action context
original = {
@@ -4358,8 +4529,15 @@ async def resume_replay(
# Stocker dans retry_pending pour le suivi
_retry_pending[resume_id] = {
"action": original,
"dispatched_action": dict(resume_action),
"retry_count": 0,
"replay_id": replay_id,
"session_id": session_id,
"machine_id": state.get("machine_id", "default"),
"dispatched_at": 0.0,
"first_dispatched_at": 0.0,
"resent_count": 0,
"last_resent_at": 0.0,
"reason": "resume_after_pause",
}
queue = _replay_queues.get(session_id, [])
@@ -4399,6 +4577,13 @@ async def cancel_replay(replay_id: str):
return {"status": "cancelled", "replay_id": replay_id, "session_id": session_id}
@app.get("/api/v1/traces/stream/replay/watchdog/metrics")
async def watchdog_metrics():
from .replay_watchdog import get_metrics_snapshot
return {"watchdog": get_metrics_snapshot()}
# =========================================================================
# Visual Replay — Résolution visuelle des cibles (module resolve_engine)
# =========================================================================
@@ -4545,10 +4730,13 @@ async def resolve_target(request: ResolveTargetRequest):
# Validation qualité en sortie de cascade : seuil de score + garde
# de proximité contre les coords enregistrées. Single point of
# insertion, n'altère pas la cascade existante.
# target_spec propagé pour relaxation contextuelle (switch_tab +
# som_element calibré, cf. resolve_engine.py 2026-05-22).
result = _validate_resolution_quality(
result,
request.fallback_x_pct,
request.fallback_y_pct,
target_spec=request.target_spec,
)
# Pré-check sémantique post-cascade : OCR sur une zone autour de la
@@ -4581,6 +4769,15 @@ async def resolve_target(request: ResolveTargetRequest):
_by_text = (request.target_spec.get("by_text") or "").strip()
if _by_text:
from agent_v0.server_v1.resolve_engine import _validate_text_at_position
# Propager la bbox SoM enregistrée (si présente) au
# pré-check OCR : pour les éléments étroits (onglets
# Notepad moderne, ~30-40px haut), le radius générique
# capture du texte voisin et rejette à tort.
# Patch 2026-05-23 — cf. inbox_codex/…_notepad-tab-ocr-precheck.
_som_bbox = (
(request.target_spec.get("som_element") or {})
.get("bbox_norm")
)
_is_valid, _observed, _ocr_ms = _validate_text_at_position(
tmp_path,
float(result.get("x_pct", 0) or 0),
@@ -4588,6 +4785,7 @@ async def resolve_target(request: ResolveTargetRequest):
_by_text,
effective_w,
effective_h,
som_bbox_norm=_som_bbox,
)
logger.info(
"[REPLAY] Pre-check OCR ACTIF : '%s' attendu @ (%.4f, %.4f) "
@@ -4600,7 +4798,16 @@ async def resolve_target(request: ResolveTargetRequest):
_is_valid,
_ocr_ms,
)
if not _is_valid:
# Patch 2026-05-23 : rejet uniquement si OCR a effectivement
# lu *autre chose* que la cible. Si observed est vide, l'OCR
# n'a rien lu (crop bbox SoM trop petit / contraste faible
# sur onglet Notepad moderne) — ambigu, on garde la
# résolution serveur. La garde drift ANCHOR-TM côté agent
# bloque les vrais faux positifs.
from agent_v0.server_v1.resolve_engine import (
_should_reject_on_text_mismatch,
)
if _should_reject_on_text_mismatch(_is_valid, _observed):
logger.warning(
"[REPLAY] Pre-check OCR REJET : '%s' attendu @ (%.4f, %.4f) "
"via %s mais OCR voit '%s' (%.0fms)",
@@ -4620,6 +4827,15 @@ async def resolve_target(request: ResolveTargetRequest):
"x_pct": None,
"y_pct": None,
}
elif not _is_valid:
# observed vide → on log mais on accepte
logger.info(
"[REPLAY] Pre-check OCR observed='' (crop trop "
"petit/contraste faible) — on garde la résolution "
"via %s (score=%s), garde drift agent protège en aval",
result.get("method", "?"),
result.get("score"),
)
# [REPLAY] log structuré de sortie résolution (après validation)
# Note: x_pct/y_pct peuvent être None quand le pré-check OCR rejette

27
agent_v0/server_v1/live_session_manager.py
View File

@@ -17,6 +17,20 @@ from typing import Any, Dict, List, Optional
logger = logging.getLogger(__name__)
def _infer_machine_id_from_session_id(session_id: str, fallback: str = "default") -> str:
"""Déduire le machine_id depuis un session_id spécial si possible.
Les heartbeats de fond de Léa utilisent `bg_<machine_id>` comme
identifiant de session. Lors d'un redémarrage serveur, ces sessions
peuvent être restaurées depuis la persistance JSON avec `machine_id`
resté à `default`. On rétablit ici l'information machine pour que les
replays ciblés retrouvent bien la session de fond active.
"""
if session_id.startswith("bg_") and len(session_id) > 3:
return session_id[3:]
return fallback
@dataclass
class LiveSessionState:
"""État d'une session active en mémoire."""
@@ -86,11 +100,18 @@ class LiveSessionManager:
def _load_persisted_sessions(self):
"""Charger les sessions sauvegardées au démarrage (JSON state files)."""
count = 0
for session_file in sorted(self._persist_dir.glob("sess_*.json")):
session_files = sorted(self._persist_dir.glob("sess_*.json"))
session_files += sorted(self._persist_dir.glob("bg_*.json"))
for session_file in session_files:
try:
with open(session_file, 'r', encoding='utf-8') as f:
data = json.load(f)
session = LiveSessionState.from_dict(data)
if session.machine_id == "default":
session.machine_id = _infer_machine_id_from_session_id(
session.session_id,
fallback=session.machine_id,
)
self._sessions[session.session_id] = session
count += 1
except Exception as e:
@@ -117,7 +138,7 @@ class LiveSessionManager:
for jsonl_file in sorted(live_dir.glob("**/live_events.jsonl")):
session_dir = jsonl_file.parent
session_id = session_dir.name
if not session_id.startswith("sess_"):
if not (session_id.startswith("sess_") or session_id.startswith("bg_")):
continue
if session_id in self._sessions:
continue
@@ -125,7 +146,7 @@ class LiveSessionManager:
# Déduire le machine_id depuis le chemin parent
parent_name = session_dir.parent.name
if parent_name == live_dir.name:
machine_id = "default"
machine_id = _infer_machine_id_from_session_id(session_id)
else:
machine_id = parent_name

972
agent_v0/server_v1/replay_engine.py
View File

File diff suppressed because it is too large Load Diff

37
agent_v0/server_v1/replay_learner.py
View File

@@ -188,7 +188,12 @@ class ReplayLearner:
"""
target_spec = action.get("target_spec", {})
by_text = target_spec.get("by_text", "")
window_title = target_spec.get("window_title", "")
window_title = (
target_spec.get("window_title", "")
or action.get("window_title", "")
or target_spec.get("expected_window_before", "")
or (target_spec.get("context_hints") or {}).get("window_title", "")
)
x_pct = correction.get("x_pct", 0.0)
y_pct = correction.get("y_pct", 0.0)
@@ -207,20 +212,36 @@ class ReplayLearner:
# Stocker dans target_memory.db pour le lookup futur
try:
from .replay_memory import get_target_memory_store
store = get_target_memory_store()
if store:
store.record_success(
screen_signature="human_correction",
from .replay_memory import memory_record_success
stored = False
if window_title:
stored = memory_record_success(
window_title=window_title,
target_spec=target_spec,
resolved_position={"x_pct": x_pct, "y_pct": y_pct},
x_pct=float(x_pct),
y_pct=float(y_pct),
method="human_supervised",
score=1.0,
confidence=1.0,
)
else:
logger.warning(
"[APPRENTISSAGE] Correction humaine non persistée : "
"window_title absent pour '%s'",
by_text,
)
if stored:
logger.info(
f"[APPRENTISSAGE] Correction stockée dans target_memory : "
f"'{by_text}' → ({x_pct:.4f}, {y_pct:.4f})"
)
elif window_title:
logger.warning(
"[APPRENTISSAGE] Correction humaine non persistée : "
"échec memory_record_success pour '%s' dans '%s'",
by_text,
window_title,
)
except Exception as e:
logger.warning(f"Learning: échec stockage target_memory: {e}")

99
agent_v0/server_v1/replay_memory.py
View File

@@ -103,15 +103,53 @@ def compute_screen_sig(window_title: str) -> str:
return hashlib.sha256(norm.encode("utf-8")).hexdigest()[:16]
def _round_float_list(values: Any, precision: int = 4) -> Optional[tuple[float, ...]]:
"""Normaliser une liste de coordonnées flottantes pour le hash mémoire."""
if not isinstance(values, (list, tuple)):
return None
out = []
for value in values:
try:
out.append(round(float(value), precision))
except (TypeError, ValueError):
return None
return tuple(out)
def _int_pair(values: Any) -> Optional[tuple[int, int]]:
"""Extraire une paire entière stable pour les hints spatiaux."""
if not isinstance(values, (list, tuple)) or len(values) < 2:
return None
try:
return int(values[0]), int(values[1])
except (TypeError, ValueError):
return None
def _should_reuse_recorded_window_relative_coords(fp: Any) -> bool:
"""Décider si on doit remplacer la mémoire apprise par la position source.
Cette réécriture n'est légitime que pour les entrées faibles de type
`position_fallback`/`v4_unknown`, où la mémoire ne contient pas une vraie
localisation visuelle robuste mais seulement un clic écran dépendant de la
résolution. Pour les méthodes visuelles apprises (template, SoM, OCR...),
réinjecter un vieux `click_relative` source crée des collisions et des
dérives sur des boutons homonymes (`Enregistrer`, `OK`, etc.).
"""
method = str(getattr(fp, "etype", "") or "").strip().lower()
return method in {"position_fallback", "v4_unknown"}
class _TargetSpecLike:
"""Adaptateur dict → objet pour `TargetMemoryStore._hash_target_spec()`.
Le hash interne de TargetMemoryStore utilise `getattr(spec, "by_role", ...)`
qui ne fonctionne pas avec un dict brut. On expose les attributs nécessaires.
On intègre aussi `resolve_order` et `vlm_description` dans `context_hints`
pour qu'ils entrent dans le hash — deux actions avec le même `by_text`
mais un `resolve_order` différent doivent avoir des hashes distincts.
On intègre aussi `resolve_order`, `vlm_description` et des indices
spatiaux (SoM, click_relative) dans `context_hints` pour qu'ils entrent
dans le hash. Sinon, deux actions `Enregistrer` dans la même fenêtre
mais à des emplacements différents collisionnent.
"""
__slots__ = ("by_role", "by_text", "by_position", "context_hints")
@@ -131,6 +169,21 @@ class _TargetSpecLike:
hints["_vlm_desc"] = str(d["vlm_description"])
if d.get("anchor_hint"):
hints["_anchor_hint"] = str(d["anchor_hint"])
som_element = d.get("som_element") or {}
som_bbox = _round_float_list(som_element.get("bbox_norm"))
if som_bbox:
hints["_som_bbox"] = som_bbox
som_center = _round_float_list(som_element.get("center_norm"), precision=5)
if som_center:
hints["_som_center"] = som_center
window_capture = d.get("window_capture") or {}
click_relative = _int_pair(window_capture.get("click_relative"))
window_size = _int_pair(window_capture.get("window_size"))
if click_relative and window_size:
hints["_window_rel"] = f"{click_relative[0]},{click_relative[1]}@{window_size[0]}x{window_size[1]}"
self.context_hints = hints
@@ -176,6 +229,46 @@ def memory_lookup(
logger.debug("memory_lookup: fingerprint bbox invalide")
return None
# Quand l'entrée mémoire provient d'un simple `position_fallback`, les
# coordonnées stockées reflètent surtout la géométrie écran source. Dans
# ce cas précis, réutiliser la position relative enregistrée dans la
# fenêtre source reste préférable si elle existe.
#
# En revanche, pour une méthode visuelle réellement apprise
# (`anchor_template`, `som_*`, `hybrid_text_direct`, ...), remplacer les
# coords mémorisées par un vieux `click_relative` crée des dérives sur
# des cibles textuelles homonymes. On garde donc les coords apprises.
window_capture = target_spec.get("window_capture") or {}
click_relative = window_capture.get("click_relative")
window_size = window_capture.get("window_size")
if (
_should_reuse_recorded_window_relative_coords(fp)
and (
isinstance(click_relative, (list, tuple))
and len(click_relative) >= 2
and isinstance(window_size, (list, tuple))
and len(window_size) >= 2
)
):
try:
rel_x = float(click_relative[0])
rel_y = float(click_relative[1])
win_w = float(window_size[0])
win_h = float(window_size[1])
if win_w > 1 and win_h > 1:
x_pct = rel_x / win_w
y_pct = rel_y / win_h
logger.info(
"memory_lookup: coords fenêtre source réutilisées "
"(click_relative=%s, window_size=%s) -> (%.4f, %.4f)",
click_relative,
window_size,
x_pct,
y_pct,
)
except (TypeError, ValueError, ZeroDivisionError):
logger.debug("memory_lookup: window_capture invalide, fallback bbox")
# Sanity check : les pourcentages doivent être dans [0, 1]
if not (0.0 <= x_pct <= 1.0 and 0.0 <= y_pct <= 1.0):
logger.warning(

9
agent_v0/server_v1/replay_verifier.py
View File

@@ -328,10 +328,11 @@ class ReplayVerifier:
),
)
# Cas 4 : Pas de changement (key_combo, wait)
# Pour les raccourcis clavier et attentes, l'absence de changement
# n'est pas forcément un problème (ex: Ctrl+C ne change pas l'écran)
if action_type in ("key_combo", "wait"):
# Cas 4 : Pas de changement (key_combo, wait, verify_screen)
# `verify_screen` côté agent n'est qu'une temporisation de stabilisation.
# Il ne doit pas exiger un NOUVEAU changement visuel sinon le setup
# boucle inutilement une fois l'application déjà ouverte.
if action_type in ("key_combo", "wait", "verify_screen"):
return VerificationResult(
verified=True,
confidence=0.4,

329
agent_v0/server_v1/replay_watchdog.py Normal file
View File

@@ -0,0 +1,329 @@
"""Replay orphan watchdog for in-flight replay actions.
This module watches `_retry_pending` and re-pushes actions that were
dispatched by the server but never acknowledged by the Windows agent.
"""
from __future__ import annotations
import asyncio
import contextlib
import logging
import os
import time
from typing import Any, Callable, Dict, List, Optional, Tuple
logger = logging.getLogger(__name__)
def _env_bool(name: str, default: str) -> bool:
return os.environ.get(name, default).strip().lower() in {
"1",
"true",
"yes",
"on",
}
def _env_float(name: str, default: float) -> float:
try:
return float(os.environ.get(name, str(default)))
except (TypeError, ValueError):
logger.warning("Watchdog: invalid env %s, fallback=%s", name, default)
return default
def _env_int(name: str, default: int) -> int:
try:
return int(os.environ.get(name, str(default)))
except (TypeError, ValueError):
logger.warning("Watchdog: invalid env %s, fallback=%s", name, default)
return default
def _env_max_resends(default: int) -> int:
raw = os.environ.get("RPA_WATCHDOG_MAX_RESENDS")
if raw is None or not str(raw).strip():
raw = os.environ.get("RPA_WATCHDOG_MAX_RETRIES")
try:
return int(raw) if raw is not None else default
except (TypeError, ValueError):
logger.warning("Watchdog: invalid max resend env, fallback=%s", default)
return default
WATCHDOG_ENABLED = _env_bool("RPA_WATCHDOG_ENABLED", "1")
WATCHDOG_SCAN_INTERVAL_S = _env_float("RPA_WATCHDOG_SCAN_INTERVAL_S", 10.0)
WATCHDOG_ORPHAN_TIMEOUT_S = _env_float("RPA_WATCHDOG_ORPHAN_TIMEOUT_S", 45.0)
WATCHDOG_MAX_RESENDS = _env_max_resends(2)
WATCHDOG_REPUSH_POSITION = (
os.environ.get("RPA_WATCHDOG_REPUSH_POSITION", "head").strip().lower()
)
_metrics_lock = asyncio.Lock()
_metrics: Dict[str, Any] = {
"orphans_detected_total": 0,
"orphans_resent_total": 0,
"orphans_giveup_total": 0,
"scans_total": 0,
"scans_failed_total": 0,
"last_scan_ts": 0.0,
"last_scan_duration_ms": 0.0,
"current_in_flight_count": 0,
"current_orphan_count": 0,
}
async def _bump(key: str, delta: int = 1) -> None:
async with _metrics_lock:
_metrics[key] = _metrics.get(key, 0) + delta
def get_metrics_snapshot() -> Dict[str, Any]:
return dict(_metrics)
SseNotifier = Callable[[str, str], None]
class ReplayWatchdog:
"""Background coroutine that re-pushes orphaned replay actions."""
def __init__(
self,
retry_pending: Dict[str, Dict[str, Any]],
replay_queues: Dict[str, List[Dict[str, Any]]],
async_lock_factory: Callable[[], Any],
sse_notifier: Optional[SseNotifier] = None,
) -> None:
self._retry_pending = retry_pending
self._replay_queues = replay_queues
self._async_lock = async_lock_factory
self._sse_notifier = sse_notifier
self._task: Optional[asyncio.Task] = None
self._stopped = asyncio.Event()
async def start(self) -> None:
if not WATCHDOG_ENABLED:
logger.info("[WATCHDOG] disabled via RPA_WATCHDOG_ENABLED=0")
return
if self._task is not None and not self._task.done():
logger.warning("[WATCHDOG] already started")
return
self._stopped.clear()
self._task = asyncio.create_task(self._run(), name="replay_watchdog")
logger.info(
"[WATCHDOG] started scan=%.1fs orphan_timeout=%.1fs max_resends=%d repush=%s",
WATCHDOG_SCAN_INTERVAL_S,
WATCHDOG_ORPHAN_TIMEOUT_S,
WATCHDOG_MAX_RESENDS,
WATCHDOG_REPUSH_POSITION,
)
async def stop(self, timeout_s: float = 5.0) -> None:
if self._task is None:
return
self._stopped.set()
self._task.cancel()
try:
await asyncio.wait_for(self._task, timeout=timeout_s)
except asyncio.CancelledError:
pass
except asyncio.TimeoutError:
logger.warning("[WATCHDOG] stop timeout after %.1fs", timeout_s)
except Exception:
logger.exception("[WATCHDOG] unexpected stop error")
self._task = None
logger.info("[WATCHDOG] stopped")
async def _run(self) -> None:
try:
while not self._stopped.is_set():
try:
await asyncio.wait_for(
self._stopped.wait(),
timeout=WATCHDOG_SCAN_INTERVAL_S,
)
break
except asyncio.TimeoutError:
pass
try:
await self._scan_once()
except Exception:
await _bump("scans_failed_total")
logger.exception("[WATCHDOG] scan failed")
except asyncio.CancelledError:
logger.info("[WATCHDOG] cancelled")
raise
finally:
logger.info("[WATCHDOG] loop terminated")
async def _scan_once(self) -> Dict[str, int]:
t0 = time.time()
await _bump("scans_total")
resent = 0
gaveup = 0
skipped = 0
in_flight = 0
orphans = 0
orphan_targets: List[Tuple[str, Dict[str, Any]]] = []
async with self._async_lock():
for action_id, info in list(self._retry_pending.items()):
dispatched_at = info.get("dispatched_at", 0.0) or 0.0
if dispatched_at <= 0:
skipped += 1
continue
age = t0 - dispatched_at
in_flight += 1
if age < WATCHDOG_ORPHAN_TIMEOUT_S:
continue
orphans += 1
orphan_targets.append((action_id, dict(info)))
for action_id, info in orphan_targets:
await _bump("orphans_detected_total")
resent_count = int(info.get("resent_count", 0) or 0)
if resent_count >= WATCHDOG_MAX_RESENDS:
async with self._async_lock():
self._retry_pending.pop(action_id, None)
age_total = t0 - float(info.get("first_dispatched_at", t0) or t0)
logger.error(
"[BUS] lea:dispatch_orphan_giveup action_id=%s resent=%d age_total=%.1fs "
"session=%s machine=%s replay=%s",
action_id,
resent_count,
age_total,
info.get("session_id", "?"),
info.get("machine_id", "?"),
info.get("replay_id", "?"),
)
gaveup += 1
await _bump("orphans_giveup_total")
continue
session_id = info.get("session_id")
machine_id = info.get("machine_id", "default")
action = info.get("dispatched_action") or info.get("action")
if not session_id or not isinstance(action, dict):
logger.warning(
"[WATCHDOG] invalid schema for %s session_id=%r action_type=%s",
action_id,
session_id,
type(action).__name__,
)
async with self._async_lock():
self._retry_pending.pop(action_id, None)
continue
async with self._async_lock():
existing = self._retry_pending.get(action_id)
if existing is None:
logger.debug(
"[WATCHDOG] %s acked between snapshot and resend; skip",
action_id,
)
continue
queue = self._replay_queues.setdefault(session_id, [])
if WATCHDOG_REPUSH_POSITION == "tail":
queue.append(dict(action))
else:
queue.insert(0, dict(action))
existing["resent_count"] = resent_count + 1
existing["last_resent_at"] = time.time()
existing["dispatched_at"] = 0.0
age_total = t0 - float(info.get("first_dispatched_at", t0) or t0)
logger.warning(
"[BUS] lea:dispatch_orphan_resent action_id=%s resent=%d/%d age=%.1fs "
"session=%s machine=%s replay=%s",
action_id,
resent_count + 1,
WATCHDOG_MAX_RESENDS,
age_total,
session_id,
machine_id,
info.get("replay_id", "?"),
)
resent += 1
await _bump("orphans_resent_total")
if self._sse_notifier is not None:
try:
self._sse_notifier(session_id, machine_id)
except Exception as exc:
logger.debug("[WATCHDOG] sse notifier failed: %s", exc)
elapsed_ms = (time.time() - t0) * 1000.0
async with _metrics_lock:
_metrics["last_scan_ts"] = t0
_metrics["last_scan_duration_ms"] = elapsed_ms
_metrics["current_in_flight_count"] = in_flight
_metrics["current_orphan_count"] = orphans
scans_total = _metrics["scans_total"]
if orphans or gaveup:
logger.info(
"[METRIC] watchdog scan=%d orphans=%d resent=%d gaveup=%d "
"in_flight=%d skipped=%d elapsed_ms=%.1f",
scans_total,
orphans,
resent,
gaveup,
in_flight,
skipped,
elapsed_ms,
)
return {
"orphans": orphans,
"resent": resent,
"gaveup": gaveup,
"skipped": skipped,
"in_flight": in_flight,
}
_singleton: Optional[ReplayWatchdog] = None
def get_or_create_watchdog(
retry_pending: Dict[str, Dict[str, Any]],
replay_queues: Dict[str, List[Dict[str, Any]]],
async_lock_factory: Callable[[], Any],
sse_notifier: Optional[SseNotifier] = None,
) -> ReplayWatchdog:
global _singleton
if _singleton is None:
_singleton = ReplayWatchdog(
retry_pending=retry_pending,
replay_queues=replay_queues,
async_lock_factory=async_lock_factory,
sse_notifier=sse_notifier,
)
return _singleton
@contextlib.asynccontextmanager
async def watchdog_lifespan(
retry_pending: Dict[str, Dict[str, Any]],
replay_queues: Dict[str, List[Dict[str, Any]]],
async_lock_factory: Callable[[], Any],
sse_notifier: Optional[SseNotifier] = None,
):
watchdog = get_or_create_watchdog(
retry_pending=retry_pending,
replay_queues=replay_queues,
async_lock_factory=async_lock_factory,
sse_notifier=sse_notifier,
)
await watchdog.start()
try:
yield watchdog
finally:
await watchdog.stop()

595
agent_v0/server_v1/resolve_engine.py
View File

@@ -243,6 +243,168 @@ def _validate_match_context(
return True
def _has_meaningful_recorded_coords(
fallback_x_pct: float,
fallback_y_pct: float,
) -> bool:
"""Indiquer si les coordonnées fallback représentent une vraie position source."""
return (
fallback_x_pct > 0.001
and fallback_y_pct > 0.001
and not (
abs(fallback_x_pct - 0.5) < 0.001
and abs(fallback_y_pct - 0.5) < 0.001
)
)
def _is_close_tab_target(target_spec: Optional[Dict[str, Any]]) -> bool:
"""Détecter une action close_tab issue du compilateur replay."""
if not isinstance(target_spec, dict):
return False
context_hints = target_spec.get("context_hints") or {}
return str((context_hints.get("interaction") or "")).strip().lower() == "close_tab"
def _get_expected_close_tab_coords(
target_spec: Optional[Dict[str, Any]],
screen_width: int,
screen_height: int,
fallback_x_pct: float = 0.0,
fallback_y_pct: float = 0.0,
) -> Optional[tuple[float, float]]:
"""Retrouver la position attendue la plus fiable pour un close_tab.
Ordre de préférence :
1. Coordonnées fallback explicites de l'action replay
2. centre SoM calibré à l'enregistrement
3. click_relative + rect fenêtre source
"""
if _has_meaningful_recorded_coords(fallback_x_pct, fallback_y_pct):
return float(fallback_x_pct), float(fallback_y_pct)
if not isinstance(target_spec, dict):
return None
som_center = (target_spec.get("som_element") or {}).get("center_norm")
if isinstance(som_center, (list, tuple)) and len(som_center) >= 2:
try:
exp_x = float(som_center[0])
exp_y = float(som_center[1])
if 0.0 <= exp_x <= 1.0 and 0.0 <= exp_y <= 1.0:
return exp_x, exp_y
except (TypeError, ValueError):
pass
window_capture = target_spec.get("window_capture") or {}
rect = window_capture.get("rect")
click_relative = window_capture.get("click_relative")
if (
isinstance(rect, (list, tuple))
and len(rect) >= 4
and isinstance(click_relative, (list, tuple))
and len(click_relative) >= 2
and screen_width > 0
and screen_height > 0
):
try:
abs_x = float(rect[0]) + float(click_relative[0])
abs_y = float(rect[1]) + float(click_relative[1])
exp_x = abs_x / float(screen_width)
exp_y = abs_y / float(screen_height)
if 0.0 <= exp_x <= 1.0 and 0.0 <= exp_y <= 1.0:
return exp_x, exp_y
except (TypeError, ValueError, ZeroDivisionError):
pass
return None
def _is_close_tab_result_plausible(
resolved_x: float,
resolved_y: float,
target_spec: Optional[Dict[str, Any]],
screen_width: int,
screen_height: int,
fallback_x_pct: float = 0.0,
fallback_y_pct: float = 0.0,
) -> bool:
"""Filtrer les faux positifs close_tab qui dérivent vers le bouton fermer."""
if not _is_close_tab_target(target_spec):
return True
expected = _get_expected_close_tab_coords(
target_spec,
screen_width,
screen_height,
fallback_x_pct=fallback_x_pct,
fallback_y_pct=fallback_y_pct,
)
if expected is None:
return True
exp_x, exp_y = expected
dx = abs(float(resolved_x) - exp_x)
dy = abs(float(resolved_y) - exp_y)
distance = (dx ** 2 + dy ** 2) ** 0.5
is_plausible = dx <= 0.18 and distance <= 0.20
if not is_plausible:
logger.warning(
"close_tab guard : résultat rejeté car trop éloigné de la zone "
"source (resolved=(%.4f, %.4f), expected=(%.4f, %.4f), "
"drift=(%.4f, %.4f), dist=%.4f)",
float(resolved_x),
float(resolved_y),
exp_x,
exp_y,
dx,
dy,
distance,
)
return is_plausible
def _is_start_button_vlm_result_plausible(
result: Dict[str, Any],
fallback_x_pct: float,
fallback_y_pct: float,
target_spec: Dict[str, Any],
max_distance: float = 0.20,
) -> bool:
"""Filtrer les faux positifs VLM sur le bouton Démarrer.
Le bouton Démarrer est un singleton système. Quand on dispose d'un vrai clic
enregistré (`fallback_*`), une localisation VLM très éloignée de cette zone
est plus probablement un faux positif qu'un vrai déplacement UI.
"""
by_role = str(target_spec.get("by_role", "") or "").strip().lower()
if by_role != "start_button":
return True
if not _has_meaningful_recorded_coords(fallback_x_pct, fallback_y_pct):
return True
if _validate_match_context(
result,
fallback_x_pct,
fallback_y_pct,
target_spec,
max_distance=max_distance,
):
return True
logger.warning(
"Start button guard : résultat VLM rejeté car trop éloigné de la "
"position enregistrée (resolved=(%.4f, %.4f), expected=(%.4f, %.4f), max=%.2f)",
float(result.get("x_pct", 0) or 0),
float(result.get("y_pct", 0) or 0),
fallback_x_pct,
fallback_y_pct,
max_distance,
)
return False
# =========================================================================
# YOLO/OmniParser — Résolution par détection d'éléments UI
# =========================================================================
@@ -1109,16 +1271,66 @@ def _resolve_by_som(
# Centre du match
match_cx = max_loc[0] + anc_w // 2
match_cy = max_loc[1] + anc_h // 2
interaction = str(
(target_spec.get("context_hints") or {}).get("interaction", "") or ""
).strip().lower()
if interaction == "close_tab":
elapsed = time.time() - t0
cx_norm = match_cx / screen_width if screen_width > 0 else 0.0
cy_norm = match_cy / screen_height if screen_height > 0 else 0.0
if _is_close_tab_result_plausible(
cx_norm,
cy_norm,
target_spec,
screen_width,
screen_height,
):
logger.info(
"SoM resolve ANCHOR exact close_tab : score=%.3f "
"centre=(%d, %d) → (%.4f, %.4f) en %.1fs",
max_score, match_cx, match_cy, cx_norm, cy_norm, elapsed,
)
return {
"resolved": True,
"method": "som_anchor_match",
"x_pct": round(cx_norm, 6),
"y_pct": round(cy_norm, 6),
"matched_element": {
"label": "close_tab_button",
"type": "visual_anchor",
"role": "som_anchor_exact",
"confidence": max_score,
},
"score": max_score,
"match_box": {
"x": int(max_loc[0]),
"y": int(max_loc[1]),
"width": int(anc_w),
"height": int(anc_h),
},
}
logger.warning(
"SoM resolve ANCHOR exact close_tab rejeté : score=%.3f "
"centre=(%d, %d) → (%.4f, %.4f), passage VLM/fallback",
max_score, match_cx, match_cy, cx_norm, cy_norm,
)
# Ne pas recycler ce faux match vers l'élément SoM le plus
# proche : pour close_tab, cela retombe facilement sur le
# bouton de fermeture de la fenêtre.
best_elem = None
else:
best_elem = None
# Trouver l'élément SomEngine le plus proche du centre du match
best_elem = None
best_dist = float("inf")
for elem in som_result.elements:
cx, cy = elem.center
dist = ((match_cx - cx) ** 2 + (match_cy - cy) ** 2) ** 0.5
if dist < best_dist:
best_dist = dist
best_elem = elem
if best_elem is None and interaction != "close_tab":
for elem in som_result.elements:
cx, cy = elem.center
dist = ((match_cx - cx) ** 2 + (match_cy - cy) ** 2) ** 0.5
if dist < best_dist:
best_dist = dist
best_elem = elem
if best_elem and best_dist < 100: # Max 100px de distance
elapsed = time.time() - t0
@@ -1584,6 +1796,49 @@ def _resolve_target_sync(
"fallback cascade legacy"
)
# ===================================================================
# Cas spécial : boutons de dialogue runtime ("Oui", "Non", "OK", ...)
# ===================================================================
# Ces boutons sont textuels, sans ancre stable, et apparaissent souvent
# au milieu d'une action déjà en cours. Si on les laisse partir dans la
# cascade générique (VLM -> SoM -> ScreenAnalyzer), on peut bloquer
# l'action principale assez longtemps pour déclencher le watchdog.
# Contrat voulu : OCR direct rapide, sinon abandon immédiat pour que le
# client essaie son fallback local par template texte.
dialog_role = str(target_spec.get("by_role", "") or "").strip().lower()
dialog_text = str(target_spec.get("by_text", "") or "").strip()
if dialog_role == "dialog_button" and dialog_text and not anchor_image_b64:
ocr_result = _resolve_by_ocr_text(
screenshot_path=screenshot_path,
target_text=dialog_text,
screen_width=screen_width,
screen_height=screen_height,
)
if ocr_result and ocr_result.get("score", 0) >= 0.80:
ocr_result["method"] = "hybrid_text_direct"
logger.info(
"Resolve dialog_button OCR-DIRECT : OK '%s' → (%.4f, %.4f) score=%.2f",
dialog_text[:40],
ocr_result.get("x_pct", 0),
ocr_result.get("y_pct", 0),
ocr_result.get("score", 0),
)
return ocr_result
logger.info(
"Resolve dialog_button OCR-only : '%s' non trouvé "
"(fenêtre='%s') — skip VLM/SoM/ScreenAnalyzer",
dialog_text[:40],
str(target_spec.get("window_title", "") or "")[:80],
)
return {
"resolved": False,
"method": "dialog_button_ocr_only",
"reason": "ocr_direct_failed_dialog_button_no_vlm",
"x_pct": fallback_x_pct,
"y_pct": fallback_y_pct,
}
# ===================================================================
# MODE STRICT (replay sessions) — Stratégie VLM-FIRST
# ===================================================================
@@ -1656,13 +1911,25 @@ def _resolve_target_sync(
screen_height=screen_height,
)
if grounding_result and grounding_result.get("resolved"):
logger.info(
"Strict resolve GROUNDING : OK (%.4f, %.4f) pour '%s'",
grounding_result.get("x_pct", 0),
grounding_result.get("y_pct", 0),
by_text_strict[:50],
if _is_close_tab_result_plausible(
float(grounding_result.get("x_pct", 0) or 0),
float(grounding_result.get("y_pct", 0) or 0),
target_spec,
screen_width,
screen_height,
fallback_x_pct=fallback_x_pct,
fallback_y_pct=fallback_y_pct,
):
logger.info(
"Strict resolve GROUNDING : OK (%.4f, %.4f) pour '%s'",
grounding_result.get("x_pct", 0),
grounding_result.get("y_pct", 0),
by_text_strict[:50],
)
return grounding_result
logger.warning(
"Strict resolve GROUNDING : résultat close_tab rejeté, passage template/VLM"
)
return grounding_result
if not by_text_strict or by_text_source not in ("ocr", "vlm"):
# Template matching pour les éléments sans texte (icônes pures)
@@ -1690,11 +1957,23 @@ def _resolve_target_sync(
abs_y = window_rect[1] + y_tm * tm_screen_h
result["x_pct"] = round(abs_x / screen_width, 6)
result["y_pct"] = round(abs_y / screen_height, 6)
logger.info(
"Strict resolve TEMPLATE : icon match (score=%.3f)",
result.get("score", 0),
if _is_close_tab_result_plausible(
float(result.get("x_pct", 0) or 0),
float(result.get("y_pct", 0) or 0),
target_spec,
screen_width,
screen_height,
fallback_x_pct=fallback_x_pct,
fallback_y_pct=fallback_y_pct,
):
logger.info(
"Strict resolve TEMPLATE : icon match (score=%.3f)",
result.get("score", 0),
)
return result
logger.warning(
"Strict resolve TEMPLATE : résultat close_tab rejeté, passage cascade suivante"
)
return result
# ---------------------------------------------------------------
# Étape 0.5 : OCR direct (hybrid_text_direct) — chemin rapide
@@ -1739,6 +2018,27 @@ def _resolve_target_sync(
by_text_strict[:40],
)
# Les boutons de dialogues runtime connus ("Oui", "Non", "OK", etc.)
# ne doivent pas partir dans la cascade lente VLM -> SoM. Si l'OCR
# direct ne les trouve pas immédiatement, on rend la main au client
# pour son fallback local par template texte, sinon on bloque l'action
# principale assez longtemps pour déclencher le watchdog.
dialog_role = str(target_spec.get("by_role", "") or "").strip().lower()
if dialog_role == "dialog_button" and by_text_strict and not anchor_image_b64:
logger.info(
"Strict resolve dialog_button : OCR-direct only pour '%s' "
"(fenêtre='%s') — skip VLM/SoM/template",
by_text_strict[:40],
str(target_spec.get("window_title", "") or "")[:80],
)
return {
"resolved": False,
"method": "dialog_button_ocr_only",
"reason": "ocr_direct_failed_dialog_button_no_vlm",
"x_pct": fallback_x_pct,
"y_pct": fallback_y_pct,
}
# ---------------------------------------------------------------
# Étape 1 : VLM Quick Find (fallback, multi-image)
# ---------------------------------------------------------------
@@ -1750,12 +2050,29 @@ def _resolve_target_sync(
)
if vlm_result and vlm_result.get("resolved"):
if vlm_result.get("score", 0) >= 0.3:
logger.info(
"Strict resolve VLM-first : VLM OK (score=%.2f) pour '%s'",
vlm_result.get("score", 0),
vlm_description[:60] if vlm_description else "(anchor)",
if _is_start_button_vlm_result_plausible(
vlm_result,
fallback_x_pct,
fallback_y_pct,
target_spec,
) and _is_close_tab_result_plausible(
float(vlm_result.get("x_pct", 0) or 0),
float(vlm_result.get("y_pct", 0) or 0),
target_spec,
screen_width,
screen_height,
fallback_x_pct=fallback_x_pct,
fallback_y_pct=fallback_y_pct,
):
logger.info(
"Strict resolve VLM-first : VLM OK (score=%.2f) pour '%s'",
vlm_result.get("score", 0),
vlm_description[:60] if vlm_description else "(anchor)",
)
return vlm_result
logger.warning(
"Strict resolve VLM-first : résultat VLM rejeté par un garde-fou, passage SoM/template"
)
return vlm_result
else:
logger.info(
"Strict resolve VLM-first : VLM score=%.2f trop bas, passage template",
@@ -1782,12 +2099,24 @@ def _resolve_target_sync(
screen_height=screen_height,
)
if som_result and som_result.get("resolved"):
logger.info(
"Strict resolve SoM+VLM : OK (score=%.2f, mark=#%s)",
som_result.get("score", 0),
som_result.get("matched_element", {}).get("som_id", "?"),
if _is_close_tab_result_plausible(
float(som_result.get("x_pct", 0) or 0),
float(som_result.get("y_pct", 0) or 0),
target_spec,
screen_width,
screen_height,
fallback_x_pct=fallback_x_pct,
fallback_y_pct=fallback_y_pct,
):
logger.info(
"Strict resolve SoM+VLM : OK (score=%.2f, mark=#%s)",
som_result.get("score", 0),
som_result.get("matched_element", {}).get("som_id", "?"),
)
return som_result
logger.warning(
"Strict resolve SoM+VLM : résultat close_tab rejeté, passage template matching"
)
return som_result
else:
logger.info("Strict resolve SoM+VLM : échoué, passage template matching")
@@ -1805,12 +2134,24 @@ def _resolve_target_sync(
score = result.get("score", 0)
# Score >= 0.95 : match quasi-parfait, pas besoin de valider le contexte
if score >= 0.95:
logger.info(
"Strict resolve VLM-first : template matching fallback OK "
"(score=%.3f >= 0.95, contexte skip — match quasi-parfait)",
score,
if _is_close_tab_result_plausible(
float(result.get("x_pct", 0) or 0),
float(result.get("y_pct", 0) or 0),
target_spec,
screen_width,
screen_height,
fallback_x_pct=fallback_x_pct,
fallback_y_pct=fallback_y_pct,
):
logger.info(
"Strict resolve VLM-first : template matching fallback OK "
"(score=%.3f >= 0.95, contexte skip — match quasi-parfait)",
score,
)
return result
logger.warning(
"Strict resolve TEMPLATE : match close_tab très fort mais hors zone source, rejeté"
)
return result
elif _validate_match_context(result, fallback_x_pct, fallback_y_pct, target_spec):
logger.info(
"Strict resolve VLM-first : template matching fallback OK "
@@ -2189,6 +2530,37 @@ def _text_match_fuzzy(expected: str, observed: str, min_token_ratio: float = 0.6
return matched / len(tokens) >= min_token_ratio
_SOM_BBOX_OCR_PADDING_PX: int = 8
_SOM_BBOX_MIN_DIM_PX: int = 12
def _should_reject_on_text_mismatch(
is_valid: bool,
observed: Optional[str],
) -> bool:
"""Décide si le pré-check OCR doit rejeter la résolution.
Patch 2026-05-23 : on distingue deux cas d'échec du fuzzy match :
- ``observed`` contient du texte (ex: ``'9 ?'``, ``'OBS Studio…'``)
→ mismatch confirmé, la cascade a probablement cliqué ailleurs
→ on rejette.
- ``observed`` est vide ou whitespace
→ l'OCR n'a rien lu (zone trop petite, texte peu contrasté,
modèle EasyOCR sous le seuil de détection). C'est ambigu :
ce n'est PAS la preuve d'un faux positif, on accepte la
résolution serveur. La garde drift ANCHOR-TM côté agent
protège en aval contre les vrais faux positifs.
Si ``is_valid=True`` → jamais de rejet (cas nominal).
"""
if is_valid:
return False
if observed is None:
return False
return bool(str(observed).strip())
def _validate_text_at_position(
screenshot_path: str,
x_pct: float,
@@ -2197,9 +2569,20 @@ def _validate_text_at_position(
screen_width: int,
screen_height: int,
radius_px: int = 280,
som_bbox_norm: Optional[List[float]] = None,
) -> tuple:
"""Pré-check sémantique : OCR sur une zone autour de (x_pct, y_pct) et
vérifie que `expected_text` y est présent (substring ou fuzzy 50%).
"""Pré-check sémantique : OCR sur une zone et vérifie que
`expected_text` y est présent (substring ou fuzzy 50%).
Zone OCR (par priorité) :
1. Si ``som_bbox_norm = [x1, y1, x2, y2]`` (normalisé 0..1) est
fourni et a une largeur/hauteur > _SOM_BBOX_MIN_DIM_PX en
pixels écran : OCR sur cette bbox élargie d'un padding court.
Plus précis pour les éléments étroits (onglets Notepad
moderne, ~30-40px haut) que le radius générique qui capture
le texte voisin (status bar, etc.).
2. Sinon : fallback historique → carré de ``radius_px`` autour
de (x_pct, y_pct).
Retourne (is_valid: bool, observed_text: str, elapsed_ms: float).
@@ -2219,16 +2602,52 @@ def _validate_text_at_position(
t0 = time.time()
img = Image.open(screenshot_path).convert("RGB")
img_w, img_h = img.size
cx = int(x_pct * screen_width)
cy = int(y_pct * screen_height)
# Saturer dans les bornes de l'image (le screenshot peut être plus
# large que la fenêtre logique — utiliser min(img_*, screen_*) en sécurité).
max_x = min(img_w, screen_width)
max_y = min(img_h, screen_height)
x1 = max(0, cx - radius_px)
y1 = max(0, cy - radius_px)
x2 = min(max_x, cx + radius_px)
y2 = min(max_y, cy + radius_px)
# --- Tentative 1 : zone OCR depuis la bbox SoM (préférée) ---
x1 = y1 = x2 = y2 = None
if (
isinstance(som_bbox_norm, (list, tuple))
and len(som_bbox_norm) == 4
):
try:
bx1, by1, bx2, by2 = (float(v) for v in som_bbox_norm)
# Tolérer ordre inversé.
bx1, bx2 = sorted((bx1, bx2))
by1, by2 = sorted((by1, by2))
# Refuser les bboxes dégénérées AVANT padding : si
# l'élément cible fait < _SOM_BBOX_MIN_DIM_PX en
# natif, c'est probablement une bbox d'apparence
# (curseur, séparateur 1px) — pas un label OCRable.
raw_w = (bx2 - bx1) * screen_width
raw_h = (by2 - by1) * screen_height
if (
raw_w >= _SOM_BBOX_MIN_DIM_PX
and raw_h >= _SOM_BBOX_MIN_DIM_PX
):
# Conversion en pixels écran + clipping et padding.
px1 = int(bx1 * screen_width) - _SOM_BBOX_OCR_PADDING_PX
py1 = int(by1 * screen_height) - _SOM_BBOX_OCR_PADDING_PX
px2 = int(bx2 * screen_width) + _SOM_BBOX_OCR_PADDING_PX
py2 = int(by2 * screen_height) + _SOM_BBOX_OCR_PADDING_PX
x1 = max(0, px1)
y1 = max(0, py1)
x2 = min(max_x, px2)
y2 = min(max_y, py2)
except (TypeError, ValueError):
# Bbox malformée : fallback silencieux sur le radius.
x1 = y1 = x2 = y2 = None
# --- Fallback : carré radius_px autour de (x_pct, y_pct) ---
if x1 is None:
cx = int(x_pct * screen_width)
cy = int(y_pct * screen_height)
x1 = max(0, cx - radius_px)
y1 = max(0, cy - radius_px)
x2 = min(max_x, cx + radius_px)
y2 = min(max_y, cy + radius_px)
if x2 - x1 < 10 or y2 - y1 < 10:
return True, "", 0.0
crop = img.crop((x1, y1, x2, y2))
@@ -2246,6 +2665,7 @@ def _validate_resolution_quality(
result: Optional[Dict[str, Any]],
fallback_x_pct: float,
fallback_y_pct: float,
target_spec: Optional[Dict[str, Any]] = None,
) -> Optional[Dict[str, Any]]:
"""Valide un résultat de résolution et le rejette s'il est peu fiable.
@@ -2263,6 +2683,16 @@ def _validate_resolution_quality(
elle n'est PAS appelée par les méthodes internes de la cascade, mais
uniquement depuis le handler HTTP `/resolve_target` après que la
cascade a produit son meilleur candidat.
Argument optionnel `target_spec` : permet d'appliquer des relaxations
contextuelles. Cas couvert (2026-05-22) : pour une cible
`context_hints.interaction == "switch_tab"` qui dispose d'un
`som_element.bbox_norm`, on abaisse le seuil des méthodes ``som_*``
de 0.75 → 0.60. Justification : (1) le focus_change pré-clic
prouve qu'on est dans la bonne fenêtre, (2) la bbox SoM a été
calibrée à l'enregistrement et reste valide, (3) les onglets
Notepad moderne sont visuellement quasi-identiques → score VLM
inévitablement lower.
"""
if not result or not isinstance(result, dict):
return result
@@ -2291,6 +2721,52 @@ def _validate_resolution_quality(
min_score = threshold
break
# Relaxation contextuelle pour switch_tab + SoM calibré (2026-05-22).
# Les onglets Notepad moderne (et apps similaires) sont visuellement
# quasi-identiques : le grounding VLM/SoM produit fréquemment un
# score 0.65-0.75, juste sous le seuil strict. Comme le contexte
# `interaction=switch_tab` + bbox SoM enregistrée + focus_change
# pré-clic confirment déjà la fenêtre et la zone, on relâche le
# seuil des méthodes som_* à 0.60 dans CE cas précis uniquement.
if (
min_score is not None
and target_spec
and method.startswith("som_")
):
context_hints = target_spec.get("context_hints") or {}
is_tab_switch = (
context_hints.get("interaction") == "switch_tab"
and target_spec.get("by_role") == "tab"
)
som_element = target_spec.get("som_element") or {}
has_calibrated_som = bool(som_element.get("bbox_norm"))
if is_tab_switch and has_calibrated_som:
relaxed = 0.60
if relaxed < min_score:
logger.info(
"[REPLAY] switch_tab + som_element calibré → seuil "
"som_* relâché %.2f → %.2f (cible='%s')",
min_score, relaxed,
target_spec.get("by_text", ""),
)
min_score = relaxed
is_close_tab = (
method == "som_anchor_match"
and str((context_hints.get("interaction") or "")).strip().lower() == "close_tab"
and not str(target_spec.get("by_text", "") or "").strip()
and bool(target_spec.get("anchor_image_base64"))
)
if is_close_tab:
relaxed = 0.70
if relaxed < min_score:
logger.info(
"[REPLAY] close_tab + anchor-only → seuil som_anchor_match "
"relâché %.2f → %.2f",
min_score, relaxed,
)
min_score = relaxed
if min_score is not None and score < min_score:
logger.warning(
"[REPLAY] Resolution REJETÉE (score trop bas) : method=%s score=%.3f < %.2f",
@@ -2306,13 +2782,40 @@ def _validate_resolution_quality(
"y_pct": fallback_y_pct,
}
if _is_close_tab_target(target_spec) and not _is_close_tab_result_plausible(
resolved_x,
resolved_y,
target_spec,
0,
0,
fallback_x_pct=fallback_x_pct,
fallback_y_pct=fallback_y_pct,
):
logger.warning(
"[REPLAY] Resolution REJETÉE (close_tab hors zone source) : "
"method=%s resolved=(%.3f, %.3f) expected=(%.3f, %.3f)",
method,
resolved_x,
resolved_y,
fallback_x_pct,
fallback_y_pct,
)
return {
"resolved": False,
"method": f"rejected_close_tab_zone_{method}",
"reason": "close_tab_out_of_recorded_zone",
"original_method": method,
"original_score": score,
"x_pct": fallback_x_pct,
"y_pct": fallback_y_pct,
}
# --- Check 2 : garde de proximité ---
# On n'applique la garde que si les coordonnées enregistrées ont un
# sens (pas des placeholders 0.5/0.5 des plans V4 ni des 0.0/0.0).
_has_recorded_coords = (
fallback_x_pct > 0.001
and fallback_y_pct > 0.001
and not (abs(fallback_x_pct - 0.5) < 0.001 and abs(fallback_y_pct - 0.5) < 0.001)
_has_recorded_coords = _has_meaningful_recorded_coords(
fallback_x_pct,
fallback_y_pct,
)
if _has_recorded_coords:
dx = abs(resolved_x - fallback_x_pct)

383
agent_v0/server_v1/stream_processor.py
View File

@@ -1025,6 +1025,345 @@ def enrich_click_from_screenshot(
return result
def _title_to_tab_label(window_title: str) -> str:
"""Réduire un titre de fenêtre en libellé d'onglet probable.
Exemples:
- "Sans titre – Bloc-notes" -> "Sans titre"
- "*test – Bloc-notes" -> "test"
"""
title = str(window_title or "").strip()
if not title:
return ""
for sep in (" – ", " - "):
if sep in title:
head = title.split(sep, 1)[0].strip()
if head:
title = head
break
return title.lstrip("*").strip()
def _split_window_title_head_suffix(window_title: str) -> tuple[str, str]:
"""Découper un titre de fenêtre en ``(head, suffix)`` si possible.
Exemples:
- ``Sans titre – Bloc-notes`` -> (``Sans titre``, ``Bloc-notes``)
- ``Page 1 - Google Chrome`` -> (``Page 1``, ``Google Chrome``)
- ``Enregistrer sous`` -> ("", "")
"""
title = str(window_title or "").strip()
if not title:
return "", ""
for sep in (" – ", " - "):
if sep in title:
head, suffix = title.split(sep, 1)
head = head.strip()
suffix = suffix.strip()
if head and suffix:
return head, suffix
return "", ""
def _looks_like_same_app_tab_switch(from_title: str, to_title: str) -> bool:
"""Vrai si la transition de focus ressemble à un vrai changement d'onglet.
On exige que les deux titres partagent un suffixe applicatif stable
(ex: ``Bloc-notes``, ``Google Chrome``). Cela exclut les dialogs
modaux same-app comme ``Enregistrer sous`` qui ne sont pas des
onglets et ne doivent pas être compilés en ``switch_tab``.
"""
from_head, from_suffix = _split_window_title_head_suffix(from_title)
to_head, to_suffix = _split_window_title_head_suffix(to_title)
if not (from_head and from_suffix and to_head and to_suffix):
return False
return from_suffix.casefold() == to_suffix.casefold()
def _infer_tab_switch_target(
raw_events: list,
click_event: Dict[str, Any],
) -> Optional[Dict[str, Any]]:
"""Détecter un clic d'onglet à partir d'une bascule de focus dans la même app.
Cas réel observé:
- fenêtre active `http...txt – Bloc-notes`
- clic dans la barre d'onglets (y relatif ~40 px)
- focus immédiat vers `Sans titre – Bloc-notes`
Dans ce cas, l'ancre image seule est trop fragile. On enrichit donc le
target_spec avec un libellé d'onglet explicite (`by_text='Sans titre'`,
`by_role='tab'`).
"""
event_type = click_event.get("type", "")
if event_type != "mouse_click":
return None
window = click_event.get("window", {})
if not isinstance(window, dict):
return None
from_title = str(window.get("title", "")).strip()
app_name = str(window.get("app_name", "")).strip().lower()
if not from_title or not app_name:
return None
# Heuristique: on ne traite que les clics très hauts dans la fenêtre,
# typiques d'une barre d'onglets / bouton de fermeture d'onglet.
window_capture = click_event.get("window_capture", {})
if not isinstance(window_capture, dict):
return None
click_relative = window_capture.get("click_relative")
if not (isinstance(click_relative, list) and len(click_relative) == 2):
return None
try:
rel_y = int(click_relative[1])
except (TypeError, ValueError):
return None
if rel_y > 90:
return None
click_ts = click_event.get("timestamp")
click_pos = click_event.get("pos") or []
match_idx = None
for idx, raw_evt in enumerate(raw_events):
event_data = raw_evt.get("event", raw_evt)
if event_data.get("type") != "mouse_click":
continue
if event_data.get("timestamp") != click_ts:
continue
if (event_data.get("pos") or []) != click_pos:
continue
match_idx = idx
break
if match_idx is None:
return None
for follow_evt in raw_events[match_idx + 1: match_idx + 7]:
follow_data = follow_evt.get("event", follow_evt)
follow_type = follow_data.get("type", "")
if follow_type in {"mouse_click", "text_input", "key_press", "key_combo"}:
# Un autre geste utilisateur est intervenu avant le focus_change :
# le focus observé n'est plus attribuable avec confiance à CE clic.
return None
if follow_type != "window_focus_change":
continue
to_info = follow_data.get("to", {})
if not isinstance(to_info, dict):
continue
if str(to_info.get("app_name", "")).strip().lower() != app_name:
continue
to_title = str(to_info.get("title", "")).strip()
if not to_title or to_title == from_title:
continue
if not _looks_like_same_app_tab_switch(from_title, to_title):
return None
follow_ts = follow_data.get("timestamp")
if (
isinstance(click_ts, (int, float))
and isinstance(follow_ts, (int, float))
and follow_ts - click_ts > 3.0
):
break
tab_label = _title_to_tab_label(to_title)
if not tab_label:
return None
return {
"by_text": tab_label,
"by_role": "tab",
"window_title": from_title,
"context_hints": {
"window_title": from_title,
"switch_to_window_title": to_title,
"interaction": "switch_tab",
},
"vlm_description": (
f"Dans la fenêtre '{from_title}', l'onglet '{tab_label}' "
"dans la barre d'onglets en haut"
),
}
return None
def _infer_close_tab_target(
raw_events: list,
click_event: Dict[str, Any],
) -> Optional[Dict[str, Any]]:
"""Détecter un clic sur le bouton fermer de l'onglet actif.
Pattern ciblé observé sur Bloc-notes moderne :
- clic très haut dans la barre d'onglets sur un titre ``*... – Bloc-notes``
- un clic suivant dans la même fenêtre
- puis focus vers ``Enregistrer sous``
Cela correspond à la fermeture d'un onglet modifié qui déclenche ensuite
le flow de sauvegarde. On enrichit le clic avec un hint sémantique pour
viser le vrai bouton ``x`` de l'onglet actif plutôt qu'un simple `yolo`.
"""
event_type = click_event.get("type", "")
if event_type != "mouse_click":
return None
window = click_event.get("window", {})
if not isinstance(window, dict):
return None
from_title = str(window.get("title", "")).strip()
app_name = str(window.get("app_name", "")).strip().lower()
if not from_title or not app_name or not from_title.startswith("*"):
return None
window_capture = click_event.get("window_capture", {})
if not isinstance(window_capture, dict):
return None
click_relative = window_capture.get("click_relative")
if not (isinstance(click_relative, list) and len(click_relative) == 2):
return None
try:
rel_y = int(click_relative[1])
except (TypeError, ValueError):
return None
if rel_y > 90:
return None
click_ts = click_event.get("timestamp")
click_pos = click_event.get("pos") or []
match_idx = None
for idx, raw_evt in enumerate(raw_events):
event_data = raw_evt.get("event", raw_evt)
if event_data.get("type") != "mouse_click":
continue
if event_data.get("timestamp") != click_ts:
continue
if (event_data.get("pos") or []) != click_pos:
continue
match_idx = idx
break
if match_idx is None:
return None
saw_follow_click_same_window = False
for follow_evt in raw_events[match_idx + 1: match_idx + 8]:
follow_data = follow_evt.get("event", follow_evt)
follow_type = follow_data.get("type", "")
if follow_type in {"text_input", "key_press", "key_combo"}:
return None
if follow_type == "mouse_click":
follow_window = follow_data.get("window", {})
if not isinstance(follow_window, dict):
return None
follow_app = str(follow_window.get("app_name", "")).strip().lower()
follow_title = str(follow_window.get("title", "")).strip()
if follow_app != app_name:
return None
if follow_title == from_title:
saw_follow_click_same_window = True
continue
return None
if follow_type != "window_focus_change" or not saw_follow_click_same_window:
continue
to_info = follow_data.get("to", {})
if not isinstance(to_info, dict):
continue
if str(to_info.get("app_name", "")).strip().lower() != app_name:
continue
to_title = str(to_info.get("title", "")).strip()
if to_title != "Enregistrer sous":
continue
follow_ts = follow_data.get("timestamp")
if (
isinstance(click_ts, (int, float))
and isinstance(follow_ts, (int, float))
and follow_ts - click_ts > 5.0
):
break
tab_label = _title_to_tab_label(from_title)
if not tab_label:
return None
return {
"by_text": "",
"by_role": "tab_close_button",
"window_title": from_title,
"context_hints": {
"window_title": from_title,
"active_tab_label": tab_label,
"interaction": "close_tab",
},
"vlm_description": (
f"Dans la fenêtre '{from_title}', le bouton x pour fermer "
f"l'onglet actif '{tab_label}' dans la barre d'onglets en haut"
),
}
return None
def _attach_expected_window_before(actions: list, raw_events: list) -> None:
"""Attacher la fenêtre attendue AVANT chaque clic en rejouant les
raw events et en conservant le dernier ``window_focus_change.to.title``.
Pourquoi : ``mouse_click.window.title`` capturé pendant
l'enregistrement peut être obsolète si une transition de fenêtre
se produit juste avant la capture (ex: dialog Windows qui s'ouvre
milliseconde avant le clic suivant). Le serveur dispose pourtant
des ``window_focus_change`` consécutifs — on s'en sert pour poser
explicitement ``expected_window_before`` sur le clic, lu en priorité
absolue par la pré-vérif côté agent.
Idempotent : si une action a déjà ``expected_window_before``, on
ne touche pas.
"""
if not actions or not raw_events:
return
last_focus_title = ""
action_idx = 0
def _next_click_idx(start: int) -> int:
i = start
while i < len(actions) and actions[i].get("type") != "click":
i += 1
return i
for raw_evt in raw_events:
ev = raw_evt.get("event", raw_evt) if isinstance(raw_evt, dict) else {}
etype = ev.get("type", "")
if etype == "window_focus_change":
to_info = ev.get("to") or {}
title = str(to_info.get("title", "") or "").strip()
if title and title != "unknown_window":
last_focus_title = title
continue
if etype != "mouse_click":
continue
action_idx = _next_click_idx(action_idx)
if action_idx >= len(actions):
return
a = actions[action_idx]
if last_focus_title and not a.get("expected_window_before"):
a["expected_window_before"] = last_focus_title
action_idx += 1
def _attach_expected_screenshots(
actions: list, raw_events: list, session_dir: Path,
) -> None:
@@ -1591,6 +1930,8 @@ def build_replay_from_raw_events(
k: v for k, v in enrichment.items()
if k != "by_position" # by_position est déjà dans x_pct/y_pct
}
if action.get("window_title") and not action["target_spec"].get("window_title"):
action["target_spec"]["window_title"] = action["window_title"]
# Ajouter les métadonnées fenêtre pour le grounding ciblé
wc = evt.get("window_capture", {})
if wc.get("rect"):
@@ -1600,6 +1941,33 @@ def build_replay_from_raw_events(
"click_relative": wc.get("click_relative"),
}
tab_switch_target = _infer_tab_switch_target(events, evt)
if tab_switch_target:
target_spec = action.setdefault("target_spec", {})
# Préférer une sémantique explicite d'onglet à un rôle brut
# `yolo`/anchor-only quand le flux brut montre une vraie
# bascule de focus dans la même application.
if not target_spec.get("by_text"):
target_spec["by_text"] = tab_switch_target["by_text"]
target_spec["by_role"] = tab_switch_target["by_role"]
target_spec["window_title"] = tab_switch_target["window_title"]
target_spec["vlm_description"] = tab_switch_target["vlm_description"]
context_hints = dict(target_spec.get("context_hints") or {})
context_hints.update(tab_switch_target["context_hints"])
target_spec["context_hints"] = context_hints
action["visual_mode"] = True
close_tab_target = _infer_close_tab_target(events, evt)
if close_tab_target:
target_spec = action.setdefault("target_spec", {})
target_spec["by_role"] = close_tab_target["by_role"]
target_spec["window_title"] = close_tab_target["window_title"]
target_spec["vlm_description"] = close_tab_target["vlm_description"]
context_hints = dict(target_spec.get("context_hints") or {})
context_hints.update(close_tab_target["context_hints"])
target_spec["context_hints"] = context_hints
action["visual_mode"] = True
elif evt_type == "text_input":
text = evt.get("text", "")
if not text:
@@ -1695,6 +2063,21 @@ def build_replay_from_raw_events(
if next_title:
result[ci]["expected_window_title"] = next_title
# ── 9b. Pré-condition fiable : expected_window_before ──
# Bug live 2026-05-22 (act_raw_c70976c8) : window.title d'un
# mouse_click peut être obsolète quand une transition de fenêtre
# (ex: ouverture dialog "Enregistrer sous") se produit juste avant
# la capture du click. Sans correction, target_spec.window_title
# reste sur l'ancien titre et la pré-vérif côté agent
# (executor.py:653) déclenche une pause supervisée à tort.
#
# On rejoue les raw events en maintenant le dernier titre vu via
# window_focus_change.to.title et on le pose comme
# expected_window_before sur chaque clic qui n'en a pas déjà un.
# Le champ est lu en priorité absolue par la pré-vérif agent, donc
# il prime sur target_spec.window_title obsolète.
_attach_expected_window_before(result, events)
# ── 10. Enrichir avec intention + expected_result via gemma4 (Critic) ──
# gemma4 analyse chaque action dans son contexte pour produire :
# - intention : ce que l'utilisateur veut accomplir