fix(qw1): bus event lea:monitor_routed + cablage offset côté executor Agent V1

Cleanup post-review QW1 :
- Émission bus lea:monitor_routed dans /replay/next (idx, source, replay_id, action_id, offset, wh)
  via logger.info "[BUS] lea:monitor_routed ..." (le serveur streaming n'a pas
  de SocketIO local, agent_chat émet déjà lea:* sur 5004 ; ici on logge en INFO
  bien lisible, prêt pour un parser/pont futur)
- Executor Agent V1 (deploy/windows_client) lit action.monitor_resolution.{offset_x, offset_y, idx}
  et applique l'offset aux coords absolues du clic/type/scroll/popup quand idx >= 0
- composite_fallback (idx=-1) : pas d'offset appliqué (backward compat mono-écran)
- Log INFO "QW1 monitor cible idx=N source=X offset=(dx,dy) — appliqué aux coords"
  émis une fois par action quand un offset non nul s'applique

Tests : baseline 95 passed (e2e + phase0_integration + stream_processor + monitor_router + grounding_offset)

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Dom
2026-05-05 23:16:06 +02:00
parent 2a51a844b9
commit fc01afa59c
2 changed files with 68 additions and 8 deletions

View File

@@ -512,6 +512,21 @@ class ActionExecutorV1:
x_pct = action.get("x_pct", 0.0) x_pct = action.get("x_pct", 0.0)
y_pct = action.get("y_pct", 0.0) y_pct = action.get("y_pct", 0.0)
# QW1 — Si le serveur a résolu un monitor cible (idx >= 0),
# appliquer son offset aux coords absolues. Pour idx == -1
# (composite_fallback), aucun offset (backward compat).
# Le calcul des coords reste percent * (width/height) du monitor[1]
# côté client (x_pct est exprimé sur l'écran physique principal).
mon_res = action.get("monitor_resolution") or {}
mon_idx = mon_res.get("idx", -1)
mon_offset_x = mon_res.get("offset_x", 0) if mon_idx >= 0 else 0
mon_offset_y = mon_res.get("offset_y", 0) if mon_idx >= 0 else 0
if mon_idx >= 0 and (mon_offset_x or mon_offset_y):
logger.info(
f"[REPLAY] QW1 monitor cible idx={mon_idx} source={mon_res.get('source')} "
f"offset=({mon_offset_x},{mon_offset_y}) — appliqué aux coords"
)
# ── Diagnostic résolution ── # ── Diagnostic résolution ──
logger.info( logger.info(
f"[REPLAY] Action {action_id} ({action_type}) — " f"[REPLAY] Action {action_id} ({action_type}) — "
@@ -578,8 +593,8 @@ class ActionExecutorV1:
print(f" [OBSERVER] Popup détectée : '{popup_label}' — fermeture") print(f" [OBSERVER] Popup détectée : '{popup_label}' — fermeture")
logger.info(f"Observer : popup '{popup_label}' détectée avant résolution") logger.info(f"Observer : popup '{popup_label}' détectée avant résolution")
if popup_coords: if popup_coords:
real_x = int(popup_coords["x_pct"] * width) real_x = int(popup_coords["x_pct"] * width) + mon_offset_x
real_y = int(popup_coords["y_pct"] * height) real_y = int(popup_coords["y_pct"] * height) + mon_offset_y
self._click((real_x, real_y), "left") self._click((real_x, real_y), "left")
time.sleep(1.0) time.sleep(1.0)
print(f" [OBSERVER] Popup fermée — reprise du flow normal") print(f" [OBSERVER] Popup fermée — reprise du flow normal")
@@ -718,8 +733,8 @@ class ActionExecutorV1:
self.notifier.replay_target_not_found(target_desc) self.notifier.replay_target_not_found(target_desc)
return result return result
real_x = int(x_pct * width) real_x = int(x_pct * width) + mon_offset_x
real_y = int(y_pct * height) real_y = int(y_pct * height) + mon_offset_y
button = action.get("button", "left") button = action.get("button", "left")
mode = "VISUAL" if result.get("visual_resolved") else "COORD" mode = "VISUAL" if result.get("visual_resolved") else "COORD"
print( print(
@@ -781,8 +796,8 @@ class ActionExecutorV1:
print(f" [TYPE] raw_keys disponibles ({len(raw_keys)} events) — replay exact") print(f" [TYPE] raw_keys disponibles ({len(raw_keys)} events) — replay exact")
# Cliquer sur le champ avant de taper (si coordonnees disponibles) # Cliquer sur le champ avant de taper (si coordonnees disponibles)
if x_pct > 0 and y_pct > 0: if x_pct > 0 and y_pct > 0:
real_x = int(x_pct * width) real_x = int(x_pct * width) + mon_offset_x
real_y = int(y_pct * height) real_y = int(y_pct * height) + mon_offset_y
print(f" [TYPE] Clic prealable sur ({real_x}, {real_y})") print(f" [TYPE] Clic prealable sur ({real_x}, {real_y})")
self._click((real_x, real_y), "left") self._click((real_x, real_y), "left")
time.sleep(0.3) time.sleep(0.3)
@@ -808,8 +823,8 @@ class ActionExecutorV1:
logger.info(f"Replay key_combo : {keys} (raw_keys={'oui' if raw_keys else 'non'})") logger.info(f"Replay key_combo : {keys} (raw_keys={'oui' if raw_keys else 'non'})")
elif action_type == "scroll": elif action_type == "scroll":
real_x = int(x_pct * width) if x_pct > 0 else int(0.5 * width) real_x = (int(x_pct * width) if x_pct > 0 else int(0.5 * width)) + mon_offset_x
real_y = int(y_pct * height) if y_pct > 0 else int(0.5 * height) real_y = (int(y_pct * height) if y_pct > 0 else int(0.5 * height)) + mon_offset_y
delta = action.get("delta", -3) delta = action.get("delta", -3)
print(f" [SCROLL] delta={delta} a ({real_x}, {real_y})") print(f" [SCROLL] delta={delta} a ({real_x}, {real_y})")
self.mouse.position = (real_x, real_y) self.mouse.position = (real_x, real_y)
@@ -1386,6 +1401,16 @@ Example: x_pct=0.50, y_pct=0.30"""
data = resp.json() data = resp.json()
action = data.get("action") action = data.get("action")
if action is None: if action is None:
# pause_for_human : afficher le message de décision à l'utilisateur
if data.get("replay_paused") and data.get("pause_message"):
msg = data["pause_message"]
print(f"[PAUSE] {msg}")
logger.info(f"Replay en pause — message : {msg}")
self.notifier.notify(
title="Léa — Validation requise",
message=msg[:250],
timeout=30,
)
return False return False
except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as e: except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as e:

View File

@@ -34,6 +34,7 @@ from .agent_registry import AgentRegistry, AgentAlreadyEnrolledError
from .stream_processor import StreamProcessor, build_replay_from_raw_events, enrich_click_from_screenshot from .stream_processor import StreamProcessor, build_replay_from_raw_events, enrich_click_from_screenshot
from .worker_stream import StreamWorker from .worker_stream import StreamWorker
from .monitor_router import resolve_target_monitor # QW1 — résolution écran cible from .monitor_router import resolve_target_monitor # QW1 — résolution écran cible
from .loop_detector import LoopDetector # QW2 — détection de boucle pendant replay
from .execution_plan_runner import ( from .execution_plan_runner import (
execution_plan_to_actions, execution_plan_to_actions,
inject_plan_into_queue, inject_plan_into_queue,
@@ -361,6 +362,18 @@ REPLAY_LOCK_FILE = _DATA_DIR / "_replay_active.lock"
processor = StreamProcessor(data_dir=str(LIVE_SESSIONS_DIR)) processor = StreamProcessor(data_dir=str(LIVE_SESSIONS_DIR))
worker = StreamWorker(live_dir=str(LIVE_SESSIONS_DIR), processor=processor) worker = StreamWorker(live_dir=str(LIVE_SESSIONS_DIR), processor=processor)
# QW2 — LoopDetector singleton lazy (utilise le CLIP embedder du processor)
_loop_detector: Optional["LoopDetector"] = None
def _get_loop_detector() -> "LoopDetector":
"""Singleton lazy — crée le LoopDetector avec le CLIP embedder du processor."""
global _loop_detector
if _loop_detector is None:
embedder = getattr(processor, "_clip_embedder", None)
_loop_detector = LoopDetector(clip_embedder=embedder)
return _loop_detector
# Registre des postes Lea enroles (table enrolled_agents dans rpa_data.db) # Registre des postes Lea enroles (table enrolled_agents dans rpa_data.db)
# Emplacement configurable via RPA_AGENTS_DB_PATH pour les tests. # Emplacement configurable via RPA_AGENTS_DB_PATH pour les tests.
_AGENTS_DB_PATH = os.environ.get( _AGENTS_DB_PATH = os.environ.get(
@@ -3166,6 +3179,28 @@ async def get_next_action(session_id: str, machine_id: str = "default"):
"h": target.h, "h": target.h,
"source": target.source, "source": target.source,
} }
# QW1 — Émission bus lea:monitor_routed (no-op si bus indisponible)
# Le serveur streaming n'a pas de SocketIO local : on logge en INFO
# bien lisible. Un consommateur (agent_chat / dashboard) peut tailer
# `journalctl -u rpa-streaming | grep '\[BUS\] lea:monitor_routed'`.
try:
_replay_id_bus = (
owning_replay.get("replay_id") if owning_replay else None
)
logger.info(
"[BUS] lea:monitor_routed replay=%s action=%s idx=%d source=%s "
"offset=(%d,%d) wh=(%d,%d)",
_replay_id_bus,
action.get("action_id"),
target.idx,
target.source,
target.offset_x,
target.offset_y,
target.w,
target.h,
)
except Exception as _e_bus:
logger.debug("emit lea:monitor_routed échec (non bloquant): %s", _e_bus)
except Exception as e: except Exception as e:
logger.debug("QW1 monitor_resolution skip (%s)", e) logger.debug("QW1 monitor_resolution skip (%s)", e)