diff --git a/agent_v0/deploy/windows_client/agent_v1/core/executor.py b/agent_v0/deploy/windows_client/agent_v1/core/executor.py index 93de6859b..0159decfc 100644 --- a/agent_v0/deploy/windows_client/agent_v1/core/executor.py +++ b/agent_v0/deploy/windows_client/agent_v1/core/executor.py @@ -512,6 +512,21 @@ class ActionExecutorV1: x_pct = action.get("x_pct", 0.0) y_pct = action.get("y_pct", 0.0) + # QW1 — Si le serveur a résolu un monitor cible (idx >= 0), + # appliquer son offset aux coords absolues. Pour idx == -1 + # (composite_fallback), aucun offset (backward compat). + # Le calcul des coords reste percent * (width/height) du monitor[1] + # côté client (x_pct est exprimé sur l'écran physique principal). + mon_res = action.get("monitor_resolution") or {} + mon_idx = mon_res.get("idx", -1) + mon_offset_x = mon_res.get("offset_x", 0) if mon_idx >= 0 else 0 + mon_offset_y = mon_res.get("offset_y", 0) if mon_idx >= 0 else 0 + if mon_idx >= 0 and (mon_offset_x or mon_offset_y): + logger.info( + f"[REPLAY] QW1 monitor cible idx={mon_idx} source={mon_res.get('source')} " + f"offset=({mon_offset_x},{mon_offset_y}) — appliqué aux coords" + ) + # ── Diagnostic résolution ── logger.info( f"[REPLAY] Action {action_id} ({action_type}) — " @@ -578,8 +593,8 @@ class ActionExecutorV1: print(f" [OBSERVER] Popup détectée : '{popup_label}' — fermeture") logger.info(f"Observer : popup '{popup_label}' détectée avant résolution") if popup_coords: - real_x = int(popup_coords["x_pct"] * width) - real_y = int(popup_coords["y_pct"] * height) + real_x = int(popup_coords["x_pct"] * width) + mon_offset_x + real_y = int(popup_coords["y_pct"] * height) + mon_offset_y self._click((real_x, real_y), "left") time.sleep(1.0) print(f" [OBSERVER] Popup fermée — reprise du flow normal") @@ -718,8 +733,8 @@ class ActionExecutorV1: self.notifier.replay_target_not_found(target_desc) return result - real_x = int(x_pct * width) - real_y = int(y_pct * height) + real_x = int(x_pct * width) + mon_offset_x + real_y = int(y_pct * height) + mon_offset_y button = action.get("button", "left") mode = "VISUAL" if result.get("visual_resolved") else "COORD" print( @@ -781,8 +796,8 @@ class ActionExecutorV1: print(f" [TYPE] raw_keys disponibles ({len(raw_keys)} events) — replay exact") # Cliquer sur le champ avant de taper (si coordonnees disponibles) if x_pct > 0 and y_pct > 0: - real_x = int(x_pct * width) - real_y = int(y_pct * height) + real_x = int(x_pct * width) + mon_offset_x + real_y = int(y_pct * height) + mon_offset_y print(f" [TYPE] Clic prealable sur ({real_x}, {real_y})") self._click((real_x, real_y), "left") time.sleep(0.3) @@ -808,8 +823,8 @@ class ActionExecutorV1: logger.info(f"Replay key_combo : {keys} (raw_keys={'oui' if raw_keys else 'non'})") elif action_type == "scroll": - real_x = int(x_pct * width) if x_pct > 0 else int(0.5 * width) - real_y = int(y_pct * height) if y_pct > 0 else int(0.5 * height) + real_x = (int(x_pct * width) if x_pct > 0 else int(0.5 * width)) + mon_offset_x + real_y = (int(y_pct * height) if y_pct > 0 else int(0.5 * height)) + mon_offset_y delta = action.get("delta", -3) print(f" [SCROLL] delta={delta} a ({real_x}, {real_y})") self.mouse.position = (real_x, real_y) @@ -1386,6 +1401,16 @@ Example: x_pct=0.50, y_pct=0.30""" data = resp.json() action = data.get("action") if action is None: + # pause_for_human : afficher le message de décision à l'utilisateur + if data.get("replay_paused") and data.get("pause_message"): + msg = data["pause_message"] + print(f"[PAUSE] {msg}") + logger.info(f"Replay en pause — message : {msg}") + self.notifier.notify( + title="Léa — Validation requise", + message=msg[:250], + timeout=30, + ) return False except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as e: diff --git a/agent_v0/server_v1/api_stream.py b/agent_v0/server_v1/api_stream.py index fa62f438b..32dba9fbf 100644 --- a/agent_v0/server_v1/api_stream.py +++ b/agent_v0/server_v1/api_stream.py @@ -34,6 +34,7 @@ from .agent_registry import AgentRegistry, AgentAlreadyEnrolledError from .stream_processor import StreamProcessor, build_replay_from_raw_events, enrich_click_from_screenshot from .worker_stream import StreamWorker from .monitor_router import resolve_target_monitor # QW1 — résolution écran cible +from .loop_detector import LoopDetector # QW2 — détection de boucle pendant replay from .execution_plan_runner import ( execution_plan_to_actions, inject_plan_into_queue, @@ -361,6 +362,18 @@ REPLAY_LOCK_FILE = _DATA_DIR / "_replay_active.lock" processor = StreamProcessor(data_dir=str(LIVE_SESSIONS_DIR)) worker = StreamWorker(live_dir=str(LIVE_SESSIONS_DIR), processor=processor) +# QW2 — LoopDetector singleton lazy (utilise le CLIP embedder du processor) +_loop_detector: Optional["LoopDetector"] = None + + +def _get_loop_detector() -> "LoopDetector": + """Singleton lazy — crée le LoopDetector avec le CLIP embedder du processor.""" + global _loop_detector + if _loop_detector is None: + embedder = getattr(processor, "_clip_embedder", None) + _loop_detector = LoopDetector(clip_embedder=embedder) + return _loop_detector + # Registre des postes Lea enroles (table enrolled_agents dans rpa_data.db) # Emplacement configurable via RPA_AGENTS_DB_PATH pour les tests. _AGENTS_DB_PATH = os.environ.get( @@ -3166,6 +3179,28 @@ async def get_next_action(session_id: str, machine_id: str = "default"): "h": target.h, "source": target.source, } + # QW1 — Émission bus lea:monitor_routed (no-op si bus indisponible) + # Le serveur streaming n'a pas de SocketIO local : on logge en INFO + # bien lisible. Un consommateur (agent_chat / dashboard) peut tailer + # `journalctl -u rpa-streaming | grep '\[BUS\] lea:monitor_routed'`. + try: + _replay_id_bus = ( + owning_replay.get("replay_id") if owning_replay else None + ) + logger.info( + "[BUS] lea:monitor_routed replay=%s action=%s idx=%d source=%s " + "offset=(%d,%d) wh=(%d,%d)", + _replay_id_bus, + action.get("action_id"), + target.idx, + target.source, + target.offset_x, + target.offset_y, + target.w, + target.h, + ) + except Exception as _e_bus: + logger.debug("emit lea:monitor_routed échec (non bloquant): %s", _e_bus) except Exception as e: logger.debug("QW1 monitor_resolution skip (%s)", e)