fix(qw1): bus event lea:monitor_routed + cablage offset côté executor Agent V1
Cleanup post-review QW1 :
- Émission bus lea:monitor_routed dans /replay/next (idx, source, replay_id, action_id, offset, wh)
via logger.info "[BUS] lea:monitor_routed ..." (le serveur streaming n'a pas
de SocketIO local, agent_chat émet déjà lea:* sur 5004 ; ici on logge en INFO
bien lisible, prêt pour un parser/pont futur)
- Executor Agent V1 (deploy/windows_client) lit action.monitor_resolution.{offset_x, offset_y, idx}
et applique l'offset aux coords absolues du clic/type/scroll/popup quand idx >= 0
- composite_fallback (idx=-1) : pas d'offset appliqué (backward compat mono-écran)
- Log INFO "QW1 monitor cible idx=N source=X offset=(dx,dy) — appliqué aux coords"
émis une fois par action quand un offset non nul s'applique
Tests : baseline 95 passed (e2e + phase0_integration + stream_processor + monitor_router + grounding_offset)
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -512,6 +512,21 @@ class ActionExecutorV1:
|
||||
x_pct = action.get("x_pct", 0.0)
|
||||
y_pct = action.get("y_pct", 0.0)
|
||||
|
||||
# QW1 — Si le serveur a résolu un monitor cible (idx >= 0),
|
||||
# appliquer son offset aux coords absolues. Pour idx == -1
|
||||
# (composite_fallback), aucun offset (backward compat).
|
||||
# Le calcul des coords reste percent * (width/height) du monitor[1]
|
||||
# côté client (x_pct est exprimé sur l'écran physique principal).
|
||||
mon_res = action.get("monitor_resolution") or {}
|
||||
mon_idx = mon_res.get("idx", -1)
|
||||
mon_offset_x = mon_res.get("offset_x", 0) if mon_idx >= 0 else 0
|
||||
mon_offset_y = mon_res.get("offset_y", 0) if mon_idx >= 0 else 0
|
||||
if mon_idx >= 0 and (mon_offset_x or mon_offset_y):
|
||||
logger.info(
|
||||
f"[REPLAY] QW1 monitor cible idx={mon_idx} source={mon_res.get('source')} "
|
||||
f"offset=({mon_offset_x},{mon_offset_y}) — appliqué aux coords"
|
||||
)
|
||||
|
||||
# ── Diagnostic résolution ──
|
||||
logger.info(
|
||||
f"[REPLAY] Action {action_id} ({action_type}) — "
|
||||
@@ -578,8 +593,8 @@ class ActionExecutorV1:
|
||||
print(f" [OBSERVER] Popup détectée : '{popup_label}' — fermeture")
|
||||
logger.info(f"Observer : popup '{popup_label}' détectée avant résolution")
|
||||
if popup_coords:
|
||||
real_x = int(popup_coords["x_pct"] * width)
|
||||
real_y = int(popup_coords["y_pct"] * height)
|
||||
real_x = int(popup_coords["x_pct"] * width) + mon_offset_x
|
||||
real_y = int(popup_coords["y_pct"] * height) + mon_offset_y
|
||||
self._click((real_x, real_y), "left")
|
||||
time.sleep(1.0)
|
||||
print(f" [OBSERVER] Popup fermée — reprise du flow normal")
|
||||
@@ -718,8 +733,8 @@ class ActionExecutorV1:
|
||||
self.notifier.replay_target_not_found(target_desc)
|
||||
return result
|
||||
|
||||
real_x = int(x_pct * width)
|
||||
real_y = int(y_pct * height)
|
||||
real_x = int(x_pct * width) + mon_offset_x
|
||||
real_y = int(y_pct * height) + mon_offset_y
|
||||
button = action.get("button", "left")
|
||||
mode = "VISUAL" if result.get("visual_resolved") else "COORD"
|
||||
print(
|
||||
@@ -781,8 +796,8 @@ class ActionExecutorV1:
|
||||
print(f" [TYPE] raw_keys disponibles ({len(raw_keys)} events) — replay exact")
|
||||
# Cliquer sur le champ avant de taper (si coordonnees disponibles)
|
||||
if x_pct > 0 and y_pct > 0:
|
||||
real_x = int(x_pct * width)
|
||||
real_y = int(y_pct * height)
|
||||
real_x = int(x_pct * width) + mon_offset_x
|
||||
real_y = int(y_pct * height) + mon_offset_y
|
||||
print(f" [TYPE] Clic prealable sur ({real_x}, {real_y})")
|
||||
self._click((real_x, real_y), "left")
|
||||
time.sleep(0.3)
|
||||
@@ -808,8 +823,8 @@ class ActionExecutorV1:
|
||||
logger.info(f"Replay key_combo : {keys} (raw_keys={'oui' if raw_keys else 'non'})")
|
||||
|
||||
elif action_type == "scroll":
|
||||
real_x = int(x_pct * width) if x_pct > 0 else int(0.5 * width)
|
||||
real_y = int(y_pct * height) if y_pct > 0 else int(0.5 * height)
|
||||
real_x = (int(x_pct * width) if x_pct > 0 else int(0.5 * width)) + mon_offset_x
|
||||
real_y = (int(y_pct * height) if y_pct > 0 else int(0.5 * height)) + mon_offset_y
|
||||
delta = action.get("delta", -3)
|
||||
print(f" [SCROLL] delta={delta} a ({real_x}, {real_y})")
|
||||
self.mouse.position = (real_x, real_y)
|
||||
@@ -1386,6 +1401,16 @@ Example: x_pct=0.50, y_pct=0.30"""
|
||||
data = resp.json()
|
||||
action = data.get("action")
|
||||
if action is None:
|
||||
# pause_for_human : afficher le message de décision à l'utilisateur
|
||||
if data.get("replay_paused") and data.get("pause_message"):
|
||||
msg = data["pause_message"]
|
||||
print(f"[PAUSE] {msg}")
|
||||
logger.info(f"Replay en pause — message : {msg}")
|
||||
self.notifier.notify(
|
||||
title="Léa — Validation requise",
|
||||
message=msg[:250],
|
||||
timeout=30,
|
||||
)
|
||||
return False
|
||||
|
||||
except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as e:
|
||||
|
||||
Reference in New Issue
Block a user