fix(agent): P0.6 guard human corrections
This commit is contained in:
@@ -3565,15 +3565,62 @@ Example: x_pct=0.50, y_pct=0.30"""
|
|||||||
|
|
||||||
monitor = self.sct.monitors[1]
|
monitor = self.sct.monitors[1]
|
||||||
screen_w, screen_h = monitor["width"], monitor["height"]
|
screen_w, screen_h = monitor["width"], monitor["height"]
|
||||||
|
screen_left = monitor.get("left", 0)
|
||||||
|
screen_top = monitor.get("top", 0)
|
||||||
|
|
||||||
|
if not (
|
||||||
|
isinstance(screen_w, int)
|
||||||
|
and isinstance(screen_h, int)
|
||||||
|
and screen_w >= 200
|
||||||
|
and screen_h >= 200
|
||||||
|
):
|
||||||
|
logger.warning(
|
||||||
|
"[APPRENTISSAGE] Monitor aberrant (%sx%s) — capture refusée",
|
||||||
|
screen_w,
|
||||||
|
screen_h,
|
||||||
|
)
|
||||||
|
return []
|
||||||
|
|
||||||
|
listener_start_ts = time.time()
|
||||||
|
drain_guard_s = 1.0
|
||||||
|
|
||||||
def _on_click(x, y, button, pressed):
|
def _on_click(x, y, button, pressed):
|
||||||
if done_event.is_set():
|
if done_event.is_set():
|
||||||
return False
|
return False
|
||||||
if pressed and button.name in ("left", "right"):
|
if pressed and button.name in ("left", "right"):
|
||||||
|
# Ignore residual low-level mouse events delivered just after
|
||||||
|
# the listener is attached. These are often synthetic agent or
|
||||||
|
# remote-desktop events, not a deliberate human correction.
|
||||||
|
if time.time() - listener_start_ts < drain_guard_s:
|
||||||
|
logger.debug(
|
||||||
|
"[APPRENTISSAGE] Clic ignoré (drain %.1fs) : (%s, %s)",
|
||||||
|
drain_guard_s,
|
||||||
|
x,
|
||||||
|
y,
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
|
rel_x = x - screen_left
|
||||||
|
rel_y = y - screen_top
|
||||||
|
if not (0 <= rel_x < screen_w and 0 <= rel_y < screen_h):
|
||||||
|
logger.warning(
|
||||||
|
"[APPRENTISSAGE] Clic ignoré hors moniteur "
|
||||||
|
"(monitor=%sx%s left=%s top=%s, raw=(%s,%s), rel=(%s,%s))",
|
||||||
|
screen_w,
|
||||||
|
screen_h,
|
||||||
|
screen_left,
|
||||||
|
screen_top,
|
||||||
|
x,
|
||||||
|
y,
|
||||||
|
rel_x,
|
||||||
|
rel_y,
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
action = {
|
action = {
|
||||||
"type": "click",
|
"type": "click",
|
||||||
"x_pct": round(x / screen_w, 6),
|
"x_pct": round(rel_x / screen_w, 6),
|
||||||
"y_pct": round(y / screen_h, 6),
|
"y_pct": round(rel_y / screen_h, 6),
|
||||||
"button": button.name,
|
"button": button.name,
|
||||||
"timestamp": time.time(),
|
"timestamp": time.time(),
|
||||||
}
|
}
|
||||||
@@ -3589,7 +3636,14 @@ Example: x_pct=0.50, y_pct=0.30"""
|
|||||||
pass
|
pass
|
||||||
actions.append(action)
|
actions.append(action)
|
||||||
last_action_time[0] = time.time()
|
last_action_time[0] = time.time()
|
||||||
logger.info(f"[APPRENTISSAGE] Clic ({x}, {y}) bouton={button.name}")
|
logger.info(
|
||||||
|
"[APPRENTISSAGE] Clic raw=(%s,%s) rel=(%s,%s) bouton=%s",
|
||||||
|
x,
|
||||||
|
y,
|
||||||
|
rel_x,
|
||||||
|
rel_y,
|
||||||
|
button.name,
|
||||||
|
)
|
||||||
|
|
||||||
def _on_key_press(key):
|
def _on_key_press(key):
|
||||||
if done_event.is_set():
|
if done_event.is_set():
|
||||||
|
|||||||
@@ -3785,9 +3785,15 @@ async def report_action_result(report: ReplayResultReport):
|
|||||||
try:
|
try:
|
||||||
corr = report.correction
|
corr = report.correction
|
||||||
target_spec = original_action.get("target_spec", {})
|
target_spec = original_action.get("target_spec", {})
|
||||||
|
log_x = corr.get("x_pct")
|
||||||
|
log_y = corr.get("y_pct")
|
||||||
|
last_click = corr.get("last_click")
|
||||||
|
if (log_x is None or log_y is None) and isinstance(last_click, dict):
|
||||||
|
log_x = last_click.get("x_pct")
|
||||||
|
log_y = last_click.get("y_pct")
|
||||||
logger.info(
|
logger.info(
|
||||||
f"[APPRENTISSAGE] Correction humaine reçue : "
|
f"[APPRENTISSAGE] Correction humaine reçue : "
|
||||||
f"({corr.get('x_pct', 0):.4f}, {corr.get('y_pct', 0):.4f}) "
|
f"({float(log_x or 0):.4f}, {float(log_y or 0):.4f}) "
|
||||||
f"pour '{target_spec.get('by_text', '?')}'"
|
f"pour '{target_spec.get('by_text', '?')}'"
|
||||||
)
|
)
|
||||||
_replay_learner.record_human_correction(
|
_replay_learner.record_human_correction(
|
||||||
|
|||||||
@@ -194,8 +194,33 @@ class ReplayLearner:
|
|||||||
or target_spec.get("expected_window_before", "")
|
or target_spec.get("expected_window_before", "")
|
||||||
or (target_spec.get("context_hints") or {}).get("window_title", "")
|
or (target_spec.get("context_hints") or {}).get("window_title", "")
|
||||||
)
|
)
|
||||||
x_pct = correction.get("x_pct", 0.0)
|
x_pct = correction.get("x_pct")
|
||||||
y_pct = correction.get("y_pct", 0.0)
|
y_pct = correction.get("y_pct")
|
||||||
|
last_click = correction.get("last_click")
|
||||||
|
if (x_pct is None or y_pct is None) and isinstance(last_click, dict):
|
||||||
|
x_pct = last_click.get("x_pct")
|
||||||
|
y_pct = last_click.get("y_pct")
|
||||||
|
|
||||||
|
try:
|
||||||
|
x_pct_f = float(x_pct)
|
||||||
|
y_pct_f = float(y_pct)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
logger.warning(
|
||||||
|
"[APPRENTISSAGE] Correction humaine non persistée : "
|
||||||
|
"aucune coordonnée clic exploitable pour '%s'",
|
||||||
|
by_text,
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
|
if not (0.0 < x_pct_f <= 1.0 and 0.0 < y_pct_f <= 1.0):
|
||||||
|
logger.warning(
|
||||||
|
"[APPRENTISSAGE] Correction humaine non persistée : "
|
||||||
|
"coordonnées hors bornes pour '%s' (%.4f, %.4f)",
|
||||||
|
by_text,
|
||||||
|
x_pct_f,
|
||||||
|
y_pct_f,
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
# Enregistrer dans le JSONL d'apprentissage
|
# Enregistrer dans le JSONL d'apprentissage
|
||||||
outcome = ActionOutcome(
|
outcome = ActionOutcome(
|
||||||
@@ -218,8 +243,8 @@ class ReplayLearner:
|
|||||||
stored = memory_record_success(
|
stored = memory_record_success(
|
||||||
window_title=window_title,
|
window_title=window_title,
|
||||||
target_spec=target_spec,
|
target_spec=target_spec,
|
||||||
x_pct=float(x_pct),
|
x_pct=x_pct_f,
|
||||||
y_pct=float(y_pct),
|
y_pct=y_pct_f,
|
||||||
method="human_supervised",
|
method="human_supervised",
|
||||||
confidence=1.0,
|
confidence=1.0,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -770,6 +770,63 @@ class TestReplayLearner:
|
|||||||
|
|
||||||
assert captured["window_title"] == "Fenêtre fallback"
|
assert captured["window_title"] == "Fenêtre fallback"
|
||||||
|
|
||||||
|
def test_record_human_correction_uses_last_click_contract(self, learner, monkeypatch):
|
||||||
|
"""Le contrat agent envoie les coordonnées dans correction.last_click."""
|
||||||
|
captured = {}
|
||||||
|
|
||||||
|
def fake_memory_record_success(**kwargs):
|
||||||
|
captured.update(kwargs)
|
||||||
|
return True
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"agent_v0.server_v1.replay_memory.memory_record_success",
|
||||||
|
fake_memory_record_success,
|
||||||
|
)
|
||||||
|
|
||||||
|
learner.record_human_correction(
|
||||||
|
session_id="s_corr_last_click",
|
||||||
|
action={
|
||||||
|
"action_id": "a_corr_last_click",
|
||||||
|
"target_spec": {"by_text": "Enregistrer", "window_title": "Bloc-notes"},
|
||||||
|
},
|
||||||
|
correction={
|
||||||
|
"actions": [{"type": "click", "x_pct": 0.33, "y_pct": 0.66}],
|
||||||
|
"last_click": {"type": "click", "x_pct": 0.33, "y_pct": 0.66},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert captured["x_pct"] == 0.33
|
||||||
|
assert captured["y_pct"] == 0.66
|
||||||
|
assert len(learner.load_session("s_corr_last_click")) == 1
|
||||||
|
|
||||||
|
def test_record_human_correction_rejects_out_of_bounds_coords(self, learner, monkeypatch):
|
||||||
|
"""Une correction hors écran ne doit pas alimenter la mémoire."""
|
||||||
|
captured = {}
|
||||||
|
|
||||||
|
def fake_memory_record_success(**kwargs):
|
||||||
|
captured.update(kwargs)
|
||||||
|
return True
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"agent_v0.server_v1.replay_memory.memory_record_success",
|
||||||
|
fake_memory_record_success,
|
||||||
|
)
|
||||||
|
|
||||||
|
learner.record_human_correction(
|
||||||
|
session_id="s_corr_oob",
|
||||||
|
action={
|
||||||
|
"action_id": "a_corr_oob",
|
||||||
|
"target_spec": {"by_text": "Enregistrer", "window_title": "Bloc-notes"},
|
||||||
|
},
|
||||||
|
correction={
|
||||||
|
"actions": [{"type": "click", "x_pct": 1.748, "y_pct": 0.135}],
|
||||||
|
"last_click": {"type": "click", "x_pct": 1.748, "y_pct": 0.135},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert captured == {}
|
||||||
|
assert learner.load_session("s_corr_oob") == []
|
||||||
|
|
||||||
|
|
||||||
# =========================================================================
|
# =========================================================================
|
||||||
# Boucle d'apprentissage : consolidation cross-workflow
|
# Boucle d'apprentissage : consolidation cross-workflow
|
||||||
|
|||||||
Reference in New Issue
Block a user