Dom/rpa_vision_v3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(agent): respect server visual reject before text fallback

Browse Source
This commit is contained in:
Dom
2026-05-24 21:10:42 +02:00
parent b1b32187ba
commit 345762330b
1 changed files with 24 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
agent_v0/agent_v1/core/executor.py
View File

@@ -2244,6 +2244,7 @@ class ActionExecutorV1:
# ---- ÉTAPE 1 : Résolution serveur (SomEngine + VLM) ---- # ---- ÉTAPE 1 : Résolution serveur (SomEngine + VLM) ----
# Le serveur comprend sémantiquement ce qu'on cherche. Pas de faux positifs. # Le serveur comprend sémantiquement ce qu'on cherche. Pas de faux positifs.
skip_text_fallback_after_server_reject = False
if server_url: if server_url:
server_result = self._server_resolve_target( server_result = self._server_resolve_target(
server_url, screenshot_b64, target_spec, server_url, screenshot_b64, target_spec,
@@ -2251,6 +2252,18 @@ class ActionExecutorV1:
) )
if server_result and server_result.get("resolved"): if server_result and server_result.get("resolved"):
return _with_metrics(server_result) return _with_metrics(server_result)
if server_result:
reason = str(server_result.get("reason") or "")
method = str(server_result.get("method") or "")
# If the server explicitly rejected a text/position candidate,
# do not replay the same broad by_text search locally. That
# bypasses semantic guards such as close-tab drift checks.
skip_text_fallback_after_server_reject = (
method.startswith("rejected_")
or reason.startswith("close_tab_")
or reason.startswith("drift_")
or "below_threshold" in reason
)
# ---- ÉTAPE 2 : Template matching local (fallback si serveur down) ---- # ---- ÉTAPE 2 : Template matching local (fallback si serveur down) ----
anchor_b64 = target_spec.get("anchor_image_base64", "") anchor_b64 = target_spec.get("anchor_image_base64", "")
@@ -2265,6 +2278,17 @@ class ActionExecutorV1:
# ---- ÉTAPE 3 : VLM local (fallback dev/test Linux) ---- # ---- ÉTAPE 3 : VLM local (fallback dev/test Linux) ----
by_text = target_spec.get("by_text", "") by_text = target_spec.get("by_text", "")
vlm_description = target_spec.get("vlm_description", "") vlm_description = target_spec.get("vlm_description", "")
if skip_text_fallback_after_server_reject and by_text:
logger.info(
"[VISUAL] Rejet serveur explicite pour '%s' — "
"skip fallback local hybrid_text_direct",
by_text,
)
print(
f" [VISUAL] Rejet serveur explicite pour '{by_text}' "
"→ pas de fallback texte local"
)
return None
if vlm_description or by_text: if vlm_description or by_text:
hybrid_result = self._hybrid_vlm_resolve( hybrid_result = self._hybrid_vlm_resolve(
screenshot_b64, target_spec, screen_width, screen_height screenshot_b64, target_spec, screen_width, screen_height