Dom/rpa_vision_v3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: anchor match sur screenshot entier + proximité élément SomEngine

Browse Source 
Le template matching du crop anchor contre les régions YOLO échouait
car l'anchor (150x150) est plus grand que les éléments détectés.
Maintenant : match sur le screenshot entier → centre du match →
élément SomEngine le plus proche (max 100px).

Fonctionne pour les icônes mais limité par la taille du crop
(150x150 de barre de titre matche à plusieurs endroits).

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Dom
2026-03-31 15:51:18 +02:00
parent 7dec3ab63a
commit 4f5c518d3a
1 changed files with 41 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

82
agent_v0/server_v1/api_stream.py
View File

@@ -3547,51 +3547,51 @@ def _resolve_by_som(
screenshot_cv = cv2.imread(screenshot_path, cv2.IMREAD_GRAYSCALE)
if anc_img is not None and screenshot_cv is not None:
best_elem = None
best_score = 0.0
# Template matching de l'anchor sur le SCREENSHOT ENTIER
# (pas sur les régions individuelles — l'anchor est souvent plus grand)
anc_h, anc_w = anc_img.shape[:2]
if screenshot_cv.shape[0] >= anc_h and screenshot_cv.shape[1] >= anc_w:
res = cv2.matchTemplate(screenshot_cv, anc_img, cv2.TM_CCOEFF_NORMED)
_, max_score, _, max_loc = cv2.minMaxLoc(res)
for elem in som_result.elements:
x1, y1, x2, y2 = elem.bbox
# Agrandir la zone de 20% pour tolérer les différences
margin_x = int((x2 - x1) * 0.2)
margin_y = int((y2 - y1) * 0.2)
rx1 = max(0, x1 - margin_x)
ry1 = max(0, y1 - margin_y)
rx2 = min(screenshot_cv.shape[1], x2 + margin_x)
ry2 = min(screenshot_cv.shape[0], y2 + margin_y)
region = screenshot_cv[ry1:ry2, rx1:rx2]
if max_score >= 0.5:
# Centre du match
match_cx = max_loc[0] + anc_w // 2
match_cy = max_loc[1] + anc_h // 2
if region.shape[0] < anc_h or region.shape[1] < anc_w:
continue
# Trouver l'élément SomEngine le plus proche du centre du match
best_elem = None
best_dist = float("inf")
for elem in som_result.elements:
cx, cy = elem.center
dist = ((match_cx - cx) ** 2 + (match_cy - cy) ** 2) ** 0.5
if dist < best_dist:
best_dist = dist
best_elem = elem
res = cv2.matchTemplate(region, anc_img, cv2.TM_CCOEFF_NORMED)
_, score, _, _ = cv2.minMaxLoc(res)
if score > best_score:
best_score = score
best_elem = elem
if best_elem and best_score >= 0.6:
elapsed = time.time() - t0
cx_norm, cy_norm = best_elem.center_norm
logger.info(
"SoM resolve ANCHOR : match crop '#%d' score=%.3f → (%.4f, %.4f) en %.1fs",
best_elem.id, best_score, cx_norm, cy_norm, elapsed,
)
return {
"resolved": True,
"method": "som_anchor_match",
"x_pct": round(cx_norm, 6),
"y_pct": round(cy_norm, 6),
"matched_element": {
"label": best_elem.label or f"icon #{best_elem.id}",
"type": best_elem.source,
"role": "som_anchor_match",
"confidence": best_score,
"som_id": best_elem.id,
},
"score": best_score,
}
if best_elem and best_dist < 100: # Max 100px de distance
elapsed = time.time() - t0
cx_norm, cy_norm = best_elem.center_norm
logger.info(
"SoM resolve ANCHOR : match crop score=%.3f → "
"elem '#%d %s' (dist=%.0fpx) → (%.4f, %.4f) en %.1fs",
max_score, best_elem.id, best_elem.label,
best_dist, cx_norm, cy_norm, elapsed,
)
return {
"resolved": True,
"method": "som_anchor_match",
"x_pct": round(cx_norm, 6),
"y_pct": round(cy_norm, 6),
"matched_element": {
"label": best_elem.label or f"icon #{best_elem.id}",
"type": best_elem.source,
"role": "som_anchor_match",
"confidence": max_score,
"som_id": best_elem.id,
},
"score": max_score,
}
except ImportError:
pass
except Exception as e: