ef5d595d98
build_replay (stream_processor.py) : - Remplir by_text depuis vision_info.text ou som_element.label - VLM identification pour les éléments sans texte (icônes) - Nettoyage du bavardage VLM (retrait préfixes courants) resolve_target (api_stream.py) : - Nouveau som_anchor_match : template matching du crop anchor vs régions YOLO - Pour les icônes sans texte (disquette, loupe, etc.) - Cascade : text match → anchor match → VLM Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
4022 lines
162 KiB
Python
4022 lines
162 KiB
Python
"""
|
||
StreamProcessor — Pont entre le streaming Agent V1 et le core pipeline RPA Vision V3.
|
||
|
||
Orchestre les composants core (ScreenAnalyzer, CLIP, FAISS, GraphBuilder)
|
||
pour traiter en temps réel les screenshots et événements reçus via fibre.
|
||
|
||
Tous les calculs GPU tournent ici (serveur RTX 5070).
|
||
"""
|
||
|
||
import base64
|
||
import hashlib
|
||
import logging
|
||
import os
|
||
import threading
|
||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||
from datetime import datetime
|
||
from pathlib import Path
|
||
from typing import Any, Dict, List, Optional
|
||
|
||
import numpy as np
|
||
|
||
from .live_session_manager import LiveSessionManager
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Filtrage des événements parasites (modificateurs seuls, text_input vide, etc.)
|
||
# Utilisé à 3 niveaux : réception (process_event), expansion (compound), et
|
||
# en amont dans GraphBuilder._find_transition_events / _build_compound_action.
|
||
# ---------------------------------------------------------------------------
|
||
|
||
_MODIFIER_ONLY_KEYS = {
|
||
"ctrl", "ctrl_l", "ctrl_r", "control", "control_l", "control_r",
|
||
"alt", "alt_l", "alt_r", "alt_gr",
|
||
"shift", "shift_l", "shift_r",
|
||
"win", "win_l", "win_r", "cmd", "cmd_l", "cmd_r",
|
||
"meta", "meta_l", "meta_r", "super", "super_l", "super_r",
|
||
}
|
||
|
||
# Mapping numpad vk codes → caractères (layout-indépendant)
|
||
_NUMPAD_VK_MAP = {
|
||
96: '0', 97: '1', 98: '2', 99: '3', 100: '4',
|
||
101: '5', 102: '6', 103: '7', 104: '8', 105: '9',
|
||
106: '*', 107: '+', 109: '-', 110: '.', 111: '/',
|
||
}
|
||
|
||
# Table de conversion des caractères de contrôle vers les touches lisibles
|
||
# (produits par certains agents qui capturent les raw keycodes)
|
||
_CONTROL_CHAR_MAP = {
|
||
'\x01': 'a', '\x02': 'b', '\x03': 'c', '\x04': 'd', '\x05': 'e',
|
||
'\x06': 'f', '\x07': 'g', '\x08': 'h', '\x09': 'i', '\x0a': 'j',
|
||
'\x0b': 'k', '\x0c': 'l', '\x0d': 'm', '\x0e': 'n', '\x0f': 'o',
|
||
'\x10': 'p', '\x11': 'q', '\x12': 'r', '\x13': 's', '\x14': 't',
|
||
'\x15': 'u', '\x16': 'v', '\x17': 'w', '\x18': 'x', '\x19': 'y',
|
||
'\x1a': 'z',
|
||
}
|
||
|
||
# Types d'événements parasites à ignorer dans les actions enrichies
|
||
_PARASITIC_ACTION_TYPES = frozenset({
|
||
'heartbeat', 'focus_change', 'window_focus_change',
|
||
'screenshot', 'status', 'ping', 'pong',
|
||
})
|
||
|
||
|
||
def _is_modifier_only(keys: list) -> bool:
|
||
"""Retourne True si la liste de touches ne contient que des modificateurs."""
|
||
if not keys:
|
||
return True
|
||
return all(k.lower() in _MODIFIER_ONLY_KEYS for k in keys)
|
||
|
||
|
||
def _sanitize_keys(keys: list) -> list:
|
||
"""Nettoyer une liste de touches : convertir les caractères de contrôle."""
|
||
cleaned = []
|
||
for k in keys:
|
||
if not k:
|
||
continue
|
||
if k in _CONTROL_CHAR_MAP:
|
||
cleaned.append(_CONTROL_CHAR_MAP[k])
|
||
else:
|
||
cleaned.append(k)
|
||
return cleaned
|
||
|
||
|
||
def _is_parasitic_event(event_data: Dict[str, Any]) -> bool:
|
||
"""Retourne True si l'événement est parasite et doit être filtré.
|
||
|
||
Événements rejetés :
|
||
- key_press / key_combo avec uniquement des modificateurs seuls
|
||
- key_press / key_combo avec liste de touches vide
|
||
- text_input avec texte vide
|
||
"""
|
||
event_type = event_data.get("type", "")
|
||
|
||
if event_type in ("key_press", "key_combo"):
|
||
keys = event_data.get("keys", event_data.get("data", {}).get("keys", []))
|
||
if not keys or _is_modifier_only(keys):
|
||
return True
|
||
|
||
elif event_type == "text_input":
|
||
text = event_data.get("text", event_data.get("data", {}).get("text", ""))
|
||
if not text:
|
||
return True
|
||
|
||
return False
|
||
|
||
|
||
def _reconstruct_text_from_raw_keys(raw_keys: list) -> str:
|
||
"""Reconstruire le texte correct à partir des vk codes des raw_keys.
|
||
|
||
Corrige les problèmes de capture AZERTY, notamment :
|
||
- Numpad / (vk=111) capturé comme char='!' → corrigé en '/'
|
||
- Numpad 0-9 (vk=96-105) capturés comme char=None → corrigés en '0'-'9'
|
||
"""
|
||
text_parts = []
|
||
for event in raw_keys:
|
||
if event.get("action") != "press":
|
||
continue
|
||
vk = event.get("vk", 0)
|
||
char = event.get("char")
|
||
kind = event.get("kind", "")
|
||
name = event.get("name", "")
|
||
|
||
# Ignorer les modificateurs (releases qui traînent dans le buffer)
|
||
if kind == "key" and name in _MODIFIER_ONLY_KEYS:
|
||
continue
|
||
|
||
# Numpad : mapping fixe (layout-indépendant)
|
||
if vk in _NUMPAD_VK_MAP:
|
||
text_parts.append(_NUMPAD_VK_MAP[vk])
|
||
# Touche normale avec caractère valide
|
||
elif char and len(char) == 1 and char.isprintable():
|
||
text_parts.append(char)
|
||
return "".join(text_parts)
|
||
|
||
|
||
def _key_combo_printable_char(keys: list) -> Optional[str]:
|
||
"""Si le key_combo produit un seul caractère imprimable, le retourner.
|
||
|
||
Exemples :
|
||
- ['ctrl', '@'] → '@' (AltGr+0 sur AZERTY, capturé comme ctrl+@)
|
||
- ['shift', 'A'] → 'A'
|
||
- ['ctrl', 'c'] → None (c'est un raccourci, pas un caractère)
|
||
- ['enter'] → None (pas un caractère imprimable)
|
||
"""
|
||
if not keys:
|
||
return None
|
||
non_modifiers = [k for k in keys if k.lower() not in _MODIFIER_ONLY_KEYS]
|
||
if len(non_modifiers) != 1:
|
||
return None
|
||
char = non_modifiers[0]
|
||
# Un seul caractère imprimable (pas un nom de touche spéciale)
|
||
if len(char) == 1 and char.isprintable():
|
||
# Vérifier que c'est pas un raccourci courant (ctrl+c, ctrl+v, etc.)
|
||
modifiers = {k.lower() for k in keys if k.lower() in _MODIFIER_ONLY_KEYS}
|
||
if modifiers <= {"shift", "shift_l", "shift_r"}:
|
||
# Shift + char = caractère majuscule/spécial → OK
|
||
return char
|
||
if "alt_gr" in modifiers or (
|
||
"ctrl" in modifiers and ("alt" in modifiers or "alt_r" in modifiers)
|
||
):
|
||
# AltGr + char = caractère spécial (@ # € etc.) → OK
|
||
return char
|
||
# Ctrl + caractère NON-alphabétique = probablement AltGr résiduel
|
||
# Sur AZERTY, AltGr+0 produit @, capturé comme ['ctrl', 'alt_gr'] + ['ctrl', '@']
|
||
# Le premier combo est filtré (modifier-only), le second a juste 'ctrl' + '@'
|
||
if "ctrl" in modifiers and not char.isalpha():
|
||
return char
|
||
# ctrl + lettre seul = raccourci (Ctrl+S, Ctrl+C) → pas un caractère
|
||
return None
|
||
return None
|
||
|
||
|
||
def _merge_consecutive_text_inputs(steps: list) -> list:
|
||
"""Fusionne les text_input consécutifs en un seul."""
|
||
merged = []
|
||
for step in steps:
|
||
if (step.get("type") in ("text_input", "type")
|
||
and merged
|
||
and merged[-1].get("type") in ("text_input", "type")):
|
||
merged[-1]["text"] = merged[-1].get("text", "") + step.get("text", "")
|
||
else:
|
||
merged.append(dict(step)) # copie pour ne pas muter l'original
|
||
return merged
|
||
|
||
|
||
def _dedup_consecutive_combos(steps: list) -> list:
|
||
"""Supprime les key_combo dupliqués consécutifs."""
|
||
deduped = []
|
||
for step in steps:
|
||
if (step.get("type") in ("key_combo", "key_press")
|
||
and deduped
|
||
and deduped[-1].get("type") in ("key_combo", "key_press")
|
||
and deduped[-1].get("keys") == step.get("keys")):
|
||
continue # Doublon → skip
|
||
deduped.append(step)
|
||
return deduped
|
||
|
||
|
||
def _filter_parasitic_steps(steps: list) -> list:
|
||
"""Supprime les steps key_combo/key_press avec uniquement des modificateurs seuls."""
|
||
return [
|
||
s for s in steps
|
||
if not (
|
||
s.get("type") in ("key_combo", "key_press")
|
||
and _is_modifier_only(s.get("keys", []))
|
||
)
|
||
]
|
||
|
||
|
||
def _ensure_min_waits(steps: list, min_wait_ms: int = 300) -> list:
|
||
"""Ajoute un wait de min_wait_ms entre les steps si aucun wait n'existe."""
|
||
if not steps:
|
||
return steps
|
||
result = [steps[0]]
|
||
for step in steps[1:]:
|
||
if result[-1].get("type") != "wait" and step.get("type") != "wait":
|
||
result.append({"type": "wait", "duration_ms": min_wait_ms})
|
||
result.append(step)
|
||
return result
|
||
|
||
|
||
def clean_compound_steps(steps: list) -> list:
|
||
"""Pipeline complet de nettoyage des steps d'une compound action.
|
||
|
||
Applique dans l'ordre :
|
||
1. Suppression des steps modificateurs seuls
|
||
2. Fusion des text_input consécutifs
|
||
3. Déduplication des key_combo consécutifs identiques
|
||
4. Ajout de waits minimum entre steps si absents
|
||
"""
|
||
cleaned = _filter_parasitic_steps(steps)
|
||
cleaned = _merge_consecutive_text_inputs(cleaned)
|
||
cleaned = _dedup_consecutive_combos(cleaned)
|
||
cleaned = _ensure_min_waits(cleaned)
|
||
return cleaned
|
||
|
||
|
||
def clean_enriched_actions(actions: list) -> list:
|
||
"""Nettoyer une liste d'actions enrichies pour éliminer le bruit de replay.
|
||
|
||
Appliqué après construction de toutes les actions enrichies (post-BFS),
|
||
travaille sur les actions au format replay (type, keys, text, etc.).
|
||
|
||
Filtres appliqués dans l'ordre :
|
||
1. Supprimer les types parasites (heartbeat, focus_change, screenshot, etc.)
|
||
2. Sanitiser les touches (caractères de contrôle → lettres)
|
||
3. Supprimer les key_combo avec uniquement des modificateurs seuls
|
||
4. Supprimer les actions type/text_input avec texte vide
|
||
5. Dédupliquer les key_combo consécutifs identiques
|
||
6. Fusionner les text_input (type) consécutifs dans la même fenêtre
|
||
7. Supprimer les waits consécutifs (garder le plus long)
|
||
"""
|
||
if not actions:
|
||
return actions
|
||
|
||
# ── Étape 1-4 : filtrer les actions parasites ──
|
||
filtered = []
|
||
for a in actions:
|
||
atype = a.get('type', '')
|
||
|
||
# Types parasites issus du streaming brut
|
||
if atype in _PARASITIC_ACTION_TYPES:
|
||
continue
|
||
|
||
# key_combo : sanitiser les touches, puis filtrer les modificateurs seuls
|
||
if atype == 'key_combo':
|
||
keys = _sanitize_keys(a.get('keys', []))
|
||
if _is_modifier_only(keys):
|
||
continue
|
||
if not keys:
|
||
continue
|
||
a = dict(a, keys=keys)
|
||
|
||
# type/text_input : supprimer si texte vide
|
||
if atype == 'type' and not a.get('text', '').strip():
|
||
continue
|
||
|
||
filtered.append(a)
|
||
|
||
# ── Étape 5 : dédupliquer les key_combo consécutifs identiques ──
|
||
deduped = []
|
||
for a in filtered:
|
||
if (deduped
|
||
and a.get('type') == 'key_combo'
|
||
and deduped[-1].get('type') == 'key_combo'
|
||
and a.get('keys') == deduped[-1].get('keys')):
|
||
continue
|
||
deduped.append(a)
|
||
|
||
# ── Étape 6 : fusionner les text_input (type) consécutifs ──
|
||
merged = []
|
||
for a in deduped:
|
||
if (merged
|
||
and a.get('type') == 'type'
|
||
and merged[-1].get('type') == 'type'
|
||
# Même fenêtre cible (ou pas de fenêtre)
|
||
and a.get('window_title', '') == merged[-1].get('window_title', '')):
|
||
merged[-1] = dict(merged[-1], text=merged[-1].get('text', '') + a.get('text', ''))
|
||
continue
|
||
merged.append(a)
|
||
|
||
# ── Étape 7 : supprimer les waits consécutifs (garder le plus long) ──
|
||
cleaned = []
|
||
for a in merged:
|
||
if (cleaned
|
||
and a.get('type') == 'wait'
|
||
and cleaned[-1].get('type') == 'wait'):
|
||
if a.get('duration_ms', 0) > cleaned[-1].get('duration_ms', 0):
|
||
cleaned[-1] = a
|
||
continue
|
||
cleaned.append(a)
|
||
|
||
return cleaned
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Replay direct depuis événements bruts (sans VLM/GraphBuilder)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
# Types d'événements ignorés lors de la construction du replay brut
|
||
_IGNORED_EVENT_TYPES = frozenset({
|
||
'heartbeat', 'focus_change', 'window_focus_change',
|
||
'screenshot', 'action_result', 'status', 'ping', 'pong',
|
||
})
|
||
|
||
# Combos de raccourcis spéciaux qui nécessitent un wait post-action
|
||
_POST_COMBO_WAITS = {
|
||
# (tuple de touches normalisées, triées en minuscule) -> wait_ms
|
||
# NB : les tuples sont sorted() alphabétiquement
|
||
('r', 'win'): 3000, # Win+R → Exécuter
|
||
('r', 'super'): 3000,
|
||
('meta', 'r'): 3000,
|
||
('enter',): 2000, # Enter (confirmation)
|
||
('return',): 2000,
|
||
('ctrl', 's'): 3000, # Ctrl+S
|
||
('ctrl', 's', 'shift'): 3000, # Ctrl+Shift+S
|
||
('alt', 'f4'): 2000, # Alt+F4
|
||
}
|
||
|
||
|
||
def _extract_screen_resolution(events: list) -> tuple:
|
||
"""Extraire la résolution d'écran depuis les métadonnées des événements.
|
||
|
||
Cherche d'abord le champ `screen_resolution` dans `screen_metadata`,
|
||
puis infère depuis les positions maximales des clics.
|
||
|
||
Returns:
|
||
Tuple (width, height).
|
||
"""
|
||
# Priorité 1 : screen_metadata.screen_resolution explicite
|
||
for evt in events:
|
||
event_data = evt.get("event", evt)
|
||
sm = event_data.get("screen_metadata", {})
|
||
sr = sm.get("screen_resolution")
|
||
if sr and isinstance(sr, (list, tuple)) and len(sr) == 2:
|
||
w, h = int(sr[0]), int(sr[1])
|
||
if w > 0 and h > 0:
|
||
return (w, h)
|
||
|
||
# Priorité 2 : inférer depuis les positions max des clics
|
||
return StreamProcessor._infer_screen_resolution(events)
|
||
|
||
|
||
def _should_cut_after_event(
|
||
event_data: dict,
|
||
saw_save_combo: bool = False,
|
||
) -> bool:
|
||
"""Retourne True si on doit couper le replay après cet événement.
|
||
|
||
Coupe quand :
|
||
- Alt+F4 (fermeture par raccourci)
|
||
- Clic dans le systray (fenêtres "unknown_window", "dépassement de capacité",
|
||
"Fenêtre de dépassement") — c'est le "C'est terminé" de Léa
|
||
- Après un Ctrl+S/Ctrl+Shift+S suivi d'un clic dans une fenêtre non-applicative
|
||
(terminal, systray, agent python, etc.)
|
||
"""
|
||
evt_type = event_data.get("type", "")
|
||
|
||
# Alt+F4
|
||
if evt_type in ("key_combo", "key_press"):
|
||
keys = event_data.get("keys", [])
|
||
keys_lower = {k.lower() for k in keys if k}
|
||
if "f4" in keys_lower and ("alt" in keys_lower or "alt_l" in keys_lower
|
||
or "alt_r" in keys_lower):
|
||
return True
|
||
|
||
# Clic dans le systray (fenêtres de fin de session)
|
||
if evt_type == "mouse_click":
|
||
window = event_data.get("window", {})
|
||
title = (window.get("title", "") if isinstance(window, dict) else "").lower()
|
||
if any(t in title for t in [
|
||
"unknown_window", "dépassement de capacité",
|
||
"fenêtre de dépassement", "overflow",
|
||
]):
|
||
return True
|
||
|
||
# Après un Ctrl+S/Ctrl+Shift+S : couper si clic dans une fenêtre
|
||
# non-applicative (terminal, agent, systray)
|
||
if saw_save_combo:
|
||
_CUT_WINDOW_PATTERNS = [
|
||
"cmd.exe", "system32", "dépassement", "unknown",
|
||
"powershell", "windowsterminal", "python.exe",
|
||
"terminal", "systray",
|
||
]
|
||
if any(t in title for t in _CUT_WINDOW_PATTERNS):
|
||
return True
|
||
# Couper aussi par app_name (plus robuste que le titre)
|
||
app_name = (window.get("app_name", "") if isinstance(window, dict) else "").lower()
|
||
if any(t in app_name for t in [
|
||
"windowsterminal", "cmd.exe", "powershell",
|
||
"python.exe", "explorer.exe",
|
||
]):
|
||
return True
|
||
|
||
return False
|
||
|
||
|
||
def _needs_post_wait(action: dict) -> int:
|
||
"""Retourne le wait en ms à insérer après cette action, ou 0."""
|
||
if action.get("type") == "key_combo":
|
||
keys = action.get("keys", [])
|
||
key_tuple = tuple(sorted(k.lower() for k in keys if k))
|
||
wait_ms = _POST_COMBO_WAITS.get(key_tuple, 0)
|
||
if wait_ms:
|
||
return wait_ms
|
||
return 0
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# VLM identification d'éléments UI (pour les éléments sans texte OCR)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def _vlm_identify_element(anchor_b64: str, window_title: str = "") -> str:
|
||
"""Demander au VLM de décrire un élément UI à partir de son crop.
|
||
|
||
Utilisé pendant le build_replay quand un élément cliqué n'a pas de
|
||
texte visible (icône YOLO sans label OCR). Le VLM décrit CE QUE c'est
|
||
(bouton, icône, menu) pour permettre la résolution sémantique au replay.
|
||
|
||
Returns:
|
||
Description courte de l'élément (ex: "search icon", "Word icon")
|
||
ou chaîne vide si le VLM n'est pas disponible.
|
||
"""
|
||
try:
|
||
import io
|
||
import tempfile
|
||
from PIL import Image
|
||
except ImportError:
|
||
return ""
|
||
|
||
try:
|
||
# Décoder le crop base64 → fichier temporaire pour le VLM
|
||
img_bytes = base64.b64decode(anchor_b64)
|
||
img = Image.open(io.BytesIO(img_bytes))
|
||
with tempfile.NamedTemporaryFile(suffix=".png", delete=False) as tmp:
|
||
img.save(tmp, format="PNG")
|
||
tmp_path = tmp.name
|
||
|
||
from core.detection.ollama_client import OllamaClient
|
||
client = OllamaClient(
|
||
endpoint="http://localhost:11434",
|
||
model="qwen3-vl:8b",
|
||
timeout=15,
|
||
)
|
||
context = f" in the window '{window_title}'" if window_title else ""
|
||
result = client.generate(
|
||
prompt=(
|
||
f"This is a cropped UI element{context}. "
|
||
"What is it? Answer with a short label (2-5 words max). "
|
||
"Examples: 'search bar icon', 'Word application icon', 'close button', "
|
||
"'file menu', 'save button'.\n"
|
||
"Answer ONLY the label, nothing else."
|
||
),
|
||
image_path=tmp_path,
|
||
system_prompt="You identify UI elements. Answer with a short label only.",
|
||
temperature=0.1,
|
||
max_tokens=20,
|
||
)
|
||
|
||
import os
|
||
os.unlink(tmp_path)
|
||
|
||
if result.get("success"):
|
||
raw = result.get("response", "").strip()
|
||
# Extraire un label court depuis la réponse (le VLM bavarde souvent)
|
||
# Retirer les préfixes courants
|
||
for prefix in (
|
||
"Based on the image, the UI element shown is a ",
|
||
"Based on the image, the UI element is a ",
|
||
"Based on the image, this is a ",
|
||
"Based on the image, it is a ",
|
||
"Based on the image, I can see ",
|
||
"Based on the image, ",
|
||
"The UI element shown is a ",
|
||
"The UI element is a ",
|
||
"The element is a ",
|
||
"This is a ", "It is a ", "It's a ", "I can see a ",
|
||
"I can see ", "A ",
|
||
):
|
||
if raw.lower().startswith(prefix.lower()):
|
||
raw = raw[len(prefix):]
|
||
break
|
||
# Prendre les 5 premiers mots utiles
|
||
words = raw.split()[:5]
|
||
label = " ".join(words).strip('",.\' ').rstrip(".")
|
||
if label and 2 <= len(label) <= 60:
|
||
logger.info("VLM identify element : '%s'", label)
|
||
return label
|
||
else:
|
||
logger.debug("VLM identify : label trop court ou vide après nettoyage (raw='%s')", raw[:80])
|
||
except Exception as e:
|
||
logger.debug("VLM identify element échoué : %s", e)
|
||
|
||
return ""
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# SomEngine — enrichissement Set-of-Mark des clics pendant le build_replay
|
||
# ---------------------------------------------------------------------------
|
||
|
||
_som_cache: Dict[str, Any] = {} # screenshot_id -> SomResult (cache build_replay)
|
||
_SOM_CACHE_MAX = 50
|
||
|
||
|
||
def _get_som_engine():
|
||
"""Singleton SomEngine partagé."""
|
||
try:
|
||
from core.detection.som_engine import get_shared_engine
|
||
return get_shared_engine()
|
||
except ImportError:
|
||
return None
|
||
|
||
|
||
def _som_identify_clicked_element(
|
||
event_data: dict,
|
||
session_dir: Optional[Path],
|
||
screen_w: int,
|
||
screen_h: int,
|
||
) -> Optional[dict]:
|
||
"""Identifier l'élément UI cliqué via SomEngine (YOLO + docTR).
|
||
|
||
Charge le full screenshot de l'événement, lance SomEngine pour détecter
|
||
tous les éléments, puis identifie celui qui se trouve sous le clic.
|
||
|
||
Returns:
|
||
Dict avec id, label, source, bbox_norm, center_norm, confidence
|
||
ou None si SomEngine indisponible ou élément non trouvé.
|
||
"""
|
||
engine = _get_som_engine()
|
||
if engine is None:
|
||
return None
|
||
|
||
if not session_dir:
|
||
return None
|
||
|
||
shots_dir = session_dir / "shots"
|
||
if not shots_dir.is_dir():
|
||
return None
|
||
|
||
# Trouver le full screenshot
|
||
screenshot_id = event_data.get("screenshot_id", "")
|
||
if not screenshot_id:
|
||
return None
|
||
|
||
full_path = shots_dir / f"{screenshot_id}_full.png"
|
||
if not full_path.is_file():
|
||
# Fallback : essayer sans le suffixe _full
|
||
full_path = shots_dir / f"{screenshot_id}.png"
|
||
if not full_path.is_file():
|
||
return None
|
||
|
||
# Vérifier le cache SomResult par (session_dir, screenshot_id)
|
||
cache_key = f"{session_dir}:{screenshot_id}"
|
||
if cache_key in _som_cache:
|
||
result = _som_cache[cache_key]
|
||
else:
|
||
try:
|
||
from PIL import Image
|
||
img = Image.open(full_path).convert("RGB")
|
||
except Exception as e:
|
||
logger.debug("SoM: impossible de charger %s : %s", full_path, e)
|
||
return None
|
||
|
||
# Lancer SomEngine
|
||
try:
|
||
result = engine.analyze(img)
|
||
except Exception as e:
|
||
logger.warning("SoM: erreur d'analyse : %s", e)
|
||
return None
|
||
|
||
# Stocker dans le cache (éléments seulement, pas l'image annotée)
|
||
from core.detection.som_engine import SomResult
|
||
cached = SomResult(
|
||
elements=result.elements,
|
||
width=result.width,
|
||
height=result.height,
|
||
analysis_time_ms=result.analysis_time_ms,
|
||
)
|
||
if len(_som_cache) >= _SOM_CACHE_MAX:
|
||
# Supprimer la plus ancienne entrée (FIFO)
|
||
oldest_key = next(iter(_som_cache))
|
||
del _som_cache[oldest_key]
|
||
_som_cache[cache_key] = cached
|
||
|
||
if not result.elements:
|
||
return None
|
||
|
||
# Trouver l'élément cliqué
|
||
pos = event_data.get("pos", [])
|
||
if not pos or len(pos) < 2:
|
||
return None
|
||
|
||
click_x, click_y = int(pos[0]), int(pos[1])
|
||
elem = result.find_element_at(click_x, click_y, margin=30)
|
||
if elem is None:
|
||
logger.debug(
|
||
"SoM: aucun élément trouvé au clic (%d, %d) parmi %d éléments",
|
||
click_x, click_y, len(result.elements),
|
||
)
|
||
return None
|
||
|
||
logger.info(
|
||
"SoM: clic (%d,%d) → élément #%d '%s' (source=%s, conf=%.2f)",
|
||
click_x, click_y, elem.id, elem.label, elem.source, elem.confidence,
|
||
)
|
||
return {
|
||
"id": elem.id,
|
||
"label": elem.label,
|
||
"source": elem.source,
|
||
"bbox_norm": list(elem.bbox_norm),
|
||
"center_norm": list(elem.center_norm),
|
||
"confidence": elem.confidence,
|
||
"element_count": len(result.elements),
|
||
}
|
||
|
||
|
||
def _load_crop_for_event(
|
||
event_data: dict,
|
||
session_dir: Optional[Path],
|
||
screen_w: int = 0,
|
||
screen_h: int = 0,
|
||
) -> Optional[str]:
|
||
"""Charger le crop de référence (anchor) associé à un événement mouse_click.
|
||
|
||
Stratégie de recherche (par priorité) :
|
||
1. vision_info.crop → extraire le nom de fichier, chercher dans session_dir/shots/
|
||
2. screenshot_id → chercher {screenshot_id}_crop.png dans session_dir/shots/
|
||
3. Timestamp → chercher le focus_XXXX.png le plus proche dans session_dir/shots/
|
||
4. Fallback → cropper le full screenshot autour de la position du clic (400x400)
|
||
|
||
Args:
|
||
event_data: Événement brut (type mouse_click).
|
||
session_dir: Répertoire de la session (contient shots/).
|
||
screen_w: Largeur écran (pour le fallback crop).
|
||
screen_h: Hauteur écran (pour le fallback crop).
|
||
|
||
Returns:
|
||
Image crop encodée en base64, ou None si rien trouvé.
|
||
"""
|
||
if not session_dir:
|
||
return None
|
||
|
||
shots_dir = Path(session_dir) / "shots"
|
||
if not shots_dir.is_dir():
|
||
return None
|
||
|
||
def _read_png_b64(path: Path) -> Optional[str]:
|
||
"""Lire un fichier PNG et retourner son contenu en base64."""
|
||
try:
|
||
if path.exists() and path.stat().st_size > 0:
|
||
return base64.b64encode(path.read_bytes()).decode("utf-8")
|
||
except Exception as e:
|
||
logger.debug("Impossible de lire le crop %s : %s", path, e)
|
||
return None
|
||
|
||
# ── Stratégie 1 : vision_info.crop (nom de fichier Windows → chercher localement) ──
|
||
vision_info = event_data.get("vision_info", {})
|
||
if isinstance(vision_info, dict):
|
||
crop_path_str = vision_info.get("crop", "")
|
||
if crop_path_str:
|
||
# Extraire le nom de fichier depuis le chemin Windows
|
||
# Ex: "C:\\rpa_vision\\...\\shots\\shot_0002_crop.png" → "shot_0002_crop.png"
|
||
crop_filename = crop_path_str.replace("\\", "/").split("/")[-1]
|
||
result = _read_png_b64(shots_dir / crop_filename)
|
||
if result:
|
||
logger.debug("Crop trouvé via vision_info : %s", crop_filename)
|
||
return result
|
||
|
||
# ── Stratégie 2 : screenshot_id → {screenshot_id}_crop.png ──
|
||
screenshot_id = event_data.get("screenshot_id", "")
|
||
if screenshot_id:
|
||
crop_filename = f"{screenshot_id}_crop.png"
|
||
result = _read_png_b64(shots_dir / crop_filename)
|
||
if result:
|
||
logger.debug("Crop trouvé via screenshot_id : %s", crop_filename)
|
||
return result
|
||
|
||
# ── Stratégie 3 : Timestamp → focus_XXXX.png le plus proche ──
|
||
evt_ts = float(event_data.get("timestamp", 0))
|
||
if evt_ts > 0:
|
||
try:
|
||
focus_files = sorted(shots_dir.glob("focus_*.png"))
|
||
if focus_files:
|
||
best_file = None
|
||
best_delta = float("inf")
|
||
for f in focus_files:
|
||
# Extraire le timestamp du nom : focus_1774437474.png
|
||
try:
|
||
ts_str = f.stem.split("_", 1)[1]
|
||
file_ts = float(ts_str)
|
||
delta = abs(file_ts - evt_ts)
|
||
if delta < best_delta:
|
||
best_delta = delta
|
||
best_file = f
|
||
except (ValueError, IndexError):
|
||
continue
|
||
# Accepter si le focus est à moins de 5 secondes du clic
|
||
if best_file and best_delta < 5.0:
|
||
result = _read_png_b64(best_file)
|
||
if result:
|
||
logger.debug(
|
||
"Crop trouvé via timestamp (focus, delta=%.1fs) : %s",
|
||
best_delta, best_file.name,
|
||
)
|
||
return result
|
||
except Exception as e:
|
||
logger.debug("Erreur recherche focus par timestamp : %s", e)
|
||
|
||
# ── Stratégie 4 : Fallback → cropper le full screenshot autour du clic ──
|
||
if screenshot_id and screen_w > 0 and screen_h > 0:
|
||
full_path = shots_dir / f"{screenshot_id}_full.png"
|
||
if full_path.exists():
|
||
try:
|
||
from PIL import Image
|
||
import io
|
||
|
||
img = Image.open(full_path)
|
||
pos = event_data.get("pos", [])
|
||
if pos and len(pos) == 2:
|
||
cx, cy = int(pos[0]), int(pos[1])
|
||
# Crop 150x150 centré sur le clic (plus discriminant, moins de bruit)
|
||
crop_size = 75
|
||
x1 = max(0, cx - crop_size)
|
||
y1 = max(0, cy - crop_size)
|
||
x2 = min(img.width, cx + crop_size)
|
||
y2 = min(img.height, cy + crop_size)
|
||
cropped = img.crop((x1, y1, x2, y2))
|
||
buf = io.BytesIO()
|
||
cropped.save(buf, format="PNG")
|
||
result = base64.b64encode(buf.getvalue()).decode("utf-8")
|
||
logger.debug(
|
||
"Crop fallback (full screenshot cropped) : %s, zone=%dx%d",
|
||
full_path.name, x2 - x1, y2 - y1,
|
||
)
|
||
return result
|
||
except ImportError:
|
||
logger.debug("PIL non disponible pour le crop fallback")
|
||
except Exception as e:
|
||
logger.debug("Erreur crop fallback depuis full screenshot : %s", e)
|
||
|
||
return None
|
||
|
||
|
||
def _attach_expected_screenshots(
|
||
actions: list, raw_events: list, session_dir: Path,
|
||
) -> None:
|
||
"""Attacher les screenshots de référence (résultat attendu) aux actions.
|
||
|
||
Pour chaque action de type click ou key_combo, cherche le screenshot
|
||
res_shot_XXXX.png (capturé 1s après l'action pendant l'enregistrement)
|
||
et l'attache comme expected_screenshot_b64.
|
||
|
||
Le screenshot est compressé en JPEG qualité 40 (~30-50 KB en b64)
|
||
pour limiter le poids de chaque action.
|
||
"""
|
||
import base64
|
||
from PIL import Image as _Image
|
||
|
||
shots_dir = session_dir / "shots"
|
||
if not shots_dir.is_dir():
|
||
return
|
||
|
||
# Mapper les screenshot_id des événements originaux aux actions
|
||
# Les événements click/key_combo ont un "screenshot_id" (ex: "shot_0003")
|
||
# Le screenshot résultat est "res_shot_0003.png"
|
||
action_idx = 0
|
||
for raw_evt in raw_events:
|
||
event_data = raw_evt.get("event", raw_evt)
|
||
screenshot_id = event_data.get("screenshot_id", "")
|
||
if not screenshot_id:
|
||
continue
|
||
|
||
evt_type = event_data.get("type", "")
|
||
if evt_type not in ("mouse_click", "key_combo", "key_press"):
|
||
continue
|
||
|
||
# Trouver l'action correspondante (même type, index croissant)
|
||
while action_idx < len(actions):
|
||
a = actions[action_idx]
|
||
a_type = a.get("type", "")
|
||
if a_type in ("click", "key_combo"):
|
||
break
|
||
action_idx += 1
|
||
else:
|
||
break # Plus d'actions
|
||
|
||
# Charger le screenshot résultat
|
||
res_file = shots_dir / f"res_{screenshot_id}.png"
|
||
if not res_file.is_file():
|
||
action_idx += 1
|
||
continue
|
||
|
||
try:
|
||
img = _Image.open(res_file)
|
||
# Redimensionner pour réduire le poids (800px de large)
|
||
if img.width > 800:
|
||
ratio = 800 / img.width
|
||
img = img.resize((800, int(img.height * ratio)), _Image.LANCZOS)
|
||
import io
|
||
buf = io.BytesIO()
|
||
img.save(buf, format="JPEG", quality=40)
|
||
b64 = base64.b64encode(buf.getvalue()).decode()
|
||
actions[action_idx]["expected_screenshot_b64"] = b64
|
||
logger.debug(
|
||
"Screenshot de référence attaché à action %d : %s (%d KB)",
|
||
action_idx, res_file.name, len(b64) // 1024,
|
||
)
|
||
except Exception as e:
|
||
logger.debug("Erreur chargement screenshot ref %s : %s", res_file, e)
|
||
|
||
action_idx += 1
|
||
|
||
|
||
def build_replay_from_raw_events(
|
||
events: list,
|
||
session_id: str = "",
|
||
session_dir: Optional[str] = None,
|
||
) -> list:
|
||
"""Construire un replay propre directement depuis les événements bruts d'une session.
|
||
|
||
Pas de dépendance au VLM, au GraphBuilder ou aux workflows.
|
||
Fonctionne immédiatement après la capture.
|
||
|
||
Pipeline de traitement :
|
||
1. Filtrer les événements parasites (heartbeat, focus_change, action_result)
|
||
2. Extraire la résolution d'écran depuis les métadonnées
|
||
3. Couper après Alt+F4 ou après des clics systray post-sauvegarde
|
||
4. Fusionner les text_input consécutifs (même séparés par <500ms)
|
||
5. Convertir en actions normalisées (coordonnées en %, waits adaptés)
|
||
6. Pour les clics : activer visual_mode et attacher le crop de référence (anchor)
|
||
7. Appliquer clean_enriched_actions() (dédup combos, sanitize, merge texte)
|
||
8. Insérer des waits contextuels après raccourcis critiques
|
||
|
||
Args:
|
||
events: Événements bruts chargés depuis live_events.jsonl.
|
||
Format : [{"session_id": ..., "event": {...}}, ...]
|
||
session_id: Identifiant de session (pour le logging).
|
||
session_dir: Répertoire de la session (contient shots/). Si fourni,
|
||
les crops de référence sont attachés aux clics pour le visual replay.
|
||
|
||
Returns:
|
||
Liste d'actions prêtes pour la queue de replay.
|
||
"""
|
||
import uuid
|
||
|
||
if not events:
|
||
return []
|
||
|
||
# Résoudre le répertoire de session pour les crops visuels
|
||
session_dir_path = Path(session_dir) if session_dir else None
|
||
if session_dir_path and not session_dir_path.is_dir():
|
||
logger.warning(
|
||
"session_dir '%s' n'existe pas — visual replay désactivé", session_dir,
|
||
)
|
||
session_dir_path = None
|
||
|
||
# ── 1. Extraire la résolution d'écran ──
|
||
screen_w, screen_h = _extract_screen_resolution(events)
|
||
logger.info(
|
||
"build_replay_from_raw_events(%s) : %d événements bruts, résolution=%dx%d, visual=%s",
|
||
session_id, len(events), screen_w, screen_h,
|
||
bool(session_dir_path),
|
||
)
|
||
|
||
# ── 2. Filtrer et normaliser les événements ──
|
||
actionable_events = []
|
||
saw_save_combo = False # Tracker Ctrl+S / Ctrl+Shift+S pour la coupure systray
|
||
for raw_evt in events:
|
||
event_data = raw_evt.get("event", raw_evt)
|
||
evt_type = event_data.get("type", "")
|
||
|
||
# Ignorer les types parasites
|
||
if evt_type in _IGNORED_EVENT_TYPES:
|
||
continue
|
||
|
||
# Tracker les raccourcis de sauvegarde (Ctrl+S, Ctrl+Shift+S)
|
||
if evt_type in ("key_combo", "key_press"):
|
||
keys = _sanitize_keys(event_data.get("keys", []))
|
||
keys_lower = {k.lower() for k in keys if k}
|
||
if "s" in keys_lower and ("ctrl" in keys_lower or "ctrl_l" in keys_lower
|
||
or "ctrl_r" in keys_lower):
|
||
saw_save_combo = True
|
||
|
||
# Vérifier la coupure AVANT d'ajouter l'événement.
|
||
# Pour les clics post-sauvegarde sur des fenêtres non-applicatives,
|
||
# on ne veut PAS inclure le clic qui déclenche la coupure.
|
||
if _should_cut_after_event(event_data, saw_save_combo=saw_save_combo):
|
||
# Alt+F4 est une action applicative → l'inclure
|
||
is_alt_f4 = False
|
||
if evt_type in ("key_combo", "key_press"):
|
||
_keys_lower = {k.lower() for k in event_data.get("keys", []) if k}
|
||
is_alt_f4 = "f4" in _keys_lower and (
|
||
"alt" in _keys_lower or "alt_l" in _keys_lower or "alt_r" in _keys_lower
|
||
)
|
||
if is_alt_f4:
|
||
actionable_events.append(event_data)
|
||
# Sinon c'est un clic parasite → ne PAS l'inclure
|
||
logger.debug(
|
||
"Coupure du replay (saw_save=%s, type=%s, included=%s)",
|
||
saw_save_combo, evt_type, is_alt_f4,
|
||
)
|
||
break
|
||
|
||
actionable_events.append(event_data)
|
||
|
||
# ── 3. Fusionner les text_input consécutifs ──
|
||
# Tous les text_input consécutifs sont fusionnés en un seul, indépendamment
|
||
# du gap temporel. L'utilisateur tape lettre par lettre mais on veut un
|
||
# seul "type" avec tout le texte dans le replay.
|
||
# Les key_combos qui produisent un caractère imprimable (ex: AltGr+0 → @)
|
||
# sont convertis en text_input pour être fusionnés avec le texte adjacent.
|
||
# Seul un changement de fenêtre (window_title différent) coupe la fusion.
|
||
merged_events = []
|
||
for evt in actionable_events:
|
||
evt_type = evt.get("type", "")
|
||
evt_ts = float(evt.get("timestamp", 0))
|
||
|
||
# Convertir les key_combos qui produisent un caractère imprimable
|
||
# en text_input pour qu'ils soient fusionnés avec le texte adjacent.
|
||
# Ex: AltGr+0 capturé comme ['ctrl', '@'] → text_input '@'
|
||
if evt_type in ("key_combo", "key_press"):
|
||
keys = _sanitize_keys(evt.get("keys", []))
|
||
printable = _key_combo_printable_char(keys)
|
||
if printable:
|
||
# Transformer en text_input pour fusion
|
||
evt = dict(evt, type="text_input", text=printable)
|
||
evt_type = "text_input"
|
||
# Pas de raw_keys pour ce caractère (sera collé via clipboard)
|
||
|
||
if evt_type == "text_input":
|
||
text = evt.get("text", "")
|
||
if not text:
|
||
continue
|
||
|
||
# Les \n et \t ne sont PAS du texte — ce sont des touches Enter/Tab
|
||
# qui doivent devenir des key_combo pour le replay
|
||
if text == "\n":
|
||
merged_events.append({
|
||
"type": "key_combo",
|
||
"keys": ["enter"],
|
||
"timestamp": evt_ts,
|
||
})
|
||
continue
|
||
if text == "\t":
|
||
merged_events.append({
|
||
"type": "key_combo",
|
||
"keys": ["tab"],
|
||
"timestamp": evt_ts,
|
||
})
|
||
continue
|
||
|
||
# Fusionner avec le précédent text_input si même application
|
||
# On compare par app_name (pas title, car le titre change pendant la frappe)
|
||
if merged_events and merged_events[-1].get("type") == "text_input":
|
||
prev_app = merged_events[-1].get("window", {}).get("app_name", "")
|
||
curr_app = evt.get("window", {}).get("app_name", "")
|
||
# Même application (ou application inconnue) → fusionner
|
||
if not prev_app or not curr_app or prev_app == curr_app:
|
||
merged_events[-1]["text"] = merged_events[-1].get("text", "") + text
|
||
merged_events[-1]["_end_ts"] = evt_ts
|
||
# Fusionner aussi les raw_keys (replay exact)
|
||
if evt.get("raw_keys"):
|
||
prev_raw = merged_events[-1].get("raw_keys", [])
|
||
merged_events[-1]["raw_keys"] = prev_raw + evt["raw_keys"]
|
||
continue
|
||
|
||
merged_events.append(dict(evt, _end_ts=evt_ts))
|
||
else:
|
||
merged_events.append(dict(evt))
|
||
|
||
# ── 3b. Reconstruire le texte correct depuis les raw_keys ──
|
||
# Les raw_keys contiennent les vk codes exacts (layout-indépendant)
|
||
# qui permettent de corriger les erreurs de capture AZERTY
|
||
# (ex: numpad / capturé comme '!' → corrigé en '/')
|
||
# ATTENTION : ne reconstruire QUE si le texte reconstruit a la même
|
||
# longueur que le texte original. Si des caractères viennent de
|
||
# key_combos convertis (ex: @ de AltGr), ils n'ont pas de raw_keys
|
||
# et la reconstruction les perdrait.
|
||
for evt in merged_events:
|
||
if evt.get("type") == "text_input" and evt.get("raw_keys"):
|
||
reconstructed = _reconstruct_text_from_raw_keys(evt["raw_keys"])
|
||
original = evt.get("text", "")
|
||
if reconstructed and len(reconstructed) == len(original):
|
||
# Même longueur → remplacement sûr (corrige les chars numpad)
|
||
evt["text"] = reconstructed
|
||
if reconstructed != original:
|
||
logger.debug(
|
||
"Texte reconstruit depuis raw_keys : '%s' → '%s'",
|
||
original[:50], reconstructed[:50],
|
||
)
|
||
elif reconstructed and len(reconstructed) < len(original):
|
||
# Longueur différente → des chars viennent de key_combos convertis
|
||
# Garder le texte original (qui inclut les chars fusionnés)
|
||
logger.debug(
|
||
"Texte non reconstruit (longueur diff) : '%s' (%d) vs '%s' (%d)",
|
||
original[:50], len(original), reconstructed[:50], len(reconstructed),
|
||
)
|
||
|
||
# ── 4. Convertir en actions replay normalisées ──
|
||
actions = []
|
||
last_ts = 0.0
|
||
|
||
for evt in merged_events:
|
||
evt_type = evt.get("type", "")
|
||
evt_ts = float(evt.get("timestamp", 0))
|
||
|
||
# Insérer un wait si pause significative (> 2s, cappé à 5s)
|
||
if last_ts > 0 and evt_ts > last_ts:
|
||
delta_ms = int((evt_ts - last_ts) * 1000)
|
||
if delta_ms > 2000:
|
||
capped_ms = min(delta_ms, 5000)
|
||
actions.append({
|
||
"action_id": f"act_raw_{uuid.uuid4().hex[:8]}",
|
||
"type": "wait",
|
||
"duration_ms": capped_ms,
|
||
})
|
||
|
||
# Mettre à jour le timestamp
|
||
end_ts = float(evt.get("_end_ts", evt_ts))
|
||
last_ts = max(last_ts, end_ts if end_ts > 0 else evt_ts)
|
||
|
||
action = {"action_id": f"act_raw_{uuid.uuid4().hex[:8]}"}
|
||
|
||
if evt_type == "mouse_click":
|
||
pos = evt.get("pos", [])
|
||
if not pos or len(pos) != 2:
|
||
continue
|
||
action["type"] = "click"
|
||
action["x_pct"] = round(pos[0] / screen_w, 6)
|
||
action["y_pct"] = round(pos[1] / screen_h, 6)
|
||
action["button"] = evt.get("button", "left")
|
||
# Enrichir avec le titre de fenêtre si disponible
|
||
window = evt.get("window", {})
|
||
if window.get("title"):
|
||
action["window_title"] = window["title"]
|
||
|
||
# ── Visual replay : attacher le crop de référence (anchor) ──
|
||
if session_dir_path:
|
||
anchor_b64 = _load_crop_for_event(
|
||
evt, session_dir_path, screen_w, screen_h,
|
||
)
|
||
if anchor_b64:
|
||
action["visual_mode"] = True
|
||
|
||
# Construire une description VLM riche pour le replay VLM-first
|
||
window_title = window.get("title", "")
|
||
x_pos, y_pos = pos[0], pos[1]
|
||
|
||
# Position relative dans l'écran
|
||
if screen_h > 0:
|
||
y_relative = (
|
||
"en bas" if y_pos / screen_h > 0.8
|
||
else "en haut" if y_pos / screen_h < 0.2
|
||
else "au milieu"
|
||
)
|
||
else:
|
||
y_relative = ""
|
||
if screen_w > 0:
|
||
x_relative = (
|
||
"à gauche" if x_pos / screen_w < 0.3
|
||
else "à droite" if x_pos / screen_w > 0.7
|
||
else "au centre"
|
||
)
|
||
else:
|
||
x_relative = ""
|
||
|
||
# Description riche pour le VLM
|
||
vlm_parts = []
|
||
if window_title:
|
||
vlm_parts.append(
|
||
f"Dans la fenêtre '{window_title}'"
|
||
)
|
||
position_desc = " ".join(
|
||
p for p in [y_relative, x_relative] if p
|
||
)
|
||
if position_desc:
|
||
vlm_parts.append(
|
||
f"l'élément cliqué se trouve {position_desc} de l'écran"
|
||
)
|
||
# Ajouter le texte visible (vision_info ou OCR)
|
||
vision_info = evt.get("vision_info", {})
|
||
if isinstance(vision_info, dict):
|
||
vis_text = vision_info.get("text", "")
|
||
vis_type = vision_info.get("type", "")
|
||
if vis_text:
|
||
vlm_parts.append(
|
||
f"le texte visible est '{vis_text}'"
|
||
)
|
||
if vis_type:
|
||
vlm_parts.append(
|
||
f"c'est un élément de type '{vis_type}'"
|
||
)
|
||
vlm_description = ", ".join(vlm_parts) if vlm_parts else ""
|
||
|
||
# ── SomEngine : identifier l'élément cliqué ──
|
||
som_elem = _som_identify_clicked_element(
|
||
evt, session_dir_path, screen_w, screen_h,
|
||
)
|
||
|
||
# Déterminer le texte de l'élément cliqué (by_text)
|
||
# Priorité : vision_info.text > som_element.label > VLM identification
|
||
element_text = ""
|
||
element_type = ""
|
||
if isinstance(vision_info, dict):
|
||
element_text = vision_info.get("text", "")
|
||
element_type = vision_info.get("type", "")
|
||
if not element_text and som_elem and som_elem.get("label"):
|
||
element_text = som_elem["label"]
|
||
|
||
# Si pas de texte (icône sans label), demander au VLM
|
||
# d'identifier CE QUE c'est à partir du crop
|
||
if not element_text and anchor_b64:
|
||
element_text = _vlm_identify_element(
|
||
anchor_b64, window_title,
|
||
)
|
||
|
||
action["target_spec"] = {
|
||
"anchor_image_base64": anchor_b64,
|
||
"by_text": element_text, # CE QUE l'élément EST
|
||
"by_role": element_type or (som_elem.get("source", "") if som_elem else ""),
|
||
"vlm_description": vlm_description,
|
||
"window_title": window_title,
|
||
"original_position": {
|
||
"x_relative": x_relative,
|
||
"y_relative": y_relative,
|
||
},
|
||
}
|
||
|
||
if som_elem:
|
||
action["target_spec"]["som_element"] = som_elem
|
||
|
||
elif evt_type == "text_input":
|
||
text = evt.get("text", "")
|
||
if not text:
|
||
continue
|
||
action["type"] = "type"
|
||
action["text"] = text
|
||
# Propager les raw_keys pour le replay exact (solution AZERTY)
|
||
if evt.get("raw_keys"):
|
||
action["raw_keys"] = evt["raw_keys"]
|
||
|
||
elif evt_type in ("key_press", "key_combo"):
|
||
keys = evt.get("keys", [])
|
||
if not keys:
|
||
key = evt.get("key", "")
|
||
if key:
|
||
keys = [key]
|
||
if not keys:
|
||
continue
|
||
keys = _sanitize_keys(keys)
|
||
if _is_modifier_only(keys):
|
||
continue
|
||
action["type"] = "key_combo"
|
||
action["keys"] = keys
|
||
# Propager les raw_keys pour le replay exact (solution AZERTY)
|
||
if evt.get("raw_keys"):
|
||
action["raw_keys"] = evt["raw_keys"]
|
||
|
||
elif evt_type == "scroll":
|
||
pos = evt.get("pos", [])
|
||
action["type"] = "scroll"
|
||
if pos and len(pos) == 2:
|
||
action["x_pct"] = round(pos[0] / screen_w, 6)
|
||
action["y_pct"] = round(pos[1] / screen_h, 6)
|
||
action["delta"] = evt.get("delta", -3)
|
||
|
||
else:
|
||
continue
|
||
|
||
actions.append(action)
|
||
|
||
# ── 5. Nettoyage global (dédup combos, sanitize, merge texte, waits) ──
|
||
actions = clean_enriched_actions(actions)
|
||
|
||
# ── 6. Insérer des waits contextuels après raccourcis critiques ──
|
||
final_actions = []
|
||
for action in actions:
|
||
final_actions.append(action)
|
||
post_wait = _needs_post_wait(action)
|
||
if post_wait > 0:
|
||
# Vérifier si un wait existe déjà juste après (sera ajouté au prochain tour)
|
||
final_actions.append({
|
||
"action_id": f"act_raw_{uuid.uuid4().hex[:8]}",
|
||
"type": "wait",
|
||
"duration_ms": post_wait,
|
||
})
|
||
|
||
# ── 7. Dernier nettoyage des waits consécutifs ──
|
||
result = []
|
||
for a in final_actions:
|
||
if (result
|
||
and a.get("type") == "wait"
|
||
and result[-1].get("type") == "wait"):
|
||
# Garder le plus long
|
||
if a.get("duration_ms", 0) > result[-1].get("duration_ms", 0):
|
||
result[-1] = a
|
||
continue
|
||
result.append(a)
|
||
|
||
# ── 8. Attacher les screenshots de référence (état attendu après action) ──
|
||
# Les screenshots res_shot_XXXX.png capturés 1s après chaque action pendant
|
||
# l'enregistrement servent de référence pour le contrôle visuel.
|
||
if session_dir_path:
|
||
_attach_expected_screenshots(result, events, session_dir_path)
|
||
|
||
# Stats visual replay
|
||
visual_clicks = sum(
|
||
1 for a in result
|
||
if a.get("type") == "click" and a.get("visual_mode")
|
||
)
|
||
total_clicks = sum(1 for a in result if a.get("type") == "click")
|
||
verified_count = sum(1 for a in result if a.get("expected_screenshot_b64"))
|
||
logger.info(
|
||
"build_replay_from_raw_events(%s) : %d actions propres produites "
|
||
"(%d/%d clics avec visual_mode, %d avec screenshot de référence)",
|
||
session_id, len(result), visual_clicks, total_clicks, verified_count,
|
||
)
|
||
return result
|
||
|
||
|
||
class StreamProcessor:
|
||
"""
|
||
Processeur de streaming qui connecte les données Agent V1 au core pipeline.
|
||
|
||
Cycle de vie :
|
||
1. register_session() — crée l'état mémoire
|
||
2. process_event() — accumule événements, extrait contexte fenêtre
|
||
3. process_screenshot() — analyse via ScreenAnalyzer + CLIP embedding
|
||
4. finalize_session() — construit le Workflow via GraphBuilder (DBSCAN)
|
||
"""
|
||
|
||
def __init__(self, data_dir: str = "data/training"):
|
||
self.data_dir = Path(data_dir)
|
||
persist_dir = str(self.data_dir / "streaming_sessions")
|
||
self.session_manager = LiveSessionManager(persist_dir=persist_dir)
|
||
self._lock = threading.Lock()
|
||
|
||
# Core components (chargés paresseusement pour éviter les imports lourds au démarrage)
|
||
self._screen_analyzer = None
|
||
self._clip_embedder = None
|
||
self._state_embedding_builder = None # P0-3 : pipeline d'embedding unifié (fusion multi-modale)
|
||
self._faiss_manager = None
|
||
self._initialized = False
|
||
|
||
# Lock pour l'accès concurrent aux données de session (screen_states, embeddings, workflows)
|
||
self._data_lock = threading.Lock()
|
||
|
||
# Lock pour l'accès FAISS (IndexFlat.add() n'est pas thread-safe)
|
||
self._faiss_lock = threading.Lock()
|
||
|
||
# Flag de suspension : quand un replay est actif, le worker se suspend
|
||
# pour libérer le GPU au resolve_target VLM du replay.
|
||
# Settée depuis api_stream.py via set_replay_flag().
|
||
self._replay_active_flag: Optional[threading.Event] = None
|
||
|
||
# Résultats d'analyse par session
|
||
self._screen_states: Dict[str, list] = {} # session_id -> List[ScreenState]
|
||
self._embeddings: Dict[str, list] = {} # session_id -> List[np.ndarray]
|
||
|
||
# Workflows construits (pour le matching)
|
||
self._workflows: Dict[str, Any] = {}
|
||
|
||
# Charger les workflows existants depuis le disque
|
||
self._load_persisted_workflows()
|
||
|
||
def _load_persisted_workflows(self):
|
||
"""Charger les workflows sauvegardés depuis le disque au démarrage.
|
||
|
||
Scanne le dossier workflows/ principal et les sous-dossiers par machine
|
||
(workflows/{machine_id}/) pour la rétrocompatibilité.
|
||
"""
|
||
workflows_dir = self.data_dir / "workflows"
|
||
if not workflows_dir.exists():
|
||
return
|
||
|
||
try:
|
||
from core.models.workflow_graph import Workflow
|
||
|
||
count = 0
|
||
# Charger les workflows du dossier racine (rétrocompatibilité)
|
||
for wf_file in sorted(workflows_dir.glob("*.json")):
|
||
try:
|
||
wf = Workflow.load_from_file(wf_file)
|
||
self._workflows[wf.workflow_id] = wf
|
||
count += 1
|
||
except Exception as e:
|
||
logger.warning(f"Impossible de charger {wf_file.name}: {e}")
|
||
|
||
# Charger les workflows des sous-dossiers par machine
|
||
for machine_dir in sorted(workflows_dir.iterdir()):
|
||
if not machine_dir.is_dir():
|
||
continue
|
||
for wf_file in sorted(machine_dir.glob("*.json")):
|
||
try:
|
||
wf = Workflow.load_from_file(wf_file)
|
||
# Stocker le machine_id dans les métadonnées du workflow
|
||
if not hasattr(wf, '_machine_id'):
|
||
wf._machine_id = machine_dir.name
|
||
self._workflows[wf.workflow_id] = wf
|
||
count += 1
|
||
except Exception as e:
|
||
logger.warning(f"Impossible de charger {wf_file.name}: {e}")
|
||
|
||
if count:
|
||
logger.info(f"{count} workflow(s) chargé(s) depuis {workflows_dir}")
|
||
except ImportError:
|
||
logger.debug("core.models.workflow_graph non disponible, skip chargement")
|
||
|
||
def set_replay_flag(self, flag: threading.Event):
|
||
"""Associer le flag de replay actif (depuis api_stream.py).
|
||
|
||
Quand ce flag est set(), reprocess_session() se suspend entre chaque
|
||
screenshot pour libérer le GPU au replay (resolve_target VLM).
|
||
"""
|
||
self._replay_active_flag = flag
|
||
logger.info("Flag de suspension replay configuré sur le StreamProcessor")
|
||
|
||
def _wait_if_replay_active(self, context: str = "") -> bool:
|
||
"""Suspendre le traitement si un replay est en cours.
|
||
|
||
Vérifie le flag _replay_active_flag et attend qu'il se clear.
|
||
Timeout de sécurité : 60s max pour éviter un blocage si le replay
|
||
plante sans clear le flag.
|
||
|
||
Args:
|
||
context: Description pour les logs (ex: "screenshot 3/10").
|
||
|
||
Returns:
|
||
True si on a dû attendre (replay était actif), False sinon.
|
||
"""
|
||
if not self._replay_active_flag or not self._replay_active_flag.is_set():
|
||
return False
|
||
|
||
import time
|
||
suspend_start = time.time()
|
||
waited = False
|
||
while self._replay_active_flag.is_set():
|
||
elapsed = time.time() - suspend_start
|
||
if elapsed > 60:
|
||
logger.warning(
|
||
f"Worker : timeout suspension (60s), reprise forcée ({context})"
|
||
)
|
||
break
|
||
if not waited:
|
||
logger.info(f"Worker suspendu — replay en cours ({context})")
|
||
waited = True
|
||
time.sleep(2)
|
||
|
||
if waited:
|
||
total_wait = time.time() - suspend_start
|
||
logger.info(
|
||
f"Worker reprend après {total_wait:.1f}s de suspension ({context})"
|
||
)
|
||
return waited
|
||
|
||
def _ensure_initialized(self):
|
||
"""Charger les composants core GPU si pas encore fait.
|
||
|
||
DÉSACTIVÉ dans le serveur HTTP : les composants GPU (ScreenAnalyzer,
|
||
CLIP, FAISS) bloquent le GIL Python et rendent le serveur non-réactif.
|
||
Ces composants sont chargés uniquement par le worker séparé (run_worker.py).
|
||
Le serveur HTTP ne fait que stocker les screenshots et distribuer les replays.
|
||
"""
|
||
if self._initialized:
|
||
return
|
||
# Marquer comme initialisé SANS charger les composants GPU
|
||
self._initialized = True
|
||
logger.info("StreamProcessor initialisé en mode LÉGER (pas de GPU, pas de VLM)")
|
||
return
|
||
|
||
with self._lock:
|
||
if self._initialized:
|
||
return
|
||
|
||
logger.info("Initialisation des composants core (GPU)...")
|
||
|
||
try:
|
||
from core.pipeline.screen_analyzer import ScreenAnalyzer
|
||
self._screen_analyzer = ScreenAnalyzer(session_id="stream_server")
|
||
logger.info(" ScreenAnalyzer prêt")
|
||
except Exception as e:
|
||
logger.error(f" Erreur init ScreenAnalyzer: {e}")
|
||
self._screen_analyzer = None
|
||
|
||
try:
|
||
from core.embedding.clip_embedder import CLIPEmbedder
|
||
self._clip_embedder = CLIPEmbedder()
|
||
logger.info(" CLIPEmbedder prêt (singleton, ne sera plus rechargé)")
|
||
except Exception as e:
|
||
logger.error(f" Erreur init CLIPEmbedder: {e}")
|
||
self._clip_embedder = None
|
||
|
||
# P0-3 : Initialiser le StateEmbeddingBuilder pour unifier l'espace d'embedding
|
||
# Utilise le même CLIPEmbedder (pas de rechargement du modèle) + FusionEngine
|
||
# pour produire des vecteurs fusionnés (image+text+title+ui) identiques à GraphBuilder
|
||
try:
|
||
from core.embedding.state_embedding_builder import StateEmbeddingBuilder
|
||
if self._clip_embedder is not None:
|
||
# Injecter le CLIPEmbedder déjà chargé pour éviter un double chargement
|
||
self._state_embedding_builder = StateEmbeddingBuilder(
|
||
embedders={
|
||
"image": self._clip_embedder,
|
||
"text": self._clip_embedder,
|
||
"title": self._clip_embedder,
|
||
"ui": self._clip_embedder,
|
||
},
|
||
output_dir=self.data_dir / "embeddings",
|
||
use_clip=False, # Pas besoin, on fournit les embedders directement
|
||
)
|
||
else:
|
||
# Fallback : laisser le builder créer son propre CLIPEmbedder
|
||
self._state_embedding_builder = StateEmbeddingBuilder(
|
||
output_dir=self.data_dir / "embeddings",
|
||
use_clip=True,
|
||
)
|
||
logger.info(" StateEmbeddingBuilder prêt (fusion multi-modale unifiée)")
|
||
except Exception as e:
|
||
logger.warning(f" StateEmbeddingBuilder non disponible, fallback CLIP pur: {e}")
|
||
self._state_embedding_builder = None
|
||
|
||
try:
|
||
from core.embedding.faiss_manager import FAISSManager
|
||
self._faiss_manager = FAISSManager(
|
||
dimensions=512,
|
||
index_type="Flat",
|
||
metric="cosine",
|
||
)
|
||
logger.info(" FAISSManager prêt (512 dims, cosine)")
|
||
except Exception as e:
|
||
logger.error(f" Erreur init FAISSManager: {e}")
|
||
self._faiss_manager = None
|
||
|
||
self._initialized = True
|
||
logger.info("Composants core initialisés.")
|
||
|
||
# =========================================================================
|
||
# Événements
|
||
# =========================================================================
|
||
|
||
def process_event(self, session_id: str, event_data: Dict[str, Any]) -> Dict[str, Any]:
|
||
"""Enregistrer un événement dans la session live.
|
||
|
||
Filtre les événements parasites à la réception :
|
||
- key_combo/key_press avec uniquement des modificateurs seuls (ctrl, alt, shift, etc.)
|
||
- key_combo/key_press avec liste de touches vide
|
||
- text_input avec texte vide
|
||
"""
|
||
if _is_parasitic_event(event_data):
|
||
logger.debug(
|
||
f"Événement parasite filtré (session {session_id}): "
|
||
f"type={event_data.get('type')}, data={event_data.get('keys', event_data.get('text', ''))}"
|
||
)
|
||
return {"status": "event_filtered", "session_id": session_id, "reason": "parasitic"}
|
||
self.session_manager.add_event(session_id, event_data)
|
||
return {"status": "event_recorded", "session_id": session_id}
|
||
|
||
# =========================================================================
|
||
# Screenshots
|
||
# =========================================================================
|
||
|
||
def process_screenshot(self, session_id: str, shot_id: str, file_path: str) -> Dict[str, Any]:
|
||
"""
|
||
Analyser un screenshot full via le core pipeline.
|
||
|
||
1. ScreenAnalyzer → ScreenState (OCR, UI detection)
|
||
2. StateEmbeddingBuilder → vecteur fusionné 512d (image+text+title+ui)
|
||
Même espace d'embedding que GraphBuilder (P0-3)
|
||
Fallback : CLIP embed_image() si StateEmbeddingBuilder échoue
|
||
3. FAISS indexation → matching temps réel
|
||
"""
|
||
self._ensure_initialized()
|
||
self.session_manager.add_screenshot(session_id, shot_id, file_path)
|
||
|
||
result = {
|
||
"shot_id": shot_id,
|
||
"session_id": session_id,
|
||
"state_id": None,
|
||
"ui_elements_count": 0,
|
||
"text_detected": 0,
|
||
"embedding_indexed": False,
|
||
"match": None,
|
||
}
|
||
|
||
# 1. Construire le ScreenState
|
||
if self._screen_analyzer is None:
|
||
logger.warning("ScreenAnalyzer non disponible, skip analyse")
|
||
return result
|
||
|
||
session = self.session_manager.get_session(session_id)
|
||
# Utiliser le mapping shot → window si disponible (reprocessing)
|
||
shot_map = getattr(session, '_shot_window_map', None) if session else None
|
||
if shot_map and shot_id in shot_map:
|
||
window_info = shot_map[shot_id]
|
||
else:
|
||
window_info = session.last_window_info if session else {}
|
||
|
||
try:
|
||
screen_state = self._screen_analyzer.analyze(
|
||
screenshot_path=file_path,
|
||
window_info=window_info,
|
||
)
|
||
result["state_id"] = screen_state.screen_state_id
|
||
result["ui_elements_count"] = len(screen_state.ui_elements)
|
||
result["text_detected"] = len(
|
||
getattr(screen_state.perception, "detected_text", [])
|
||
)
|
||
|
||
# Stocker le ScreenState pour le build final
|
||
with self._data_lock:
|
||
if session_id not in self._screen_states:
|
||
self._screen_states[session_id] = []
|
||
self._screen_states[session_id].append(screen_state)
|
||
|
||
logger.info(
|
||
f"Screenshot analysé: {shot_id} | "
|
||
f"{result['ui_elements_count']} UI elements, "
|
||
f"{result['text_detected']} textes"
|
||
)
|
||
except Exception as e:
|
||
logger.error(f"Erreur analyse screenshot {shot_id}: {e}")
|
||
return result
|
||
|
||
# 2. Construire l'embedding fusionné via StateEmbeddingBuilder (P0-3)
|
||
# Utilise le même pipeline que GraphBuilder : fusion image+text+title+ui
|
||
# pour garantir que les vecteurs FAISS sont dans le même espace d'embedding
|
||
embedding_vector = None
|
||
|
||
if self._state_embedding_builder is not None:
|
||
try:
|
||
state_embedding = self._state_embedding_builder.build(screen_state)
|
||
# Récupérer le vecteur fusionné depuis le StateEmbedding
|
||
fused_vec = state_embedding.get_vector()
|
||
if fused_vec is not None:
|
||
embedding_vector = fused_vec.astype(np.float32)
|
||
logger.debug(
|
||
f"Embedding fusionné multi-modal calculé pour {shot_id} "
|
||
f"(dim={embedding_vector.shape[0]})"
|
||
)
|
||
except Exception as e:
|
||
logger.warning(
|
||
f"StateEmbeddingBuilder échoué pour {shot_id}: {e}, "
|
||
f"fallback sur CLIP pur"
|
||
)
|
||
|
||
# Fallback : utiliser le CLIPEmbedder singleton (embedding image seul)
|
||
if embedding_vector is None and self._clip_embedder is not None:
|
||
try:
|
||
from PIL import Image
|
||
pil_image = Image.open(file_path)
|
||
embedding_vector = self._clip_embedder.embed_image(pil_image)
|
||
except Exception as e:
|
||
logger.debug(f"CLIP embedding échoué: {e}")
|
||
|
||
if embedding_vector is not None:
|
||
# Stocker pour le build final
|
||
with self._data_lock:
|
||
if session_id not in self._embeddings:
|
||
self._embeddings[session_id] = []
|
||
self._embeddings[session_id].append(embedding_vector)
|
||
|
||
# 3. Indexer dans FAISS (protégé par _faiss_lock car IndexFlat.add n'est pas thread-safe)
|
||
if self._faiss_manager is not None:
|
||
try:
|
||
with self._faiss_lock:
|
||
self._faiss_manager.add_embedding(
|
||
embedding_id=screen_state.screen_state_id,
|
||
vector=embedding_vector,
|
||
metadata={
|
||
"session_id": session_id,
|
||
"shot_id": shot_id,
|
||
"window_title": window_info.get("title", ""),
|
||
},
|
||
)
|
||
result["embedding_indexed"] = True
|
||
except Exception as e:
|
||
logger.error(f"Erreur FAISS indexation: {e}")
|
||
|
||
# 4. Matching temps réel contre les workflows connus
|
||
with self._data_lock:
|
||
has_workflows = bool(self._workflows)
|
||
if embedding_vector is not None and has_workflows:
|
||
result["match"] = self._try_match(embedding_vector)
|
||
|
||
return result
|
||
|
||
def process_crop(self, session_id: str, shot_id: str, file_path: str) -> Dict[str, Any]:
|
||
"""
|
||
Enregistrer un crop (400x400). Pas d'analyse ScreenAnalyzer
|
||
(un crop est un fragment, pas un écran complet).
|
||
"""
|
||
self.session_manager.add_screenshot(session_id, shot_id, file_path)
|
||
return {"status": "crop_stored", "shot_id": shot_id}
|
||
|
||
# =========================================================================
|
||
# Finalisation
|
||
# =========================================================================
|
||
|
||
def finalize_session(self, session_id: str) -> Dict[str, Any]:
|
||
"""
|
||
Construire un Workflow depuis les données accumulées.
|
||
|
||
Utilise le GraphBuilder du core avec les ScreenStates et embeddings
|
||
collectés pendant le streaming.
|
||
"""
|
||
self._ensure_initialized()
|
||
|
||
session = self.session_manager.finalize(session_id)
|
||
if not session:
|
||
return {"error": f"Session {session_id} non trouvée"}
|
||
|
||
with self._data_lock:
|
||
states = list(self._screen_states.get(session_id, []))
|
||
embeddings = list(self._embeddings.get(session_id, []))
|
||
|
||
if len(states) < 2:
|
||
logger.warning(
|
||
f"Session {session_id}: seulement {len(states)} states, "
|
||
f"pas assez pour construire un workflow"
|
||
)
|
||
return {
|
||
"session_id": session_id,
|
||
"status": "insufficient_data",
|
||
"states_count": len(states),
|
||
"min_required": 2,
|
||
}
|
||
|
||
# Convertir en RawSession pour le GraphBuilder
|
||
raw_dict = self.session_manager.to_raw_session(session_id)
|
||
if not raw_dict:
|
||
return {"error": "Conversion RawSession échouée"}
|
||
|
||
try:
|
||
from core.models.raw_session import RawSession
|
||
raw_session = RawSession.from_dict(raw_dict)
|
||
except Exception as e:
|
||
logger.error(f"Erreur construction RawSession: {e}")
|
||
# Fallback : construire manuellement
|
||
try:
|
||
raw_session = self._build_raw_session_fallback(session, raw_dict)
|
||
except Exception as e2:
|
||
return {"error": f"Erreur RawSession: {e2}"}
|
||
|
||
# Construire le workflow via GraphBuilder
|
||
try:
|
||
from core.graph.graph_builder import GraphBuilder
|
||
|
||
n = len(states)
|
||
min_reps = 1 if n < 6 else 2 if n <= 30 else min(3, n // 10)
|
||
|
||
builder = GraphBuilder(
|
||
embedding_builder=self._state_embedding_builder, # Réutiliser le même modèle CLIP
|
||
min_pattern_repetitions=min_reps,
|
||
clustering_eps=0.08,
|
||
clustering_min_samples=2,
|
||
)
|
||
|
||
# Nommer le workflow intelligemment à partir des titres de fenêtre
|
||
workflow_name = self._generate_workflow_name(session_id)
|
||
|
||
# Récupérer les embeddings pré-calculés pendant le streaming
|
||
with self._data_lock:
|
||
precomputed_embs = list(self._embeddings.get(session_id, []))
|
||
|
||
# Injecter les ScreenStates et embeddings pré-calculés pour éviter
|
||
# de re-analyser et de recalculer les embeddings (triple calcul)
|
||
workflow = builder.build_from_session(
|
||
raw_session,
|
||
workflow_name=workflow_name,
|
||
precomputed_states=states,
|
||
precomputed_embeddings=precomputed_embs if len(precomputed_embs) == len(states) else None,
|
||
)
|
||
|
||
with self._data_lock:
|
||
self._workflows[workflow.workflow_id] = workflow
|
||
|
||
# Persister sur disque (dans le dossier de la machine source)
|
||
machine_id = session.machine_id if hasattr(session, 'machine_id') else "default"
|
||
saved_path = self._persist_workflow(workflow, session_id, machine_id=machine_id)
|
||
# Stocker le machine_id dans le workflow pour le filtrage
|
||
workflow._machine_id = machine_id
|
||
|
||
# Récupérer les métadonnées applicatives de la session
|
||
session_state = self.session_manager.get_session(session_id)
|
||
app_context = {}
|
||
if session_state:
|
||
app_context = {
|
||
"window_titles": dict(session_state.window_titles_seen),
|
||
"app_names": dict(session_state.app_names_seen),
|
||
"primary_app": sorted(
|
||
session_state.app_names_seen.items(),
|
||
key=lambda x: -x[1]
|
||
)[0][0] if session_state.app_names_seen else None,
|
||
"multi_app": len(session_state.app_names_seen) >= 3,
|
||
}
|
||
|
||
result = {
|
||
"session_id": session_id,
|
||
"machine_id": machine_id,
|
||
"status": "workflow_built",
|
||
"workflow_id": workflow.workflow_id,
|
||
"workflow_name": workflow_name,
|
||
"nodes": len(workflow.nodes),
|
||
"edges": len(workflow.edges),
|
||
"states_analyzed": len(states),
|
||
"embeddings_indexed": len(embeddings),
|
||
"saved_path": str(saved_path) if saved_path else None,
|
||
"app_context": app_context,
|
||
}
|
||
|
||
logger.info(
|
||
f"Workflow construit: '{workflow_name}' ({workflow.workflow_id}) | "
|
||
f"{result['nodes']} nodes, {result['edges']} edges"
|
||
+ (f" | apps: {list(app_context.get('app_names', {}).keys())}" if app_context.get('app_names') else "")
|
||
)
|
||
|
||
# Libérer la mémoire des données de session (peuvent être lourdes)
|
||
self._cleanup_session_data(session_id)
|
||
|
||
return result
|
||
|
||
except Exception as e:
|
||
logger.error(f"Erreur construction workflow: {e}")
|
||
return {"error": f"GraphBuilder: {e}", "session_id": session_id}
|
||
|
||
# =========================================================================
|
||
# Matching
|
||
# =========================================================================
|
||
|
||
def _try_match(self, embedding_vector: np.ndarray) -> Optional[Dict[str, Any]]:
|
||
"""Matcher un embedding contre les workflows connus."""
|
||
if self._faiss_manager is None or self._faiss_manager.index.ntotal == 0:
|
||
return None
|
||
|
||
try:
|
||
results = self._faiss_manager.search_similar(
|
||
query_vector=embedding_vector,
|
||
k=1,
|
||
min_similarity=0.85,
|
||
)
|
||
if results:
|
||
best = results[0]
|
||
return {
|
||
"matched_id": best.embedding_id,
|
||
"similarity": round(best.similarity, 4),
|
||
"metadata": best.metadata,
|
||
}
|
||
except Exception as e:
|
||
logger.debug(f"Erreur matching: {e}")
|
||
|
||
return None
|
||
|
||
# =========================================================================
|
||
# Retraitement (appelé par le SessionWorker)
|
||
# =========================================================================
|
||
|
||
def reprocess_session(
|
||
self,
|
||
session_id: str,
|
||
progress_callback=None,
|
||
) -> Dict[str, Any]:
|
||
"""Retraiter une session finalisée : analyser tous les screenshots puis construire le workflow.
|
||
|
||
Utilisé par le SessionWorker pour traiter les sessions en arrière-plan.
|
||
Cherche les fichiers shot_*_full.png sur disque, les analyse un par un
|
||
via process_screenshot(), puis appelle finalize_session() pour construire
|
||
le workflow.
|
||
|
||
Args:
|
||
session_id: Identifiant de la session à retraiter.
|
||
progress_callback: Callable(session_id, current, total, shot_id) pour la progression.
|
||
|
||
Returns:
|
||
Dict avec le résultat de finalize_session() ou un dict d'erreur.
|
||
"""
|
||
logger.info(f"Retraitement de la session {session_id}")
|
||
|
||
# Trouver le dossier de la session sur disque
|
||
# Les screenshots peuvent être dans :
|
||
# - data/training/live_sessions/{session_id}/shots/
|
||
# - data/training/live_sessions/{machine_id}/{session_id}/shots/
|
||
session_dir = self._find_session_dir(session_id)
|
||
if not session_dir:
|
||
return {"error": f"Dossier session {session_id} introuvable sur disque"}
|
||
|
||
shots_dir = session_dir / "shots"
|
||
if not shots_dir.exists():
|
||
return {"error": f"Dossier shots/ introuvable pour {session_id}"}
|
||
|
||
# Lister les screenshots full (shot_XXXX_full.png), triés par nom
|
||
all_shots = sorted(shots_dir.glob("shot_*_full.png"))
|
||
if not all_shots:
|
||
return {
|
||
"error": f"Aucun screenshot shot_*_full.png trouvé dans {shots_dir}",
|
||
"session_id": session_id,
|
||
}
|
||
|
||
# Sélection intelligente : ne garder que les screenshots significatifs
|
||
# pour éviter d'analyser des captures redondantes (~identiques)
|
||
key_shots = self._select_key_screenshots(session_id, all_shots)
|
||
total_all = len(all_shots)
|
||
total = len(key_shots)
|
||
logger.info(
|
||
f"Screenshots sélectionnés : {total}/{total_all} "
|
||
f"(déduplication perceptuelle) dans {shots_dir}"
|
||
)
|
||
|
||
# S'assurer que la session est enregistrée dans le session_manager
|
||
self.session_manager.get_or_create(session_id)
|
||
|
||
# Restaurer les window_info depuis live_events.jsonl
|
||
# pour que ScreenAnalyzer crée des ScreenStates avec les bons titres de fenêtre
|
||
self._restore_window_events(session_id, session_dir)
|
||
|
||
# Nettoyer les données en mémoire (au cas où un traitement précédent a échoué)
|
||
with self._data_lock:
|
||
self._screen_states.pop(session_id, None)
|
||
self._embeddings.pop(session_id, None)
|
||
|
||
# Analyser les screenshots en parallèle (2 workers max pour la VRAM)
|
||
# Chaque process_screenshot() est indépendant : ScreenAnalyzer + CLIP + FAISS
|
||
# Les structures partagées (_screen_states, _embeddings) sont protégées par _data_lock
|
||
max_parallel = min(2, total)
|
||
errors = 0
|
||
processed_count = 0
|
||
|
||
def _analyze_one(shot_file, index=0):
|
||
"""Analyse un screenshot, retourne (shot_id, result_or_error).
|
||
|
||
Vérifie avant chaque analyse si un replay est en cours (flag GPU).
|
||
Si oui, attend que le replay termine avant de lancer le VLM.
|
||
"""
|
||
shot_id = shot_file.stem
|
||
# Suspendre si un replay est en cours (libérer le GPU)
|
||
self._wait_if_replay_active(
|
||
context=f"screenshot {index + 1}/{total} ({shot_id})"
|
||
)
|
||
try:
|
||
res = self.process_screenshot(session_id, shot_id, str(shot_file))
|
||
return (shot_id, res, None)
|
||
except Exception as e:
|
||
return (shot_id, None, str(e))
|
||
|
||
if total <= 1:
|
||
# Un seul screenshot — pas besoin de pool
|
||
for shot_file in key_shots:
|
||
shot_id, res, err = _analyze_one(shot_file, index=0)
|
||
processed_count += 1
|
||
if progress_callback:
|
||
try:
|
||
progress_callback(session_id, processed_count, total, shot_id)
|
||
except Exception:
|
||
pass
|
||
if err:
|
||
logger.error(f"Erreur analyse screenshot {shot_id}: {err}")
|
||
errors += 1
|
||
elif res and res.get("state_id") is None:
|
||
logger.warning(f"Screenshot {shot_id} : analyse échouée (pas de state_id)")
|
||
errors += 1
|
||
else:
|
||
# Traitement parallèle — 2 screenshots simultanés
|
||
# Note : _analyze_one vérifie le flag replay avant chaque VLM call,
|
||
# donc les workers se suspendent automatiquement si un replay est en cours.
|
||
logger.info(f"Traitement parallèle : {max_parallel} workers pour {total} screenshots")
|
||
with ThreadPoolExecutor(max_workers=max_parallel, thread_name_prefix="vlm") as pool:
|
||
futures = {pool.submit(_analyze_one, sf, i): sf for i, sf in enumerate(key_shots)}
|
||
for future in as_completed(futures):
|
||
shot_id, res, err = future.result()
|
||
processed_count += 1
|
||
if progress_callback:
|
||
try:
|
||
progress_callback(session_id, processed_count, total, shot_id)
|
||
except Exception:
|
||
pass
|
||
if err:
|
||
logger.error(f"Erreur analyse screenshot {shot_id}: {err}")
|
||
errors += 1
|
||
elif res and res.get("state_id") is None:
|
||
logger.warning(f"Screenshot {shot_id} : analyse échouée (pas de state_id)")
|
||
errors += 1
|
||
|
||
# Vérifier combien de states ont été produits
|
||
with self._data_lock:
|
||
states_count = len(self._screen_states.get(session_id, []))
|
||
|
||
logger.info(
|
||
f"Session {session_id} : {states_count}/{total} screenshots analysés "
|
||
f"({errors} erreurs, {total_all - total} skippés par dédup)"
|
||
)
|
||
|
||
# Construire le workflow via finalize_session()
|
||
# Note: finalize() du session_manager a déjà été appelé quand la session
|
||
# a été marquée comme finalisée. On n'a pas besoin de le refaire.
|
||
# finalize_session() utilise les screen_states accumulés.
|
||
result = self.finalize_session(session_id)
|
||
return result
|
||
|
||
def _select_key_screenshots(self, session_id: str, shot_paths: List[Path]) -> List[Path]:
|
||
"""Sélectionner uniquement les screenshots significatifs pour éviter les analyses redondantes.
|
||
|
||
Critères :
|
||
1. Garder le premier et le dernier screenshot (toujours)
|
||
2. Comparer chaque screenshot au précédent via hash perceptuel (32x32 grayscale)
|
||
3. Si l'image est identique au précédent → skip (même écran, pas de changement)
|
||
4. Privilégier les screenshots d'action (shot_*_full) vs heartbeat
|
||
|
||
Réduit typiquement 12 screenshots à 3-4 screenshots utiles.
|
||
"""
|
||
if len(shot_paths) <= 2:
|
||
return list(shot_paths)
|
||
|
||
from PIL import Image
|
||
|
||
selected = []
|
||
last_hash = None
|
||
|
||
for path in shot_paths:
|
||
basename = os.path.basename(str(path))
|
||
|
||
# Les screenshots d'action sont prioritaires
|
||
is_action = 'shot_' in basename and '_full' in basename
|
||
|
||
# Hash perceptuel : redimensionner à 32x32 en niveaux de gris
|
||
# Assez discriminant pour détecter les changements d'état de l'UI
|
||
try:
|
||
img = Image.open(str(path)).resize((32, 32)).convert('L')
|
||
current_hash = hashlib.md5(img.tobytes()).hexdigest()
|
||
except Exception as e:
|
||
logger.debug(f"Impossible de hasher {basename}: {e}")
|
||
# En cas d'erreur, inclure le screenshot par sécurité
|
||
selected.append(path)
|
||
continue
|
||
|
||
# Inclure si : premier screenshot, hash différent, ou screenshot d'action
|
||
if last_hash is None or current_hash != last_hash:
|
||
selected.append(path)
|
||
last_hash = current_hash
|
||
elif is_action:
|
||
# Action mais visuellement identique — skip quand même
|
||
# car l'état de l'écran n'a pas changé
|
||
logger.debug(f"Screenshot d'action {basename} identique au précédent, skip")
|
||
|
||
# Garantir que le premier et le dernier sont toujours inclus
|
||
if shot_paths[0] not in selected:
|
||
selected.insert(0, shot_paths[0])
|
||
if shot_paths[-1] not in selected:
|
||
selected.append(shot_paths[-1])
|
||
|
||
return selected
|
||
|
||
def _restore_window_events(self, session_id: str, session_dir: Path):
|
||
"""Restaurer les window_info depuis live_events.jsonl lors du retraitement.
|
||
|
||
Crée un mapping chronologique timestamp → window_info pour que chaque
|
||
screenshot soit associé au bon titre de fenêtre. Stocké dans
|
||
session._shot_window_map : {shot_number → window_info}.
|
||
|
||
Le mapping est utilisé par process_screenshot() via last_window_info
|
||
qui est mis à jour avant chaque shot.
|
||
"""
|
||
import json as _json
|
||
import re
|
||
|
||
events_file = session_dir / "live_events.jsonl"
|
||
if not events_file.exists():
|
||
logger.debug(f"Pas de live_events.jsonl pour {session_id}")
|
||
return
|
||
|
||
try:
|
||
# Collecter tous les window_focus_change avec leur timestamp
|
||
window_changes = [] # [(timestamp, {"title": ..., "app_name": ...})]
|
||
|
||
for line in events_file.read_text().splitlines():
|
||
if not line.strip():
|
||
continue
|
||
try:
|
||
evt = _json.loads(line)
|
||
except _json.JSONDecodeError:
|
||
continue
|
||
|
||
event_data = evt.get("event", evt)
|
||
evt_type = event_data.get("type", "")
|
||
ts = float(event_data.get("timestamp", evt.get("timestamp", 0)))
|
||
|
||
if evt_type == "window_focus_change":
|
||
to_info = event_data.get("to") or event_data.get("window") or {}
|
||
title = to_info.get("title", "")
|
||
if title:
|
||
window_changes.append((ts, {
|
||
"title": title,
|
||
"app_name": to_info.get("app_name", "unknown"),
|
||
}))
|
||
|
||
if not window_changes:
|
||
logger.debug(f"Pas de window_focus_change dans {session_id}")
|
||
return
|
||
|
||
window_changes.sort(key=lambda x: x[0])
|
||
|
||
# Mapper chaque shot_XXXX aux window_info par timestamp
|
||
shots_dir = session_dir / "shots"
|
||
shot_files = sorted(shots_dir.glob("shot_*_full.png"))
|
||
|
||
# Extraire le timestamp du nom de fichier (shot_XXXX_full.png → XXXX = index)
|
||
# ou du mtime du fichier
|
||
shot_window_map = {}
|
||
for shot_file in shot_files:
|
||
shot_ts = shot_file.stat().st_mtime
|
||
# Trouver le dernier window_change avant ce shot
|
||
best_window = window_changes[0][1] # Premier par défaut
|
||
for wc_ts, wc_info in window_changes:
|
||
if wc_ts <= shot_ts:
|
||
best_window = wc_info
|
||
else:
|
||
break
|
||
shot_window_map[shot_file.stem] = best_window
|
||
|
||
# Stocker dans la session pour que process_screenshot() puisse l'utiliser
|
||
session = self.session_manager.get_session(session_id)
|
||
if session:
|
||
session._shot_window_map = shot_window_map
|
||
# Mettre le dernier titre connu comme fallback
|
||
session.last_window_info = window_changes[-1][1]
|
||
|
||
titles_seen = set(w.get("title", "") for w in shot_window_map.values())
|
||
logger.info(
|
||
f"Window events restaurés pour {session_id}: "
|
||
f"{len(shot_window_map)} shots mappés, "
|
||
f"{len(titles_seen)} titres uniques: {titles_seen}"
|
||
)
|
||
|
||
except Exception as e:
|
||
logger.warning(f"Erreur restauration window events pour {session_id}: {e}")
|
||
|
||
def _find_session_dir(self, session_id: str) -> Optional[Path]:
|
||
"""Trouver le dossier d'une session sur disque.
|
||
|
||
Cherche dans :
|
||
1. data/training/live_sessions/{session_id}/
|
||
2. data/training/live_sessions/{machine_id}/{session_id}/ (multi-machine)
|
||
"""
|
||
# Chemin direct
|
||
direct = self.data_dir / session_id
|
||
if direct.is_dir() and (direct / "shots").exists():
|
||
return direct
|
||
|
||
# Chercher dans les sous-dossiers (machine_id)
|
||
parent = self.data_dir
|
||
if parent.exists():
|
||
for subdir in parent.iterdir():
|
||
if subdir.is_dir():
|
||
candidate = subdir / session_id
|
||
if candidate.is_dir() and (candidate / "shots").exists():
|
||
return candidate
|
||
|
||
# Chercher aussi dans le parent du data_dir (cas où data_dir = streaming_sessions)
|
||
parent_parent = self.data_dir.parent
|
||
if parent_parent.exists() and parent_parent != self.data_dir:
|
||
direct2 = parent_parent / session_id
|
||
if direct2.is_dir() and (direct2 / "shots").exists():
|
||
return direct2
|
||
for subdir in parent_parent.iterdir():
|
||
if subdir.is_dir() and subdir.name != self.data_dir.name:
|
||
candidate = subdir / session_id
|
||
if candidate.is_dir() and (candidate / "shots").exists():
|
||
return candidate
|
||
|
||
return None
|
||
|
||
def find_pending_sessions(self) -> List[str]:
|
||
"""Trouver les sessions finalisées qui n'ont pas encore été traitées.
|
||
|
||
Une session est "pending" si :
|
||
- Elle est marquée comme finalisée dans le session_manager
|
||
- Elle a 0 ScreenStates en mémoire (jamais analysée ou analyse perdue)
|
||
- Elle a des screenshots full sur disque
|
||
|
||
Returns:
|
||
Liste de session_ids à traiter.
|
||
"""
|
||
pending = []
|
||
for sid in self.session_manager.session_ids:
|
||
session = self.session_manager.get_session(sid)
|
||
if session is None:
|
||
continue
|
||
if not session.finalized:
|
||
continue
|
||
|
||
# Vérifier si des states existent déjà
|
||
with self._data_lock:
|
||
states_count = len(self._screen_states.get(sid, []))
|
||
if states_count > 0:
|
||
continue
|
||
|
||
# Vérifier si un workflow existe déjà pour cette session
|
||
# 1. Check en mémoire (attribut _source_session)
|
||
with self._data_lock:
|
||
has_workflow = any(
|
||
getattr(wf, '_source_session', None) == sid
|
||
for wf in self._workflows.values()
|
||
)
|
||
if has_workflow:
|
||
continue
|
||
|
||
# 2. Check sur disque (metadata.source_session_id dans les fichiers JSON)
|
||
if self._workflow_exists_on_disk(sid):
|
||
continue
|
||
|
||
# Vérifier qu'il y a des screenshots full sur disque
|
||
session_dir = self._find_session_dir(sid)
|
||
if session_dir:
|
||
shots_dir = session_dir / "shots"
|
||
if shots_dir.exists():
|
||
full_shots = list(shots_dir.glob("shot_*_full.png"))
|
||
if full_shots:
|
||
logger.info(
|
||
f"Session pending trouvée : {sid} "
|
||
f"({len(full_shots)} screenshots full)"
|
||
)
|
||
pending.append(sid)
|
||
|
||
return pending
|
||
|
||
def _workflow_exists_on_disk(self, session_id: str) -> bool:
|
||
"""Vérifier si un workflow a déjà été produit pour cette session.
|
||
|
||
Parcourt les fichiers JSON dans data/training/workflows/ et cherche
|
||
source_session_id dans les métadonnées.
|
||
Utilise un cache pour éviter de re-lire les fichiers à chaque appel.
|
||
"""
|
||
import json as _json
|
||
|
||
if not hasattr(self, '_processed_sessions_cache'):
|
||
# Construire le cache au premier appel
|
||
self._processed_sessions_cache = set()
|
||
workflows_dir = self.data_dir / "workflows"
|
||
if workflows_dir.exists():
|
||
for wf_file in workflows_dir.rglob("*.json"):
|
||
try:
|
||
with open(wf_file, 'r') as f:
|
||
data = _json.load(f)
|
||
src = data.get('metadata', {}).get('source_session_id', '')
|
||
if src:
|
||
self._processed_sessions_cache.add(src)
|
||
except Exception:
|
||
continue
|
||
logger.info(
|
||
f"Cache sessions traitées : {len(self._processed_sessions_cache)} workflows existants"
|
||
)
|
||
|
||
return session_id in self._processed_sessions_cache
|
||
|
||
def _cleanup_session_data(self, session_id: str):
|
||
"""Libérer la mémoire des ScreenStates et embeddings après finalization."""
|
||
with self._data_lock:
|
||
states = self._screen_states.pop(session_id, [])
|
||
embeddings = self._embeddings.pop(session_id, [])
|
||
logger.info(
|
||
f"Mémoire libérée pour {session_id}: "
|
||
f"{len(states)} states, {len(embeddings)} embeddings"
|
||
)
|
||
|
||
# =========================================================================
|
||
# Helpers
|
||
# =========================================================================
|
||
|
||
def _generate_workflow_name(self, session_id: str) -> str:
|
||
"""
|
||
Générer un nom de tâche lisible et humain à partir des titres de fenêtre.
|
||
|
||
Analyse les titres vus pendant la session pour extraire :
|
||
- L'application principale (la plus fréquente)
|
||
- Le contexte documentaire (après le tiret dans le titre)
|
||
- Une description d'action déduite du contexte
|
||
|
||
Exemples de résultats :
|
||
"Chrome - Facturation DPI" → "Chrome — Facturation DPI"
|
||
"Excel - Budget_2026.xlsx" → "Excel — Budget 2026"
|
||
3 apps → "Chrome, Excel et Word"
|
||
Aucun contexte → "Tâche du 17 mars à 14h"
|
||
"""
|
||
import re
|
||
|
||
session = self.session_manager.get_session(session_id)
|
||
if not session:
|
||
return self._fallback_task_name()
|
||
|
||
titles = session.window_titles_seen
|
||
apps = session.app_names_seen
|
||
|
||
if not titles and not apps:
|
||
return self._fallback_task_name()
|
||
|
||
# Trier par fréquence décroissante
|
||
sorted_titles = sorted(titles.items(), key=lambda x: -x[1])
|
||
sorted_apps = sorted(apps.items(), key=lambda x: -x[1])
|
||
|
||
# Extraire le nom d'app depuis le titre le plus fréquent
|
||
primary_title = sorted_titles[0][0] if sorted_titles else ""
|
||
primary_app = sorted_apps[0][0] if sorted_apps else ""
|
||
|
||
# Nettoyer le nom d'application pour l'affichage humain
|
||
app_display = self._humanize_app_name(primary_app) if primary_app else ""
|
||
|
||
# Extraire la partie contextuelle du titre (après/avant le séparateur)
|
||
context_part = ""
|
||
for sep in [" - ", " — ", " – ", " | ", ": "]:
|
||
if sep in primary_title:
|
||
parts = primary_title.split(sep)
|
||
if len(parts) >= 2:
|
||
candidates = [p.strip() for p in parts]
|
||
app_lower = primary_app.lower()
|
||
context_candidates = [
|
||
c for c in candidates
|
||
if app_lower not in c.lower()
|
||
and c.lower() not in app_lower
|
||
]
|
||
if context_candidates:
|
||
context_part = context_candidates[0]
|
||
else:
|
||
context_part = candidates[0]
|
||
break
|
||
|
||
# Construire le nom lisible
|
||
distinct_apps = [a for a, _ in sorted_apps if a.lower() not in ("unknown", "explorer")]
|
||
|
||
if len(distinct_apps) >= 3:
|
||
# Multi-app : "Chrome, Excel et Word"
|
||
app_names = [self._humanize_app_name(a) for a in distinct_apps[:3]]
|
||
if len(app_names) == 3:
|
||
name = f"{app_names[0]}, {app_names[1]} et {app_names[2]}"
|
||
else:
|
||
name = " et ".join(app_names)
|
||
elif context_part:
|
||
# Nettoyer le contexte pour le rendre lisible
|
||
clean_context = re.sub(r'[<>:"/\\|?*\[\]]', '', context_part)
|
||
# Retirer les extensions de fichier courantes
|
||
clean_context = re.sub(r'\.(xlsx?|csv|docx?|pdf|txt)$', '', clean_context, flags=re.IGNORECASE)
|
||
# Remplacer les underscores par des espaces
|
||
clean_context = clean_context.replace('_', ' ').strip()[:40]
|
||
if app_display:
|
||
name = f"{app_display} \u2014 {clean_context}"
|
||
else:
|
||
name = clean_context
|
||
elif app_display:
|
||
name = f"{app_display} \u2014 session"
|
||
else:
|
||
name = self._fallback_task_name()
|
||
|
||
# Dédoublonner si une tâche avec ce nom existe déjà
|
||
base_name = name
|
||
counter = 1
|
||
with self._data_lock:
|
||
existing_names = {
|
||
getattr(w, 'name', '') for w in self._workflows.values()
|
||
}
|
||
while name in existing_names:
|
||
counter += 1
|
||
name = f"{base_name} ({counter})"
|
||
|
||
return name
|
||
|
||
@staticmethod
|
||
def _fallback_task_name() -> str:
|
||
"""Générer un nom de tâche par défaut basé sur la date et l'heure."""
|
||
now = datetime.now()
|
||
# Noms de mois en français
|
||
mois = [
|
||
"", "janvier", "février", "mars", "avril", "mai", "juin",
|
||
"juillet", "août", "septembre", "octobre", "novembre", "décembre"
|
||
]
|
||
return f"Tâche du {now.day} {mois[now.month]} à {now.hour}h{now.minute:02d}"
|
||
|
||
@staticmethod
|
||
def _humanize_app_name(app_name: str) -> str:
|
||
"""Convertir un nom d'application technique en nom lisible.
|
||
|
||
Exemples :
|
||
"notepad.exe" → "Bloc-notes"
|
||
"chrome.exe" → "Chrome"
|
||
"WindowsTerminal" → "Terminal"
|
||
"""
|
||
import re
|
||
# Supprimer l'extension .exe et les chemins
|
||
name = app_name.split("\\")[-1].split("/")[-1]
|
||
name = re.sub(r'\.exe$', '', name, flags=re.IGNORECASE).strip()
|
||
|
||
# Dictionnaire de noms humains pour les applications courantes
|
||
app_human_names = {
|
||
"notepad": "Bloc-notes",
|
||
"notepad++": "Notepad++",
|
||
"chrome": "Chrome",
|
||
"msedge": "Edge",
|
||
"firefox": "Firefox",
|
||
"explorer": "Explorateur",
|
||
"windowsterminal": "Terminal",
|
||
"cmd": "Invite de commandes",
|
||
"powershell": "PowerShell",
|
||
"excel": "Excel",
|
||
"winword": "Word",
|
||
"powerpnt": "PowerPoint",
|
||
"outlook": "Outlook",
|
||
"teams": "Teams",
|
||
"code": "VS Code",
|
||
"searchhost": "Recherche",
|
||
"applicationframehost": "Application",
|
||
"calc": "Calculatrice",
|
||
"mspaint": "Paint",
|
||
"snippingtool": "Capture d'écran",
|
||
}
|
||
|
||
name_lower = name.lower()
|
||
if name_lower in app_human_names:
|
||
return app_human_names[name_lower]
|
||
|
||
# Capitaliser le nom si pas dans le dictionnaire
|
||
return name.capitalize() if name else "Application"
|
||
|
||
@staticmethod
|
||
def _clean_app_name(app_name: str) -> str:
|
||
"""Nettoyer un nom d'application pour l'utiliser dans un nom de workflow."""
|
||
import re
|
||
# Supprimer l'extension .exe et les chemins
|
||
name = app_name.split("\\")[-1].split("/")[-1]
|
||
name = re.sub(r'\.exe$', '', name, flags=re.IGNORECASE)
|
||
# Capitaliser
|
||
name = name.strip().capitalize()
|
||
# Supprimer les caractères spéciaux
|
||
name = re.sub(r'[^a-zA-Z0-9àâäéèêëïîôùûüÿçÀÂÄÉÈÊËÏÎÔÙÛÜŸÇ_]', '', name)
|
||
return name or "App"
|
||
|
||
def _persist_workflow(self, workflow, session_id: str, machine_id: str = "default") -> Optional[Path]:
|
||
"""Sauvegarder le workflow JSON sur disque.
|
||
|
||
Les workflows sont sauvegardés dans un sous-dossier par machine :
|
||
data/training/workflows/{machine_id}/wf_xxx.json
|
||
|
||
Cela permet de distinguer les workflows appris sur des machines différentes.
|
||
"""
|
||
try:
|
||
# Dossier par machine (ou racine pour "default")
|
||
if machine_id and machine_id != "default":
|
||
workflows_dir = self.data_dir / "workflows" / machine_id
|
||
else:
|
||
workflows_dir = self.data_dir / "workflows"
|
||
workflows_dir.mkdir(parents=True, exist_ok=True)
|
||
filepath = workflows_dir / f"{workflow.workflow_id}.json"
|
||
# Stocker le session_id source et machine_id dans les métadonnées
|
||
if hasattr(workflow, 'metadata') and isinstance(workflow.metadata, dict):
|
||
workflow.metadata['source_session_id'] = session_id
|
||
workflow.metadata['machine_id'] = machine_id
|
||
if not hasattr(workflow, '_machine_id'):
|
||
workflow._machine_id = machine_id
|
||
workflow._source_session = session_id
|
||
workflow.save_to_file(filepath)
|
||
logger.info(f"Workflow sauvegardé: {filepath} (session={session_id}, machine={machine_id})")
|
||
return filepath
|
||
except Exception as e:
|
||
logger.error(f"Erreur sauvegarde workflow {session_id}: {e}")
|
||
return None
|
||
|
||
def _build_raw_session_fallback(self, session, raw_dict):
|
||
"""Construire un RawSession manuellement si from_dict échoue."""
|
||
from core.models.raw_session import RawSession, Event, Screenshot, RawWindowContext
|
||
|
||
events = []
|
||
for evt_dict in raw_dict.get("events", []):
|
||
window_data = evt_dict.get("window", {"title": "", "app_name": "unknown"})
|
||
window = RawWindowContext(
|
||
title=window_data.get("title", ""),
|
||
app_name=window_data.get("app_name", "unknown"),
|
||
)
|
||
events.append(Event(
|
||
t=evt_dict.get("t", 0.0),
|
||
type=evt_dict.get("type", "unknown"),
|
||
window=window,
|
||
data={k: v for k, v in evt_dict.items()
|
||
if k not in ("t", "type", "window", "screenshot_id")},
|
||
screenshot_id=evt_dict.get("screenshot_id"),
|
||
))
|
||
|
||
screenshots = []
|
||
for ss_dict in raw_dict.get("screenshots", []):
|
||
screenshots.append(Screenshot(
|
||
screenshot_id=ss_dict["screenshot_id"],
|
||
relative_path=ss_dict.get("relative_path", ss_dict.get("path", "")),
|
||
captured_at=ss_dict.get("captured_at", datetime.now().isoformat()),
|
||
))
|
||
|
||
return RawSession(
|
||
session_id=session.session_id,
|
||
agent_version="agent_v1_stream",
|
||
environment=raw_dict.get("environment", {}),
|
||
user=raw_dict.get("user", {"id": "remote_agent"}),
|
||
context=raw_dict.get("context", {}),
|
||
started_at=session.created_at,
|
||
ended_at=datetime.now(),
|
||
events=events,
|
||
screenshots=screenshots,
|
||
)
|
||
|
||
def list_sessions(self, machine_id: Optional[str] = None) -> List[Dict[str, Any]]:
|
||
"""Lister les sessions avec leur état.
|
||
|
||
Args:
|
||
machine_id: Si fourni, filtre par machine. Si None, retourne toutes les sessions.
|
||
"""
|
||
sessions = []
|
||
for sid in self.session_manager.session_ids:
|
||
session = self.session_manager.get_session(sid)
|
||
if session is None:
|
||
continue
|
||
# Filtre par machine si demandé
|
||
if machine_id and session.machine_id != machine_id:
|
||
continue
|
||
with self._data_lock:
|
||
states_count = len(self._screen_states.get(sid, []))
|
||
embeddings_count = len(self._embeddings.get(sid, []))
|
||
sessions.append({
|
||
"session_id": session.session_id,
|
||
"machine_id": session.machine_id,
|
||
"events_count": len(session.events),
|
||
"screenshots_count": len(session.shot_paths),
|
||
"states_count": states_count,
|
||
"embeddings_count": embeddings_count,
|
||
"last_window": session.last_window_info,
|
||
"created_at": session.created_at.isoformat(),
|
||
"last_activity": session.last_activity.isoformat(),
|
||
"finalized": session.finalized,
|
||
})
|
||
return sessions
|
||
|
||
def list_workflows(self, machine_id: Optional[str] = None) -> List[Dict[str, Any]]:
|
||
"""Lister les workflows construits.
|
||
|
||
Args:
|
||
machine_id: Si fourni, filtre par machine. Si None, retourne tous les workflows.
|
||
"""
|
||
with self._data_lock:
|
||
workflows_snapshot = list(self._workflows.items())
|
||
result = []
|
||
for wf_id, wf in workflows_snapshot:
|
||
wf_machine = getattr(wf, '_machine_id', 'default')
|
||
# Filtre par machine si demandé
|
||
if machine_id and wf_machine != machine_id:
|
||
continue
|
||
result.append({
|
||
"workflow_id": wf_id,
|
||
"machine_id": wf_machine,
|
||
"nodes": len(wf.nodes) if hasattr(wf, "nodes") else 0,
|
||
"edges": len(wf.edges) if hasattr(wf, "edges") else 0,
|
||
"name": getattr(wf, "name", wf_id),
|
||
})
|
||
return result
|
||
|
||
def reload_workflows(self) -> int:
|
||
"""Recharger les workflows depuis le disque.
|
||
|
||
Utile après qu'un nouveau workflow a été exporté depuis le VWB
|
||
ou appris par le streaming. Retourne le nombre de workflows chargés.
|
||
"""
|
||
with self._data_lock:
|
||
self._workflows.clear()
|
||
self._load_persisted_workflows()
|
||
with self._data_lock:
|
||
count = len(self._workflows)
|
||
logger.info("Workflows rechargés depuis le disque : %d", count)
|
||
return count
|
||
|
||
# =========================================================================
|
||
# Extraction d'actions enrichies depuis un workflow appris
|
||
# =========================================================================
|
||
|
||
def extract_enriched_actions(
|
||
self,
|
||
workflow,
|
||
params: Optional[Dict[str, Any]] = None,
|
||
) -> List[Dict[str, Any]]:
|
||
"""Extraire les actions enrichies d'un workflow appris (nodes + edges + events).
|
||
|
||
Parcourt le graphe en BFS depuis les entry_nodes et construit pour chaque edge
|
||
une action enrichie contenant :
|
||
- Les coordonnées normalisées (x_pct, y_pct) depuis les events originaux
|
||
- Les infos de ciblage visuel (by_text, by_role, window_title)
|
||
- Les identifiants de nodes (from_node, to_node) pour le pre-check et post-check
|
||
- Le flag visual_mode=True pour activer la résolution visuelle côté agent
|
||
|
||
Args:
|
||
workflow: Objet Workflow ou dict brut avec nodes/edges
|
||
params: Paramètres de substitution (variables ${var})
|
||
|
||
Returns:
|
||
Liste d'actions enrichies prêtes pour la queue de replay, ou liste vide
|
||
si le workflow n'a pas d'edges exploitables.
|
||
"""
|
||
import uuid
|
||
from collections import defaultdict
|
||
|
||
params = params or {}
|
||
|
||
# Accéder aux données du workflow (objet ou dict)
|
||
if hasattr(workflow, 'edges'):
|
||
edges = workflow.edges
|
||
entry_nodes = workflow.entry_nodes if hasattr(workflow, 'entry_nodes') else []
|
||
nodes_list = workflow.nodes if hasattr(workflow, 'nodes') else []
|
||
elif isinstance(workflow, dict):
|
||
edges = workflow.get('edges', [])
|
||
entry_nodes = workflow.get('entry_nodes', [])
|
||
nodes_list = workflow.get('nodes', [])
|
||
else:
|
||
return []
|
||
|
||
if not edges:
|
||
return []
|
||
|
||
# Index des nodes par ID
|
||
node_index = {}
|
||
for n in nodes_list:
|
||
nid = n.node_id if hasattr(n, 'node_id') else n.get('node_id', '')
|
||
node_index[nid] = n
|
||
|
||
# Index des edges sortants par node
|
||
outgoing: Dict[str, list] = defaultdict(list)
|
||
for edge in edges:
|
||
fn = edge.from_node if hasattr(edge, 'from_node') else edge.get('from_node', '')
|
||
outgoing[fn].append(edge)
|
||
|
||
# Trouver les événements originaux de la session source
|
||
original_events = self._load_original_events_for_workflow(workflow)
|
||
|
||
# Trouver le dossier de la session source (pour les crops/anchors)
|
||
source_session_dir = self._find_session_dir_for_workflow(workflow)
|
||
|
||
# Inférer la résolution d'écran depuis les positions maximales des events
|
||
inferred_resolution = self._infer_screen_resolution(original_events)
|
||
|
||
# BFS depuis les entry_nodes
|
||
if not entry_nodes:
|
||
# Fallback : premier node des edges
|
||
first_edge = edges[0]
|
||
fn = first_edge.from_node if hasattr(first_edge, 'from_node') else first_edge.get('from_node', '')
|
||
entry_nodes = [fn]
|
||
|
||
visited = set()
|
||
queue = list(entry_nodes)
|
||
ordered_edges = []
|
||
|
||
while queue:
|
||
node_id = queue.pop(0)
|
||
if node_id in visited:
|
||
continue
|
||
visited.add(node_id)
|
||
|
||
for edge in outgoing.get(node_id, []):
|
||
ordered_edges.append(edge)
|
||
tn = edge.to_node if hasattr(edge, 'to_node') else edge.get('to_node', '')
|
||
if tn not in visited:
|
||
queue.append(tn)
|
||
|
||
# Construire les actions enrichies depuis les edges ordonnés
|
||
actions = []
|
||
for edge in ordered_edges:
|
||
enriched = self._edge_to_enriched_action(
|
||
edge, node_index, original_events, params, inferred_resolution,
|
||
source_session_dir,
|
||
)
|
||
if enriched:
|
||
actions.extend(enriched)
|
||
|
||
# Nettoyage global : éliminer les actions parasites, fusionner les
|
||
# text_input consécutifs, dédupliquer les key_combo, etc.
|
||
raw_count = len(actions)
|
||
actions = clean_enriched_actions(actions)
|
||
|
||
logger.info(
|
||
"Actions enrichies extraites : %d actions (nettoyées depuis %d brutes) "
|
||
"depuis %d edges (events originaux : %d)",
|
||
len(actions), raw_count, len(ordered_edges), len(original_events),
|
||
)
|
||
return actions
|
||
|
||
def _load_original_events_for_workflow(self, workflow) -> List[Dict[str, Any]]:
|
||
"""Charger les événements originaux (live_events.jsonl) liés à un workflow.
|
||
|
||
Stratégie de recherche :
|
||
1. metadata.source_session_id (si le workflow le stocke)
|
||
2. Parcourir les sessions existantes pour trouver un match temporel
|
||
3. Utiliser le workflow_id comme hint (parfois contient le session_id)
|
||
|
||
Returns:
|
||
Liste d'events bruts (dicts), ou liste vide si introuvable.
|
||
"""
|
||
import json
|
||
|
||
# Stratégie 1 : metadata.source_session_id
|
||
metadata = workflow.metadata if hasattr(workflow, 'metadata') else (
|
||
workflow.get('metadata', {}) if isinstance(workflow, dict) else {}
|
||
)
|
||
source_sid = metadata.get('source_session_id', '')
|
||
|
||
if source_sid:
|
||
events = self._load_events_from_session(source_sid)
|
||
if events:
|
||
return events
|
||
|
||
# Stratégie 2 : workflow_id peut contenir ou être un session_id
|
||
wf_id = workflow.workflow_id if hasattr(workflow, 'workflow_id') else (
|
||
workflow.get('workflow_id', '') if isinstance(workflow, dict) else ''
|
||
)
|
||
if wf_id.startswith('sess_'):
|
||
events = self._load_events_from_session(wf_id)
|
||
if events:
|
||
return events
|
||
|
||
# Stratégie 3 : chercher les sessions les plus proches temporellement
|
||
created_at = None
|
||
if hasattr(workflow, 'created_at'):
|
||
created_at = workflow.created_at
|
||
elif isinstance(workflow, dict) and 'created_at' in workflow:
|
||
try:
|
||
from datetime import datetime
|
||
created_at = datetime.fromisoformat(workflow['created_at'])
|
||
except (ValueError, TypeError):
|
||
pass
|
||
|
||
if created_at:
|
||
events = self._find_closest_session_events(created_at)
|
||
if events:
|
||
return events
|
||
|
||
return []
|
||
|
||
def _find_session_dir_for_workflow(self, workflow) -> Optional[Path]:
|
||
"""Trouver le dossier de la session source associée à un workflow.
|
||
|
||
Utilise la même logique de recherche que _load_original_events_for_workflow
|
||
mais retourne le chemin du dossier au lieu des événements. Nécessaire pour
|
||
accéder aux crops (anchor images) stockés dans {session_dir}/shots/.
|
||
|
||
Returns:
|
||
Path du dossier session, ou None si introuvable.
|
||
"""
|
||
# Stratégie 1 : metadata.source_session_id
|
||
metadata = workflow.metadata if hasattr(workflow, 'metadata') else (
|
||
workflow.get('metadata', {}) if isinstance(workflow, dict) else {}
|
||
)
|
||
source_sid = metadata.get('source_session_id', '')
|
||
|
||
if source_sid:
|
||
session_dir = self._find_session_dir(source_sid)
|
||
if session_dir:
|
||
return session_dir
|
||
|
||
# Stratégie 2 : workflow_id peut contenir ou être un session_id
|
||
wf_id = workflow.workflow_id if hasattr(workflow, 'workflow_id') else (
|
||
workflow.get('workflow_id', '') if isinstance(workflow, dict) else ''
|
||
)
|
||
if wf_id.startswith('sess_'):
|
||
session_dir = self._find_session_dir(wf_id)
|
||
if session_dir:
|
||
return session_dir
|
||
|
||
# Stratégie 3 : chercher la session la plus proche temporellement
|
||
created_at = None
|
||
if hasattr(workflow, 'created_at'):
|
||
created_at = workflow.created_at
|
||
elif isinstance(workflow, dict) and 'created_at' in workflow:
|
||
try:
|
||
from datetime import datetime as _dt
|
||
created_at = _dt.fromisoformat(workflow['created_at'])
|
||
except (ValueError, TypeError):
|
||
pass
|
||
|
||
if created_at:
|
||
session_dir = self._find_closest_session_dir(created_at)
|
||
if session_dir:
|
||
return session_dir
|
||
|
||
return None
|
||
|
||
def _find_closest_session_dir(self, workflow_created_at) -> Optional[Path]:
|
||
"""Trouver le dossier de la session la plus proche temporellement.
|
||
|
||
Même logique que _find_closest_session_events mais retourne le Path
|
||
du dossier au lieu de charger les événements.
|
||
|
||
Returns:
|
||
Path du dossier session, ou None si aucun match dans les 10 minutes.
|
||
"""
|
||
from datetime import datetime as _dt
|
||
|
||
best_dir = None
|
||
best_delta = float('inf')
|
||
|
||
search_dirs = [self.data_dir]
|
||
if self.data_dir.exists():
|
||
for subdir in self.data_dir.iterdir():
|
||
if subdir.is_dir() and not subdir.name.startswith('.'):
|
||
search_dirs.append(subdir)
|
||
|
||
for search_dir in search_dirs:
|
||
if not search_dir.exists():
|
||
continue
|
||
for session_dir in search_dir.iterdir():
|
||
if not session_dir.is_dir():
|
||
continue
|
||
name = session_dir.name
|
||
if not name.startswith('sess_'):
|
||
continue
|
||
try:
|
||
ts_part = name.split('_')[1]
|
||
session_dt = _dt.strptime(ts_part, '%Y%m%dT%H%M%S')
|
||
delta = abs((workflow_created_at - session_dt).total_seconds())
|
||
if delta < best_delta:
|
||
best_delta = delta
|
||
best_dir = session_dir
|
||
except (IndexError, ValueError):
|
||
continue
|
||
|
||
if best_dir and best_delta < 600:
|
||
return best_dir
|
||
|
||
return None
|
||
|
||
def _load_anchor_crop(
|
||
self,
|
||
matched_event: Dict[str, Any],
|
||
session_dir: Path,
|
||
) -> Optional[str]:
|
||
"""Charger le crop de référence (anchor image) pour un événement clic.
|
||
|
||
Cherche le crop dans le dossier shots/ de la session source, en utilisant
|
||
le screenshot_id de l'événement original. Si le crop n'existe pas, tente
|
||
de le recréer à partir du screenshot full en croppant autour de la position
|
||
du clic.
|
||
|
||
Args:
|
||
matched_event: Événement original (dict avec screenshot_id et pos)
|
||
session_dir: Dossier de la session source
|
||
|
||
Returns:
|
||
Image crop encodée en base64, ou None si introuvable.
|
||
"""
|
||
import base64
|
||
|
||
screenshot_id = matched_event.get('screenshot_id', '')
|
||
if not screenshot_id:
|
||
return None
|
||
|
||
shots_dir = session_dir / "shots"
|
||
if not shots_dir.exists():
|
||
return None
|
||
|
||
# Stratégie 1 : crop déjà capturé par l'agent (shot_XXXX_crop.png)
|
||
crop_path = shots_dir / f"{screenshot_id}_crop.png"
|
||
if crop_path.exists():
|
||
try:
|
||
crop_b64 = base64.b64encode(crop_path.read_bytes()).decode()
|
||
logger.debug("Anchor crop chargé : %s", crop_path.name)
|
||
return crop_b64
|
||
except Exception as e:
|
||
logger.warning("Erreur lecture crop %s : %s", crop_path, e)
|
||
|
||
# Stratégie 1b : crop en JPEG (compression possible côté agent)
|
||
crop_jpg = shots_dir / f"{screenshot_id}_crop.jpg"
|
||
if crop_jpg.exists():
|
||
try:
|
||
crop_b64 = base64.b64encode(crop_jpg.read_bytes()).decode()
|
||
logger.debug("Anchor crop JPEG chargé : %s", crop_jpg.name)
|
||
return crop_b64
|
||
except Exception as e:
|
||
logger.warning("Erreur lecture crop JPEG %s : %s", crop_jpg, e)
|
||
|
||
# Stratégie 2 : cropper le full screenshot autour de la position du clic
|
||
full_path = shots_dir / f"{screenshot_id}_full.png"
|
||
if not full_path.exists():
|
||
full_path = shots_dir / f"{screenshot_id}_full.jpg"
|
||
if not full_path.exists():
|
||
return None
|
||
|
||
pos = matched_event.get('pos', [])
|
||
if not pos or len(pos) < 2:
|
||
return None
|
||
|
||
try:
|
||
from PIL import Image
|
||
import io
|
||
|
||
img = Image.open(full_path)
|
||
x, y = int(pos[0]), int(pos[1])
|
||
|
||
# Crop 400x400 centré sur le clic (même taille que le captor)
|
||
crop_size = 200 # demi-côté
|
||
left = max(0, x - crop_size)
|
||
top = max(0, y - crop_size)
|
||
right = min(img.width, x + crop_size)
|
||
bottom = min(img.height, y + crop_size)
|
||
|
||
crop = img.crop((left, top, right, bottom))
|
||
|
||
buf = io.BytesIO()
|
||
crop.save(buf, format="PNG", optimize=True)
|
||
crop_b64 = base64.b64encode(buf.getvalue()).decode()
|
||
|
||
logger.debug(
|
||
"Anchor crop généré depuis %s (pos=%s, crop=%dx%d)",
|
||
full_path.name, pos, crop.width, crop.height,
|
||
)
|
||
return crop_b64
|
||
except Exception as e:
|
||
logger.warning("Erreur génération crop depuis %s : %s", full_path, e)
|
||
return None
|
||
|
||
def _load_events_from_session(self, session_id: str) -> List[Dict[str, Any]]:
|
||
"""Charger les événements depuis le live_events.jsonl d'une session."""
|
||
import json
|
||
|
||
session_dir = self._find_session_dir(session_id)
|
||
if not session_dir:
|
||
return []
|
||
|
||
events_file = session_dir / "live_events.jsonl"
|
||
if not events_file.exists():
|
||
return []
|
||
|
||
events = []
|
||
try:
|
||
with open(events_file, 'r', encoding='utf-8') as f:
|
||
for line in f:
|
||
line = line.strip()
|
||
if not line:
|
||
continue
|
||
try:
|
||
data = json.loads(line)
|
||
event = data.get('event', data)
|
||
events.append(event)
|
||
except json.JSONDecodeError:
|
||
continue
|
||
except Exception as e:
|
||
logger.warning("Erreur lecture events %s : %s", events_file, e)
|
||
|
||
return events
|
||
|
||
def _find_closest_session_events(self, workflow_created_at) -> List[Dict[str, Any]]:
|
||
"""Trouver la session la plus proche temporellement du workflow.
|
||
|
||
Parcourt les dossiers de sessions sur disque et compare les dates
|
||
de création (encodées dans le nom du dossier sess_YYYYMMDDTHHMMSS_xxx).
|
||
"""
|
||
import json
|
||
from datetime import datetime
|
||
|
||
best_dir = None
|
||
best_delta = float('inf')
|
||
|
||
# Chercher dans data_dir et ses sous-dossiers (machine_id)
|
||
search_dirs = [self.data_dir]
|
||
if self.data_dir.exists():
|
||
for subdir in self.data_dir.iterdir():
|
||
if subdir.is_dir() and not subdir.name.startswith('.'):
|
||
search_dirs.append(subdir)
|
||
|
||
for search_dir in search_dirs:
|
||
for session_dir in search_dir.iterdir():
|
||
if not session_dir.is_dir():
|
||
continue
|
||
name = session_dir.name
|
||
if not name.startswith('sess_'):
|
||
continue
|
||
# Extraire le timestamp du nom : sess_YYYYMMDDTHHMMSS_xxx
|
||
try:
|
||
ts_part = name.split('_')[1] # YYYYMMDDTHHMMSS
|
||
session_dt = datetime.strptime(ts_part, '%Y%m%dT%H%M%S')
|
||
delta = abs((workflow_created_at - session_dt).total_seconds())
|
||
if delta < best_delta:
|
||
best_delta = delta
|
||
best_dir = session_dir
|
||
except (IndexError, ValueError):
|
||
continue
|
||
|
||
if best_dir and best_delta < 600: # Max 10 minutes d'écart
|
||
events_file = best_dir / "live_events.jsonl"
|
||
if events_file.exists():
|
||
events = []
|
||
try:
|
||
with open(events_file, 'r', encoding='utf-8') as f:
|
||
for line in f:
|
||
line = line.strip()
|
||
if not line:
|
||
continue
|
||
try:
|
||
data = json.loads(line)
|
||
event = data.get('event', data)
|
||
events.append(event)
|
||
except json.JSONDecodeError:
|
||
continue
|
||
if events:
|
||
logger.info(
|
||
"Events originaux trouvés dans %s (delta=%ds, %d events)",
|
||
best_dir.name, int(best_delta), len(events),
|
||
)
|
||
return events
|
||
except Exception as e:
|
||
logger.warning("Erreur lecture events %s : %s", events_file, e)
|
||
|
||
return []
|
||
|
||
def _find_session_wide_search(
|
||
self,
|
||
workflow,
|
||
return_dir: bool = False,
|
||
):
|
||
"""Recherche élargie de la session source d'un workflow.
|
||
|
||
Utilisé quand ``_load_original_events_for_workflow`` échoue (pas de
|
||
``source_session_id`` et fenêtre temporelle de 10 min trop serrée).
|
||
|
||
Élargit la fenêtre à 3 heures et utilise le ``_machine_id`` du workflow
|
||
pour filtrer les candidats.
|
||
|
||
Args:
|
||
workflow: Objet Workflow ou dict.
|
||
return_dir: Si True, retourne aussi le Path du dossier session.
|
||
|
||
Returns:
|
||
Si ``return_dir`` est False : (events_list, session_dir_path)
|
||
Si ``return_dir`` est True : (events_list, session_dir_path)
|
||
"""
|
||
import json as _json
|
||
from datetime import datetime as _dt
|
||
|
||
created_at = None
|
||
if hasattr(workflow, 'created_at'):
|
||
created_at = workflow.created_at
|
||
elif isinstance(workflow, dict) and 'created_at' in workflow:
|
||
try:
|
||
created_at = _dt.fromisoformat(workflow['created_at'])
|
||
except (ValueError, TypeError):
|
||
pass
|
||
|
||
if not created_at:
|
||
return ([], None)
|
||
|
||
# Machine_id du workflow (peut être dans l'attribut privé ou dans les métadonnées)
|
||
machine_id = getattr(workflow, '_machine_id', None)
|
||
if not machine_id:
|
||
metadata = workflow.metadata if hasattr(workflow, 'metadata') else (
|
||
workflow.get('metadata', {}) if isinstance(workflow, dict) else {}
|
||
)
|
||
machine_id = metadata.get('machine_id', '')
|
||
|
||
best_dir = None
|
||
best_delta = float('inf')
|
||
max_delta = 10800 # 3 heures
|
||
|
||
search_dirs = [self.data_dir]
|
||
if self.data_dir.exists():
|
||
for subdir in self.data_dir.iterdir():
|
||
if subdir.is_dir() and not subdir.name.startswith('.'):
|
||
search_dirs.append(subdir)
|
||
|
||
for search_dir in search_dirs:
|
||
if not search_dir.exists():
|
||
continue
|
||
for session_dir in search_dir.iterdir():
|
||
if not session_dir.is_dir():
|
||
continue
|
||
name = session_dir.name
|
||
if not name.startswith('sess_'):
|
||
continue
|
||
|
||
# Filtrer par machine_id si connu : le session_dir parent doit contenir
|
||
# le machine_id, ou être directement dans data_dir
|
||
if machine_id and machine_id != "default":
|
||
parent_name = search_dir.name
|
||
# Accepter si le parent est le machine_id ou si c'est le data_dir racine
|
||
if parent_name != machine_id and search_dir != self.data_dir:
|
||
continue
|
||
|
||
try:
|
||
ts_part = name.split('_')[1]
|
||
session_dt = _dt.strptime(ts_part, '%Y%m%dT%H%M%S')
|
||
delta = abs((created_at - session_dt).total_seconds())
|
||
if delta < best_delta:
|
||
best_delta = delta
|
||
best_dir = session_dir
|
||
except (IndexError, ValueError):
|
||
continue
|
||
|
||
if best_dir and best_delta < max_delta:
|
||
events_file = best_dir / "live_events.jsonl"
|
||
if events_file.exists():
|
||
events = []
|
||
try:
|
||
with open(events_file, 'r', encoding='utf-8') as f:
|
||
for line in f:
|
||
line = line.strip()
|
||
if not line:
|
||
continue
|
||
try:
|
||
data = _json.loads(line)
|
||
event = data.get('event', data)
|
||
events.append(event)
|
||
except _json.JSONDecodeError:
|
||
continue
|
||
if events:
|
||
logger.info(
|
||
"Recherche élargie : session trouvée dans %s "
|
||
"(delta=%ds, %d events, machine=%s)",
|
||
best_dir.name, int(best_delta), len(events),
|
||
machine_id or "?",
|
||
)
|
||
return (events, best_dir)
|
||
except Exception as e:
|
||
logger.warning("Erreur lecture events %s : %s", events_file, e)
|
||
|
||
return ([], None)
|
||
|
||
@staticmethod
|
||
def _infer_screen_resolution(events: List[Dict[str, Any]]) -> tuple:
|
||
"""Inférer la résolution d'écran depuis les positions maximales des events.
|
||
|
||
Analyse les coordonnées de tous les clics pour estimer la résolution
|
||
de l'écran source. Si un clic a x=1800 ou y=1500, la résolution
|
||
est au moins 1800+marge x 1500+marge.
|
||
|
||
Utilise une heuristique : arrondir vers la résolution standard la plus
|
||
proche parmi les plus courantes (1920x1080, 2560x1440, 2560x1600,
|
||
3840x2160, 1366x768, 1280x720).
|
||
|
||
Returns:
|
||
Tuple (width, height) de la résolution inférée, ou (1920, 1080) par défaut.
|
||
"""
|
||
# Résolutions standard connues
|
||
STANDARD_RESOLUTIONS = [
|
||
(1280, 720), (1366, 768), (1440, 900), (1600, 900),
|
||
(1920, 1080), (1920, 1200), (2560, 1440), (2560, 1600),
|
||
(3440, 1440), (3840, 2160),
|
||
]
|
||
|
||
max_x = 0
|
||
max_y = 0
|
||
for evt in events:
|
||
pos = evt.get('pos', [])
|
||
if pos and len(pos) == 2:
|
||
max_x = max(max_x, pos[0])
|
||
max_y = max(max_y, pos[1])
|
||
|
||
if max_x == 0 and max_y == 0:
|
||
return (1920, 1080)
|
||
|
||
# Trouver la résolution standard minimale qui contient tous les clics
|
||
for w, h in STANDARD_RESOLUTIONS:
|
||
if w >= max_x and h >= max_y:
|
||
return (w, h)
|
||
|
||
# Si aucune résolution standard ne convient, arrondir vers le haut
|
||
# par paliers de 100 pixels
|
||
inferred_w = ((max_x // 100) + 1) * 100
|
||
inferred_h = ((max_y // 100) + 1) * 100
|
||
return (inferred_w, inferred_h)
|
||
|
||
def _edge_to_enriched_action(
|
||
self,
|
||
edge,
|
||
node_index: Dict[str, Any],
|
||
original_events: List[Dict[str, Any]],
|
||
params: Dict[str, Any],
|
||
inferred_resolution: tuple = (1920, 1080),
|
||
source_session_dir: Optional[Path] = None,
|
||
) -> List[Dict[str, Any]]:
|
||
"""Convertir un edge de workflow en action(s) enrichie(s).
|
||
|
||
Enrichit chaque action avec :
|
||
- Coordonnées normalisées depuis les événements originaux
|
||
- Infos de ciblage visuel (by_text, by_role, window_title)
|
||
- Anchor image (crop) pour le template matching visuel
|
||
- Flag visual_mode pour la résolution visuelle côté agent
|
||
- Identifiants from_node/to_node pour pre-check et post-conditions
|
||
|
||
Args:
|
||
edge: WorkflowEdge (objet ou dict)
|
||
node_index: Index des nodes par ID
|
||
original_events: Événements originaux de la session
|
||
params: Variables de substitution
|
||
inferred_resolution: Résolution écran inférée
|
||
source_session_dir: Dossier de la session source (pour les crops/anchors)
|
||
|
||
Returns:
|
||
Liste d'actions enrichies (1 pour un edge simple, N pour un compound)
|
||
"""
|
||
import uuid
|
||
|
||
# Extraire les données de l'edge (objet ou dict)
|
||
if hasattr(edge, 'edge_id'):
|
||
edge_id = edge.edge_id
|
||
from_node = edge.from_node
|
||
to_node = edge.to_node
|
||
action_obj = edge.action
|
||
edge_metadata = edge.metadata if hasattr(edge, 'metadata') else {}
|
||
else:
|
||
edge_id = edge.get('edge_id', '')
|
||
from_node = edge.get('from_node', '')
|
||
to_node = edge.get('to_node', '')
|
||
action_obj = edge.get('action', {})
|
||
edge_metadata = edge.get('metadata', {})
|
||
|
||
# Extraire les données de l'action
|
||
if hasattr(action_obj, 'type'):
|
||
action_type = action_obj.type
|
||
target = action_obj.target
|
||
action_params = action_obj.parameters or {}
|
||
elif isinstance(action_obj, dict):
|
||
action_type = action_obj.get('type', 'unknown')
|
||
target = action_obj.get('target', {})
|
||
action_params = action_obj.get('parameters', {})
|
||
else:
|
||
return []
|
||
|
||
# Extraire les infos du target
|
||
if hasattr(target, 'by_role'):
|
||
by_role = target.by_role or ''
|
||
by_text = target.by_text or ''
|
||
by_position = target.by_position
|
||
elif isinstance(target, dict):
|
||
by_role = target.get('by_role', '') or ''
|
||
by_text = target.get('by_text', '') or ''
|
||
by_position = target.get('by_position')
|
||
else:
|
||
by_role = ''
|
||
by_text = ''
|
||
by_position = None
|
||
|
||
# Données du node source (pour pre-check et window_title)
|
||
source_node = node_index.get(from_node)
|
||
target_node = node_index.get(to_node)
|
||
|
||
window_title = self._extract_window_title(source_node)
|
||
target_window_title = self._extract_window_title(target_node)
|
||
|
||
# Chercher l'événement original correspondant à cet edge
|
||
matched_event = self._match_edge_to_event(
|
||
edge_metadata, action_type, action_params, original_events
|
||
)
|
||
|
||
# Construire les coordonnées par ordre de priorité :
|
||
# 1. by_position du target (explicite, fiable)
|
||
# 2. position dans action_params (set par GraphBuilder depuis l'event original)
|
||
# 3. matched_event (recherche dans les events de la session - moins fiable)
|
||
# 4. (0, 0) → sera résolu visuellement par l'agent
|
||
x_pct = 0.0
|
||
y_pct = 0.0
|
||
text = ''
|
||
keys = []
|
||
button = action_params.get('button', 'left') if isinstance(action_params, dict) else 'left'
|
||
|
||
# Priorité 1 : by_position explicite du target
|
||
if by_position and isinstance(by_position, (list, tuple)) and len(by_position) == 2:
|
||
px, py = by_position
|
||
if px <= 1.0 and py <= 1.0:
|
||
x_pct = px
|
||
y_pct = py
|
||
elif px > 0 or py > 0:
|
||
rw = (action_params.get('ref_width', 1920) or 1920) if isinstance(action_params, dict) else 1920
|
||
rh = (action_params.get('ref_height', 1080) or 1080) if isinstance(action_params, dict) else 1080
|
||
x_pct = round(px / rw, 6)
|
||
y_pct = round(py / rh, 6)
|
||
|
||
# Priorité 2 : position dans action_params (de GraphBuilder)
|
||
if x_pct == 0.0 and y_pct == 0.0 and isinstance(action_params, dict):
|
||
pos = action_params.get('position', [])
|
||
if pos and len(pos) == 2 and (pos[0] > 0 or pos[1] > 0):
|
||
rw = (action_params.get('ref_width') or inferred_resolution[0]) if isinstance(action_params, dict) else inferred_resolution[0]
|
||
rh = (action_params.get('ref_height') or inferred_resolution[1]) if isinstance(action_params, dict) else inferred_resolution[1]
|
||
x_pct = round(pos[0] / rw, 6)
|
||
y_pct = round(pos[1] / rh, 6)
|
||
|
||
# Priorité 3 : matched_event (session la plus proche)
|
||
if x_pct == 0.0 and y_pct == 0.0 and matched_event:
|
||
pos = matched_event.get('pos', [])
|
||
if pos and len(pos) == 2:
|
||
ref_width = matched_event.get('screen_width') or inferred_resolution[0]
|
||
ref_height = matched_event.get('screen_height') or inferred_resolution[1]
|
||
x_pct = round(pos[0] / ref_width, 6)
|
||
y_pct = round(pos[1] / ref_height, 6)
|
||
|
||
# Sécurité : clamper à [0, 1]
|
||
x_pct = max(0.0, min(1.0, x_pct))
|
||
y_pct = max(0.0, min(1.0, y_pct))
|
||
|
||
# Texte et touches : action_params d'abord, matched_event en complément
|
||
if isinstance(action_params, dict):
|
||
text = action_params.get('text', '')
|
||
keys = action_params.get('keys', [])
|
||
if not text and matched_event:
|
||
text = matched_event.get('text', '')
|
||
if not keys and matched_event:
|
||
keys = matched_event.get('keys', [])
|
||
if matched_event:
|
||
button = matched_event.get('button', button)
|
||
# Enrichir le window_title si absent
|
||
event_window = matched_event.get('window', {})
|
||
if not window_title and event_window:
|
||
window_title = event_window.get('title', '')
|
||
|
||
# Sanitiser les touches : convertir les caractères de contrôle
|
||
if keys:
|
||
keys = _sanitize_keys(keys)
|
||
# Si ne reste que des modificateurs seuls → action parasite, skip
|
||
if _is_modifier_only(keys):
|
||
return []
|
||
|
||
# Substitution de variables dans le texte
|
||
if text and params:
|
||
text = self._substitute_vars(text, params, action_params)
|
||
|
||
# Déterminer le type d'action normalisé
|
||
if action_type == 'mouse_click':
|
||
norm_type = 'click'
|
||
elif action_type == 'text_input':
|
||
norm_type = 'type'
|
||
elif action_type == 'key_press':
|
||
norm_type = 'key_combo'
|
||
elif action_type == 'compound':
|
||
# Décomposer les compound en sous-actions
|
||
steps = action_params.get('steps', []) if isinstance(action_params, dict) else []
|
||
return self._expand_compound_enriched(
|
||
steps, edge_id, from_node, to_node, window_title, params
|
||
)
|
||
elif action_type in ('unknown', 'unknown_element'):
|
||
# Actions "unknown" : essayer de deviner depuis l'événement original
|
||
if matched_event:
|
||
evt_type = matched_event.get('type', '')
|
||
if evt_type == 'mouse_click':
|
||
norm_type = 'click'
|
||
elif evt_type == 'text_input':
|
||
norm_type = 'type'
|
||
elif evt_type == 'key_press':
|
||
norm_type = 'key_combo'
|
||
else:
|
||
# Event trouvé mais type non reconnu : défaut click
|
||
norm_type = 'click'
|
||
logger.debug(
|
||
"Edge %s : action unknown, event type=%s -> défaut click",
|
||
edge_id, evt_type,
|
||
)
|
||
else:
|
||
# Pas d'événement original : défaut click (la transition entre
|
||
# deux états est presque toujours causée par un clic)
|
||
norm_type = 'click'
|
||
logger.debug(
|
||
"Edge %s : action unknown, pas d'event original -> défaut click",
|
||
edge_id,
|
||
)
|
||
else:
|
||
norm_type = action_type
|
||
|
||
# Construire le target_spec pour la résolution visuelle
|
||
target_spec = {}
|
||
if by_text and by_text not in ('', 'null', 'None'):
|
||
target_spec['by_text'] = by_text
|
||
if by_role and by_role not in ('', 'unknown', 'unknown_element', 'null'):
|
||
target_spec['by_role'] = by_role
|
||
if window_title:
|
||
target_spec['window_title'] = window_title
|
||
|
||
# Enrichir le target_spec avec les textes du node source (OCR)
|
||
source_texts = self._extract_required_texts(source_node)
|
||
if source_texts:
|
||
target_spec['context_hints'] = {'screen_texts': source_texts[:3]}
|
||
|
||
# Enrichir avec l'anchor image (crop de référence) pour les clics
|
||
if norm_type == 'click' and matched_event and source_session_dir:
|
||
anchor_b64 = self._load_anchor_crop(matched_event, source_session_dir)
|
||
if anchor_b64:
|
||
target_spec['anchor_image_base64'] = anchor_b64
|
||
logger.debug(
|
||
"Anchor image chargée pour edge %s (screenshot_id=%s)",
|
||
edge_id, matched_event.get('screenshot_id', '?'),
|
||
)
|
||
|
||
# Construire l'action enrichie
|
||
action = {
|
||
'action_id': f'act_{uuid.uuid4().hex[:8]}',
|
||
'type': norm_type,
|
||
'edge_id': edge_id,
|
||
'from_node': from_node,
|
||
'to_node': to_node,
|
||
'x_pct': x_pct,
|
||
'y_pct': y_pct,
|
||
'window_title': window_title,
|
||
}
|
||
|
||
# Ajouter les champs spécifiques au type d'action
|
||
if norm_type == 'click':
|
||
action['button'] = button
|
||
elif norm_type == 'type':
|
||
action['text'] = text
|
||
elif norm_type == 'key_combo':
|
||
action['keys'] = keys
|
||
|
||
# Activer la résolution visuelle si on a des critères sémantiques
|
||
# OU si les coordonnées sont nulles (nécessite une résolution)
|
||
if target_spec or (x_pct == 0.0 and y_pct == 0.0 and norm_type == 'click'):
|
||
action['visual_mode'] = True
|
||
action['target_spec'] = target_spec
|
||
|
||
return [action]
|
||
|
||
def _match_edge_to_event(
|
||
self,
|
||
edge_metadata: Dict[str, Any],
|
||
action_type: str,
|
||
action_params: Dict[str, Any],
|
||
original_events: List[Dict[str, Any]],
|
||
) -> Optional[Dict[str, Any]]:
|
||
"""Trouver l'événement original correspondant à un edge.
|
||
|
||
Stratégie de matching :
|
||
1. Par type d'action (mouse_click, text_input, key_press)
|
||
2. Par position approximative (si position dans action_params)
|
||
3. Par ordre chronologique (premier événement non-matché du bon type)
|
||
|
||
Returns:
|
||
L'événement matché ou None.
|
||
"""
|
||
if not original_events:
|
||
return None
|
||
|
||
# Type d'événement attendu
|
||
expected_types = set()
|
||
if action_type in ('mouse_click', 'click', 'unknown'):
|
||
expected_types.add('mouse_click')
|
||
if action_type in ('text_input', 'type', 'unknown'):
|
||
expected_types.add('text_input')
|
||
if action_type in ('key_press', 'key_combo', 'unknown'):
|
||
expected_types.add('key_press')
|
||
|
||
if not expected_types:
|
||
expected_types = {'mouse_click', 'text_input', 'key_press'}
|
||
|
||
# Filtrer les événements du bon type
|
||
candidates = [
|
||
e for e in original_events
|
||
if e.get('type', '') in expected_types
|
||
]
|
||
|
||
if not candidates:
|
||
return None
|
||
|
||
# Si on a une position dans action_params, chercher l'événement le plus proche
|
||
ref_pos = (action_params.get('position', []) if isinstance(action_params, dict) else [])
|
||
if ref_pos and len(ref_pos) == 2 and ref_pos[0] > 0:
|
||
best_event = None
|
||
best_dist = float('inf')
|
||
for evt in candidates:
|
||
evt_pos = evt.get('pos', [])
|
||
if evt_pos and len(evt_pos) == 2:
|
||
dx = ref_pos[0] - evt_pos[0]
|
||
dy = ref_pos[1] - evt_pos[1]
|
||
dist = (dx * dx + dy * dy) ** 0.5
|
||
if dist < best_dist:
|
||
best_dist = dist
|
||
best_event = evt
|
||
if best_event and best_dist < 200: # Max 200px d'écart
|
||
return best_event
|
||
|
||
# Fallback : le premier événement du bon type
|
||
# On utilise created_from_event du edge metadata comme hint
|
||
created_from = edge_metadata.get('created_from_event', '')
|
||
if created_from:
|
||
for evt in candidates:
|
||
if evt.get('type') == created_from:
|
||
return evt
|
||
|
||
return candidates[0] if candidates else None
|
||
|
||
def _extract_window_title(self, node) -> str:
|
||
"""Extraire le titre de fenêtre depuis un node (objet ou dict)."""
|
||
if node is None:
|
||
return ''
|
||
if hasattr(node, 'template'):
|
||
tpl = node.template
|
||
if tpl and hasattr(tpl, 'window') and tpl.window:
|
||
return tpl.window.title_contains or tpl.window.title_pattern or ''
|
||
elif isinstance(node, dict):
|
||
template = node.get('template', {})
|
||
if isinstance(template, dict):
|
||
window = template.get('window', {})
|
||
if isinstance(window, dict):
|
||
return window.get('title_contains', '') or window.get('title_pattern', '') or ''
|
||
return ''
|
||
|
||
def _extract_required_texts(self, node) -> List[str]:
|
||
"""Extraire les textes requis depuis le template d'un node."""
|
||
if node is None:
|
||
return []
|
||
if hasattr(node, 'template'):
|
||
tpl = node.template
|
||
if tpl and hasattr(tpl, 'text') and tpl.text:
|
||
texts = tpl.text.required_texts or []
|
||
# Filtrer les textes trop courts ou trop longs
|
||
return [t for t in texts if 3 <= len(t) <= 80]
|
||
elif isinstance(node, dict):
|
||
template = node.get('template', {})
|
||
if isinstance(template, dict):
|
||
text_spec = template.get('text', {})
|
||
if isinstance(text_spec, dict):
|
||
texts = text_spec.get('required_texts', [])
|
||
return [t for t in texts if isinstance(t, str) and 3 <= len(t) <= 80]
|
||
return []
|
||
|
||
@staticmethod
|
||
def _substitute_vars(text: str, params: Dict[str, Any], action_params: Dict[str, Any]) -> str:
|
||
"""Substituer les variables ${var} dans un texte."""
|
||
import re
|
||
defaults = action_params.get('defaults', {}) if isinstance(action_params, dict) else {}
|
||
|
||
def replacer(match):
|
||
var_name = match.group(1)
|
||
return str(params.get(var_name, defaults.get(var_name, match.group(0))))
|
||
|
||
return re.sub(r'\$\{(\w+)\}', replacer, text)
|
||
|
||
def _expand_compound_enriched(
|
||
self,
|
||
steps: List[Dict[str, Any]],
|
||
edge_id: str,
|
||
from_node: str,
|
||
to_node: str,
|
||
window_title: str,
|
||
params: Dict[str, Any],
|
||
) -> List[Dict[str, Any]]:
|
||
"""Décomposer un compound en actions enrichies individuelles.
|
||
|
||
Applique le nettoyage des steps parasites avant expansion :
|
||
- Suppression des modificateurs seuls (ctrl, alt, shift, etc.)
|
||
- Fusion des text_input consécutifs
|
||
- Déduplication des key_combo consécutifs identiques
|
||
|
||
Supporte les types de steps produits par GraphBuilder._build_compound_action() :
|
||
- mouse_click / click : clic souris avec x_pct/y_pct ou position/ref_*
|
||
- text_input / type : saisie de texte
|
||
- key_press / key_combo : combinaison de touches
|
||
- wait : pause entre actions
|
||
"""
|
||
import uuid
|
||
|
||
# Nettoyage des steps parasites avant expansion
|
||
steps = clean_compound_steps(steps)
|
||
|
||
actions = []
|
||
for step in steps:
|
||
step_type = step.get('type', 'unknown')
|
||
action = {
|
||
'action_id': f'act_{uuid.uuid4().hex[:8]}',
|
||
'edge_id': edge_id,
|
||
'from_node': from_node,
|
||
'to_node': to_node,
|
||
'window_title': window_title,
|
||
}
|
||
if step_type in ('key_press', 'key_combo'):
|
||
action['type'] = 'key_combo'
|
||
keys = step.get('keys', [])
|
||
if not keys and step.get('key'):
|
||
keys = [step['key']]
|
||
action['keys'] = keys
|
||
elif step_type in ('text_input', 'type'):
|
||
action['type'] = 'type'
|
||
text = step.get('text', '')
|
||
text = self._substitute_vars(text, params, step)
|
||
action['text'] = text
|
||
elif step_type == 'wait':
|
||
action['type'] = 'wait'
|
||
action['duration_ms'] = step.get('duration_ms', 500)
|
||
elif step_type in ('mouse_click', 'click'):
|
||
action['type'] = 'click'
|
||
# Coordonnées normalisées directes (x_pct/y_pct)
|
||
x_pct = step.get('x_pct', 0.0)
|
||
y_pct = step.get('y_pct', 0.0)
|
||
# Fallback : calculer depuis position absolue + résolution de référence
|
||
if x_pct == 0.0 and y_pct == 0.0:
|
||
pos = step.get('position', [])
|
||
if pos and len(pos) == 2 and (pos[0] > 0 or pos[1] > 0):
|
||
rw = step.get('ref_width', 1920) or 1920
|
||
rh = step.get('ref_height', 1080) or 1080
|
||
x_pct = round(pos[0] / rw, 6)
|
||
y_pct = round(pos[1] / rh, 6)
|
||
action['x_pct'] = x_pct
|
||
action['y_pct'] = y_pct
|
||
action['button'] = step.get('button', 'left')
|
||
else:
|
||
continue
|
||
actions.append(action)
|
||
|
||
# Nettoyage post-expansion des actions enrichies
|
||
return clean_enriched_actions(actions)
|
||
|
||
# =========================================================================
|
||
# Replay hybride : événements bruts + structure workflow
|
||
# =========================================================================
|
||
|
||
def build_hybrid_replay(
|
||
self,
|
||
workflow,
|
||
session_id: Optional[str] = None,
|
||
) -> List[Dict[str, Any]]:
|
||
"""Construire un replay hybride combinant événements bruts et structure workflow.
|
||
|
||
Le replay hybride utilise :
|
||
- Les événements bruts (live_events.jsonl) comme SOURCE D'ACTIONS
|
||
- La structure du workflow (nodes) comme STRUCTURE DE VÉRIFICATION
|
||
|
||
Les événements sont groupés par transition de node. Entre chaque groupe,
|
||
une action ``verify_screen`` est insérée pour vérifier que l'écran
|
||
correspond bien au node attendu avant de continuer.
|
||
|
||
Args:
|
||
workflow: Objet Workflow ou dict brut avec nodes/edges.
|
||
session_id: Identifiant de session explicite (optionnel, sinon
|
||
déduit depuis les métadonnées du workflow).
|
||
|
||
Returns:
|
||
Liste d'actions propres prêtes pour la queue de replay, avec des
|
||
``verify_screen`` intercalés entre les groupes de transition.
|
||
"""
|
||
import uuid
|
||
|
||
# 1. Charger les événements bruts
|
||
original_events = self._load_original_events_for_workflow(workflow)
|
||
|
||
# Si la recherche standard échoue, élargir la fenêtre temporelle.
|
||
# Le SessionWorker peut construire le workflow 1-2h après la capture
|
||
# (analyse VLM longue), donc la fenêtre de 10 min est trop serrée.
|
||
if not original_events:
|
||
original_events, session_dir_hint = self._find_session_wide_search(workflow)
|
||
else:
|
||
session_dir_hint = None
|
||
|
||
if not original_events:
|
||
logger.warning("build_hybrid_replay : aucun événement brut trouvé")
|
||
return []
|
||
|
||
# 2. Trouver le dossier de la session source
|
||
session_dir = session_dir_hint or self._find_session_dir_for_workflow(workflow)
|
||
if not session_dir:
|
||
# Fallback : chercher avec une fenêtre large
|
||
if not session_dir_hint:
|
||
_, session_dir = self._find_session_wide_search(workflow, return_dir=True)
|
||
if not session_dir:
|
||
logger.warning("build_hybrid_replay : dossier session introuvable")
|
||
return []
|
||
|
||
# 3. Mapper les screenshots aux nodes du workflow
|
||
node_timeline = self._map_screenshots_to_nodes(session_dir, workflow)
|
||
if not node_timeline:
|
||
logger.warning(
|
||
"build_hybrid_replay : impossible de mapper les screenshots aux nodes, "
|
||
"fallback sur extract_enriched_actions"
|
||
)
|
||
return []
|
||
|
||
# 4. Grouper les événements par transition de node
|
||
groups = self._group_events_by_transition(original_events, node_timeline)
|
||
if not groups:
|
||
logger.warning("build_hybrid_replay : aucun groupe de transition construit")
|
||
return []
|
||
|
||
# 5. Inférer la résolution d'écran
|
||
screen_w, screen_h = self._infer_screen_resolution(original_events)
|
||
|
||
# 6. Construire la liste d'actions
|
||
actions = []
|
||
for group_idx, group in enumerate(groups):
|
||
group_events = group["events"]
|
||
to_node = group["to_node"]
|
||
|
||
# Convertir les événements bruts en actions de replay
|
||
group_actions = self._events_to_replay_actions(
|
||
group_events, screen_w, screen_h, group_idx,
|
||
)
|
||
|
||
# Nettoyer le groupe (filtrer parasites, fusionner texte, dédupliquer combos, waits)
|
||
group_actions = clean_enriched_actions(group_actions)
|
||
|
||
actions.extend(group_actions)
|
||
|
||
# Insérer une vérification visuelle après chaque groupe
|
||
if to_node:
|
||
actions.append({
|
||
"action_id": f"act_verify_{uuid.uuid4().hex[:8]}",
|
||
"type": "verify_screen",
|
||
"expected_node": to_node,
|
||
"timeout_ms": 5000,
|
||
"group": group_idx,
|
||
})
|
||
|
||
logger.info(
|
||
"Replay hybride construit : %d actions (%d groupes, %d events bruts, "
|
||
"résolution=%dx%d)",
|
||
len(actions), len(groups), len(original_events), screen_w, screen_h,
|
||
)
|
||
return actions
|
||
|
||
def _map_screenshots_to_nodes(
|
||
self,
|
||
session_dir: Path,
|
||
workflow,
|
||
) -> List[tuple]:
|
||
"""Mapper chaque screenshot de la session au node du workflow correspondant.
|
||
|
||
Utilise les window_focus_change events pour associer un timestamp à un
|
||
window_title, puis le window_title au node (via node.template.window.title_contains).
|
||
|
||
Returns:
|
||
Liste de (timestamp, node_id) triée chronologiquement, ou liste vide
|
||
si le mapping est impossible.
|
||
"""
|
||
import json as _json
|
||
|
||
# Extraire les titres de fenêtre depuis les nodes du workflow
|
||
if hasattr(workflow, 'nodes'):
|
||
nodes = workflow.nodes
|
||
elif isinstance(workflow, dict):
|
||
nodes = workflow.get('nodes', [])
|
||
else:
|
||
return []
|
||
|
||
# Construire un index title_fragment → node_id
|
||
# Un node peut avoir un title_contains comme "Bloc-notes" ou "Sans titre"
|
||
title_to_node = {}
|
||
node_order = [] # Ordre topologique (par index)
|
||
for node in nodes:
|
||
nid = node.node_id if hasattr(node, 'node_id') else node.get('node_id', '')
|
||
node_order.append(nid)
|
||
win_title = self._extract_window_title(node)
|
||
if win_title:
|
||
title_to_node[win_title.lower()] = nid
|
||
|
||
if not title_to_node:
|
||
logger.debug("Aucun titre de fenêtre dans les nodes du workflow")
|
||
return []
|
||
|
||
# Charger les changements de fenêtre depuis live_events.jsonl
|
||
events_file = session_dir / "live_events.jsonl"
|
||
if not events_file.exists():
|
||
return []
|
||
|
||
window_changes = [] # [(timestamp, window_title)]
|
||
try:
|
||
for line in events_file.read_text(encoding="utf-8").splitlines():
|
||
if not line.strip():
|
||
continue
|
||
try:
|
||
evt = _json.loads(line)
|
||
except _json.JSONDecodeError:
|
||
continue
|
||
|
||
event_data = evt.get("event", evt)
|
||
evt_type = event_data.get("type", "")
|
||
ts = float(event_data.get("timestamp", evt.get("timestamp", 0)))
|
||
|
||
if evt_type == "window_focus_change":
|
||
to_info = event_data.get("to") or event_data.get("window") or {}
|
||
title = to_info.get("title", "")
|
||
if title:
|
||
window_changes.append((ts, title))
|
||
except Exception as e:
|
||
logger.warning("Erreur lecture events pour mapping screenshots : %s", e)
|
||
return []
|
||
|
||
if not window_changes:
|
||
# Fallback : utiliser les timestamps des screenshots (mtime) et les
|
||
# mapper aux nodes dans l'ordre du workflow
|
||
return self._map_screenshots_by_order(session_dir, node_order)
|
||
|
||
# Associer chaque changement de fenêtre au node correspondant
|
||
timeline = []
|
||
last_node_id = None
|
||
for ts, title in sorted(window_changes, key=lambda x: x[0]):
|
||
matched_node = self._match_title_to_node(title, title_to_node)
|
||
if matched_node and matched_node != last_node_id:
|
||
timeline.append((ts, matched_node))
|
||
last_node_id = matched_node
|
||
|
||
# Si timeline est vide, essayer un mapping plus souple avec les screenshots
|
||
if not timeline:
|
||
return self._map_screenshots_by_order(session_dir, node_order)
|
||
|
||
logger.info(
|
||
"Timeline node mappée : %d transitions (%s)",
|
||
len(timeline),
|
||
" → ".join(nid for _, nid in timeline),
|
||
)
|
||
return timeline
|
||
|
||
def _match_title_to_node(self, window_title: str, title_to_node: dict) -> Optional[str]:
|
||
"""Matcher un titre de fenêtre à un node via les fragments de titre.
|
||
|
||
Le matching est insensible à la casse et cherche si le fragment du node
|
||
est contenu dans le titre de la fenêtre.
|
||
|
||
Returns:
|
||
Le node_id correspondant, ou None.
|
||
"""
|
||
title_lower = window_title.lower()
|
||
best_match = None
|
||
best_len = 0
|
||
for fragment, node_id in title_to_node.items():
|
||
if fragment in title_lower and len(fragment) > best_len:
|
||
best_match = node_id
|
||
best_len = len(fragment)
|
||
return best_match
|
||
|
||
def _map_screenshots_by_order(
|
||
self,
|
||
session_dir: Path,
|
||
node_order: List[str],
|
||
) -> List[tuple]:
|
||
"""Fallback : mapper les screenshots aux nodes dans l'ordre topologique.
|
||
|
||
Utilisé quand les window_focus_change ne sont pas disponibles. Distribue
|
||
les screenshots uniformément entre les nodes.
|
||
|
||
Returns:
|
||
Liste de (timestamp, node_id).
|
||
"""
|
||
shots_dir = session_dir / "shots"
|
||
if not shots_dir.exists() or not node_order:
|
||
return []
|
||
|
||
shot_files = sorted(shots_dir.glob("shot_*_full.png"))
|
||
if not shot_files:
|
||
return []
|
||
|
||
timeline = []
|
||
shots_per_node = max(1, len(shot_files) // max(1, len(node_order)))
|
||
|
||
for i, node_id in enumerate(node_order):
|
||
shot_idx = min(i * shots_per_node, len(shot_files) - 1)
|
||
ts = shot_files[shot_idx].stat().st_mtime
|
||
timeline.append((ts, node_id))
|
||
|
||
return timeline
|
||
|
||
def _group_events_by_transition(
|
||
self,
|
||
events: List[Dict[str, Any]],
|
||
node_timeline: List[tuple],
|
||
) -> List[Dict[str, Any]]:
|
||
"""Grouper les événements bruts par transition de node.
|
||
|
||
Pour chaque événement, détermine dans quel intervalle de transition il
|
||
se situe (entre quel changement de node et le suivant).
|
||
|
||
Args:
|
||
events: Liste d'événements bruts (dicts avec ``timestamp`` et ``type``).
|
||
node_timeline: Liste de (timestamp, node_id) triée chronologiquement.
|
||
|
||
Returns:
|
||
Liste de groupes :
|
||
``[{"from_node": "node_000", "to_node": "node_001", "events": [...]}, ...]``
|
||
"""
|
||
if not node_timeline or not events:
|
||
return []
|
||
|
||
# Construire les intervalles de transition
|
||
# Chaque transition va du node_timeline[i] au node_timeline[i+1]
|
||
groups = []
|
||
for i in range(len(node_timeline)):
|
||
from_node = node_timeline[i][1]
|
||
to_node = node_timeline[i + 1][1] if i + 1 < len(node_timeline) else ""
|
||
start_ts = node_timeline[i][0]
|
||
end_ts = node_timeline[i + 1][0] if i + 1 < len(node_timeline) else float("inf")
|
||
|
||
# Collecter les événements dans cet intervalle
|
||
group_events = []
|
||
for evt in events:
|
||
evt_ts = float(evt.get("timestamp", 0))
|
||
evt_type = evt.get("type", "")
|
||
|
||
# Ignorer les événements non-actionnables
|
||
if evt_type in _PARASITIC_ACTION_TYPES:
|
||
continue
|
||
if evt_type in ("window_focus_change", "screenshot", "heartbeat"):
|
||
continue
|
||
|
||
if start_ts <= evt_ts < end_ts:
|
||
group_events.append(evt)
|
||
|
||
if group_events:
|
||
groups.append({
|
||
"from_node": from_node,
|
||
"to_node": to_node,
|
||
"events": group_events,
|
||
})
|
||
|
||
# Si aucun événement n'a de timestamp ou tout est tombé dans les parasites,
|
||
# essayer un groupement séquentiel sans timestamp
|
||
if not groups:
|
||
groups = self._group_events_sequential(events, node_timeline)
|
||
|
||
return groups
|
||
|
||
def _group_events_sequential(
|
||
self,
|
||
events: List[Dict[str, Any]],
|
||
node_timeline: List[tuple],
|
||
) -> List[Dict[str, Any]]:
|
||
"""Groupement séquentiel des événements quand les timestamps ne matchent pas.
|
||
|
||
Distribue les événements actionnables entre les transitions du workflow
|
||
de manière proportionnelle.
|
||
|
||
Returns:
|
||
Liste de groupes comme ``_group_events_by_transition``.
|
||
"""
|
||
# Filtrer les événements actionnables
|
||
actionable_types = {
|
||
"mouse_click", "text_input", "key_press", "key_combo", "scroll",
|
||
}
|
||
actionable = [
|
||
e for e in events
|
||
if e.get("type", "") in actionable_types
|
||
]
|
||
|
||
if not actionable or len(node_timeline) < 2:
|
||
# Un seul node → tout dans un seul groupe
|
||
if actionable and node_timeline:
|
||
return [{
|
||
"from_node": node_timeline[0][1],
|
||
"to_node": node_timeline[-1][1] if len(node_timeline) > 1 else "",
|
||
"events": actionable,
|
||
}]
|
||
return []
|
||
|
||
# Distribuer proportionnellement
|
||
n_transitions = len(node_timeline) - 1
|
||
events_per_group = max(1, len(actionable) // n_transitions)
|
||
groups = []
|
||
|
||
for i in range(n_transitions):
|
||
start_idx = i * events_per_group
|
||
end_idx = (i + 1) * events_per_group if i < n_transitions - 1 else len(actionable)
|
||
group_events = actionable[start_idx:end_idx]
|
||
if group_events:
|
||
groups.append({
|
||
"from_node": node_timeline[i][1],
|
||
"to_node": node_timeline[i + 1][1],
|
||
"events": group_events,
|
||
})
|
||
|
||
return groups
|
||
|
||
def _events_to_replay_actions(
|
||
self,
|
||
events: List[Dict[str, Any]],
|
||
screen_w: int,
|
||
screen_h: int,
|
||
group_idx: int,
|
||
) -> List[Dict[str, Any]]:
|
||
"""Convertir une liste d'événements bruts en actions de replay normalisées.
|
||
|
||
Pré-fusionne les text_input consécutifs (frappes individuelles) en un seul
|
||
bloc de texte avant de les convertir en actions avec timing. Cela évite
|
||
d'avoir des dizaines d'actions ``type`` d'un seul caractère.
|
||
|
||
Ajoute un ``wait`` entre les actions de types différents quand le délai
|
||
naturel est significatif (> 2s = pause de réflexion de l'utilisateur).
|
||
|
||
Args:
|
||
events: Événements bruts d'un groupe.
|
||
screen_w: Largeur de l'écran source (pixels).
|
||
screen_h: Hauteur de l'écran source (pixels).
|
||
group_idx: Index du groupe de transition.
|
||
|
||
Returns:
|
||
Liste d'actions normalisées pour le replay.
|
||
"""
|
||
import uuid
|
||
|
||
# ── Phase 1 : pré-fusionner les événements text_input consécutifs ──
|
||
# Les frappes clavier produisent un text_input par caractère. On les
|
||
# fusionne en un seul événement avec le texte complet et le timestamp
|
||
# du premier caractère.
|
||
merged_events = []
|
||
for evt in events:
|
||
evt_type = evt.get("type", "")
|
||
|
||
# Filtrer les événements non-actionnables
|
||
if evt_type in _PARASITIC_ACTION_TYPES:
|
||
continue
|
||
if evt_type in ("window_focus_change", "screenshot", "heartbeat"):
|
||
continue
|
||
|
||
if evt_type == "text_input":
|
||
text = evt.get("text", "")
|
||
if not text:
|
||
continue
|
||
# Fusionner avec le précédent si c'est aussi un text_input
|
||
if merged_events and merged_events[-1].get("type") == "text_input":
|
||
merged_events[-1]["text"] = merged_events[-1].get("text", "") + text
|
||
# Garder le timestamp le plus récent pour le calcul du delta suivant
|
||
merged_events[-1]["_end_ts"] = float(evt.get("timestamp", 0))
|
||
continue
|
||
|
||
merged_events.append(dict(evt))
|
||
|
||
# ── Phase 2 : convertir les événements fusionnés en actions replay ──
|
||
actions = []
|
||
last_action_end_ts = 0.0
|
||
|
||
for evt in merged_events:
|
||
evt_type = evt.get("type", "")
|
||
evt_ts = float(evt.get("timestamp", 0))
|
||
|
||
# Calculer le délai entre la fin de la dernière action et le début
|
||
# de celle-ci. Les waits ne sont insérés que pour les pauses
|
||
# significatives (> 2s), pas entre chaque frappe.
|
||
if last_action_end_ts > 0 and evt_ts > last_action_end_ts:
|
||
delta_ms = int((evt_ts - last_action_end_ts) * 1000)
|
||
if delta_ms > 2000:
|
||
capped_ms = min(delta_ms, 5000)
|
||
actions.append({
|
||
"action_id": f"act_wait_{uuid.uuid4().hex[:8]}",
|
||
"type": "wait",
|
||
"duration_ms": capped_ms,
|
||
"group": group_idx,
|
||
})
|
||
|
||
# Mettre à jour le timestamp de fin
|
||
end_ts = float(evt.get("_end_ts", evt_ts))
|
||
if end_ts > 0:
|
||
last_action_end_ts = end_ts
|
||
elif evt_ts > 0:
|
||
last_action_end_ts = evt_ts
|
||
|
||
action = {
|
||
"action_id": f"act_hyb_{uuid.uuid4().hex[:8]}",
|
||
"group": group_idx,
|
||
}
|
||
|
||
if evt_type == "mouse_click":
|
||
pos = evt.get("pos", [])
|
||
if pos and len(pos) == 2:
|
||
action["type"] = "click"
|
||
action["x_pct"] = round(pos[0] / screen_w, 6)
|
||
action["y_pct"] = round(pos[1] / screen_h, 6)
|
||
action["button"] = evt.get("button", "left")
|
||
else:
|
||
continue
|
||
|
||
elif evt_type == "text_input":
|
||
text = evt.get("text", "")
|
||
if not text:
|
||
continue
|
||
action["type"] = "type"
|
||
action["text"] = text
|
||
|
||
elif evt_type in ("key_press", "key_combo"):
|
||
keys = evt.get("keys", [])
|
||
if not keys:
|
||
key = evt.get("key", "")
|
||
if key:
|
||
keys = [key]
|
||
if not keys:
|
||
continue
|
||
keys = _sanitize_keys(keys)
|
||
if _is_modifier_only(keys):
|
||
continue
|
||
action["type"] = "key_combo"
|
||
action["keys"] = keys
|
||
|
||
elif evt_type == "scroll":
|
||
pos = evt.get("pos", [])
|
||
action["type"] = "scroll"
|
||
if pos and len(pos) == 2:
|
||
action["x_pct"] = round(pos[0] / screen_w, 6)
|
||
action["y_pct"] = round(pos[1] / screen_h, 6)
|
||
action["delta"] = evt.get("delta", -3)
|
||
|
||
else:
|
||
continue
|
||
|
||
actions.append(action)
|
||
|
||
return actions
|
||
|
||
@property
|
||
def stats(self) -> Dict[str, Any]:
|
||
"""Statistiques du processeur."""
|
||
with self._data_lock:
|
||
total_workflows = len(self._workflows)
|
||
return {
|
||
"active_sessions": self.session_manager.active_session_count,
|
||
"total_sessions": len(self.session_manager.session_ids),
|
||
"total_workflows": total_workflows,
|
||
"faiss_vectors": self._faiss_manager.index.ntotal if self._faiss_manager else 0,
|
||
"initialized": self._initialized,
|
||
}
|