feat: replay visuel VLM-first, worker séparé, package Léa, AZERTY, sécurité HTTPS
Pipeline replay visuel : - VLM-first : l'agent appelle Ollama directement pour trouver les éléments - Template matching en fallback (seuil strict 0.90) - Stop immédiat si élément non trouvé (pas de clic blind) - Replay depuis session brute (/replay-session) sans attendre le VLM - Vérification post-action (screenshot hash avant/après) - Gestion des popups (Enter/Escape/Tab+Enter) Worker VLM séparé : - run_worker.py : process distinct du serveur HTTP - Communication par fichiers (_worker_queue.txt + _replay_active.lock) - Le serveur HTTP ne fait plus jamais de VLM → toujours réactif - Service systemd rpa-worker.service Capture clavier : - raw_keys (vk + press/release) pour replay exact indépendant du layout - Fix AZERTY : ToUnicodeEx + AltGr detection - Enter capturé comme \n, Tab comme \t - Filtrage modificateurs seuls (Ctrl/Alt/Shift parasites) - Fusion text_input consécutifs, dédup key_combo Sécurité & Internet : - HTTPS Let's Encrypt (lea.labs + vwb.labs.laurinebazin.design) - Token API fixe dans .env.local - HTTP Basic Auth sur VWB - Security headers (HSTS, CSP, nosniff) - CORS domaines publics, plus de wildcard Infrastructure : - DPI awareness (SetProcessDpiAwareness) Python + Rust - Métadonnées système (dpi_scale, window_bounds, monitors, os_theme) - Template matching multi-scale [0.5, 2.0] - Résolution dynamique (plus de hardcode 1920x1080) - VLM prefill fix (47x speedup, 3.5s au lieu de 180s) Modules : - core/auth/ : credential vault (Fernet AES), TOTP (RFC 6238), auth handler - core/federation/ : LearningPack export/import anonymisé, FAISS global - deploy/ : package Léa (config.txt, Lea.bat, install.bat, LISEZMOI.txt) UX : - Filtrage OS (VWB + Chat montrent que les workflows de l'OS courant) - Bibliothèque persistante (cache local + SQLite) - Clustering hybride (titre fenêtre + DBSCAN) - EdgeConstraints + PostConditions peuplés - GraphBuilder compound actions (toutes les frappes) Agent Rust : - Token Bearer auth (network.rs) - sysinfo.rs (DPI, résolution, window bounds via Win32 API) - config.txt lu automatiquement - Support Chrome/Brave/Firefox (pas que Edge) Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Dom
@@ -135,27 +135,48 @@ class ContextLevel:
|
||||
|
||||
@dataclass
|
||||
class WindowContext:
|
||||
"""Contexte de fenêtre"""
|
||||
"""Contexte de fenêtre avec métadonnées d'environnement graphique"""
|
||||
app_name: str
|
||||
window_title: str
|
||||
screen_resolution: List[int]
|
||||
workspace: str = "main"
|
||||
|
||||
monitor_index: int = 0 # Index du moniteur (0 = principal)
|
||||
dpi_scale: int = 100 # Facteur DPI en % (100 = normal, 150 = haute résolution)
|
||||
window_bounds: Optional[List[int]] = None # [x, y, width, height] de la fenêtre
|
||||
monitors: Optional[List[Dict[str, int]]] = None # Liste des moniteurs [{width, height, x, y}]
|
||||
os_theme: str = "unknown" # "light", "dark", "unknown"
|
||||
os_language: str = "unknown" # Code langue (fr, en, de...)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
result = {
|
||||
"app_name": self.app_name,
|
||||
"window_title": self.window_title,
|
||||
"screen_resolution": self.screen_resolution,
|
||||
"workspace": self.workspace
|
||||
"workspace": self.workspace,
|
||||
"monitor_index": self.monitor_index,
|
||||
"dpi_scale": self.dpi_scale,
|
||||
"os_theme": self.os_theme,
|
||||
"os_language": self.os_language,
|
||||
}
|
||||
|
||||
if self.window_bounds is not None:
|
||||
result["window_bounds"] = self.window_bounds
|
||||
if self.monitors is not None:
|
||||
result["monitors"] = self.monitors
|
||||
return result
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> 'WindowContext':
|
||||
return cls(
|
||||
app_name=data["app_name"],
|
||||
window_title=data["window_title"],
|
||||
screen_resolution=data["screen_resolution"],
|
||||
workspace=data.get("workspace", "main")
|
||||
workspace=data.get("workspace", "main"),
|
||||
monitor_index=data.get("monitor_index", 0),
|
||||
dpi_scale=data.get("dpi_scale", 100),
|
||||
window_bounds=data.get("window_bounds"),
|
||||
monitors=data.get("monitors"),
|
||||
os_theme=data.get("os_theme", "unknown"),
|
||||
os_language=data.get("os_language", "unknown"),
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -304,7 +304,7 @@ class ScreenTemplate:
|
||||
|
||||
# Vérifier contraintes de fenêtre
|
||||
if hasattr(screen_state, 'window'):
|
||||
window_title = getattr(screen_state.window, 'title', '')
|
||||
window_title = getattr(screen_state.window, 'window_title', '')
|
||||
process = getattr(screen_state.window, 'process', '')
|
||||
if not self.window.matches(window_title, process):
|
||||
return False, 0.0
|
||||
@@ -672,24 +672,94 @@ class Action:
|
||||
|
||||
@dataclass
|
||||
class EdgeConstraints:
|
||||
"""Contraintes pour l'exécution d'un edge"""
|
||||
"""Contraintes pour l'exécution d'un edge (pré-conditions avant l'action)"""
|
||||
pre_conditions: Dict[str, Any] = field(default_factory=dict)
|
||||
required_confidence: float = 0.8
|
||||
max_wait_time_ms: int = 5000
|
||||
|
||||
|
||||
# Contraintes enrichies extraites du node source
|
||||
window: Optional[WindowConstraint] = None
|
||||
text: Optional[TextConstraint] = None
|
||||
min_source_similarity: float = 0.80
|
||||
required_app_name: Optional[str] = None
|
||||
required_window_title: Optional[str] = None
|
||||
|
||||
def check_preconditions(
|
||||
self, window_title: str = "", app_name: str = "",
|
||||
detected_texts: Optional[List[str]] = None,
|
||||
source_similarity: float = 1.0,
|
||||
) -> Tuple[bool, str]:
|
||||
"""
|
||||
Vérifier si toutes les pré-conditions sont satisfaites.
|
||||
|
||||
Returns:
|
||||
(ok: bool, reason: str)
|
||||
"""
|
||||
# Vérifier similarité minimale avec le node source
|
||||
if source_similarity < self.min_source_similarity:
|
||||
return False, (
|
||||
f"Similarité source insuffisante: {source_similarity:.2f} "
|
||||
f"< {self.min_source_similarity:.2f}"
|
||||
)
|
||||
|
||||
# Vérifier titre de fenêtre
|
||||
if self.required_window_title and window_title:
|
||||
if self.required_window_title not in window_title:
|
||||
return False, (
|
||||
f"Titre de fenêtre incorrect: '{window_title}' "
|
||||
f"ne contient pas '{self.required_window_title}'"
|
||||
)
|
||||
|
||||
# Vérifier nom d'application
|
||||
if self.required_app_name and app_name:
|
||||
if self.required_app_name.lower() not in app_name.lower():
|
||||
return False, (
|
||||
f"Application incorrecte: '{app_name}' "
|
||||
f"ne correspond pas à '{self.required_app_name}'"
|
||||
)
|
||||
|
||||
# Vérifier contrainte de fenêtre (objet WindowConstraint)
|
||||
if self.window:
|
||||
if not self.window.matches(window_title, app_name):
|
||||
return False, f"Contrainte de fenêtre non satisfaite"
|
||||
|
||||
# Vérifier contrainte de texte (objet TextConstraint)
|
||||
if self.text and detected_texts is not None:
|
||||
if not self.text.matches(detected_texts):
|
||||
return False, f"Contrainte de texte non satisfaite"
|
||||
|
||||
return True, "OK"
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"pre_conditions": self.pre_conditions,
|
||||
"required_confidence": self.required_confidence,
|
||||
"max_wait_time_ms": self.max_wait_time_ms
|
||||
"max_wait_time_ms": self.max_wait_time_ms,
|
||||
"window": self.window.to_dict() if self.window else None,
|
||||
"text": self.text.to_dict() if self.text else None,
|
||||
"min_source_similarity": self.min_source_similarity,
|
||||
"required_app_name": self.required_app_name,
|
||||
"required_window_title": self.required_window_title,
|
||||
}
|
||||
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> 'EdgeConstraints':
|
||||
window = None
|
||||
if data.get("window"):
|
||||
window = WindowConstraint.from_dict(data["window"])
|
||||
text = None
|
||||
if data.get("text"):
|
||||
text = TextConstraint.from_dict(data["text"])
|
||||
|
||||
return cls(
|
||||
pre_conditions=data.get("pre_conditions", {}),
|
||||
required_confidence=data.get("required_confidence", 0.8),
|
||||
max_wait_time_ms=data.get("max_wait_time_ms", 5000)
|
||||
max_wait_time_ms=data.get("max_wait_time_ms", 5000),
|
||||
window=window,
|
||||
text=text,
|
||||
min_source_similarity=data.get("min_source_similarity", 0.80),
|
||||
required_app_name=data.get("required_app_name"),
|
||||
required_window_title=data.get("required_window_title"),
|
||||
)
|
||||
|
||||
|
||||
@@ -709,23 +779,101 @@ class PostConditionCheck:
|
||||
@dataclass
|
||||
class PostConditions:
|
||||
"""Post-conditions attendues après exécution - Fiche #9"""
|
||||
# (garde tes champs existants si tu en as déjà, et ajoute ceux-ci)
|
||||
|
||||
|
||||
success_mode: str = "all" # "all" | "any"
|
||||
timeout_ms: int = 2500
|
||||
poll_ms: int = 200
|
||||
|
||||
|
||||
success: List[PostConditionCheck] = field(default_factory=list)
|
||||
fail_fast: List[PostConditionCheck] = field(default_factory=list)
|
||||
|
||||
|
||||
retries: int = 2 # nb de tentatives après échec post-conditions
|
||||
backoff_ms: int = 150 # 150, 300, 600...
|
||||
|
||||
|
||||
# Contraintes enrichies extraites du node cible
|
||||
expected_window_title: Optional[str] = None
|
||||
expected_app_name: Optional[str] = None
|
||||
min_target_similarity: float = 0.80
|
||||
|
||||
# Legacy fields (garde compatibilité)
|
||||
expected_node: Optional[str] = None # Node attendu après action
|
||||
window_change_expected: bool = False
|
||||
new_ui_elements_expected: List[str] = field(default_factory=list)
|
||||
|
||||
|
||||
def check_postconditions(
|
||||
self, window_title: str = "", app_name: str = "",
|
||||
detected_texts: Optional[List[str]] = None,
|
||||
target_similarity: float = 1.0,
|
||||
) -> Tuple[bool, str]:
|
||||
"""
|
||||
Vérifier si les post-conditions sont satisfaites après l'action.
|
||||
|
||||
Returns:
|
||||
(ok: bool, reason: str)
|
||||
"""
|
||||
# Vérifier similarité minimale avec le node cible
|
||||
if target_similarity < self.min_target_similarity:
|
||||
return False, (
|
||||
f"Similarité cible insuffisante: {target_similarity:.2f} "
|
||||
f"< {self.min_target_similarity:.2f}"
|
||||
)
|
||||
|
||||
# Vérifier titre de fenêtre attendu
|
||||
if self.expected_window_title and window_title:
|
||||
if self.expected_window_title not in window_title:
|
||||
return False, (
|
||||
f"Titre de fenêtre post-action incorrect: '{window_title}' "
|
||||
f"ne contient pas '{self.expected_window_title}'"
|
||||
)
|
||||
|
||||
# Vérifier application attendue
|
||||
if self.expected_app_name and app_name:
|
||||
if self.expected_app_name.lower() not in app_name.lower():
|
||||
return False, (
|
||||
f"Application post-action incorrecte: '{app_name}' "
|
||||
f"ne correspond pas à '{self.expected_app_name}'"
|
||||
)
|
||||
|
||||
# Vérifier les checks de succès (PostConditionCheck)
|
||||
if self.success:
|
||||
results = []
|
||||
for check in self.success:
|
||||
ok = self._evaluate_check(check, window_title, detected_texts or [])
|
||||
results.append(ok)
|
||||
|
||||
if self.success_mode == "all" and not all(results):
|
||||
return False, "Certaines post-conditions de succès non satisfaites"
|
||||
if self.success_mode == "any" and not any(results):
|
||||
return False, "Aucune post-condition de succès satisfaite"
|
||||
|
||||
# Vérifier fail_fast (si un pattern d'erreur est détecté, échec immédiat)
|
||||
if self.fail_fast and detected_texts:
|
||||
for check in self.fail_fast:
|
||||
if self._evaluate_check(check, window_title, detected_texts):
|
||||
return False, (
|
||||
f"Condition d'échec détectée: {check.kind}={check.value}"
|
||||
)
|
||||
|
||||
return True, "OK"
|
||||
|
||||
@staticmethod
|
||||
def _evaluate_check(
|
||||
check: PostConditionCheck,
|
||||
window_title: str,
|
||||
detected_texts: List[str],
|
||||
) -> bool:
|
||||
"""Évaluer un PostConditionCheck individuel."""
|
||||
texts_lower = [t.lower() for t in detected_texts]
|
||||
|
||||
if check.kind == "text_present":
|
||||
return any(check.value.lower() in t for t in texts_lower) if check.value else False
|
||||
elif check.kind == "text_absent":
|
||||
return not any(check.value.lower() in t for t in texts_lower) if check.value else True
|
||||
elif check.kind == "window_title_contains":
|
||||
return check.value.lower() in window_title.lower() if check.value else False
|
||||
# Autres types de checks non gérés ici → considérés comme OK
|
||||
return True
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"success_mode": self.success_mode,
|
||||
@@ -735,24 +883,28 @@ class PostConditions:
|
||||
"fail_fast": [{"kind": c.kind, "value": c.value, "target": c.target.to_dict() if c.target else None} for c in self.fail_fast],
|
||||
"retries": self.retries,
|
||||
"backoff_ms": self.backoff_ms,
|
||||
# Contraintes enrichies
|
||||
"expected_window_title": self.expected_window_title,
|
||||
"expected_app_name": self.expected_app_name,
|
||||
"min_target_similarity": self.min_target_similarity,
|
||||
# Legacy
|
||||
"expected_node": self.expected_node,
|
||||
"window_change_expected": self.window_change_expected,
|
||||
"new_ui_elements_expected": self.new_ui_elements_expected
|
||||
}
|
||||
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> 'PostConditions':
|
||||
success_checks = []
|
||||
for c in data.get("success", []):
|
||||
target = TargetSpec.from_dict(c["target"]) if c.get("target") else None
|
||||
success_checks.append(PostConditionCheck(kind=c["kind"], value=c.get("value"), target=target))
|
||||
|
||||
|
||||
fail_fast_checks = []
|
||||
for c in data.get("fail_fast", []):
|
||||
target = TargetSpec.from_dict(c["target"]) if c.get("target") else None
|
||||
fail_fast_checks.append(PostConditionCheck(kind=c["kind"], value=c.get("value"), target=target))
|
||||
|
||||
|
||||
return cls(
|
||||
success_mode=data.get("success_mode", "all"),
|
||||
timeout_ms=data.get("timeout_ms", 2500),
|
||||
@@ -761,6 +913,10 @@ class PostConditions:
|
||||
fail_fast=fail_fast_checks,
|
||||
retries=data.get("retries", 2),
|
||||
backoff_ms=data.get("backoff_ms", 150),
|
||||
# Contraintes enrichies
|
||||
expected_window_title=data.get("expected_window_title"),
|
||||
expected_app_name=data.get("expected_app_name"),
|
||||
min_target_similarity=data.get("min_target_similarity", 0.80),
|
||||
# Legacy
|
||||
expected_node=data.get("expected_node"),
|
||||
window_change_expected=data.get("window_change_expected", False),
|
||||
|
||||
Reference in New Issue
Block a user