feat: chat unifié, GestureCatalog, Copilot, Léa UI, extraction données, vérification replay
Refonte majeure du système Agent Chat et ajout de nombreux modules : - Chat unifié : suppression du dual Workflows/Agent Libre, tout passe par /api/chat avec résolution en 3 niveaux (workflow → geste → "montre-moi") - GestureCatalog : 38 raccourcis clavier universels Windows avec matching sémantique, substitution automatique dans les replays, et endpoint /api/gestures - Mode Copilot : exécution pas-à-pas des workflows avec validation humaine via WebSocket (approve/skip/abort) avant chaque action - Léa UI (agent_v0/lea_ui/) : interface PyQt5 pour Windows avec overlay transparent pour feedback visuel pendant le replay - Data Extraction (core/extraction/) : moteur d'extraction visuelle de données (OCR + VLM → SQLite), avec schémas YAML et export CSV/Excel - ReplayVerifier (agent_v0/server_v1/) : vérification post-action par comparaison de screenshots, avec logique de retry (max 3) - IntentParser durci : meilleur fallback regex, type GREETING, patterns améliorés - Dashboard : nouvelles pages gestures, streaming, extractions - Tests : 63 tests GestureCatalog, 47 tests extraction, corrections tests existants - Dépréciation : /api/agent/plan et /api/agent/execute retournent HTTP 410, suppression du code hardcodé _plan_to_replay_actions Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Dom
315
scripts/record_and_build.py
Executable file
315
scripts/record_and_build.py
Executable file
@@ -0,0 +1,315 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
record_and_build.py — Script démo Phase 1
|
||||
|
||||
Enregistre une session RPA (screenshots + événements clavier/souris)
|
||||
puis construit un Workflow automatiquement via le GraphBuilder.
|
||||
|
||||
Usage:
|
||||
# Enregistrer une session (Ctrl+C pour arrêter)
|
||||
python scripts/record_and_build.py record --name "login_workflow"
|
||||
|
||||
# Construire un workflow depuis une session existante
|
||||
python scripts/record_and_build.py build --session data/training/sessions/session_xxx
|
||||
|
||||
# Enregistrer ET construire
|
||||
python scripts/record_and_build.py full --name "login_workflow"
|
||||
|
||||
# Lister les sessions enregistrées
|
||||
python scripts/record_and_build.py list
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import signal
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
# Ajouter la racine du projet au path
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
# Vérifier que le venv est activé (éviter les erreurs silencieuses)
|
||||
_venv_path = ROOT / ".venv" / "bin" / "python"
|
||||
if _venv_path.exists() and _venv_path.resolve() != Path(sys.executable).resolve():
|
||||
print(f"ATTENTION : le venv n'est pas activé.")
|
||||
print(f" Lancer avec : source .venv/bin/activate && python {' '.join(sys.argv)}")
|
||||
print(f" Ou directement : .venv/bin/python {' '.join(sys.argv)}")
|
||||
sys.exit(1)
|
||||
|
||||
from core.models.raw_session import RawSession
|
||||
from core.capture.session_recorder import SessionRecorder
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
|
||||
datefmt="%H:%M:%S",
|
||||
)
|
||||
logger = logging.getLogger("record_and_build")
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Commandes
|
||||
# =========================================================================
|
||||
|
||||
|
||||
def cmd_record(args) -> str:
|
||||
"""Enregistrer une session."""
|
||||
recorder = SessionRecorder(
|
||||
output_dir=args.output_dir,
|
||||
screenshot_on_click=True,
|
||||
screenshot_interval_ms=args.interval,
|
||||
)
|
||||
|
||||
session_id = recorder.start(workflow_name=args.name)
|
||||
|
||||
print(f"\n{'='*60}")
|
||||
print(f" Enregistrement en cours : {session_id}")
|
||||
print(f" Workflow : {args.name or '(non nommé)'}")
|
||||
print(f" Screenshots sur clic : oui")
|
||||
if args.interval > 0:
|
||||
print(f" Screenshots périodiques : {args.interval}ms")
|
||||
print(f"{'='*60}")
|
||||
print(f" Effectuez vos actions... Appuyez sur Ctrl+C pour arrêter.")
|
||||
print(f"{'='*60}\n")
|
||||
|
||||
# Afficher les events en temps réel
|
||||
def on_event(raw_event):
|
||||
etype = raw_event.get("type", "?")
|
||||
t = raw_event.get("t", 0)
|
||||
if etype == "mouse_click":
|
||||
pos = raw_event.get("pos", [0, 0])
|
||||
btn = raw_event.get("button", "?")
|
||||
print(f" [{t:7.2f}s] CLIC {btn} @ ({pos[0]}, {pos[1]})")
|
||||
elif etype in ("key_press",):
|
||||
keys = raw_event.get("keys", [])
|
||||
print(f" [{t:7.2f}s] TOUCHE {' '.join(keys)}")
|
||||
|
||||
def on_screenshot(path):
|
||||
count = recorder.screenshot_count
|
||||
print(f" -> Screenshot #{count} sauvegardé")
|
||||
|
||||
recorder._on_event = on_event
|
||||
recorder._on_screenshot = on_screenshot
|
||||
|
||||
# Attendre Ctrl+C
|
||||
stop_event = False
|
||||
|
||||
def signal_handler(sig, frame):
|
||||
nonlocal stop_event
|
||||
stop_event = True
|
||||
|
||||
signal.signal(signal.SIGINT, signal_handler)
|
||||
|
||||
while not stop_event:
|
||||
time.sleep(0.2)
|
||||
|
||||
session = recorder.stop()
|
||||
|
||||
print(f"\n{'='*60}")
|
||||
print(f" Session terminée : {session.session_id}")
|
||||
print(f" Events : {len(session.events)}")
|
||||
print(f" Screenshots: {len(session.screenshots)}")
|
||||
print(f" Durée : {(session.ended_at - session.started_at).total_seconds():.1f}s")
|
||||
session_dir = Path(args.output_dir) / session.session_id
|
||||
print(f" Dossier : {session_dir}")
|
||||
print(f"{'='*60}\n")
|
||||
|
||||
return str(session_dir)
|
||||
|
||||
|
||||
def cmd_build(args) -> None:
|
||||
"""Construire un workflow depuis une session existante."""
|
||||
session_dir = Path(args.session)
|
||||
|
||||
# Trouver le fichier JSON de session
|
||||
json_files = list(session_dir.glob("*.json"))
|
||||
if not json_files:
|
||||
print(f"Erreur : aucun fichier JSON trouvé dans {session_dir}")
|
||||
sys.exit(1)
|
||||
|
||||
session_path = json_files[0]
|
||||
print(f"Chargement de la session : {session_path}")
|
||||
|
||||
session = RawSession.load_from_file(session_path)
|
||||
print(
|
||||
f"Session {session.session_id} : "
|
||||
f"{len(session.events)} events, {len(session.screenshots)} screenshots"
|
||||
)
|
||||
|
||||
if not session.screenshots:
|
||||
print("Erreur : la session n'a aucun screenshot. Impossible de construire un workflow.")
|
||||
sys.exit(1)
|
||||
|
||||
# Construire le workflow
|
||||
print("\nConstruction du workflow...")
|
||||
print(" Initialisation des composants (CLIP, FAISS, DBSCAN)...")
|
||||
|
||||
from core.graph.graph_builder import GraphBuilder
|
||||
|
||||
# min_pattern_repetitions adaptatif :
|
||||
# < 10 screenshots → 2 (exploration)
|
||||
# 10-30 screenshots → 3
|
||||
# > 30 screenshots → min(5, n//10)
|
||||
n = len(session.screenshots)
|
||||
if n < 10:
|
||||
min_reps = 2
|
||||
elif n <= 30:
|
||||
min_reps = 3
|
||||
else:
|
||||
min_reps = min(5, n // 10)
|
||||
|
||||
builder = GraphBuilder(
|
||||
min_pattern_repetitions=min_reps,
|
||||
clustering_eps=0.15,
|
||||
clustering_min_samples=2,
|
||||
enable_quality_validation=True,
|
||||
)
|
||||
print(f" min_pattern_repetitions={min_reps} (pour {n} screenshots)")
|
||||
|
||||
workflow_name = args.name or f"workflow_{session.session_id}"
|
||||
workflow = builder.build_from_session(session, workflow_name)
|
||||
|
||||
print(f"\n{'='*60}")
|
||||
print(f" Workflow construit : {workflow.name}")
|
||||
print(f" Nodes : {len(workflow.nodes)}")
|
||||
print(f" Edges : {len(workflow.edges)}")
|
||||
print(f" État : {workflow.learning_state}")
|
||||
|
||||
if workflow.metadata and "quality_report" in workflow.metadata:
|
||||
qr = workflow.metadata["quality_report"]
|
||||
print(f" Qualité: {qr.get('overall_score', 0):.2%}")
|
||||
print(f" Prod OK: {qr.get('is_production_ready', False)}")
|
||||
|
||||
print(f"\n Nodes :")
|
||||
for node in workflow.nodes:
|
||||
title = ""
|
||||
if node.template and node.template.window:
|
||||
title = node.template.window.title_pattern or ""
|
||||
obs = node.metadata.get("observation_count", "?")
|
||||
print(f" - {node.node_id}: {node.name} [{title}] ({obs} obs)")
|
||||
|
||||
print(f"\n Edges :")
|
||||
for edge in workflow.edges:
|
||||
action_type = edge.action.type if edge.action else "?"
|
||||
count = edge.stats.execution_count if edge.stats else 0
|
||||
print(f" - {edge.from_node} → {edge.to_node} [{action_type}] (×{count})")
|
||||
|
||||
# Sauvegarder le workflow
|
||||
workflow_path = session_dir / f"{workflow.workflow_id}.json"
|
||||
try:
|
||||
with open(workflow_path, "w", encoding="utf-8") as f:
|
||||
f.write(workflow.to_json())
|
||||
print(f"\n Workflow sauvegardé : {workflow_path}")
|
||||
except Exception as e:
|
||||
print(f"\n Erreur sauvegarde : {e}")
|
||||
|
||||
print(f"{'='*60}\n")
|
||||
|
||||
|
||||
def cmd_full(args) -> None:
|
||||
"""Enregistrer puis construire."""
|
||||
session_dir = cmd_record(args)
|
||||
args.session = session_dir
|
||||
cmd_build(args)
|
||||
|
||||
|
||||
def cmd_list(args) -> None:
|
||||
"""Lister les sessions enregistrées."""
|
||||
sessions_dir = Path(args.output_dir)
|
||||
if not sessions_dir.exists():
|
||||
print(f"Aucune session trouvée dans {sessions_dir}")
|
||||
return
|
||||
|
||||
sessions = []
|
||||
for d in sorted(sessions_dir.iterdir()):
|
||||
if not d.is_dir():
|
||||
continue
|
||||
json_files = list(d.glob("*.json"))
|
||||
if json_files:
|
||||
try:
|
||||
session = RawSession.load_from_file(json_files[0])
|
||||
sessions.append(session)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not sessions:
|
||||
print("Aucune session trouvée.")
|
||||
return
|
||||
|
||||
print(f"\n{'='*70}")
|
||||
print(f" Sessions enregistrées ({len(sessions)})")
|
||||
print(f"{'='*70}")
|
||||
for s in sessions:
|
||||
duration = ""
|
||||
if s.ended_at and s.started_at:
|
||||
duration = f"{(s.ended_at - s.started_at).total_seconds():.0f}s"
|
||||
wf = s.context.get("workflow", "")
|
||||
print(
|
||||
f" {s.session_id} | {len(s.events):3d} events "
|
||||
f"| {len(s.screenshots):3d} screenshots | {duration:>5s} | {wf}"
|
||||
)
|
||||
print(f"{'='*70}\n")
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Main
|
||||
# =========================================================================
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Enregistre des sessions RPA et construit des workflows",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output-dir",
|
||||
default="data/training/sessions",
|
||||
help="Répertoire de sortie (défaut: data/training/sessions)",
|
||||
)
|
||||
|
||||
sub = parser.add_subparsers(dest="command", help="Commande")
|
||||
|
||||
# record
|
||||
p_record = sub.add_parser("record", help="Enregistrer une session")
|
||||
p_record.add_argument("--name", default="", help="Nom du workflow")
|
||||
p_record.add_argument(
|
||||
"--interval", type=int, default=0,
|
||||
help="Intervalle de capture périodique en ms (0=désactivé)",
|
||||
)
|
||||
|
||||
# build
|
||||
p_build = sub.add_parser("build", help="Construire un workflow depuis une session")
|
||||
p_build.add_argument("--session", required=True, help="Chemin vers le dossier de session")
|
||||
p_build.add_argument("--name", default="", help="Nom du workflow")
|
||||
|
||||
# full
|
||||
p_full = sub.add_parser("full", help="Enregistrer + construire")
|
||||
p_full.add_argument("--name", default="", help="Nom du workflow")
|
||||
p_full.add_argument(
|
||||
"--interval", type=int, default=0,
|
||||
help="Intervalle de capture périodique en ms (0=désactivé)",
|
||||
)
|
||||
|
||||
# list
|
||||
sub.add_parser("list", help="Lister les sessions enregistrées")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.command == "record":
|
||||
cmd_record(args)
|
||||
elif args.command == "build":
|
||||
cmd_build(args)
|
||||
elif args.command == "full":
|
||||
cmd_full(args)
|
||||
elif args.command == "list":
|
||||
cmd_list(args)
|
||||
else:
|
||||
parser.print_help()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user