feat: chat unifié, GestureCatalog, Copilot, Léa UI, extraction données, vérification replay
Refonte majeure du système Agent Chat et ajout de nombreux modules : - Chat unifié : suppression du dual Workflows/Agent Libre, tout passe par /api/chat avec résolution en 3 niveaux (workflow → geste → "montre-moi") - GestureCatalog : 38 raccourcis clavier universels Windows avec matching sémantique, substitution automatique dans les replays, et endpoint /api/gestures - Mode Copilot : exécution pas-à-pas des workflows avec validation humaine via WebSocket (approve/skip/abort) avant chaque action - Léa UI (agent_v0/lea_ui/) : interface PyQt5 pour Windows avec overlay transparent pour feedback visuel pendant le replay - Data Extraction (core/extraction/) : moteur d'extraction visuelle de données (OCR + VLM → SQLite), avec schémas YAML et export CSV/Excel - ReplayVerifier (agent_v0/server_v1/) : vérification post-action par comparaison de screenshots, avec logique de retry (max 3) - IntentParser durci : meilleur fallback regex, type GREETING, patterns améliorés - Dashboard : nouvelles pages gestures, streaming, extractions - Tests : 63 tests GestureCatalog, 47 tests extraction, corrections tests existants - Dépréciation : /api/agent/plan et /api/agent/execute retournent HTTP 410, suppression du code hardcodé _plan_to_replay_actions Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Dom
@@ -0,0 +1,382 @@
|
||||
"""
|
||||
GraphToVisual Converter — Convertit un core Workflow en VisualWorkflow VWB.
|
||||
|
||||
Inverse du VisualToGraphConverter : prend un Workflow (issu du GraphBuilder
|
||||
ou de l'exécution streaming) et produit un VisualWorkflow affichable
|
||||
dans le Visual Workflow Builder.
|
||||
|
||||
Cas d'usage :
|
||||
- Workflow appris par streaming → affichage/review dans le VWB
|
||||
- Import d'un workflow core pour édition manuelle
|
||||
- Mode validation humaine : voir et corriger un workflow auto-généré
|
||||
"""
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
from datetime import datetime
|
||||
|
||||
# Ajouter le chemin racine pour les imports core
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent))
|
||||
|
||||
from core.models.workflow_graph import (
|
||||
Workflow,
|
||||
WorkflowNode,
|
||||
WorkflowEdge,
|
||||
)
|
||||
|
||||
from models.visual_workflow import (
|
||||
VisualWorkflow,
|
||||
VisualNode,
|
||||
VisualEdge,
|
||||
Position,
|
||||
Size,
|
||||
Port,
|
||||
EdgeStyle,
|
||||
EdgeCondition,
|
||||
Variable,
|
||||
WorkflowSettings,
|
||||
)
|
||||
|
||||
|
||||
class GraphToVisualConverter:
|
||||
"""
|
||||
Convertit un core Workflow en VisualWorkflow VWB.
|
||||
|
||||
Le layout automatique place les nodes en grille verticale
|
||||
(de haut en bas), avec les branches condition sur les côtés.
|
||||
"""
|
||||
|
||||
# Mapping inverse : action_type (core) → visual_type (VWB)
|
||||
ACTION_TO_NODE_TYPE = {
|
||||
'mouse_click': 'click',
|
||||
'text_input': 'type',
|
||||
'wait': 'wait',
|
||||
'navigate': 'navigate',
|
||||
'extract_data': 'extract',
|
||||
'set_variable': 'variable',
|
||||
'evaluate_condition': 'condition',
|
||||
'execute_loop': 'loop',
|
||||
'key_press': 'validate',
|
||||
'scroll': 'scroll',
|
||||
'screenshot': 'screenshot',
|
||||
'transform_data': 'transform',
|
||||
'api_call': 'api',
|
||||
'database_query': 'database',
|
||||
'workflow_start': 'start',
|
||||
'workflow_end': 'end',
|
||||
}
|
||||
|
||||
# Couleurs par type de node
|
||||
NODE_COLORS = {
|
||||
'click': '#3B82F6',
|
||||
'type': '#8B5CF6',
|
||||
'wait': '#F59E0B',
|
||||
'navigate': '#10B981',
|
||||
'extract': '#06B6D4',
|
||||
'variable': '#6366F1',
|
||||
'condition': '#EF4444',
|
||||
'loop': '#F97316',
|
||||
'validate': '#14B8A6',
|
||||
'scroll': '#64748B',
|
||||
'screenshot': '#EC4899',
|
||||
'start': '#22C55E',
|
||||
'end': '#EF4444',
|
||||
}
|
||||
|
||||
# Dimensions par défaut
|
||||
DEFAULT_NODE_WIDTH = 200
|
||||
DEFAULT_NODE_HEIGHT = 80
|
||||
VERTICAL_SPACING = 120
|
||||
HORIZONTAL_SPACING = 280
|
||||
START_X = 400
|
||||
START_Y = 80
|
||||
|
||||
def __init__(self):
|
||||
self.warnings: List[str] = []
|
||||
|
||||
def convert(self, workflow: Workflow) -> VisualWorkflow:
|
||||
"""
|
||||
Convertit un core Workflow en VisualWorkflow.
|
||||
|
||||
Args:
|
||||
workflow: Le Workflow core (issu de GraphBuilder ou load_from_file)
|
||||
|
||||
Returns:
|
||||
VisualWorkflow prêt à être affiché dans le VWB
|
||||
"""
|
||||
self.warnings = []
|
||||
|
||||
# Convertir les nodes avec layout automatique
|
||||
visual_nodes = self._convert_nodes(workflow)
|
||||
|
||||
# Convertir les edges
|
||||
visual_edges = self._convert_edges(workflow)
|
||||
|
||||
# Construire le VisualWorkflow
|
||||
now = datetime.now()
|
||||
vw = VisualWorkflow(
|
||||
id=workflow.workflow_id,
|
||||
name=workflow.name or f"Workflow {workflow.workflow_id}",
|
||||
description=workflow.description or "Workflow importé depuis le core pipeline",
|
||||
version="1.0.0",
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
created_by="graph_to_visual_converter",
|
||||
nodes=visual_nodes,
|
||||
edges=visual_edges,
|
||||
variables=[],
|
||||
settings=WorkflowSettings(),
|
||||
tags=workflow.metadata.get('tags', []) if workflow.metadata else [],
|
||||
category=workflow.metadata.get('category', 'imported') if workflow.metadata else 'imported',
|
||||
is_template=False,
|
||||
)
|
||||
|
||||
return vw
|
||||
|
||||
def _convert_nodes(self, workflow: Workflow) -> List[VisualNode]:
|
||||
"""Convertit les WorkflowNodes en VisualNodes avec layout automatique."""
|
||||
visual_nodes = []
|
||||
|
||||
# Déterminer l'ordre topologique pour le layout
|
||||
ordered_ids = self._topological_order(workflow)
|
||||
|
||||
for idx, node_id in enumerate(ordered_ids):
|
||||
node = self._find_node(workflow, node_id)
|
||||
if node is None:
|
||||
continue
|
||||
|
||||
vnode = self._convert_node(node, idx, workflow)
|
||||
visual_nodes.append(vnode)
|
||||
|
||||
return visual_nodes
|
||||
|
||||
def _convert_node(self, node: WorkflowNode, index: int, workflow: Workflow) -> VisualNode:
|
||||
"""Convertit un seul WorkflowNode en VisualNode."""
|
||||
|
||||
# Déterminer le type visuel
|
||||
visual_type = self._infer_visual_type(node)
|
||||
|
||||
# Position (layout vertical simple)
|
||||
pos = self._compute_position(index, visual_type)
|
||||
|
||||
# Extraire les paramètres depuis le node core
|
||||
parameters = self._extract_parameters(node)
|
||||
|
||||
# Déterminer les ports
|
||||
input_ports, output_ports = self._create_ports(visual_type)
|
||||
|
||||
# Label
|
||||
label = node.name or node.node_id
|
||||
|
||||
# Couleur
|
||||
color = self.NODE_COLORS.get(visual_type, '#64748B')
|
||||
|
||||
return VisualNode(
|
||||
id=node.node_id,
|
||||
type=visual_type,
|
||||
position=pos,
|
||||
size=Size(width=self.DEFAULT_NODE_WIDTH, height=self.DEFAULT_NODE_HEIGHT),
|
||||
parameters=parameters,
|
||||
input_ports=input_ports,
|
||||
output_ports=output_ports,
|
||||
label=label,
|
||||
description=node.description or "",
|
||||
color=color,
|
||||
)
|
||||
|
||||
def _convert_edges(self, workflow: Workflow) -> List[VisualEdge]:
|
||||
"""Convertit les WorkflowEdges en VisualEdges."""
|
||||
visual_edges = []
|
||||
|
||||
for edge in workflow.edges:
|
||||
vedge = self._convert_edge(edge)
|
||||
visual_edges.append(vedge)
|
||||
|
||||
return visual_edges
|
||||
|
||||
def _convert_edge(self, edge: WorkflowEdge) -> VisualEdge:
|
||||
"""Convertit un seul WorkflowEdge en VisualEdge."""
|
||||
|
||||
# Déterminer les ports source/target
|
||||
source_port = "out"
|
||||
target_port = "in"
|
||||
|
||||
# Si l'edge a des métadonnées visuelles (aller-retour via le converter)
|
||||
if edge.metadata:
|
||||
source_port = edge.metadata.get('source_port', 'out')
|
||||
target_port = edge.metadata.get('target_port', 'in')
|
||||
|
||||
# Condition sur l'edge
|
||||
condition = None
|
||||
if edge.constraints and edge.constraints.pre_conditions:
|
||||
pre = edge.constraints.pre_conditions
|
||||
if 'condition_result' in pre:
|
||||
branch = 'true' if pre['condition_result'] else 'false'
|
||||
source_port = f"out_{branch}"
|
||||
condition = EdgeCondition(
|
||||
type='expression',
|
||||
expression=f"result == {branch}"
|
||||
)
|
||||
elif 'expression' in pre:
|
||||
condition = EdgeCondition(
|
||||
type='expression',
|
||||
expression=pre['expression']
|
||||
)
|
||||
|
||||
# Style
|
||||
style = EdgeStyle(color=None, width=2, dashed=bool(condition))
|
||||
|
||||
return VisualEdge(
|
||||
id=edge.edge_id,
|
||||
source=edge.from_node,
|
||||
target=edge.to_node,
|
||||
source_port=source_port,
|
||||
target_port=target_port,
|
||||
condition=condition,
|
||||
style=style,
|
||||
)
|
||||
|
||||
# =========================================================================
|
||||
# Helpers
|
||||
# =========================================================================
|
||||
|
||||
def _infer_visual_type(self, node: WorkflowNode) -> str:
|
||||
"""Déterminer le type visuel VWB depuis un WorkflowNode."""
|
||||
|
||||
# 1. Vérifier les métadonnées (si le node a déjà un visual_type)
|
||||
if node.metadata and 'visual_type' in node.metadata:
|
||||
return node.metadata['visual_type']
|
||||
|
||||
# 2. Chercher dans les edges sortants le type d'action
|
||||
# (le type d'action est sur l'edge dans le modèle core)
|
||||
# On ne peut pas le faire ici sans le workflow complet,
|
||||
# donc on utilise le node_type ou le label
|
||||
|
||||
# 3. Déduire depuis le node_type
|
||||
if hasattr(node, 'node_type') and node.node_type:
|
||||
reverse = self.ACTION_TO_NODE_TYPE.get(node.node_type)
|
||||
if reverse:
|
||||
return reverse
|
||||
|
||||
# 4. Heuristiques sur le nom/label
|
||||
name_lower = (node.name or "").lower()
|
||||
if any(k in name_lower for k in ['clic', 'click', 'bouton']):
|
||||
return 'click'
|
||||
if any(k in name_lower for k in ['saisie', 'type', 'input', 'texte']):
|
||||
return 'type'
|
||||
if any(k in name_lower for k in ['attente', 'wait', 'pause']):
|
||||
return 'wait'
|
||||
if 'start' in name_lower or 'début' in name_lower:
|
||||
return 'start'
|
||||
if 'end' in name_lower or 'fin' in name_lower:
|
||||
return 'end'
|
||||
|
||||
# 5. Défaut
|
||||
return 'click'
|
||||
|
||||
def _extract_parameters(self, node: WorkflowNode) -> Dict[str, Any]:
|
||||
"""Extraire les paramètres depuis un WorkflowNode."""
|
||||
params: Dict[str, Any] = {}
|
||||
|
||||
# Métadonnées visuelles (aller-retour)
|
||||
if node.metadata and 'parameters' in node.metadata:
|
||||
params.update(node.metadata['parameters'])
|
||||
|
||||
# Informations du template
|
||||
if node.template:
|
||||
if node.template.window and node.template.window.title_pattern:
|
||||
params['window_title'] = node.template.window.title_pattern
|
||||
if node.template.text and node.template.text.required_texts:
|
||||
params['text_patterns'] = node.template.text.required_texts
|
||||
|
||||
return params
|
||||
|
||||
def _create_ports(self, visual_type: str) -> tuple:
|
||||
"""Créer les ports par défaut pour un type de node."""
|
||||
input_ports = [Port(id="in", name="Entrée", type="input")]
|
||||
|
||||
if visual_type == 'condition':
|
||||
output_ports = [
|
||||
Port(id="out_true", name="Vrai", type="output"),
|
||||
Port(id="out_false", name="Faux", type="output"),
|
||||
]
|
||||
elif visual_type == 'loop':
|
||||
output_ports = [
|
||||
Port(id="out_body", name="Corps", type="output"),
|
||||
Port(id="out_exit", name="Sortie", type="output"),
|
||||
]
|
||||
elif visual_type == 'start':
|
||||
input_ports = []
|
||||
output_ports = [Port(id="out", name="Sortie", type="output")]
|
||||
elif visual_type == 'end':
|
||||
output_ports = []
|
||||
else:
|
||||
output_ports = [Port(id="out", name="Sortie", type="output")]
|
||||
|
||||
return input_ports, output_ports
|
||||
|
||||
def _compute_position(self, index: int, visual_type: str) -> Position:
|
||||
"""Calculer la position d'un node dans le layout vertical."""
|
||||
x = self.START_X
|
||||
y = self.START_Y + index * self.VERTICAL_SPACING
|
||||
|
||||
# Décaler les conditions légèrement à droite
|
||||
if visual_type == 'condition':
|
||||
x += 20
|
||||
|
||||
return Position(x=x, y=y)
|
||||
|
||||
def _topological_order(self, workflow: Workflow) -> List[str]:
|
||||
"""Ordre topologique des nodes (entry → end)."""
|
||||
# Construire le graphe d'adjacence
|
||||
adj: Dict[str, List[str]] = {}
|
||||
in_degree: Dict[str, int] = {}
|
||||
|
||||
all_ids = {n.node_id for n in workflow.nodes}
|
||||
for nid in all_ids:
|
||||
adj[nid] = []
|
||||
in_degree[nid] = 0
|
||||
|
||||
for edge in workflow.edges:
|
||||
if edge.from_node in adj and edge.to_node in in_degree:
|
||||
adj[edge.from_node].append(edge.to_node)
|
||||
in_degree[edge.to_node] += 1
|
||||
|
||||
# BFS Kahn
|
||||
queue = [nid for nid in all_ids if in_degree[nid] == 0]
|
||||
|
||||
# Prioriser les entry_nodes
|
||||
if workflow.entry_nodes:
|
||||
entries = [e for e in workflow.entry_nodes if e in all_ids]
|
||||
others = [q for q in queue if q not in entries]
|
||||
queue = entries + others
|
||||
|
||||
result = []
|
||||
while queue:
|
||||
node = queue.pop(0)
|
||||
result.append(node)
|
||||
for neighbor in adj.get(node, []):
|
||||
in_degree[neighbor] -= 1
|
||||
if in_degree[neighbor] == 0:
|
||||
queue.append(neighbor)
|
||||
|
||||
# Ajouter les nodes orphelins (pas atteints)
|
||||
for nid in all_ids:
|
||||
if nid not in result:
|
||||
result.append(nid)
|
||||
|
||||
return result
|
||||
|
||||
def _find_node(self, workflow: Workflow, node_id: str) -> Optional[WorkflowNode]:
|
||||
"""Trouver un node par ID."""
|
||||
for n in workflow.nodes:
|
||||
if n.node_id == node_id:
|
||||
return n
|
||||
return None
|
||||
|
||||
|
||||
def convert_graph_to_visual(workflow: Workflow) -> VisualWorkflow:
|
||||
"""Fonction utilitaire pour convertir un Workflow en VisualWorkflow."""
|
||||
converter = GraphToVisualConverter()
|
||||
return converter.convert(workflow)
|
||||
Reference in New Issue
Block a user