Le mode "Agent Libre" envoyait les actions localement (Linux) au lieu du PC Windows. Maintenant les plans LLM sont convertis en actions normalisées et envoyés au streaming server via POST /replay/raw. L'Agent V1 les exécute sur la bonne machine. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1682 lines
58 KiB
Python
1682 lines
58 KiB
Python
#!/usr/bin/env python3
|
||
"""
|
||
RPA Vision V3 - Agent Chat
|
||
|
||
Interface conversationnelle pour communiquer avec le système RPA.
|
||
Style "Spotlight/Alfred" - minimaliste et efficace.
|
||
|
||
Composants intégrés:
|
||
- IntentParser: Compréhension des intentions utilisateur
|
||
- ConfirmationLoop: Validation avant actions critiques
|
||
- ResponseGenerator: Réponses en langage naturel
|
||
- ConversationManager: Contexte multi-tour
|
||
|
||
Usage:
|
||
python agent_chat/app.py
|
||
|
||
Puis ouvrir: http://localhost:5004
|
||
|
||
Auteur: Dom - Janvier 2026
|
||
"""
|
||
|
||
import asyncio
|
||
import json
|
||
import logging
|
||
import os
|
||
import sys
|
||
from pathlib import Path
|
||
from datetime import datetime
|
||
from typing import Dict, Any, List, Optional
|
||
|
||
import requests as http_requests # Pour les appels au streaming server
|
||
|
||
from flask import Flask, render_template, request, jsonify
|
||
from flask_socketio import SocketIO, emit
|
||
|
||
# Add project root to path
|
||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||
|
||
from core.workflow import SemanticMatcher, VariableManager
|
||
|
||
# Import des composants conversationnels
|
||
from .intent_parser import IntentParser, IntentType, get_intent_parser
|
||
from .confirmation import ConfirmationLoop, ConfirmationStatus, RiskLevel, get_confirmation_loop
|
||
from .response_generator import ResponseGenerator, get_response_generator
|
||
from .conversation_manager import ConversationManager, get_conversation_manager
|
||
from .autonomous_planner import AutonomousPlanner, get_autonomous_planner, ExecutionPlan
|
||
|
||
# GPU Resource Manager (optional)
|
||
try:
|
||
from core.gpu import get_gpu_resource_manager, ExecutionMode
|
||
GPU_AVAILABLE = True
|
||
except ImportError:
|
||
GPU_AVAILABLE = False
|
||
|
||
# Execution components (optional - pour exécution réelle)
|
||
try:
|
||
from core.execution import ActionExecutor, TargetResolver, ErrorHandler
|
||
from core.execution.execution_loop import ExecutionLoop, ExecutionMode as ExecMode, ExecutionState
|
||
from core.pipeline.workflow_pipeline import WorkflowPipeline
|
||
from core.capture import ScreenCapturer
|
||
EXECUTION_AVAILABLE = True
|
||
except ImportError as e:
|
||
logger.warning(f"Composants d'exécution non disponibles: {e}")
|
||
EXECUTION_AVAILABLE = False
|
||
|
||
logging.basicConfig(level=logging.INFO)
|
||
logger = logging.getLogger(__name__)
|
||
|
||
app = Flask(__name__)
|
||
app.config['SECRET_KEY'] = 'rpa-vision-v3-secret'
|
||
socketio = SocketIO(app, cors_allowed_origins="*")
|
||
|
||
# Global state
|
||
matcher: Optional[SemanticMatcher] = None
|
||
gpu_manager = None
|
||
intent_parser: Optional[IntentParser] = None
|
||
confirmation_loop: Optional[ConfirmationLoop] = None
|
||
response_generator: Optional[ResponseGenerator] = None
|
||
conversation_manager: Optional[ConversationManager] = None
|
||
autonomous_planner: Optional[AutonomousPlanner] = None
|
||
|
||
# Execution components
|
||
workflow_pipeline = None
|
||
action_executor = None
|
||
execution_loop = None
|
||
screen_capturer = None
|
||
|
||
# URL du streaming server (Agent V1) pour l'exécution distante
|
||
STREAMING_SERVER_URL = os.environ.get(
|
||
"RPA_STREAMING_URL", "http://localhost:5005"
|
||
)
|
||
|
||
execution_status = {
|
||
"running": False,
|
||
"workflow": None,
|
||
"progress": 0,
|
||
"message": "",
|
||
"can_minimize": True
|
||
}
|
||
command_history: List[Dict[str, Any]] = []
|
||
|
||
|
||
def init_system():
|
||
"""Initialiser tous les composants du système."""
|
||
global matcher, gpu_manager
|
||
global intent_parser, confirmation_loop, response_generator, conversation_manager
|
||
global autonomous_planner
|
||
|
||
# 1. SemanticMatcher — multi-répertoires (P0-6) + matching LLM (P0-7)
|
||
# Scan data/workflows/ + data/training/workflows/ + data/training/live_sessions/workflows/
|
||
try:
|
||
matcher = SemanticMatcher(
|
||
workflows_dir=None, # None = scan tous les répertoires par défaut
|
||
use_llm=True, # Matching sémantique via Ollama (P0-7)
|
||
llm_model="qwen2.5:7b",
|
||
)
|
||
dirs_info = matcher.get_directories()
|
||
dirs_summary = ", ".join(
|
||
f"{d['path']}({d['workflow_count']})" for d in dirs_info if d['exists']
|
||
)
|
||
logger.info(
|
||
f"✓ SemanticMatcher: {len(matcher.get_all_workflows())} workflows "
|
||
f"[{dirs_summary}]"
|
||
)
|
||
except Exception as e:
|
||
logger.error(f"✗ SemanticMatcher: {e}")
|
||
matcher = None
|
||
|
||
# 2. GPU Resource Manager
|
||
if GPU_AVAILABLE:
|
||
try:
|
||
gpu_manager = get_gpu_resource_manager()
|
||
logger.info("✓ GPU Resource Manager connected")
|
||
except Exception as e:
|
||
logger.warning(f"⚠ GPU Resource Manager: {e}")
|
||
gpu_manager = None
|
||
|
||
# 3. Composants conversationnels
|
||
try:
|
||
intent_parser = get_intent_parser(use_llm=True) # LLM activé (Ollama)
|
||
confirmation_loop = get_confirmation_loop()
|
||
response_generator = get_response_generator()
|
||
conversation_manager = get_conversation_manager()
|
||
|
||
# Injecter les workflows dans l'intent_parser pour contexte LLM
|
||
if matcher and intent_parser:
|
||
workflows_for_llm = [
|
||
{"name": wf.name, "description": wf.description, "tags": wf.tags}
|
||
for wf in matcher.get_all_workflows()
|
||
]
|
||
intent_parser.set_workflows(workflows_for_llm)
|
||
|
||
logger.info("✓ Composants conversationnels initialisés (LLM activé)")
|
||
except Exception as e:
|
||
logger.error(f"✗ Composants conversationnels: {e}")
|
||
# Fallback aux composants de base
|
||
intent_parser = IntentParser(use_llm=False)
|
||
confirmation_loop = ConfirmationLoop()
|
||
response_generator = ResponseGenerator()
|
||
conversation_manager = ConversationManager()
|
||
|
||
# 4. Composants d'exécution réelle
|
||
global workflow_pipeline, action_executor, execution_loop, screen_capturer
|
||
if EXECUTION_AVAILABLE:
|
||
try:
|
||
# Pipeline de workflow (matching + actions)
|
||
workflow_pipeline = WorkflowPipeline()
|
||
logger.info("✓ WorkflowPipeline initialisé")
|
||
|
||
# Capture d'écran
|
||
screen_capturer = ScreenCapturer()
|
||
logger.info("✓ ScreenCapturer initialisé")
|
||
|
||
# Résolveur de cibles et gestionnaire d'erreurs
|
||
target_resolver = TargetResolver()
|
||
error_handler = ErrorHandler()
|
||
|
||
# Exécuteur d'actions
|
||
action_executor = ActionExecutor(
|
||
target_resolver=target_resolver,
|
||
error_handler=error_handler,
|
||
verify_postconditions=True
|
||
)
|
||
logger.info("✓ ActionExecutor initialisé")
|
||
|
||
# Boucle d'exécution (pour mode automatique)
|
||
execution_loop = ExecutionLoop(
|
||
pipeline=workflow_pipeline,
|
||
action_executor=action_executor,
|
||
screen_capturer=screen_capturer
|
||
)
|
||
logger.info("✓ ExecutionLoop initialisé")
|
||
|
||
except Exception as e:
|
||
logger.warning(f"⚠ Composants d'exécution partiels: {e}")
|
||
# Mode dégradé: simulation uniquement
|
||
workflow_pipeline = None
|
||
action_executor = None
|
||
execution_loop = None
|
||
else:
|
||
logger.info("ℹ Mode simulation (composants d'exécution non disponibles)")
|
||
|
||
# 5. Autonomous Planner (Agent Libre)
|
||
try:
|
||
autonomous_planner = get_autonomous_planner(llm_model="qwen2.5:7b")
|
||
|
||
# Configurer les callbacks pour l'exécution
|
||
if screen_capturer:
|
||
autonomous_planner.set_screen_capturer(screen_capturer.capture)
|
||
|
||
def progress_callback(data):
|
||
socketio.emit('agent_progress', data)
|
||
|
||
autonomous_planner.set_progress_callback(progress_callback)
|
||
|
||
logger.info(f"✓ AutonomousPlanner initialisé (LLM: {autonomous_planner.llm_available})")
|
||
except Exception as e:
|
||
logger.warning(f"⚠ AutonomousPlanner: {e}")
|
||
autonomous_planner = None
|
||
|
||
|
||
# =============================================================================
|
||
# Routes Web
|
||
# =============================================================================
|
||
|
||
@app.route('/')
|
||
def index():
|
||
"""Page principale - nouvelle interface chat."""
|
||
return render_template('chat.html')
|
||
|
||
|
||
@app.route('/classic')
|
||
def classic():
|
||
"""Ancienne interface (fallback)."""
|
||
return render_template('command.html')
|
||
|
||
|
||
@app.route('/api/status')
|
||
def api_status():
|
||
"""Statut complet du système."""
|
||
workflows_count = len(matcher.get_all_workflows()) if matcher else 0
|
||
|
||
# GPU Status
|
||
gpu_status = None
|
||
if gpu_manager:
|
||
try:
|
||
status = gpu_manager.get_status()
|
||
gpu_status = {
|
||
"mode": status.execution_mode.value,
|
||
"vlm_state": status.vlm_state.value,
|
||
"vlm_model": status.vlm_model,
|
||
"clip_device": status.clip_device,
|
||
"degraded": status.degraded_mode
|
||
}
|
||
if status.vram:
|
||
gpu_status["vram"] = {
|
||
"used_mb": status.vram.used_mb,
|
||
"total_mb": status.vram.total_mb,
|
||
"percent": round(status.vram.used_mb / status.vram.total_mb * 100, 1) if status.vram.total_mb > 0 else 0
|
||
}
|
||
except Exception as e:
|
||
logger.warning(f"GPU status error: {e}")
|
||
|
||
# Ollama Status
|
||
ollama_status = None
|
||
try:
|
||
import requests
|
||
response = requests.get("http://localhost:11434/api/tags", timeout=2)
|
||
if response.status_code == 200:
|
||
models = response.json().get('models', [])
|
||
ollama_status = {
|
||
"available": True,
|
||
"models_count": len(models)
|
||
}
|
||
except:
|
||
ollama_status = {"available": False}
|
||
|
||
return jsonify({
|
||
"status": "online",
|
||
"workflows_count": workflows_count,
|
||
"execution": execution_status,
|
||
"gpu": gpu_status,
|
||
"ollama": ollama_status
|
||
})
|
||
|
||
|
||
@app.route('/api/workflows')
|
||
def api_workflows():
|
||
"""Liste des workflows (tous répertoires confondus)."""
|
||
if not matcher:
|
||
return jsonify({"workflows": [], "directories": []})
|
||
|
||
workflows = []
|
||
for wf in matcher.get_all_workflows():
|
||
workflows.append({
|
||
"id": wf.workflow_id,
|
||
"name": wf.name,
|
||
"description": wf.description,
|
||
"tags": wf.tags,
|
||
"source": wf.source_dir,
|
||
})
|
||
|
||
return jsonify({
|
||
"workflows": workflows,
|
||
"directories": matcher.get_directories(),
|
||
})
|
||
|
||
|
||
@app.route('/api/workflows/refresh', methods=['POST'])
|
||
def api_workflows_refresh():
|
||
"""
|
||
Forcer le rechargement des workflows depuis tous les répertoires.
|
||
|
||
Utile après qu'un nouveau workflow a été appris par le StreamProcessor.
|
||
"""
|
||
if not matcher:
|
||
return jsonify({"success": False, "error": "SemanticMatcher non initialisé"})
|
||
|
||
try:
|
||
count = matcher.reload_workflows()
|
||
|
||
# Re-injecter les workflows dans l'intent_parser (contexte LLM)
|
||
if intent_parser:
|
||
workflows_for_llm = [
|
||
{"name": wf.name, "description": wf.description, "tags": wf.tags}
|
||
for wf in matcher.get_all_workflows()
|
||
]
|
||
intent_parser.set_workflows(workflows_for_llm)
|
||
|
||
return jsonify({
|
||
"success": True,
|
||
"workflows_count": count,
|
||
"directories": matcher.get_directories(),
|
||
})
|
||
except Exception as e:
|
||
logger.error(f"Erreur rechargement workflows: {e}")
|
||
return jsonify({"success": False, "error": str(e)})
|
||
|
||
|
||
@app.route('/api/search', methods=['POST'])
|
||
def api_search():
|
||
"""Rechercher des workflows."""
|
||
data = request.json
|
||
query = data.get('query', '')
|
||
|
||
if not matcher or not query:
|
||
return jsonify({"matches": []})
|
||
|
||
matches = matcher.find_workflows(query, limit=5, min_confidence=0.2)
|
||
|
||
results = []
|
||
for m in matches:
|
||
results.append({
|
||
"workflow_id": m.workflow_id,
|
||
"workflow_name": m.workflow_name,
|
||
"confidence": m.confidence,
|
||
"extracted_params": m.extracted_params,
|
||
"match_reason": m.match_reason
|
||
})
|
||
|
||
return jsonify({"matches": results})
|
||
|
||
|
||
@app.route('/api/execute', methods=['POST'])
|
||
def api_execute():
|
||
"""Exécuter une commande."""
|
||
global execution_status
|
||
|
||
data = request.json
|
||
command = data.get('command', '')
|
||
params = data.get('params', {})
|
||
|
||
if not matcher or not command:
|
||
return jsonify({"success": False, "error": "Invalid command"})
|
||
|
||
# Trouver le workflow
|
||
match = matcher.find_workflow(command, min_confidence=0.2)
|
||
|
||
if not match:
|
||
return jsonify({
|
||
"success": False,
|
||
"error": "Aucun workflow correspondant trouvé"
|
||
})
|
||
|
||
# Combiner les paramètres
|
||
all_params = {**match.extracted_params, **params}
|
||
|
||
# Enregistrer dans l'historique
|
||
command_history.append({
|
||
"timestamp": datetime.now().isoformat(),
|
||
"command": command,
|
||
"workflow": match.workflow_name,
|
||
"params": all_params,
|
||
"status": "started"
|
||
})
|
||
|
||
# Mettre à jour le statut
|
||
execution_status = {
|
||
"running": True,
|
||
"workflow": match.workflow_name,
|
||
"progress": 0,
|
||
"message": "Démarrage..."
|
||
}
|
||
|
||
# Notifier via WebSocket
|
||
socketio.emit('execution_started', {
|
||
"workflow": match.workflow_name,
|
||
"params": all_params
|
||
})
|
||
|
||
# Exécuter le workflow en arrière-plan
|
||
socketio.start_background_task(execute_workflow, match, all_params)
|
||
|
||
return jsonify({
|
||
"success": True,
|
||
"workflow": match.workflow_name,
|
||
"params": all_params,
|
||
"confidence": match.confidence
|
||
})
|
||
|
||
|
||
@app.route('/api/history')
|
||
def api_history():
|
||
"""Historique des commandes."""
|
||
return jsonify({"history": command_history[-20:]})
|
||
|
||
|
||
@app.route('/api/chat', methods=['POST'])
|
||
def api_chat():
|
||
"""
|
||
Endpoint conversationnel principal.
|
||
|
||
Utilise le flux complet:
|
||
1. IntentParser: Analyse l'intention
|
||
2. ConversationManager: Gère le contexte multi-tour
|
||
3. ConfirmationLoop: Valide les actions sensibles
|
||
4. ResponseGenerator: Génère la réponse
|
||
"""
|
||
data = request.json
|
||
message = data.get('message', '').strip()
|
||
session_id = data.get('session_id')
|
||
|
||
if not message:
|
||
return jsonify({"error": "Message vide"}), 400
|
||
|
||
# 1. Obtenir ou créer la session
|
||
session = conversation_manager.get_or_create_session(session_id=session_id)
|
||
|
||
# 2. Parser l'intention
|
||
intent = intent_parser.parse(message)
|
||
|
||
# 3. Résoudre les références anaphoriques (ex: "le même", "celui-ci")
|
||
intent = conversation_manager.resolve_references(session, intent)
|
||
|
||
# 4. Construire le contexte
|
||
context = conversation_manager.get_context_summary(session)
|
||
context["execution_status"] = execution_status
|
||
|
||
# 5. Traiter selon le type d'intention
|
||
result = {}
|
||
action_taken = None
|
||
|
||
if intent.intent_type == IntentType.CONFIRM:
|
||
# Confirmer une action en attente
|
||
pending = conversation_manager.get_pending_confirmation(session)
|
||
if pending:
|
||
confirmation_loop.confirm(pending.id)
|
||
conversation_manager.clear_pending_confirmation(session)
|
||
result = {"confirmed": True, "workflow": pending.workflow_name}
|
||
action_taken = "confirmed"
|
||
|
||
# Lancer l'exécution
|
||
socketio.start_background_task(
|
||
execute_workflow_from_confirmation, pending, session.session_id
|
||
)
|
||
else:
|
||
result = {"confirmed": False}
|
||
|
||
elif intent.intent_type == IntentType.DENY:
|
||
# Refuser une action en attente
|
||
pending = conversation_manager.get_pending_confirmation(session)
|
||
if pending:
|
||
confirmation_loop.deny(pending.id)
|
||
conversation_manager.clear_pending_confirmation(session)
|
||
result = {"denied": True}
|
||
action_taken = "denied"
|
||
|
||
elif intent.intent_type == IntentType.EXECUTE:
|
||
# Exécuter un workflow
|
||
if matcher and intent.workflow_hint:
|
||
match = matcher.find_workflow(intent.workflow_hint, min_confidence=0.2)
|
||
|
||
if match:
|
||
# Évaluer le risque
|
||
risk = confirmation_loop.evaluate_risk(
|
||
match.workflow_name,
|
||
{**match.extracted_params, **intent.parameters}
|
||
)
|
||
|
||
if confirmation_loop.requires_confirmation(risk):
|
||
# Créer une demande de confirmation
|
||
conf = confirmation_loop.create_confirmation_request(
|
||
workflow_name=match.workflow_name,
|
||
parameters={**match.extracted_params, **intent.parameters},
|
||
action_type="execute",
|
||
risk_level=risk
|
||
)
|
||
conversation_manager.set_pending_confirmation(session, conf)
|
||
|
||
# Générer la réponse de confirmation
|
||
response = response_generator.generate_confirmation_request(conf)
|
||
result = {"needs_confirmation": True, "confirmation": conf.to_dict()}
|
||
action_taken = "confirmation_requested"
|
||
|
||
else:
|
||
# Exécuter directement
|
||
all_params = {**match.extracted_params, **intent.parameters}
|
||
result = {
|
||
"success": True,
|
||
"workflow": match.workflow_name,
|
||
"params": all_params,
|
||
"confidence": match.confidence
|
||
}
|
||
action_taken = "executed"
|
||
|
||
socketio.start_background_task(execute_workflow, match, all_params)
|
||
else:
|
||
result = {"not_found": True, "query": intent.workflow_hint}
|
||
else:
|
||
result = {"error": "Pas de workflow spécifié"}
|
||
|
||
elif intent.intent_type == IntentType.LIST:
|
||
# Lister les workflows avec métadonnées enrichies
|
||
if matcher:
|
||
workflows = [
|
||
{
|
||
"name": wf.name,
|
||
"description": wf.description or "Workflow appris automatiquement",
|
||
"tags": wf.tags[:3] if wf.tags else [],
|
||
"id": wf.workflow_id,
|
||
}
|
||
for wf in matcher.get_all_workflows()
|
||
]
|
||
result = {"workflows": workflows}
|
||
else:
|
||
result = {"workflows": []}
|
||
action_taken = "listed"
|
||
|
||
elif intent.intent_type == IntentType.STATUS:
|
||
result = {"execution": execution_status}
|
||
action_taken = "status_checked"
|
||
|
||
elif intent.intent_type == IntentType.CANCEL:
|
||
if execution_status.get("running"):
|
||
execution_status["running"] = False
|
||
execution_status["message"] = "Annulé"
|
||
result = {"cancelled": True}
|
||
else:
|
||
result = {"cancelled": False}
|
||
action_taken = "cancelled"
|
||
|
||
elif intent.intent_type == IntentType.HISTORY:
|
||
result = {"history": command_history[-10:]}
|
||
action_taken = "history_shown"
|
||
|
||
elif intent.intent_type == IntentType.QUERY:
|
||
# Rechercher des infos sur le workflow demandé
|
||
topic = intent.workflow_hint or intent.raw_query
|
||
if matcher and topic:
|
||
# Tenter de trouver le workflow correspondant
|
||
match = matcher.find_workflow(topic, min_confidence=0.3)
|
||
if match:
|
||
help_text = matcher.get_workflow_help(match.workflow_id)
|
||
result = {"answer": help_text}
|
||
else:
|
||
# Peut-être une question générale sur les workflows dispo
|
||
all_wf = matcher.get_all_workflows()
|
||
if all_wf:
|
||
wf_list = "\n".join(
|
||
f"• **{wf.name}**: {wf.description or 'Pas de description'}"
|
||
for wf in all_wf[:10]
|
||
)
|
||
result = {
|
||
"answer": f"Je n'ai pas trouvé de workflow '{topic}', "
|
||
f"mais voici les workflows disponibles :\n{wf_list}"
|
||
}
|
||
else:
|
||
result = {}
|
||
else:
|
||
result = {}
|
||
action_taken = "query_answered"
|
||
|
||
elif intent.intent_type == IntentType.HELP:
|
||
result = {}
|
||
action_taken = "help_shown"
|
||
|
||
elif intent.clarification_needed:
|
||
result = {"clarification_needed": True}
|
||
action_taken = "clarification_requested"
|
||
|
||
# 6. Générer la réponse (si pas déjà fait pour confirmation)
|
||
if action_taken != "confirmation_requested":
|
||
response = response_generator.generate(intent, context, result)
|
||
|
||
# 7. Enregistrer le tour dans la conversation
|
||
conversation_manager.add_turn(
|
||
session=session,
|
||
user_message=message,
|
||
intent=intent,
|
||
response=response.message,
|
||
action_taken=action_taken,
|
||
result=result
|
||
)
|
||
|
||
# 8. Retourner la réponse
|
||
return jsonify({
|
||
"session_id": session.session_id,
|
||
"intent": intent.to_dict(),
|
||
"response": response.to_dict(),
|
||
"result": result,
|
||
"context": {
|
||
"current_workflow": session.context.current_workflow,
|
||
"has_pending_confirmation": session.context.pending_confirmation is not None
|
||
}
|
||
})
|
||
|
||
|
||
def execute_workflow_from_confirmation(confirmation, session_id):
|
||
"""Exécuter un workflow après confirmation."""
|
||
global execution_status
|
||
|
||
if not matcher:
|
||
return
|
||
|
||
# Trouver le workflow
|
||
match = matcher.find_workflow(confirmation.workflow_name, min_confidence=0.1)
|
||
if not match:
|
||
return
|
||
|
||
# Utiliser les paramètres confirmés (ou modifiés)
|
||
params = confirmation.modified_parameters or confirmation.parameters
|
||
|
||
# IMPORTANT: Marquer l'exécution comme active
|
||
execution_status["running"] = True
|
||
execution_status["workflow"] = confirmation.workflow_name
|
||
execution_status["progress"] = 0
|
||
execution_status["message"] = "Démarrage..."
|
||
|
||
execute_workflow(match, params)
|
||
|
||
|
||
@app.route('/api/gpu/<action>', methods=['POST'])
|
||
def api_gpu_action(action):
|
||
"""Contrôler le GPU Resource Manager."""
|
||
if not gpu_manager:
|
||
return jsonify({"success": False, "error": "GPU Manager non disponible"})
|
||
|
||
async def do_action():
|
||
if action == "load-vlm":
|
||
return await gpu_manager.ensure_vlm_loaded()
|
||
elif action == "unload-vlm":
|
||
return await gpu_manager.ensure_vlm_unloaded()
|
||
elif action == "recording":
|
||
await gpu_manager.set_execution_mode(ExecutionMode.RECORDING)
|
||
return True
|
||
elif action == "autopilot":
|
||
await gpu_manager.set_execution_mode(ExecutionMode.AUTOPILOT)
|
||
return True
|
||
elif action == "idle":
|
||
await gpu_manager.set_execution_mode(ExecutionMode.IDLE)
|
||
return True
|
||
return False
|
||
|
||
try:
|
||
loop = asyncio.new_event_loop()
|
||
asyncio.set_event_loop(loop)
|
||
result = loop.run_until_complete(do_action())
|
||
loop.close()
|
||
|
||
return jsonify({"success": result, "action": action})
|
||
except Exception as e:
|
||
return jsonify({"success": False, "error": str(e)})
|
||
|
||
|
||
@app.route('/api/llm/status')
|
||
def api_llm_status():
|
||
"""Statut du LLM (Ollama)."""
|
||
status = {
|
||
"enabled": intent_parser.use_llm if intent_parser else False,
|
||
"available": intent_parser.llm_available if intent_parser else False,
|
||
"model": intent_parser.llm_model if intent_parser else None,
|
||
"endpoint": intent_parser.llm_endpoint if intent_parser else None,
|
||
"workflows_loaded": len(intent_parser._workflows_cache) if intent_parser else 0
|
||
}
|
||
|
||
# Lister les modèles Ollama disponibles
|
||
try:
|
||
import requests
|
||
response = requests.get("http://localhost:11434/api/tags", timeout=2)
|
||
if response.status_code == 200:
|
||
models = response.json().get('models', [])
|
||
status["available_models"] = [m["name"] for m in models]
|
||
except:
|
||
status["available_models"] = []
|
||
|
||
return jsonify(status)
|
||
|
||
|
||
@app.route('/api/llm/model', methods=['POST'])
|
||
def api_llm_set_model():
|
||
"""Changer le modèle LLM."""
|
||
data = request.json
|
||
model = data.get('model')
|
||
|
||
if not model:
|
||
return jsonify({"success": False, "error": "Modèle non spécifié"})
|
||
|
||
if intent_parser:
|
||
intent_parser.llm_model = model
|
||
intent_parser._check_llm_availability()
|
||
return jsonify({
|
||
"success": True,
|
||
"model": model,
|
||
"available": intent_parser.llm_available
|
||
})
|
||
|
||
return jsonify({"success": False, "error": "IntentParser non initialisé"})
|
||
|
||
|
||
# =============================================================================
|
||
# API Agent Libre (Autonomous Mode)
|
||
# =============================================================================
|
||
|
||
@app.route('/api/agent/plan', methods=['POST'])
|
||
def api_agent_plan():
|
||
"""
|
||
Génère un plan d'exécution pour une tâche en langage naturel.
|
||
|
||
Le mode "Agent Libre" permet d'exécuter des tâches sans workflow pré-enregistré.
|
||
Le LLM (Qwen) décompose la demande en étapes d'actions.
|
||
"""
|
||
if not autonomous_planner:
|
||
return jsonify({"error": "Agent autonome non disponible"}), 503
|
||
|
||
data = request.json
|
||
user_request = data.get('request', '').strip()
|
||
|
||
if not user_request:
|
||
return jsonify({"error": "Requête vide"}), 400
|
||
|
||
try:
|
||
# Contexte optionnel (écran actuel, etc.)
|
||
context = data.get('context', {})
|
||
|
||
# Générer le plan
|
||
plan = autonomous_planner.plan(user_request, context)
|
||
|
||
return jsonify({
|
||
"success": True,
|
||
"plan": {
|
||
"task": plan.task_description,
|
||
"steps": [
|
||
{
|
||
"step": s.step_number,
|
||
"action": s.action_type.value,
|
||
"description": s.description,
|
||
"target": s.target,
|
||
"params": s.parameters,
|
||
"expected_result": s.expected_result
|
||
}
|
||
for s in plan.steps
|
||
],
|
||
"estimated_seconds": plan.estimated_duration_seconds,
|
||
"risk_level": plan.risk_level,
|
||
"requires_confirmation": plan.requires_confirmation
|
||
},
|
||
"llm_available": autonomous_planner.llm_available
|
||
})
|
||
|
||
except Exception as e:
|
||
logger.error(f"Agent plan error: {e}")
|
||
return jsonify({"error": str(e)}), 500
|
||
|
||
|
||
@app.route('/api/agent/execute', methods=['POST'])
|
||
def api_agent_execute():
|
||
"""
|
||
Exécute un plan d'agent autonome.
|
||
|
||
Attend un objet plan (généré par /api/agent/plan) et l'exécute étape par étape.
|
||
"""
|
||
if not autonomous_planner:
|
||
return jsonify({"error": "Agent autonome non disponible"}), 503
|
||
|
||
data = request.json
|
||
plan_data = data.get('plan')
|
||
|
||
if not plan_data:
|
||
return jsonify({"error": "Plan manquant"}), 400
|
||
|
||
try:
|
||
# Reconstruire le plan depuis les données
|
||
from .autonomous_planner import PlannedAction, ActionType
|
||
|
||
steps = []
|
||
for step_data in plan_data.get('steps', []):
|
||
action_type_str = step_data.get('action', 'click')
|
||
action_type_map = {
|
||
'open_app': ActionType.OPEN_APP,
|
||
'open_url': ActionType.OPEN_URL,
|
||
'click': ActionType.CLICK,
|
||
'type_text': ActionType.TYPE_TEXT,
|
||
'hotkey': ActionType.HOTKEY,
|
||
'scroll': ActionType.SCROLL,
|
||
'wait': ActionType.WAIT,
|
||
'screenshot': ActionType.SCREENSHOT
|
||
}
|
||
|
||
steps.append(PlannedAction(
|
||
step_number=step_data.get('step', len(steps) + 1),
|
||
action_type=action_type_map.get(action_type_str, ActionType.CLICK),
|
||
description=step_data.get('description', ''),
|
||
target=step_data.get('target'),
|
||
parameters=step_data.get('params', {}),
|
||
expected_result=step_data.get('expected_result')
|
||
))
|
||
|
||
plan = ExecutionPlan(
|
||
task_description=plan_data.get('task', ''),
|
||
steps=steps,
|
||
estimated_duration_seconds=plan_data.get('estimated_seconds', 30),
|
||
risk_level=plan_data.get('risk_level', 'low')
|
||
)
|
||
|
||
# Exécuter en arrière-plan
|
||
socketio.start_background_task(execute_agent_plan, plan)
|
||
|
||
return jsonify({
|
||
"success": True,
|
||
"message": "Exécution démarrée",
|
||
"steps_count": len(steps)
|
||
})
|
||
|
||
except Exception as e:
|
||
logger.error(f"Agent execute error: {e}")
|
||
return jsonify({"error": str(e)}), 500
|
||
|
||
|
||
@app.route('/api/agent/status')
|
||
def api_agent_status():
|
||
"""Statut de l'agent autonome."""
|
||
return jsonify({
|
||
"available": autonomous_planner is not None,
|
||
"llm_available": autonomous_planner.llm_available if autonomous_planner else False,
|
||
"llm_model": autonomous_planner.llm_model if autonomous_planner else None
|
||
})
|
||
|
||
|
||
def execute_agent_plan(plan: ExecutionPlan):
|
||
"""Exécute un plan d'agent sur la machine distante via le streaming server."""
|
||
|
||
try:
|
||
# Convertir le plan LLM en actions normalisées pour l'Agent V1
|
||
actions = _plan_to_replay_actions(plan)
|
||
|
||
if not actions:
|
||
socketio.emit('execution_completed', {
|
||
"success": False,
|
||
"workflow": plan.task_description,
|
||
"message": "Aucune action convertible dans ce plan."
|
||
})
|
||
return
|
||
|
||
# Envoyer au streaming server pour exécution sur le PC cible
|
||
resp = http_requests.post(
|
||
f"{STREAMING_SERVER_URL}/api/v1/traces/stream/replay/raw",
|
||
json={
|
||
"actions": actions,
|
||
"session_id": "", # Auto-détection
|
||
"task_description": plan.task_description,
|
||
},
|
||
timeout=15,
|
||
)
|
||
|
||
if resp.status_code == 200:
|
||
data = resp.json()
|
||
replay_id = data.get("replay_id", "")
|
||
total = data.get("total_actions", len(actions))
|
||
|
||
socketio.emit('agent_execution_started', {
|
||
"workflow": plan.task_description,
|
||
"message": f"Exécution démarrée sur le PC cible ({total} actions)",
|
||
"replay_id": replay_id,
|
||
})
|
||
|
||
# Suivre la progression
|
||
_poll_replay_progress(replay_id, plan.task_description, total)
|
||
|
||
else:
|
||
error = resp.text[:200]
|
||
logger.error(f"Streaming server refus: HTTP {resp.status_code}: {error}")
|
||
socketio.emit('execution_completed', {
|
||
"success": False,
|
||
"workflow": plan.task_description,
|
||
"message": f"Erreur serveur: {error}"
|
||
})
|
||
|
||
except http_requests.ConnectionError:
|
||
logger.error("Streaming server non disponible pour l'agent libre")
|
||
socketio.emit('execution_completed', {
|
||
"success": False,
|
||
"workflow": plan.task_description,
|
||
"message": "Le serveur de streaming n'est pas disponible. "
|
||
"Vérifiez qu'il tourne sur le port 5005."
|
||
})
|
||
except Exception as e:
|
||
logger.error(f"Agent execution error: {e}")
|
||
socketio.emit('execution_completed', {
|
||
"success": False,
|
||
"workflow": plan.task_description,
|
||
"message": f"Erreur: {str(e)}"
|
||
})
|
||
|
||
|
||
def _plan_to_replay_actions(plan: ExecutionPlan) -> list:
|
||
"""Convertir un ExecutionPlan LLM en actions normalisées pour l'Agent V1."""
|
||
import uuid as _uuid
|
||
from .autonomous_planner import ActionType
|
||
|
||
actions = []
|
||
for step in plan.steps:
|
||
action = {"action_id": f"act_free_{_uuid.uuid4().hex[:6]}"}
|
||
|
||
if step.action_type == ActionType.OPEN_URL:
|
||
url = step.parameters.get("url", "")
|
||
# Ouvrir le navigateur : touche Windows, taper le navigateur, Enter, puis naviguer
|
||
actions.append({
|
||
**action,
|
||
"type": "key_combo",
|
||
"keys": ["super"],
|
||
})
|
||
actions.append({
|
||
"action_id": f"act_free_{_uuid.uuid4().hex[:6]}",
|
||
"type": "wait",
|
||
"duration_ms": 800,
|
||
})
|
||
actions.append({
|
||
"action_id": f"act_free_{_uuid.uuid4().hex[:6]}",
|
||
"type": "type",
|
||
"text": "chrome",
|
||
})
|
||
actions.append({
|
||
"action_id": f"act_free_{_uuid.uuid4().hex[:6]}",
|
||
"type": "key_combo",
|
||
"keys": ["enter"],
|
||
})
|
||
actions.append({
|
||
"action_id": f"act_free_{_uuid.uuid4().hex[:6]}",
|
||
"type": "wait",
|
||
"duration_ms": 2000,
|
||
})
|
||
# Focus barre d'adresse + taper URL
|
||
actions.append({
|
||
"action_id": f"act_free_{_uuid.uuid4().hex[:6]}",
|
||
"type": "key_combo",
|
||
"keys": ["ctrl", "l"],
|
||
})
|
||
actions.append({
|
||
"action_id": f"act_free_{_uuid.uuid4().hex[:6]}",
|
||
"type": "wait",
|
||
"duration_ms": 300,
|
||
})
|
||
actions.append({
|
||
"action_id": f"act_free_{_uuid.uuid4().hex[:6]}",
|
||
"type": "type",
|
||
"text": url,
|
||
})
|
||
actions.append({
|
||
"action_id": f"act_free_{_uuid.uuid4().hex[:6]}",
|
||
"type": "key_combo",
|
||
"keys": ["enter"],
|
||
})
|
||
actions.append({
|
||
"action_id": f"act_free_{_uuid.uuid4().hex[:6]}",
|
||
"type": "wait",
|
||
"duration_ms": 3000,
|
||
})
|
||
continue
|
||
|
||
elif step.action_type == ActionType.OPEN_APP:
|
||
app_name = step.parameters.get("app_name", "")
|
||
actions.append({**action, "type": "key_combo", "keys": ["super"]})
|
||
actions.append({
|
||
"action_id": f"act_free_{_uuid.uuid4().hex[:6]}",
|
||
"type": "wait", "duration_ms": 800,
|
||
})
|
||
actions.append({
|
||
"action_id": f"act_free_{_uuid.uuid4().hex[:6]}",
|
||
"type": "type", "text": app_name,
|
||
})
|
||
actions.append({
|
||
"action_id": f"act_free_{_uuid.uuid4().hex[:6]}",
|
||
"type": "key_combo", "keys": ["enter"],
|
||
})
|
||
actions.append({
|
||
"action_id": f"act_free_{_uuid.uuid4().hex[:6]}",
|
||
"type": "wait", "duration_ms": 2000,
|
||
})
|
||
continue
|
||
|
||
elif step.action_type == ActionType.TYPE_TEXT:
|
||
text = step.parameters.get("text", "")
|
||
action["type"] = "type"
|
||
action["text"] = text
|
||
# Si un target est spécifié, activer la résolution visuelle
|
||
if step.target:
|
||
action["visual_mode"] = True
|
||
action["target_spec"] = {"by_text": step.target}
|
||
|
||
elif step.action_type == ActionType.CLICK:
|
||
action["type"] = "click"
|
||
action["x_pct"] = 0.5
|
||
action["y_pct"] = 0.5
|
||
action["button"] = "left"
|
||
if step.target:
|
||
action["visual_mode"] = True
|
||
action["target_spec"] = {"by_text": step.target}
|
||
|
||
elif step.action_type == ActionType.HOTKEY:
|
||
keys_str = step.parameters.get("keys", "")
|
||
if isinstance(keys_str, str):
|
||
keys = [k.strip() for k in keys_str.split("+")]
|
||
else:
|
||
keys = keys_str
|
||
action["type"] = "key_combo"
|
||
action["keys"] = keys
|
||
|
||
elif step.action_type == ActionType.SCROLL:
|
||
direction = step.parameters.get("direction", "down")
|
||
amount = step.parameters.get("amount", 3)
|
||
action["type"] = "scroll"
|
||
action["delta"] = -amount if direction == "down" else amount
|
||
|
||
elif step.action_type == ActionType.WAIT:
|
||
seconds = step.parameters.get("seconds", 2)
|
||
action["type"] = "wait"
|
||
action["duration_ms"] = int(seconds * 1000)
|
||
|
||
elif step.action_type == ActionType.SCREENSHOT:
|
||
# Skip — l'Agent V1 capture déjà automatiquement
|
||
continue
|
||
|
||
else:
|
||
continue
|
||
|
||
actions.append(action)
|
||
|
||
return actions
|
||
|
||
|
||
@app.route('/api/help')
|
||
def api_help():
|
||
"""Aide et mode d'emploi."""
|
||
help_content = {
|
||
"title": "RPA Vision V3 - Mode d'emploi",
|
||
"sections": [
|
||
{
|
||
"title": "🎯 Commandes en langage naturel",
|
||
"content": """
|
||
Tapez simplement ce que vous voulez faire en français ou anglais.
|
||
Le système trouvera automatiquement le workflow correspondant.
|
||
|
||
**Exemples :**
|
||
- "facturer le client Acme"
|
||
- "exporter le rapport en PDF"
|
||
- "créer une facture pour Client ABC"
|
||
- "facturer les clients de A à Z"
|
||
"""
|
||
},
|
||
{
|
||
"title": "📋 Paramètres",
|
||
"content": """
|
||
Les paramètres sont extraits automatiquement de votre commande.
|
||
Vous pouvez aussi les spécifier manuellement dans le formulaire.
|
||
|
||
**Paramètres courants :**
|
||
- `client` : Nom du client
|
||
- `format` : Format d'export (pdf, excel)
|
||
- `start`, `end` : Plage de valeurs
|
||
"""
|
||
},
|
||
{
|
||
"title": "⌨️ Raccourcis clavier",
|
||
"content": """
|
||
- `Entrée` : Exécuter la commande
|
||
- `Échap` : Annuler / Fermer
|
||
- `↑` / `↓` : Naviguer dans l'historique
|
||
- `Ctrl+M` : Minimiser l'interface
|
||
"""
|
||
},
|
||
{
|
||
"title": "🔄 Pendant l'exécution",
|
||
"content": """
|
||
L'interface peut être minimisée pendant l'exécution.
|
||
Le workflow s'exécute en arrière-plan.
|
||
Vous serez notifié à la fin de l'exécution.
|
||
"""
|
||
}
|
||
]
|
||
}
|
||
|
||
return jsonify(help_content)
|
||
|
||
|
||
# =============================================================================
|
||
# WebSocket Events
|
||
# =============================================================================
|
||
|
||
@socketio.on('connect')
|
||
def handle_connect():
|
||
"""Client connecté."""
|
||
logger.info("Client connected")
|
||
emit('status', execution_status)
|
||
|
||
|
||
@socketio.on('disconnect')
|
||
def handle_disconnect():
|
||
"""Client déconnecté."""
|
||
logger.info("Client disconnected")
|
||
|
||
|
||
@socketio.on('cancel_execution')
|
||
def handle_cancel():
|
||
"""Annuler l'exécution."""
|
||
global execution_status
|
||
execution_status["running"] = False
|
||
execution_status["message"] = "Annulé"
|
||
emit('execution_cancelled', {}, broadcast=True)
|
||
|
||
|
||
# =============================================================================
|
||
# Exécution de workflow
|
||
# =============================================================================
|
||
|
||
def _try_streaming_server_replay(workflow_id: str, params: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
||
"""
|
||
Tenter d'exécuter un workflow via le streaming server (Agent V1).
|
||
|
||
POST http://localhost:5005/api/v1/traces/stream/replay
|
||
avec workflow_id et params.
|
||
|
||
Returns:
|
||
dict avec le résultat si succès.
|
||
dict avec {"error": ...} si le serveur est UP mais refuse (ex: pas de session).
|
||
None si le serveur est injoignable (connexion refusée, timeout).
|
||
"""
|
||
try:
|
||
resp = http_requests.post(
|
||
f"{STREAMING_SERVER_URL}/api/v1/traces/stream/replay",
|
||
json={
|
||
"workflow_id": workflow_id,
|
||
"session_id": "", # Vide = auto-détection de la session Agent V1 active
|
||
"params": params or {},
|
||
},
|
||
timeout=15,
|
||
)
|
||
if resp.status_code == 200:
|
||
data = resp.json()
|
||
logger.info(f"Workflow {workflow_id} envoyé au streaming server: {data}")
|
||
return data
|
||
else:
|
||
# Le serveur est UP mais refuse — renvoyer l'erreur pour éviter le fallback local
|
||
error_detail = resp.text[:200]
|
||
logger.warning(
|
||
f"Streaming server refus (HTTP {resp.status_code}): {error_detail}"
|
||
)
|
||
return {"error": error_detail, "status_code": resp.status_code}
|
||
except http_requests.ConnectionError:
|
||
logger.debug("Streaming server non disponible (connexion refusée)")
|
||
except http_requests.Timeout:
|
||
logger.debug("Streaming server timeout")
|
||
except Exception as e:
|
||
logger.debug(f"Erreur streaming server: {e}")
|
||
|
||
return None
|
||
|
||
|
||
def _poll_replay_progress(replay_id: str, workflow_name: str, total_actions: int):
|
||
"""Suivre la progression d'un replay distant via polling."""
|
||
import time
|
||
|
||
max_wait = 120 # 2 minutes max
|
||
poll_interval = 2.0
|
||
elapsed = 0
|
||
|
||
while elapsed < max_wait and execution_status.get("running"):
|
||
time.sleep(poll_interval)
|
||
elapsed += poll_interval
|
||
|
||
try:
|
||
resp = http_requests.get(
|
||
f"{STREAMING_SERVER_URL}/api/v1/traces/stream/replay/{replay_id}",
|
||
timeout=3,
|
||
)
|
||
if not resp.ok:
|
||
continue
|
||
|
||
data = resp.json()
|
||
status = data.get("status", "unknown")
|
||
completed = data.get("completed_actions", 0)
|
||
failed = data.get("failed_actions", 0)
|
||
progress = int(10 + (completed / max(total_actions, 1)) * 80)
|
||
|
||
socketio.emit('execution_progress', {
|
||
"progress": progress,
|
||
"step": f"Action {completed}/{total_actions} exécutée",
|
||
"current": completed,
|
||
"total": total_actions,
|
||
})
|
||
|
||
if status == "completed":
|
||
finish_execution(
|
||
workflow_name, failed == 0,
|
||
f"Replay terminé : {completed} actions exécutées"
|
||
+ (f", {failed} échecs" if failed else "")
|
||
)
|
||
return
|
||
elif status == "failed":
|
||
finish_execution(
|
||
workflow_name, False,
|
||
f"Replay échoué : {completed}/{total_actions} actions, {failed} échecs"
|
||
)
|
||
return
|
||
|
||
except Exception as e:
|
||
logger.debug(f"Poll replay progress: {e}")
|
||
|
||
# Timeout
|
||
if execution_status.get("running"):
|
||
finish_execution(
|
||
workflow_name, False,
|
||
f"Timeout — replay toujours en cours après {max_wait}s"
|
||
)
|
||
|
||
|
||
def execute_workflow(match, params):
|
||
"""
|
||
Exécuter un workflow — tente d'abord le streaming server,
|
||
puis fallback sur l'exécution locale.
|
||
"""
|
||
global execution_status
|
||
|
||
import time
|
||
|
||
# Tenter l'exécution via le streaming server (Agent V1 distant)
|
||
replay_result = _try_streaming_server_replay(match.workflow_id, params)
|
||
if replay_result:
|
||
# Vérifier si c'est une erreur du serveur (UP mais pas de session/workflow)
|
||
if "error" in replay_result:
|
||
error_msg = replay_result["error"]
|
||
status_code = replay_result.get("status_code", 0)
|
||
if status_code == 404 and "session" in error_msg.lower():
|
||
# Pas de session Agent V1 active — NE PAS faire de fallback local
|
||
finish_execution(
|
||
match.workflow_name, False,
|
||
"Aucun Agent V1 connecté. Lancez l'agent sur le PC cible "
|
||
"et démarrez une session d'abord."
|
||
)
|
||
elif status_code == 404:
|
||
finish_execution(
|
||
match.workflow_name, False,
|
||
f"Workflow '{match.workflow_id}' non trouvé sur le serveur de streaming."
|
||
)
|
||
else:
|
||
finish_execution(
|
||
match.workflow_name, False,
|
||
f"Erreur serveur streaming : {error_msg}"
|
||
)
|
||
return
|
||
|
||
# Le streaming server a accepté le replay
|
||
total_actions = replay_result.get("total_actions", 1)
|
||
target_session = replay_result.get("session_id", "?")
|
||
execution_status["running"] = True
|
||
execution_status["workflow"] = match.workflow_name
|
||
execution_status["progress"] = 10
|
||
execution_status["message"] = f"Envoyé à l'Agent V1 ({target_session})"
|
||
|
||
socketio.emit('execution_progress', {
|
||
"progress": 10,
|
||
"step": f"Replay envoyé à l'Agent V1 — {total_actions} actions en attente",
|
||
"current": 0,
|
||
"total": total_actions,
|
||
})
|
||
|
||
# Suivre la progression du replay (polling toutes les 2s, max 120s)
|
||
replay_id = replay_result.get("replay_id")
|
||
if replay_id:
|
||
socketio.start_background_task(
|
||
_poll_replay_progress, replay_id, match.workflow_name, total_actions
|
||
)
|
||
else:
|
||
finish_execution(
|
||
match.workflow_name, True,
|
||
f"Replay envoyé ({total_actions} actions)"
|
||
)
|
||
return
|
||
|
||
# Fallback : exécution locale (seulement si le streaming server est injoignable)
|
||
logger.info("Streaming server injoignable, exécution locale")
|
||
|
||
execution_status["running"] = True
|
||
execution_status["workflow"] = match.workflow_name
|
||
execution_status["progress"] = 0
|
||
execution_status["message"] = "Démarrage (exécution locale)..."
|
||
|
||
try:
|
||
# Charger le workflow
|
||
with open(match.workflow_path, 'r') as f:
|
||
workflow_data = json.load(f)
|
||
|
||
# Créer le VariableManager et injecter les paramètres
|
||
var_manager = VariableManager()
|
||
var_manager.set_variables(params)
|
||
|
||
# Substituer les variables
|
||
workflow_data = var_manager.substitute_dict(workflow_data)
|
||
|
||
# Obtenir les étapes (edges)
|
||
edges = workflow_data.get("edges", [])
|
||
total_steps = len(edges) if edges else 1
|
||
|
||
# Étape 1: Initialisation
|
||
update_progress(10, "Initialisation", 1, total_steps + 2)
|
||
time.sleep(0.3)
|
||
|
||
# Étape 2: Préparation GPU (si disponible)
|
||
if gpu_manager and GPU_AVAILABLE:
|
||
update_progress(15, "Préparation GPU...", 2, total_steps + 2)
|
||
try:
|
||
loop = asyncio.new_event_loop()
|
||
asyncio.set_event_loop(loop)
|
||
loop.run_until_complete(gpu_manager.set_execution_mode(ExecutionMode.AUTOPILOT))
|
||
loop.close()
|
||
except Exception as e:
|
||
logger.warning(f"GPU mode change failed: {e}")
|
||
|
||
# Vérifier si l'exécution réelle est disponible
|
||
use_real_execution = (
|
||
EXECUTION_AVAILABLE and
|
||
action_executor is not None and
|
||
screen_capturer is not None
|
||
)
|
||
|
||
if use_real_execution:
|
||
logger.info(f"🚀 Exécution RÉELLE du workflow: {match.workflow_name}")
|
||
_execute_workflow_real(workflow_data, edges, total_steps, params)
|
||
else:
|
||
logger.info(f"🎭 Exécution SIMULÉE du workflow: {match.workflow_name}")
|
||
_execute_workflow_simulated(edges, total_steps)
|
||
|
||
# Finalisation
|
||
if execution_status["running"]:
|
||
update_progress(95, "Finalisation...", total_steps + 1, total_steps + 2)
|
||
time.sleep(0.2)
|
||
finish_execution(match.workflow_name, True, "Workflow terminé avec succès")
|
||
|
||
except Exception as e:
|
||
logger.error(f"Execution error: {e}")
|
||
import traceback
|
||
traceback.print_exc()
|
||
finish_execution(match.workflow_name, False, f"Erreur: {str(e)}")
|
||
|
||
|
||
def _execute_workflow_real(workflow_data, edges, total_steps, params):
|
||
"""Exécution réelle avec ActionExecutor."""
|
||
import time
|
||
import tempfile
|
||
import os
|
||
from PIL import Image
|
||
|
||
# Capturer l'écran initial
|
||
update_progress(20, "Capture écran initial...", 2, total_steps + 2)
|
||
|
||
try:
|
||
screenshot_array = screen_capturer.capture()
|
||
if screenshot_array is None:
|
||
raise Exception("Capture retourne None")
|
||
|
||
# Sauvegarder le screenshot dans un fichier temporaire
|
||
temp_dir = Path("data/temp")
|
||
temp_dir.mkdir(parents=True, exist_ok=True)
|
||
screenshot_path = str(temp_dir / f"capture_{datetime.now().strftime('%H%M%S')}.png")
|
||
|
||
# Convertir numpy array en image et sauvegarder
|
||
img = Image.fromarray(screenshot_array)
|
||
img.save(screenshot_path)
|
||
logger.info(f"📸 Screenshot capturé: {screenshot_path}")
|
||
except Exception as e:
|
||
logger.warning(f"Capture écran échouée: {e}, utilisation mode dégradé")
|
||
_execute_workflow_simulated(edges, total_steps)
|
||
return
|
||
|
||
# Créer le ScreenState complet pour l'exécution
|
||
try:
|
||
from core.models.screen_state import (
|
||
ScreenState, WindowContext, RawLevel, PerceptionLevel,
|
||
ContextLevel, EmbeddingRef
|
||
)
|
||
|
||
file_size = os.path.getsize(screenshot_path) if os.path.exists(screenshot_path) else 0
|
||
|
||
screen_state = ScreenState(
|
||
screen_state_id=f"agent_{datetime.now().strftime('%Y%m%d_%H%M%S')}",
|
||
timestamp=datetime.now(),
|
||
session_id="agent_chat",
|
||
window=WindowContext(
|
||
app_name="unknown",
|
||
window_title="Unknown",
|
||
screen_resolution=[1920, 1080],
|
||
workspace="main"
|
||
),
|
||
raw=RawLevel(
|
||
screenshot_path=screenshot_path,
|
||
capture_method="agent_chat",
|
||
file_size_bytes=file_size
|
||
),
|
||
perception=PerceptionLevel(
|
||
embedding=EmbeddingRef(provider="", vector_id="", dimensions=512),
|
||
detected_text=[],
|
||
text_detection_method="none",
|
||
confidence_avg=0.0
|
||
),
|
||
context=ContextLevel(),
|
||
ui_elements=[]
|
||
)
|
||
logger.info("✓ ScreenState complet créé")
|
||
except Exception as e:
|
||
logger.warning(f"Création ScreenState échouée: {e}, utilisation mode simulé")
|
||
_execute_workflow_simulated(edges, total_steps)
|
||
return
|
||
|
||
# Exécuter chaque edge avec ActionExecutor
|
||
success_count = 0
|
||
fail_count = 0
|
||
|
||
for i, edge in enumerate(edges):
|
||
if not execution_status["running"]:
|
||
logger.info("⏹️ Exécution annulée par l'utilisateur")
|
||
break
|
||
|
||
action = edge.get("action", {})
|
||
action_type = action.get("type", "unknown")
|
||
|
||
progress = int(20 + (i + 1) / total_steps * 70)
|
||
step_name = f"Étape {i+1}/{total_steps}: {action_type}"
|
||
|
||
update_progress(progress, step_name, i + 3, total_steps + 2)
|
||
logger.info(f"▶️ Exécution: {step_name}")
|
||
|
||
try:
|
||
# Créer un objet Edge compatible avec ActionExecutor
|
||
workflow_edge = _create_workflow_edge(edge, params)
|
||
|
||
# Exécuter l'action réelle
|
||
result = action_executor.execute_edge(workflow_edge, screen_state)
|
||
|
||
if result.status.value == "success":
|
||
success_count += 1
|
||
logger.info(f"✅ {step_name} - Succès")
|
||
|
||
# Recapturer l'écran après chaque action réussie
|
||
try:
|
||
time.sleep(0.3) # Petit délai pour laisser l'UI se mettre à jour
|
||
new_screenshot = screen_capturer.capture()
|
||
if new_screenshot is not None:
|
||
new_path = str(temp_dir / f"capture_{datetime.now().strftime('%H%M%S_%f')}.png")
|
||
Image.fromarray(new_screenshot).save(new_path)
|
||
screen_state.raw.screenshot_path = new_path
|
||
screen_state.raw.file_size_bytes = os.path.getsize(new_path)
|
||
except Exception as recapture_err:
|
||
logger.debug(f"Recapture échouée: {recapture_err}")
|
||
pass # Continuer même si la recapture échoue
|
||
else:
|
||
fail_count += 1
|
||
logger.warning(f"⚠️ {step_name} - {result.status.value}: {result.message}")
|
||
|
||
# Continuer malgré l'échec (mode best-effort)
|
||
if fail_count >= 3:
|
||
logger.error("❌ Trop d'échecs, arrêt de l'exécution")
|
||
break
|
||
|
||
except Exception as e:
|
||
fail_count += 1
|
||
logger.error(f"❌ Erreur lors de {step_name}: {e}")
|
||
|
||
if fail_count >= 3:
|
||
logger.error("❌ Trop d'erreurs, arrêt de l'exécution")
|
||
break
|
||
|
||
logger.info(f"📊 Résultat: {success_count} succès, {fail_count} échecs sur {total_steps} étapes")
|
||
|
||
|
||
def _execute_workflow_simulated(edges, total_steps):
|
||
"""Exécution simulée (fallback)."""
|
||
import time
|
||
|
||
for i, edge in enumerate(edges):
|
||
if not execution_status["running"]:
|
||
break
|
||
|
||
action = edge.get("action", {})
|
||
action_type = action.get("type", "unknown")
|
||
|
||
progress = int(20 + (i + 1) / total_steps * 70)
|
||
step_name = f"Étape {i+1}: {action_type} (simulé)"
|
||
|
||
update_progress(progress, step_name, i + 3, total_steps + 2)
|
||
|
||
# Simulation avec délai
|
||
time.sleep(0.5)
|
||
logger.info(f"🎭 Simulé: {step_name}")
|
||
|
||
|
||
def _create_workflow_edge(edge_dict, params):
|
||
"""Créer un objet WorkflowEdge depuis un dictionnaire."""
|
||
from dataclasses import dataclass, field
|
||
from typing import Optional, Dict, Any, List
|
||
|
||
# Importer les vraies classes du système
|
||
try:
|
||
from core.models.workflow_graph import Action, TargetSpec, WorkflowEdge, EdgeConstraints, PostConditions, ActionType
|
||
USE_REAL_CLASSES = True
|
||
except ImportError:
|
||
USE_REAL_CLASSES = False
|
||
|
||
action_dict = edge_dict.get("action", {})
|
||
target_dict = action_dict.get("target", {})
|
||
|
||
# Substituer les paramètres dans le texte cible
|
||
target_text = target_dict.get("text", "")
|
||
if target_text and isinstance(target_text, str):
|
||
for key, val in params.items():
|
||
target_text = target_text.replace(f"{{{{{key}}}}}", str(val)) # {{key}}
|
||
target_text = target_text.replace(f"${{{key}}}", str(val)) # ${key}
|
||
target_text = target_text.replace(f"${key}", str(val)) # $key
|
||
|
||
if USE_REAL_CLASSES:
|
||
# Créer le TargetSpec avec les vraies classes
|
||
target_spec = TargetSpec(
|
||
by_role=target_dict.get("role"),
|
||
by_text=target_text if target_text else target_dict.get("text"),
|
||
by_position=target_dict.get("position"),
|
||
)
|
||
|
||
# Convertir le type d'action en ActionType Enum
|
||
action_type_str = action_dict.get("type", "mouse_click")
|
||
try:
|
||
action_type = ActionType(action_type_str)
|
||
except ValueError:
|
||
# Fallback si le type n'existe pas dans l'enum
|
||
action_type = ActionType.MOUSE_CLICK
|
||
|
||
# Créer l'Action
|
||
action = Action(
|
||
type=action_type,
|
||
target=target_spec,
|
||
parameters=action_dict.get("parameters", {})
|
||
)
|
||
|
||
# Créer le WorkflowEdge
|
||
return WorkflowEdge(
|
||
edge_id=edge_dict.get("edge_id", edge_dict.get("id", f"edge_{id(edge_dict)}")),
|
||
from_node=edge_dict.get("source", edge_dict.get("from_node", "")),
|
||
to_node=edge_dict.get("target", edge_dict.get("to_node", "")),
|
||
action=action,
|
||
constraints=EdgeConstraints(),
|
||
post_conditions=PostConditions()
|
||
)
|
||
else:
|
||
# Fallback: classes simples
|
||
@dataclass
|
||
class SimpleTargetSpec:
|
||
by_role: Optional[str] = None
|
||
by_text: Optional[str] = None
|
||
by_position: Optional[tuple] = None
|
||
selection_policy: str = "first"
|
||
fallback_strategy: str = "visual_similarity"
|
||
embedding_ref: Optional[Any] = None
|
||
context_hints: Dict[str, Any] = field(default_factory=dict)
|
||
hard_constraints: Dict[str, Any] = field(default_factory=dict)
|
||
weights: Dict[str, float] = field(default_factory=dict)
|
||
|
||
@dataclass
|
||
class SimpleAction:
|
||
type: str
|
||
target: SimpleTargetSpec
|
||
parameters: Dict[str, Any] = field(default_factory=dict)
|
||
|
||
@dataclass
|
||
class SimpleWorkflowEdge:
|
||
edge_id: str
|
||
from_node: str
|
||
to_node: str
|
||
action: SimpleAction
|
||
constraints: Any = None
|
||
post_conditions: Any = None
|
||
|
||
target_spec = SimpleTargetSpec(
|
||
by_role=target_dict.get("role"),
|
||
by_text=target_text if target_text else target_dict.get("text"),
|
||
)
|
||
|
||
action = SimpleAction(
|
||
type=action_dict.get("type", "unknown"),
|
||
target=target_spec,
|
||
parameters=action_dict.get("parameters", {})
|
||
)
|
||
|
||
return SimpleWorkflowEdge(
|
||
edge_id=edge_dict.get("edge_id", edge_dict.get("id", f"edge_{id(edge_dict)}")),
|
||
from_node=edge_dict.get("source", edge_dict.get("from_node", "")),
|
||
to_node=edge_dict.get("target", edge_dict.get("to_node", "")),
|
||
action=action,
|
||
constraints=None,
|
||
post_conditions=None
|
||
)
|
||
|
||
|
||
def update_progress(progress: int, message: str, current: int, total: int):
|
||
"""Mettre à jour la progression."""
|
||
global execution_status
|
||
|
||
execution_status["progress"] = progress
|
||
execution_status["message"] = message
|
||
|
||
socketio.emit('execution_progress', {
|
||
"progress": progress,
|
||
"step": message,
|
||
"current": current,
|
||
"total": total
|
||
})
|
||
|
||
|
||
def finish_execution(workflow_name: str, success: bool, message: str):
|
||
"""Terminer l'exécution."""
|
||
global execution_status
|
||
|
||
execution_status["running"] = False
|
||
execution_status["progress"] = 100 if success else 0
|
||
execution_status["message"] = message
|
||
|
||
# Mettre à jour l'historique
|
||
if command_history:
|
||
command_history[-1]["status"] = "completed" if success else "failed"
|
||
|
||
socketio.emit('execution_completed', {
|
||
"workflow": workflow_name,
|
||
"success": success,
|
||
"message": message
|
||
})
|
||
|
||
|
||
# =============================================================================
|
||
# Main
|
||
# =============================================================================
|
||
|
||
if __name__ == '__main__':
|
||
init_system()
|
||
|
||
print("""
|
||
╔════════════════════════════════════════════════════════════╗
|
||
║ RPA Vision V3 - Interface de Commande ║
|
||
║ ║
|
||
║ 🌐 http://localhost:5004 ║
|
||
║ ║
|
||
║ Ctrl+C pour arrêter ║
|
||
╚════════════════════════════════════════════════════════════╝
|
||
""")
|
||
|
||
socketio.run(app, host='0.0.0.0', port=5004, debug=False, allow_unsafe_werkzeug=True)
|