fix(agent_chat): Corriger intégration exécution réelle
- Importer les vraies classes Action, TargetSpec, WorkflowEdge, ActionType - Convertir le type d'action en ActionType Enum au lieu de string - Créer un ScreenState complet avec tous les niveaux (raw, perception, context) - Corriger _serialize_state dans error_handler.py pour accès compatibles - Ajouter import os pour manipulation des fichiers - Sauvegarder les screenshots dans data/temp/ L'exécution réelle fonctionne maintenant - les erreurs "Target not found" sont attendues car il faut une vraie interface utilisateur à l'écran. Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Dom
@@ -22,6 +22,7 @@ Auteur: Dom - Janvier 2026
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
@@ -517,6 +518,12 @@ def execute_workflow_from_confirmation(confirmation, session_id):
|
||||
# Utiliser les paramètres confirmés (ou modifiés)
|
||||
params = confirmation.modified_parameters or confirmation.parameters
|
||||
|
||||
# IMPORTANT: Marquer l'exécution comme active
|
||||
execution_status["running"] = True
|
||||
execution_status["workflow"] = confirmation.workflow_name
|
||||
execution_status["progress"] = 0
|
||||
execution_status["message"] = "Démarrage..."
|
||||
|
||||
execute_workflow(match, params)
|
||||
|
||||
|
||||
@@ -704,31 +711,70 @@ def execute_workflow(match, params):
|
||||
def _execute_workflow_real(workflow_data, edges, total_steps, params):
|
||||
"""Exécution réelle avec ActionExecutor."""
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
import tempfile
|
||||
import os
|
||||
from PIL import Image
|
||||
|
||||
# Capturer l'écran initial
|
||||
update_progress(20, "Capture écran initial...", 2, total_steps + 2)
|
||||
|
||||
try:
|
||||
screenshot_path = screen_capturer.capture()
|
||||
screenshot_array = screen_capturer.capture()
|
||||
if screenshot_array is None:
|
||||
raise Exception("Capture retourne None")
|
||||
|
||||
# Sauvegarder le screenshot dans un fichier temporaire
|
||||
temp_dir = Path("data/temp")
|
||||
temp_dir.mkdir(parents=True, exist_ok=True)
|
||||
screenshot_path = str(temp_dir / f"capture_{datetime.now().strftime('%H%M%S')}.png")
|
||||
|
||||
# Convertir numpy array en image et sauvegarder
|
||||
img = Image.fromarray(screenshot_array)
|
||||
img.save(screenshot_path)
|
||||
logger.info(f"📸 Screenshot capturé: {screenshot_path}")
|
||||
except Exception as e:
|
||||
logger.warning(f"Capture écran échouée: {e}, utilisation mode dégradé")
|
||||
_execute_workflow_simulated(edges, total_steps)
|
||||
return
|
||||
|
||||
# Créer le ScreenState pour l'exécution
|
||||
# Créer le ScreenState complet pour l'exécution
|
||||
try:
|
||||
from core.models import ScreenState
|
||||
screen_state = ScreenState.from_screenshot(screenshot_path)
|
||||
from core.models.screen_state import (
|
||||
ScreenState, WindowContext, RawLevel, PerceptionLevel,
|
||||
ContextLevel, EmbeddingRef
|
||||
)
|
||||
|
||||
file_size = os.path.getsize(screenshot_path) if os.path.exists(screenshot_path) else 0
|
||||
|
||||
screen_state = ScreenState(
|
||||
screen_state_id=f"agent_{datetime.now().strftime('%Y%m%d_%H%M%S')}",
|
||||
timestamp=datetime.now(),
|
||||
session_id="agent_chat",
|
||||
window=WindowContext(
|
||||
app_name="unknown",
|
||||
window_title="Unknown",
|
||||
screen_resolution=[1920, 1080],
|
||||
workspace="main"
|
||||
),
|
||||
raw=RawLevel(
|
||||
screenshot_path=screenshot_path,
|
||||
capture_method="agent_chat",
|
||||
file_size_bytes=file_size
|
||||
),
|
||||
perception=PerceptionLevel(
|
||||
embedding=EmbeddingRef(provider="", vector_id="", dimensions=512),
|
||||
detected_text=[],
|
||||
text_detection_method="none",
|
||||
confidence_avg=0.0
|
||||
),
|
||||
context=ContextLevel(),
|
||||
ui_elements=[]
|
||||
)
|
||||
logger.info("✓ ScreenState complet créé")
|
||||
except Exception as e:
|
||||
logger.warning(f"Création ScreenState échouée: {e}")
|
||||
# Créer un ScreenState minimal
|
||||
screen_state = type('ScreenState', (), {
|
||||
'screenshot_path': screenshot_path,
|
||||
'detected_elements': [],
|
||||
'timestamp': datetime.now()
|
||||
})()
|
||||
logger.warning(f"Création ScreenState échouée: {e}, utilisation mode simulé")
|
||||
_execute_workflow_simulated(edges, total_steps)
|
||||
return
|
||||
|
||||
# Exécuter chaque edge avec ActionExecutor
|
||||
success_count = 0
|
||||
@@ -762,9 +808,14 @@ def _execute_workflow_real(workflow_data, edges, total_steps, params):
|
||||
# Recapturer l'écran après chaque action réussie
|
||||
try:
|
||||
time.sleep(0.3) # Petit délai pour laisser l'UI se mettre à jour
|
||||
screenshot_path = screen_capturer.capture()
|
||||
screen_state = ScreenState.from_screenshot(screenshot_path)
|
||||
except:
|
||||
new_screenshot = screen_capturer.capture()
|
||||
if new_screenshot is not None:
|
||||
new_path = str(temp_dir / f"capture_{datetime.now().strftime('%H%M%S_%f')}.png")
|
||||
Image.fromarray(new_screenshot).save(new_path)
|
||||
screen_state.raw.screenshot_path = new_path
|
||||
screen_state.raw.file_size_bytes = os.path.getsize(new_path)
|
||||
except Exception as recapture_err:
|
||||
logger.debug(f"Recapture échouée: {recapture_err}")
|
||||
pass # Continuer même si la recapture échoue
|
||||
else:
|
||||
fail_count += 1
|
||||
@@ -812,46 +863,104 @@ def _create_workflow_edge(edge_dict, params):
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Optional, Dict, Any, List
|
||||
|
||||
@dataclass
|
||||
class Action:
|
||||
type: str
|
||||
target: Optional[Dict] = None
|
||||
value: Optional[str] = None
|
||||
parameters: Dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
@dataclass
|
||||
class WorkflowEdge:
|
||||
id: str
|
||||
source: str
|
||||
target: str
|
||||
action: Action
|
||||
pre_conditions: List[Dict] = field(default_factory=list)
|
||||
post_conditions: List[Dict] = field(default_factory=list)
|
||||
# Importer les vraies classes du système
|
||||
try:
|
||||
from core.models.workflow_graph import Action, TargetSpec, WorkflowEdge, EdgeConstraints, PostConditions, ActionType
|
||||
USE_REAL_CLASSES = True
|
||||
except ImportError:
|
||||
USE_REAL_CLASSES = False
|
||||
|
||||
action_dict = edge_dict.get("action", {})
|
||||
target_dict = action_dict.get("target", {})
|
||||
|
||||
# Substituer les paramètres dans l'action
|
||||
action_value = action_dict.get("value", "")
|
||||
if action_value and isinstance(action_value, str):
|
||||
# Substituer les paramètres dans le texte cible
|
||||
target_text = target_dict.get("text", "")
|
||||
if target_text and isinstance(target_text, str):
|
||||
for key, val in params.items():
|
||||
action_value = action_value.replace(f"${{{key}}}", str(val))
|
||||
action_value = action_value.replace(f"${key}", str(val))
|
||||
target_text = target_text.replace(f"{{{{{key}}}}}", str(val)) # {{key}}
|
||||
target_text = target_text.replace(f"${{{key}}}", str(val)) # ${key}
|
||||
target_text = target_text.replace(f"${key}", str(val)) # $key
|
||||
|
||||
action = Action(
|
||||
type=action_dict.get("type", "unknown"),
|
||||
target=action_dict.get("target"),
|
||||
value=action_value,
|
||||
parameters=action_dict.get("parameters", {})
|
||||
)
|
||||
if USE_REAL_CLASSES:
|
||||
# Créer le TargetSpec avec les vraies classes
|
||||
target_spec = TargetSpec(
|
||||
by_role=target_dict.get("role"),
|
||||
by_text=target_text if target_text else target_dict.get("text"),
|
||||
by_position=target_dict.get("position"),
|
||||
)
|
||||
|
||||
return WorkflowEdge(
|
||||
id=edge_dict.get("id", f"edge_{id(edge_dict)}"),
|
||||
source=edge_dict.get("source", ""),
|
||||
target=edge_dict.get("target", ""),
|
||||
action=action,
|
||||
pre_conditions=edge_dict.get("pre_conditions", []),
|
||||
post_conditions=edge_dict.get("post_conditions", [])
|
||||
)
|
||||
# Convertir le type d'action en ActionType Enum
|
||||
action_type_str = action_dict.get("type", "mouse_click")
|
||||
try:
|
||||
action_type = ActionType(action_type_str)
|
||||
except ValueError:
|
||||
# Fallback si le type n'existe pas dans l'enum
|
||||
action_type = ActionType.MOUSE_CLICK
|
||||
|
||||
# Créer l'Action
|
||||
action = Action(
|
||||
type=action_type,
|
||||
target=target_spec,
|
||||
parameters=action_dict.get("parameters", {})
|
||||
)
|
||||
|
||||
# Créer le WorkflowEdge
|
||||
return WorkflowEdge(
|
||||
edge_id=edge_dict.get("edge_id", edge_dict.get("id", f"edge_{id(edge_dict)}")),
|
||||
from_node=edge_dict.get("source", edge_dict.get("from_node", "")),
|
||||
to_node=edge_dict.get("target", edge_dict.get("to_node", "")),
|
||||
action=action,
|
||||
constraints=EdgeConstraints(),
|
||||
post_conditions=PostConditions()
|
||||
)
|
||||
else:
|
||||
# Fallback: classes simples
|
||||
@dataclass
|
||||
class SimpleTargetSpec:
|
||||
by_role: Optional[str] = None
|
||||
by_text: Optional[str] = None
|
||||
by_position: Optional[tuple] = None
|
||||
selection_policy: str = "first"
|
||||
fallback_strategy: str = "visual_similarity"
|
||||
embedding_ref: Optional[Any] = None
|
||||
context_hints: Dict[str, Any] = field(default_factory=dict)
|
||||
hard_constraints: Dict[str, Any] = field(default_factory=dict)
|
||||
weights: Dict[str, float] = field(default_factory=dict)
|
||||
|
||||
@dataclass
|
||||
class SimpleAction:
|
||||
type: str
|
||||
target: SimpleTargetSpec
|
||||
parameters: Dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
@dataclass
|
||||
class SimpleWorkflowEdge:
|
||||
edge_id: str
|
||||
from_node: str
|
||||
to_node: str
|
||||
action: SimpleAction
|
||||
constraints: Any = None
|
||||
post_conditions: Any = None
|
||||
|
||||
target_spec = SimpleTargetSpec(
|
||||
by_role=target_dict.get("role"),
|
||||
by_text=target_text if target_text else target_dict.get("text"),
|
||||
)
|
||||
|
||||
action = SimpleAction(
|
||||
type=action_dict.get("type", "unknown"),
|
||||
target=target_spec,
|
||||
parameters=action_dict.get("parameters", {})
|
||||
)
|
||||
|
||||
return SimpleWorkflowEdge(
|
||||
edge_id=edge_dict.get("edge_id", edge_dict.get("id", f"edge_{id(edge_dict)}")),
|
||||
from_node=edge_dict.get("source", edge_dict.get("from_node", "")),
|
||||
to_node=edge_dict.get("target", edge_dict.get("to_node", "")),
|
||||
action=action,
|
||||
constraints=None,
|
||||
post_conditions=None
|
||||
)
|
||||
|
||||
|
||||
def update_progress(progress: int, message: str, current: int, total: int):
|
||||
|
||||
Reference in New Issue
Block a user