feat: boucle ORA (observe→raisonne→agit) avec vérification post-action
Some checks failed
security-audit / Bandit (scan statique) (push) Successful in 13s
security-audit / pip-audit (CVE dépendances) (push) Successful in 11s
security-audit / Scan secrets (grep) (push) Successful in 8s
tests / Lint (ruff + black) (push) Successful in 14s
tests / Tests unitaires (sans GPU) (push) Failing after 15s
tests / Tests sécurité (critique) (push) Has been skipped
Some checks failed
security-audit / Bandit (scan statique) (push) Successful in 13s
security-audit / pip-audit (CVE dépendances) (push) Successful in 11s
security-audit / Scan secrets (grep) (push) Successful in 8s
tests / Lint (ruff + black) (push) Successful in 14s
tests / Tests unitaires (sans GPU) (push) Failing after 15s
tests / Tests sécurité (critique) (push) Has been skipped
Nouveau module core/execution/observe_reason_act.py (794 lignes) : - ORALoop : boucle unifiée pour workflow VWB et instructions - observe() : capture écran + pHash + titre fenêtre - reason_workflow_step() : mappe step VWB → Decision (sans VLM) - act() : template matching → find_element → pyautogui - verify() : Level 1 pHash + Level 2 VLM conditionnel - run_workflow() : boucle complète avec retries et callbacks Nouveau mode execution_mode='verified' dans execute.py : - run_workflow_verified() utilise ORALoop - Modes basic/intelligent/debug inchangés (zéro risque) Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -1346,6 +1346,182 @@ def execute_action(action_type: str, params: dict) -> dict:
|
||||
return {'success': False, 'error': str(e)}
|
||||
|
||||
|
||||
def run_workflow_verified(execution_id: str, workflow_id: str, app):
|
||||
"""
|
||||
Thread d'exécution en mode 'verified' — boucle ORA.
|
||||
|
||||
Charge les steps du workflow depuis la BDD, construit les params
|
||||
comme execute_workflow_thread, puis délègue à ORALoop.run_workflow().
|
||||
|
||||
NE modifie PAS execute_workflow_thread : les modes basic/intelligent/debug
|
||||
continuent de fonctionner exactement comme avant.
|
||||
"""
|
||||
global _execution_state
|
||||
|
||||
with app.app_context():
|
||||
try:
|
||||
execution = Execution.query.get(execution_id)
|
||||
workflow = Workflow.query.get(workflow_id)
|
||||
|
||||
if not execution or not workflow:
|
||||
logger.error("[ORA] Workflow ou exécution non trouvé")
|
||||
return
|
||||
|
||||
steps_db = workflow.steps.order_by(Step.order).all()
|
||||
execution.total_steps = len(steps_db)
|
||||
execution.status = 'running'
|
||||
execution.started_at = datetime.utcnow()
|
||||
db.session.commit()
|
||||
|
||||
logger.info(f"🚀 [ORA] Démarrage workflow vérifié {workflow_id}: {len(steps_db)} étapes")
|
||||
|
||||
# --- Construire les params pour chaque step (même logique que execute_workflow_thread) ---
|
||||
ora_steps = []
|
||||
for step in steps_db:
|
||||
params = step.parameters or {}
|
||||
|
||||
# Charger l'ancre visuelle si présente
|
||||
if step.anchor_id:
|
||||
anchor = VisualAnchor.query.get(step.anchor_id)
|
||||
if anchor:
|
||||
anchor_image_path = anchor.thumbnail_path or anchor.image_path
|
||||
if anchor_image_path and os.path.exists(anchor_image_path):
|
||||
with open(anchor_image_path, 'rb') as f:
|
||||
image_base64 = base64.b64encode(f.read()).decode('utf-8')
|
||||
else:
|
||||
image_base64 = None
|
||||
|
||||
anchor_data = {
|
||||
'anchor_id': anchor.id,
|
||||
'screenshot': image_base64,
|
||||
'bounding_box': {
|
||||
'x': anchor.bbox_x,
|
||||
'y': anchor.bbox_y,
|
||||
'width': anchor.bbox_width,
|
||||
'height': anchor.bbox_height,
|
||||
},
|
||||
'metadata': {
|
||||
'screen_resolution': {
|
||||
'width': anchor.screen_width,
|
||||
'height': anchor.screen_height,
|
||||
}
|
||||
},
|
||||
}
|
||||
if anchor.target_text:
|
||||
anchor_data['target_text'] = anchor.target_text
|
||||
if anchor.ocr_description:
|
||||
anchor_data['description'] = anchor.ocr_description
|
||||
|
||||
params['visual_anchor'] = anchor_data
|
||||
|
||||
# Injecter le label pour le grounding
|
||||
if step.label:
|
||||
params['_step_label'] = step.label
|
||||
|
||||
ora_steps.append({
|
||||
'action_type': step.action_type,
|
||||
'parameters': params,
|
||||
'visual_anchor': params.get('visual_anchor', {}),
|
||||
'label': step.label or step.action_type,
|
||||
'_full_params': params, # Pour les passthrough
|
||||
'_step_id': step.id,
|
||||
})
|
||||
|
||||
# --- Créer et lancer la boucle ORA ---
|
||||
from core.execution.observe_reason_act import ORALoop
|
||||
|
||||
ora = ORALoop(max_retries=2, max_steps=50, verify_level='auto')
|
||||
ora._variables = _execution_state.get('variables', {})
|
||||
|
||||
# Créer les ExecutionStep en amont pour le suivi
|
||||
step_results_map = {}
|
||||
for idx, step in enumerate(steps_db):
|
||||
step_result = ExecutionStep(
|
||||
execution_id=execution_id,
|
||||
step_id=step.id,
|
||||
status='pending',
|
||||
)
|
||||
db.session.add(step_result)
|
||||
step_results_map[idx] = step_result
|
||||
db.session.commit()
|
||||
|
||||
def on_progress(step_index, total, verification):
|
||||
"""Callback de progression — met à jour la BDD."""
|
||||
try:
|
||||
sr = step_results_map.get(step_index - 1)
|
||||
if sr:
|
||||
sr.status = 'success' if verification.success else 'error'
|
||||
sr.started_at = sr.started_at or datetime.utcnow()
|
||||
sr.ended_at = datetime.utcnow()
|
||||
if sr.started_at:
|
||||
sr.duration_ms = int((sr.ended_at - sr.started_at).total_seconds() * 1000)
|
||||
sr.output = {
|
||||
'change_level': verification.change_level,
|
||||
'matches_expected': verification.matches_expected,
|
||||
'detail': verification.detail,
|
||||
'mode': 'verified',
|
||||
}
|
||||
if verification.success:
|
||||
execution.completed_steps = step_index
|
||||
else:
|
||||
execution.failed_steps = (execution.failed_steps or 0) + 1
|
||||
sr.error_message = verification.detail
|
||||
execution.current_step_index = step_index
|
||||
db.session.commit()
|
||||
except Exception as e:
|
||||
logger.warning(f"[ORA/progress] Erreur BDD: {e}")
|
||||
|
||||
# Exécuter
|
||||
loop_result = ora.run_workflow(
|
||||
steps=ora_steps,
|
||||
on_progress=on_progress,
|
||||
execute_action_fn=execute_action,
|
||||
)
|
||||
|
||||
# Synchroniser les variables runtime
|
||||
_execution_state['variables'] = ora._variables
|
||||
|
||||
# Finaliser
|
||||
if loop_result.success:
|
||||
execution.status = 'completed'
|
||||
else:
|
||||
execution.status = 'error'
|
||||
execution.error_message = loop_result.reason
|
||||
# Marquer le step en échec
|
||||
failed_sr = step_results_map.get(loop_result.steps_completed)
|
||||
if failed_sr and failed_sr.status == 'pending':
|
||||
failed_sr.status = 'error'
|
||||
failed_sr.error_message = loop_result.reason
|
||||
failed_sr.ended_at = datetime.utcnow()
|
||||
|
||||
execution.ended_at = datetime.utcnow()
|
||||
execution.completed_steps = loop_result.steps_completed
|
||||
db.session.commit()
|
||||
|
||||
logger.info(
|
||||
f"{'✅' if loop_result.success else '❌'} [ORA] Workflow terminé: "
|
||||
f"{loop_result.steps_completed}/{loop_result.total_steps} "
|
||||
f"({execution.status}) — {loop_result.reason}"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"❌ [ORA] Erreur fatale: {e}", exc_info=True)
|
||||
try:
|
||||
execution = Execution.query.get(execution_id)
|
||||
if execution:
|
||||
execution.status = 'error'
|
||||
execution.error_message = f"Erreur fatale ORA: {str(e)}"
|
||||
execution.ended_at = datetime.utcnow()
|
||||
db.session.commit()
|
||||
except Exception as db_err:
|
||||
logger.warning(f"[ORA] DB cleanup error: {db_err}")
|
||||
|
||||
finally:
|
||||
with _execution_lock:
|
||||
_execution_state['is_running'] = False
|
||||
_execution_state['current_execution_id'] = None
|
||||
|
||||
|
||||
@api_v3_bp.route('/execute/start', methods=['POST'])
|
||||
def start_execution():
|
||||
"""
|
||||
@@ -1371,7 +1547,7 @@ def start_execution():
|
||||
minimize_browser = data.get('minimize_browser', True) # Activé par défaut
|
||||
|
||||
# Valider le mode
|
||||
if execution_mode not in ['basic', 'intelligent', 'debug']:
|
||||
if execution_mode not in ['basic', 'intelligent', 'debug', 'verified']:
|
||||
execution_mode = 'basic'
|
||||
|
||||
# Utiliser le workflow actif si non spécifié
|
||||
@@ -1430,15 +1606,22 @@ def start_execution():
|
||||
from flask import current_app
|
||||
app = current_app._get_current_object()
|
||||
|
||||
thread = threading.Thread(
|
||||
target=execute_workflow_thread,
|
||||
args=(execution.id, workflow_id, app)
|
||||
)
|
||||
if execution_mode == 'verified':
|
||||
# Mode ORA : boucle observe→raisonne→agit avec vérification
|
||||
thread = threading.Thread(
|
||||
target=run_workflow_verified,
|
||||
args=(execution.id, workflow_id, app)
|
||||
)
|
||||
else:
|
||||
thread = threading.Thread(
|
||||
target=execute_workflow_thread,
|
||||
args=(execution.id, workflow_id, app)
|
||||
)
|
||||
thread.daemon = True
|
||||
thread.start()
|
||||
_execution_state['thread'] = thread
|
||||
|
||||
print(f"🚀 [API v3] Exécution lancée: {execution.id}")
|
||||
print(f"🚀 [API v3] Exécution lancée: {execution.id} (mode={execution_mode})")
|
||||
|
||||
return jsonify({
|
||||
'success': True,
|
||||
|
||||
Reference in New Issue
Block a user