feat: boucle ORA (observe→raisonne→agit) avec vérification post-action
Some checks failed
security-audit / Bandit (scan statique) (push) Successful in 13s
security-audit / pip-audit (CVE dépendances) (push) Successful in 11s
security-audit / Scan secrets (grep) (push) Successful in 8s
tests / Lint (ruff + black) (push) Successful in 14s
tests / Tests unitaires (sans GPU) (push) Failing after 15s
tests / Tests sécurité (critique) (push) Has been skipped

Nouveau module core/execution/observe_reason_act.py (794 lignes) :
- ORALoop : boucle unifiée pour workflow VWB et instructions
- observe() : capture écran + pHash + titre fenêtre
- reason_workflow_step() : mappe step VWB → Decision (sans VLM)
- act() : template matching → find_element → pyautogui
- verify() : Level 1 pHash + Level 2 VLM conditionnel
- run_workflow() : boucle complète avec retries et callbacks

Nouveau mode execution_mode='verified' dans execute.py :
- run_workflow_verified() utilise ORALoop
- Modes basic/intelligent/debug inchangés (zéro risque)

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Dom
2026-04-22 09:02:54 +02:00
parent 5027ed9a23
commit 0c5fffe951
3 changed files with 990 additions and 6 deletions

View File

@@ -1346,6 +1346,182 @@ def execute_action(action_type: str, params: dict) -> dict:
return {'success': False, 'error': str(e)}
def run_workflow_verified(execution_id: str, workflow_id: str, app):
"""
Thread d'exécution en mode 'verified' — boucle ORA.
Charge les steps du workflow depuis la BDD, construit les params
comme execute_workflow_thread, puis délègue à ORALoop.run_workflow().
NE modifie PAS execute_workflow_thread : les modes basic/intelligent/debug
continuent de fonctionner exactement comme avant.
"""
global _execution_state
with app.app_context():
try:
execution = Execution.query.get(execution_id)
workflow = Workflow.query.get(workflow_id)
if not execution or not workflow:
logger.error("[ORA] Workflow ou exécution non trouvé")
return
steps_db = workflow.steps.order_by(Step.order).all()
execution.total_steps = len(steps_db)
execution.status = 'running'
execution.started_at = datetime.utcnow()
db.session.commit()
logger.info(f"🚀 [ORA] Démarrage workflow vérifié {workflow_id}: {len(steps_db)} étapes")
# --- Construire les params pour chaque step (même logique que execute_workflow_thread) ---
ora_steps = []
for step in steps_db:
params = step.parameters or {}
# Charger l'ancre visuelle si présente
if step.anchor_id:
anchor = VisualAnchor.query.get(step.anchor_id)
if anchor:
anchor_image_path = anchor.thumbnail_path or anchor.image_path
if anchor_image_path and os.path.exists(anchor_image_path):
with open(anchor_image_path, 'rb') as f:
image_base64 = base64.b64encode(f.read()).decode('utf-8')
else:
image_base64 = None
anchor_data = {
'anchor_id': anchor.id,
'screenshot': image_base64,
'bounding_box': {
'x': anchor.bbox_x,
'y': anchor.bbox_y,
'width': anchor.bbox_width,
'height': anchor.bbox_height,
},
'metadata': {
'screen_resolution': {
'width': anchor.screen_width,
'height': anchor.screen_height,
}
},
}
if anchor.target_text:
anchor_data['target_text'] = anchor.target_text
if anchor.ocr_description:
anchor_data['description'] = anchor.ocr_description
params['visual_anchor'] = anchor_data
# Injecter le label pour le grounding
if step.label:
params['_step_label'] = step.label
ora_steps.append({
'action_type': step.action_type,
'parameters': params,
'visual_anchor': params.get('visual_anchor', {}),
'label': step.label or step.action_type,
'_full_params': params, # Pour les passthrough
'_step_id': step.id,
})
# --- Créer et lancer la boucle ORA ---
from core.execution.observe_reason_act import ORALoop
ora = ORALoop(max_retries=2, max_steps=50, verify_level='auto')
ora._variables = _execution_state.get('variables', {})
# Créer les ExecutionStep en amont pour le suivi
step_results_map = {}
for idx, step in enumerate(steps_db):
step_result = ExecutionStep(
execution_id=execution_id,
step_id=step.id,
status='pending',
)
db.session.add(step_result)
step_results_map[idx] = step_result
db.session.commit()
def on_progress(step_index, total, verification):
"""Callback de progression — met à jour la BDD."""
try:
sr = step_results_map.get(step_index - 1)
if sr:
sr.status = 'success' if verification.success else 'error'
sr.started_at = sr.started_at or datetime.utcnow()
sr.ended_at = datetime.utcnow()
if sr.started_at:
sr.duration_ms = int((sr.ended_at - sr.started_at).total_seconds() * 1000)
sr.output = {
'change_level': verification.change_level,
'matches_expected': verification.matches_expected,
'detail': verification.detail,
'mode': 'verified',
}
if verification.success:
execution.completed_steps = step_index
else:
execution.failed_steps = (execution.failed_steps or 0) + 1
sr.error_message = verification.detail
execution.current_step_index = step_index
db.session.commit()
except Exception as e:
logger.warning(f"[ORA/progress] Erreur BDD: {e}")
# Exécuter
loop_result = ora.run_workflow(
steps=ora_steps,
on_progress=on_progress,
execute_action_fn=execute_action,
)
# Synchroniser les variables runtime
_execution_state['variables'] = ora._variables
# Finaliser
if loop_result.success:
execution.status = 'completed'
else:
execution.status = 'error'
execution.error_message = loop_result.reason
# Marquer le step en échec
failed_sr = step_results_map.get(loop_result.steps_completed)
if failed_sr and failed_sr.status == 'pending':
failed_sr.status = 'error'
failed_sr.error_message = loop_result.reason
failed_sr.ended_at = datetime.utcnow()
execution.ended_at = datetime.utcnow()
execution.completed_steps = loop_result.steps_completed
db.session.commit()
logger.info(
f"{'' if loop_result.success else ''} [ORA] Workflow terminé: "
f"{loop_result.steps_completed}/{loop_result.total_steps} "
f"({execution.status}) — {loop_result.reason}"
)
except Exception as e:
logger.error(f"❌ [ORA] Erreur fatale: {e}", exc_info=True)
try:
execution = Execution.query.get(execution_id)
if execution:
execution.status = 'error'
execution.error_message = f"Erreur fatale ORA: {str(e)}"
execution.ended_at = datetime.utcnow()
db.session.commit()
except Exception as db_err:
logger.warning(f"[ORA] DB cleanup error: {db_err}")
finally:
with _execution_lock:
_execution_state['is_running'] = False
_execution_state['current_execution_id'] = None
@api_v3_bp.route('/execute/start', methods=['POST'])
def start_execution():
"""
@@ -1371,7 +1547,7 @@ def start_execution():
minimize_browser = data.get('minimize_browser', True) # Activé par défaut
# Valider le mode
if execution_mode not in ['basic', 'intelligent', 'debug']:
if execution_mode not in ['basic', 'intelligent', 'debug', 'verified']:
execution_mode = 'basic'
# Utiliser le workflow actif si non spécifié
@@ -1430,15 +1606,22 @@ def start_execution():
from flask import current_app
app = current_app._get_current_object()
thread = threading.Thread(
target=execute_workflow_thread,
args=(execution.id, workflow_id, app)
)
if execution_mode == 'verified':
# Mode ORA : boucle observe→raisonne→agit avec vérification
thread = threading.Thread(
target=run_workflow_verified,
args=(execution.id, workflow_id, app)
)
else:
thread = threading.Thread(
target=execute_workflow_thread,
args=(execution.id, workflow_id, app)
)
thread.daemon = True
thread.start()
_execution_state['thread'] = thread
print(f"🚀 [API v3] Exécution lancée: {execution.id}")
print(f"🚀 [API v3] Exécution lancée: {execution.id} (mode={execution_mode})")
return jsonify({
'success': True,