Dom/rpa_vision_v3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(vwb): log supervised competence verdicts

Browse Source
This commit is contained in:
Dom
2026-05-29 18:36:06 +02:00
parent 7ad260d02f
commit aba849324a
18 changed files with 1082 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

61
visual_workflow_builder/backend/api/lea_competences.py Normal file
View File

@@ -0,0 +1,61 @@
"""Lea competence supervision endpoints exposed by the VWB backend."""
from __future__ import annotations
from flask import Blueprint, jsonify, request
from core.competences.verdicts import (
CompetenceVerdictError,
iter_competence_verdicts,
store_competence_verdict,
)
lea_competences_bp = Blueprint(
"lea_competences",
__name__,
url_prefix="/api/v1/lea/competences",
)
@lea_competences_bp.route("/<competence_id>/verdict", methods=["POST"])
def submit_competence_verdict(competence_id: str):
"""Persist one supervised human verdict without touching YAML."""
payload = request.get_json(silent=True) or {}
try:
verdict = store_competence_verdict(competence_id, payload)
except KeyError:
return jsonify({
"success": False,
"error": f"Competence '{competence_id}' introuvable",
}), 404
except CompetenceVerdictError as exc:
return jsonify({
"success": False,
"error": str(exc),
"write_back_enabled": False,
"yaml_write": False,
}), 400
return jsonify({
"success": True,
"competence_id": competence_id,
"verdict": verdict,
"duplicate": verdict["duplicate"],
"write_back_enabled": False,
"yaml_write": False,
}), 200 if verdict["duplicate"] else 201
@lea_competences_bp.route("/<competence_id>/verdicts", methods=["GET"])
def list_competence_verdicts(competence_id: str):
"""Return logged supervised verdicts for one competence."""
return jsonify({
"success": True,
"competence_id": competence_id,
"verdicts": iter_competence_verdicts(competence_id=competence_id),
"write_back_enabled": False,
"yaml_write": False,
})

91
visual_workflow_builder/backend/api_v3/execute.py
View File

@@ -103,6 +103,33 @@ def _execute_wait_for_state(params: Dict[str, Any]) -> dict:
output = result.to_dict()
if result.matched:
return {"success": True, "output": output}
if params.get("supervised_popup_detection", False):
detected_popup = None
try:
detected_popup = _check_screen_for_patterns()
except Exception as exc:
print(f"⚠️ [wait_for_state] Detection popup indisponible: {exc}")
from visual_workflow_builder.backend.services.supervised_popup_guard import (
build_unexpected_popup_pause,
)
pause_payload = build_unexpected_popup_pause(
detected_popup,
expected_state=expected_state,
competence_id=str(params.get("competence_id") or ""),
source_method_id=str(params.get("source_method_id") or ""),
)
if pause_payload:
return {
"success": True,
"needs_human_pause": True,
"output": {
"wait_for_state": output,
"human_pause": pause_payload,
"write_back_enabled": False,
},
}
return {
"success": False,
"error": "Etat attendu non observe avant timeout",
@@ -110,6 +137,23 @@ def _execute_wait_for_state(params: Dict[str, Any]) -> dict:
}
def _execute_pause_for_human(params: Dict[str, Any]) -> dict:
return {
"success": True,
"needs_human_pause": True,
"output": {
"needs_human": True,
"pause_reason": "supervised_pause",
"message": params.get("message", "Validation humaine requise"),
"phase": params.get("phase"),
"verdict_required": bool(params.get("verdict_required", False)),
"verdict_endpoint": params.get("verdict_endpoint"),
"competence_id": params.get("competence_id"),
"write_back_enabled": False,
},
}
def minimize_active_window():
"""Minimise le navigateur VWB et active la fenêtre suivante (VM, app cible)."""
try:
@@ -168,7 +212,8 @@ _execution_state = {
'pending_action': None, # Action en attente de choix utilisateur
'candidates': [], # Candidats proposés
'user_choice': None, # Choix de l'utilisateur (coordonnées ou 'skip' ou 'static')
'current_step_info': None # Info sur l'étape en cours pour affichage
'current_step_info': None, # Info sur l'étape en cours pour affichage
'human_pause': None,
}
@@ -307,6 +352,35 @@ def execute_workflow_thread(execution_id: str, workflow_id: str, app):
params['_step_label'] = step.label
result = execute_action(step.action_type, params)
if result.get('needs_human_pause'):
pause_payload = result.get('output', {})
print(f"⏸️ [Supervision] Pause humaine étape {index + 1}: {pause_payload}")
with _execution_lock:
_execution_state['is_paused'] = True
_execution_state['human_pause'] = pause_payload
_execution_state['current_step_info'] = {
'index': index,
'total': len(steps),
'step_id': step.id,
'action_type': step.action_type,
'label': step.label,
'human_pause': pause_payload,
}
execution.status = 'paused'
db.session.commit()
while _execution_state['is_paused'] and not _execution_state['should_stop']:
time.sleep(0.1)
if _execution_state['should_stop']:
execution.status = 'cancelled'
break
with _execution_lock:
_execution_state['human_pause'] = None
execution.status = 'running'
db.session.commit()
# === SELF-HEALING INTERACTIF ===
# Si l'action nécessite un choix utilisateur, attendre
if result.get('needs_user_choice'):
@@ -456,6 +530,7 @@ def execute_workflow_thread(execution_id: str, workflow_id: str, app):
with _execution_lock:
_execution_state['is_running'] = False
_execution_state['current_execution_id'] = None
_execution_state['human_pause'] = None
def execute_ai_analyze(params: dict) -> dict:
@@ -1198,6 +1273,9 @@ def execute_action(action_type: str, params: dict) -> dict:
elif action_type == 'wait_for_state':
return _execute_wait_for_state(params)
elif action_type == 'pause_for_human':
return _execute_pause_for_human(params)
elif action_type == 'keyboard_shortcut':
keys = params.get('keys', [])
if not keys:
@@ -1631,6 +1709,7 @@ def start_execution():
_execution_state['current_execution_id'] = execution.id
_execution_state['execution_mode'] = execution_mode
_execution_state['variables'] = {} # Reset des variables
_execution_state['human_pause'] = None
print(f"🎯 [API v3] Mode d'exécution: {execution_mode}")
@@ -1790,7 +1869,15 @@ def get_execution_status():
# Self-healing interactif
'waiting_for_choice': _execution_state.get('waiting_for_choice', False),
'candidates': _execution_state.get('candidates', []) if _execution_state.get('waiting_for_choice') else [],
'current_step_info': _execution_state.get('current_step_info') if _execution_state.get('waiting_for_choice') else None
'current_step_info': (
_execution_state.get('current_step_info')
if (
_execution_state.get('waiting_for_choice')
or _execution_state.get('human_pause')
)
else None
),
'human_pause': _execution_state.get('human_pause'),
})

7
visual_workflow_builder/backend/app.py
View File

@@ -170,6 +170,13 @@ try:
except ImportError as e:
print(f"⚠️ Blueprint correction_packs désactivé: {e}")
try:
from api.lea_competences import lea_competences_bp
app.register_blueprint(lea_competences_bp)
print("✅ Blueprint lea_competences enregistré")
except ImportError as e:
print(f"⚠️ Blueprint lea_competences désactivé: {e}")
try:
from api.coaching_sessions import coaching_sessions_bp
app.register_blueprint(coaching_sessions_bp, url_prefix='/api/coaching-sessions')

165
visual_workflow_builder/backend/catalog_routes_v2_vlm.py
View File

@@ -1253,6 +1253,108 @@ def preview_lea_competence_workflow(competence_id: str):
}), 500
def _catalog_paused_result(
*,
action_type: str,
step_id: str,
output_data: Dict[str, Any],
start_time: datetime,
) -> Any:
end_time = datetime.now()
execution_time_ms = (end_time - start_time).total_seconds() * 1000
return jsonify({
"success": True,
"result": {
"action_id": f"paused_{action_type}_{step_id}",
"step_id": step_id,
"status": "paused",
"execution_time_ms": execution_time_ms,
"output_data": output_data,
"evidence_list": [],
"error": None,
},
})
def _execute_catalog_wait_for_state(
*,
parameters: Dict[str, Any],
action_type: str,
step_id: str,
start_time: datetime,
) -> Any:
from visual_workflow_builder.backend.services.wait_for_state import (
wait_for_expected_state,
)
expected_state = parameters.get('expected_state') or {}
timeout_ms = int(parameters.get('timeout_ms', 5000))
poll_interval_ms = int(parameters.get('poll_interval_ms', 250))
evidence_required = parameters.get('evidence_required', 'window_or_process')
wait_result = wait_for_expected_state(
expected_state=expected_state,
timeout_ms=timeout_ms,
poll_interval_ms=poll_interval_ms,
evidence_required=evidence_required,
)
output_data = wait_result.to_dict()
if (
not wait_result.matched
and parameters.get('supervised_popup_detection', False)
):
detected_popup = None
try:
from core.execution.input_handler import check_screen_for_patterns
detected_popup = check_screen_for_patterns()
except Exception as popup_error:
print(f"⚠️ [wait_for_state] Detection popup indisponible: {popup_error}")
from visual_workflow_builder.backend.services.supervised_popup_guard import (
build_unexpected_popup_pause,
)
pause_payload = build_unexpected_popup_pause(
detected_popup,
expected_state=expected_state,
competence_id=str(parameters.get('competence_id') or ''),
source_method_id=str(parameters.get('source_method_id') or ''),
)
if pause_payload:
return _catalog_paused_result(
action_type=action_type,
step_id=step_id,
start_time=start_time,
output_data={
"needs_human": True,
"wait_for_state": output_data,
"human_pause": pause_payload,
"write_back_enabled": False,
},
)
end_time = datetime.now()
execution_time_ms = (end_time - start_time).total_seconds() * 1000
result_message = (
'Etat attendu observe'
if wait_result.matched
else 'Etat attendu non observe avant timeout'
)
return jsonify({
"success": True,
"result": {
"action_id": f"direct_{action_type}_{step_id}",
"step_id": step_id,
"status": "success" if wait_result.matched else "failed",
"execution_time_ms": execution_time_ms,
"output_data": output_data,
"evidence_list": [],
"error": None if wait_result.matched else {"message": result_message},
},
})
def get_screen_capturer():
"""
Obtient l'instance du ScreenCapturer (initialisation paresseuse).
@@ -2096,6 +2198,33 @@ def execute_action():
if action_type == "test_competence" or action_type.startswith("lea_competence_"):
return _execute_lea_competence_action(data, action_type, step_id, parameters)
request_start_time = datetime.now()
if action_type == "pause_for_human":
return _catalog_paused_result(
action_type=action_type,
step_id=step_id,
start_time=request_start_time,
output_data={
"needs_human": True,
"pause_reason": "supervised_pause",
"message": parameters.get("message", "Validation humaine requise"),
"phase": parameters.get("phase"),
"verdict_required": bool(parameters.get("verdict_required", False)),
"verdict_endpoint": parameters.get("verdict_endpoint"),
"competence_id": parameters.get("competence_id"),
"write_back_enabled": False,
},
)
if action_type == "wait_for_state":
return _execute_catalog_wait_for_state(
parameters=parameters,
action_type=action_type,
step_id=step_id,
start_time=request_start_time,
)
# LOG DEBUG - Voir ce qui arrive du frontend
print(f"\n{'='*60}")
print(f"🔥 REQUÊTE EXECUTE REÇUE:")
@@ -2768,6 +2897,40 @@ def execute_action():
if wait_result.matched
else 'Etat attendu non observe avant timeout'
)
if (
not wait_result.matched
and parameters.get('supervised_popup_detection', False)
):
detected_popup = None
try:
from core.execution.input_handler import check_screen_for_patterns
detected_popup = check_screen_for_patterns()
except Exception as popup_error:
print(f"⚠️ [wait_for_state] Detection popup indisponible: {popup_error}")
from visual_workflow_builder.backend.services.supervised_popup_guard import (
build_unexpected_popup_pause,
)
pause_payload = build_unexpected_popup_pause(
detected_popup,
expected_state=expected_state,
competence_id=str(parameters.get('competence_id') or ''),
source_method_id=str(parameters.get('source_method_id') or ''),
)
if pause_payload:
return _catalog_paused_result(
action_type=action_type,
step_id=step_id,
start_time=start_time,
output_data={
"needs_human": True,
"wait_for_state": direct_output_data,
"human_pause": pause_payload,
"write_back_enabled": False,
},
)
elif action_type in ['focus', 'focus_anchor', 'focaliser']:
# focus_anchor: chercher l'ancre et cliquer dessus pour donner le focus
@@ -2845,7 +3008,7 @@ def execute_action():
result_message = 'Coordonnées invalides - ancre non trouvée'
execution_success = False
elif action_type in ['hotkey', 'raccourci']:
elif action_type in ['hotkey', 'raccourci', 'keyboard_shortcut']:
keys = parameters.get('keys', parameters.get('touches', []))
if keys:
print(f"⌨️ [Direct] Raccourci: {'+'.join(keys)}")

31
visual_workflow_builder/backend/contracts/action_contracts.py
View File

@@ -202,7 +202,17 @@ VWB_ACTION_CONTRACTS: Dict[str, ActionContract] = {
action_type="wait_for_state",
description="Attendre qu'un etat semantique d'ecran soit observe",
required_params=["expected_state"],
optional_params=["timeout_ms", "poll_interval_ms", "evidence_required"],
optional_params=[
"timeout_ms",
"poll_interval_ms",
"evidence_required",
"supervised_popup_detection",
"popup_policy",
"competence_id",
"source",
"source_method_id",
"write_back_enabled",
],
param_validators={
"expected_state": lambda p: isinstance(p, dict) and any(
p.get(key)
@@ -214,6 +224,25 @@ VWB_ACTION_CONTRACTS: Dict[str, ActionContract] = {
)
}
),
"pause_for_human": ActionContract(
action_type="pause_for_human",
description="Suspendre l'execution pour une validation humaine",
required_params=["message"],
optional_params=[
"phase",
"verdict_required",
"verdict_endpoint",
"competence_id",
"source",
"write_back_enabled",
"intention",
"attendu",
"demande",
"safety_level",
"safety_checks",
],
param_validators={"message": lambda p: bool(p and isinstance(p, str))}
),
# --- ACTIONS DE SCROLL ---
"scroll_to_anchor": ActionContract(

4
visual_workflow_builder/backend/services/competence_vwb_preview.py
View File

@@ -49,6 +49,7 @@ def competence_yaml_to_vwb_preview(
"review_status": "preview",
"tags": ["lea", "competence", competence.learning_state],
"competence_id": competence.id,
"verdict_endpoint": f"/api/v1/lea/competences/{competence.id}/verdict",
"learning_state": competence.learning_state,
"source_path": competence.source_path,
"readonly": True,
@@ -120,6 +121,8 @@ def _method_to_vwb_step(
"timeout_ms": int(params.get("timeout_ms", 5000)),
"poll_interval_ms": int(params.get("poll_interval_ms", 250)),
"evidence_required": params.get("evidence_required", "window_or_process"),
"supervised_popup_detection": True,
"popup_policy": "pause_only",
**source_params,
}
return _step(
@@ -185,6 +188,7 @@ def _pause_step(
"message": message,
"phase": phase,
"verdict_required": verdict_required,
"verdict_endpoint": f"/api/v1/lea/competences/{competence.id}/verdict",
"competence_id": competence.id,
"source": f"lea_competence:{competence.id}",
"write_back_enabled": False,

69
visual_workflow_builder/backend/services/supervised_popup_guard.py Normal file
View File

@@ -0,0 +1,69 @@
"""Helpers for supervised popup detection without auto-resolution."""
from __future__ import annotations
from typing import Any, Dict, Mapping, Optional
def build_unexpected_popup_pause(
detected_popup: Optional[Mapping[str, Any]],
*,
expected_state: Mapping[str, Any],
competence_id: str = "",
source_method_id: str = "",
) -> Optional[Dict[str, Any]]:
"""Return a human-pause payload when a detected popup is not expected."""
if not detected_popup:
return None
popup = dict(detected_popup)
popup_title = _popup_title(popup)
if popup_title and _title_is_expected(popup_title, expected_state):
return None
competence_label = f" pour {competence_id}" if competence_id else ""
title_label = popup_title or str(popup.get("pattern") or "popup inconnue")
return {
"needs_human": True,
"pause_reason": "unexpected_popup",
"message": (
f"Popup inattendue detectee{competence_label}: '{title_label}'. "
"Mode supervise: aucune resolution automatique n'est lancee."
),
"detected_popup": popup,
"expected_state": dict(expected_state or {}),
"competence_id": competence_id,
"source_method_id": source_method_id,
"auto_resolution": False,
"write_back_enabled": False,
}
def _popup_title(popup: Mapping[str, Any]) -> str:
for key in ("title", "window_title", "target", "pattern", "name"):
value = popup.get(key)
if isinstance(value, str) and value.strip():
return value.strip()
return ""
def _title_is_expected(title: str, expected_state: Mapping[str, Any]) -> bool:
normalized = _norm(title)
exact = expected_state.get("window_title_in")
if isinstance(exact, str):
exact = [exact]
if isinstance(exact, list) and any(_norm(candidate) == normalized for candidate in exact):
return True
contains = expected_state.get("window_title_contains")
if isinstance(contains, str):
contains = [contains]
if isinstance(contains, list) and any(_norm(candidate) in normalized for candidate in contains):
return True
return False
def _norm(value: Any) -> str:
return str(value or "").strip().casefold()

27
visual_workflow_builder/frontend_v4/src/App.tsx
View File

@@ -106,6 +106,24 @@ function App() {
setRuntimeVariables(status.variables as Record<string, unknown>);
}
const localExecution = status.execution;
if (status.human_pause && localExecution) {
const rawPause = status.human_pause as any;
const pause = rawPause.human_pause || rawPause;
setAppState((prev) => prev ? ({
...prev,
execution: {
...localExecution,
pause_message: pause.message || 'Validation humaine requise',
pause_reason: pause.pause_reason || 'supervised_pause',
safety_checks: [],
verdict_required: Boolean(pause.verdict_required),
verdict_endpoint: pause.verdict_endpoint,
competence_id: pause.competence_id,
},
}) : prev);
}
// Self-healing interactif: detecter si on attend un choix utilisateur
if (status.waiting_for_choice && status.candidates) {
setHealingCandidates(status.candidates);
@@ -640,12 +658,19 @@ function App() {
safety_checks vide, PauseDialog rend la bulle simple legacy. */}
{(appState?.execution?.status === 'paused_need_help' ||
(appState?.execution?.status === 'paused' &&
(appState?.execution?.safety_checks?.length ?? 0) > 0)) && (
(
(appState?.execution?.safety_checks?.length ?? 0) > 0 ||
Boolean(appState?.execution?.pause_message)
))) && (
<div className="pause-dialog-overlay">
<PauseDialog
pauseMessage={appState.execution.pause_message || 'Validation requise'}
pauseReason={appState.execution.pause_reason}
safetyChecks={appState.execution.safety_checks || []}
verdictRequired={appState.execution.verdict_required}
verdictEndpoint={appState.execution.verdict_endpoint}
competenceId={appState.execution.competence_id}
executionId={appState.execution.id}
onResume={async (ackIds) => {
const replayId = appState.execution?.replay_id || appState.execution?.id;
if (replayId) {

85
visual_workflow_builder/frontend_v4/src/components/PauseDialog.tsx
View File

@@ -14,6 +14,10 @@ interface Props {
pauseMessage: string;
pauseReason?: string;
safetyChecks: SafetyCheck[];
verdictRequired?: boolean;
verdictEndpoint?: string;
competenceId?: string;
executionId?: string;
onResume: (acknowledgedIds: string[]) => Promise<void>;
onCancel: () => void;
}
@@ -22,6 +26,10 @@ export default function PauseDialog({
pauseMessage,
pauseReason,
safetyChecks,
verdictRequired = false,
verdictEndpoint,
competenceId,
executionId,
onResume,
onCancel,
}: Props) {
@@ -54,6 +62,57 @@ export default function PauseDialog({
}
};
const newVerdictId = (): string => {
if (window.crypto?.randomUUID) {
return window.crypto.randomUUID();
}
return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, (char) => {
const value = Math.floor(Math.random() * 16);
const resolved = char === 'x' ? value : (value & 0x3) | 0x8;
return resolved.toString(16);
});
};
const submitVerdict = async (verdictKind: 'valid' | 'invalid' | 'inconclusive') => {
if (!verdictEndpoint || !competenceId) return;
setSubmitting(true);
setError(null);
try {
const response = await fetch(verdictEndpoint, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
verdict_id: newVerdictId(),
verdict_kind: verdictKind,
verdict_by: 'human:dom',
context_signature: {
machine_id: `browser:${window.navigator.platform || 'unknown'}`,
screen_state_initial: '',
screen_state_after_action: '',
},
evidence: {
execution_id: executionId || '',
pause_reason: pauseReason || '',
},
source: {
frontend: 'vwb_v4',
execution_id: executionId || '',
},
comments: `Verdict humain VWB: ${verdictKind}`,
}),
});
if (!response.ok) {
const data = await response.json().catch(() => ({}));
throw new Error(data.error || response.statusText);
}
await onResume([]);
} catch (e: any) {
setError(e?.message || 'Erreur lors du verdict');
} finally {
setSubmitting(false);
}
};
// Backward compat : pas de checks -> bulle simple legacy
if (safetyChecks.length === 0) {
return (
@@ -61,6 +120,19 @@ export default function PauseDialog({
<p>{pauseMessage}</p>
{pauseReason && <small className="pause-reason">Raison : {pauseReason}</small>}
<div className="pause-actions">
{verdictRequired && verdictEndpoint && competenceId && (
<>
<button onClick={() => submitVerdict('valid')} disabled={submitting}>
Valide
</button>
<button onClick={() => submitVerdict('invalid')} disabled={submitting}>
Invalide
</button>
<button onClick={() => submitVerdict('inconclusive')} disabled={submitting}>
Incertain
</button>
</>
)}
<button onClick={() => onResume([])} disabled={submitting}>
Continuer
</button>
@@ -110,6 +182,19 @@ export default function PauseDialog({
{error && <div className="pause-error">{error}</div>}
<div className="pause-actions">
{verdictRequired && verdictEndpoint && competenceId && (
<>
<button onClick={() => submitVerdict('valid')} disabled={submitting}>
Valide
</button>
<button onClick={() => submitVerdict('invalid')} disabled={submitting}>
Invalide
</button>
<button onClick={() => submitVerdict('inconclusive')} disabled={submitting}>
Incertain
</button>
</>
)}
<button
onClick={handleResume}
disabled={!allRequiredOK || submitting}

2
visual_workflow_builder/frontend_v4/src/services/api.ts
View File

@@ -191,7 +191,9 @@ export async function getExecutionStatus(): Promise<{
total: number;
original_bbox?: { x: number; y: number; width: number; height: number };
error?: string;
human_pause?: Record<string, unknown>;
};
human_pause?: Record<string, unknown> | null;
}> {
return request('GET', '/execute/status');
}

3
visual_workflow_builder/frontend_v4/src/types.ts
View File

@@ -350,6 +350,9 @@ export interface Execution {
pause_reason?: string;
pause_message?: string;
safety_checks?: SafetyCheck[];
verdict_required?: boolean;
verdict_endpoint?: string;
competence_id?: string;
// ID du replay (utile pour appeler /replay/resume avec acknowledged_check_ids)
replay_id?: string;
}