Dom/rpa_vision_v3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(vwb): log supervised competence verdicts

Browse Source
This commit is contained in:
Dom
2026-05-29 18:36:06 +02:00
parent 7ad260d02f
commit aba849324a
18 changed files with 1082 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
core/competences/__init__.py
View File

@@ -10,12 +10,20 @@ from .replay import (
build_competence_replay_payload,
find_competence,
)
from .verdicts import (
CompetenceVerdictError,
iter_competence_verdicts,
store_competence_verdict,
)
__all__ = [
"CompetenceSummary",
"CompetenceVerdictError",
"build_competence_replay_actions",
"build_competence_replay_payload",
"find_competence",
"iter_competence_verdicts",
"load_competence_catalog_actions",
"load_competences",
"store_competence_verdict",
]

181
core/competences/verdicts.py Normal file
View File

@@ -0,0 +1,181 @@
"""Persist supervised human verdicts for Lea competences."""
from __future__ import annotations
import json
import uuid
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Dict, Iterable, Optional
from .catalog import DEFAULT_COMPETENCE_ROOT, REPO_ROOT
from .replay import find_competence
DEFAULT_VERDICT_LOG = REPO_ROOT / "data" / "competence_verdicts" / "verdicts.jsonl"
VALID_VERDICT_KINDS = {"valid", "invalid", "inconclusive"}
SCHEMA_VERSION = "lea_competence_verdict.v1"
class CompetenceVerdictError(ValueError):
"""Raised when a supervised verdict payload is invalid."""
def store_competence_verdict(
competence_id: str,
payload: Dict[str, Any],
*,
log_path: Path | str = DEFAULT_VERDICT_LOG,
competence_root: Path | str = DEFAULT_COMPETENCE_ROOT,
states: Optional[Iterable[str]] = None,
now: Optional[datetime] = None,
) -> Dict[str, Any]:
"""Validate and append one supervised verdict.
The function is idempotent on ``verdict_id``. If the same verdict was
already logged for the same competence, the stored record is returned with
``duplicate=True`` and the log is left untouched.
"""
if not isinstance(payload, dict):
raise CompetenceVerdictError("Payload verdict invalide")
competence = find_competence(competence_id, root=competence_root, states=states)
log = Path(log_path)
verdict_id = _required_text(payload, "verdict_id")
_validate_uuid(verdict_id)
for existing in iter_competence_verdicts(log_path=log):
if existing.get("verdict_id") != verdict_id:
continue
if existing.get("competence_id") != competence_id:
raise CompetenceVerdictError(
f"verdict_id deja utilise pour {existing.get('competence_id')}"
)
duplicate = dict(existing)
duplicate["duplicate"] = True
return duplicate
verdict_kind = _required_text(payload, "verdict_kind")
if verdict_kind not in VALID_VERDICT_KINDS:
raise CompetenceVerdictError(
"verdict_kind doit etre valid, invalid ou inconclusive"
)
verdict_at = _timestamp(payload.get("verdict_at"), now=now)
context_signature = _context_signature(payload.get("context_signature"))
evidence = _mapping(payload.get("evidence"), field="evidence")
source = _mapping(payload.get("source"), field="source")
record = {
"schema_version": SCHEMA_VERSION,
"verdict_id": verdict_id,
"competence_id": competence.id,
"competence_source_path": competence.source_path,
"learning_state": competence.learning_state,
"verdict_kind": verdict_kind,
"verdict_at": verdict_at,
"verdict_by": str(payload.get("verdict_by") or "human:dom"),
"context_signature": context_signature,
"evidence": evidence,
"comments": str(payload.get("comments") or ""),
"source": source,
"write_back_enabled": False,
"yaml_write": False,
"duplicate": False,
}
_append_jsonl(log, record)
return record
def iter_competence_verdicts(
*,
log_path: Path | str = DEFAULT_VERDICT_LOG,
competence_id: Optional[str] = None,
) -> list[Dict[str, Any]]:
"""Load logged verdict records, skipping malformed historical lines."""
log = Path(log_path)
if not log.exists():
return []
records: list[Dict[str, Any]] = []
with log.open("r", encoding="utf-8") as handle:
for line in handle:
line = line.strip()
if not line:
continue
try:
record = json.loads(line)
except json.JSONDecodeError:
continue
if not isinstance(record, dict):
continue
if competence_id and record.get("competence_id") != competence_id:
continue
records.append(record)
return records
def _required_text(payload: Dict[str, Any], key: str) -> str:
value = payload.get(key)
if not isinstance(value, str) or not value.strip():
raise CompetenceVerdictError(f"{key} requis")
return value.strip()
def _validate_uuid(value: str) -> None:
try:
parsed = uuid.UUID(value, version=4)
except ValueError as exc:
raise CompetenceVerdictError("verdict_id doit etre un UUID v4") from exc
if str(parsed) != value.lower():
raise CompetenceVerdictError("verdict_id UUID v4 invalide")
def _timestamp(value: Any, *, now: Optional[datetime]) -> str:
if value is None:
timestamp = now or datetime.now(timezone.utc)
elif isinstance(value, datetime):
timestamp = value
elif isinstance(value, str) and value.strip():
text = value.strip()
try:
parsed = datetime.fromisoformat(text.replace("Z", "+00:00"))
except ValueError as exc:
raise CompetenceVerdictError("verdict_at doit etre ISO 8601") from exc
timestamp = parsed
else:
raise CompetenceVerdictError("verdict_at doit etre ISO 8601")
if timestamp.tzinfo is None:
timestamp = timestamp.replace(tzinfo=timezone.utc)
return timestamp.astimezone(timezone.utc).isoformat()
def _context_signature(value: Any) -> Dict[str, Any]:
context = _mapping(value, field="context_signature")
machine_id = context.get("machine_id")
if not isinstance(machine_id, str) or not machine_id.strip():
raise CompetenceVerdictError("context_signature.machine_id requis")
normalized = dict(context)
normalized["machine_id"] = machine_id.strip()
normalized.setdefault("screen_state_initial", "")
normalized.setdefault("screen_state_after_action", "")
return normalized
def _mapping(value: Any, *, field: str) -> Dict[str, Any]:
if value is None:
return {}
if not isinstance(value, dict):
raise CompetenceVerdictError(f"{field} doit etre un objet")
return dict(value)
def _append_jsonl(log_path: Path, record: Dict[str, Any]) -> None:
log_path.parent.mkdir(parents=True, exist_ok=True)
with log_path.open("a", encoding="utf-8") as handle:
handle.write(json.dumps(record, ensure_ascii=False, sort_keys=True))
handle.write("\n")

8
tests/unit/test_competence_to_vwb_preview.py
View File

@@ -16,6 +16,9 @@ def test_competence_to_vwb_preview_key_win_r_steps():
assert preview["readonly"] is True
assert preview["write_back_enabled"] is False
assert preview["workflow"]["competence_id"] == "key_win_r_wait_explorer_exe"
assert preview["workflow"]["verdict_endpoint"] == (
"/api/v1/lea/competences/key_win_r_wait_explorer_exe/verdict"
)
assert _by_type(preview) == [
"pause_for_human",
"keyboard_shortcut",
@@ -47,6 +50,8 @@ def test_wait_for_state_expected_state_is_preserved():
assert params["timeout_ms"] == 5000
assert params["poll_interval_ms"] == 250
assert params["evidence_required"] == "window_or_process"
assert params["supervised_popup_detection"] is True
assert params["popup_policy"] == "pause_only"
def test_pause_for_human_before_and_after_are_supervision_only():
@@ -59,6 +64,9 @@ def test_pause_for_human_before_and_after_are_supervision_only():
assert after["parameters"]["phase"] == "after"
assert after["parameters"]["verdict_required"] is True
assert after["parameters"]["write_back_enabled"] is False
assert after["parameters"]["verdict_endpoint"] == (
"/api/v1/lea/competences/key_win_r_wait_explorer_exe/verdict"
)
def test_adapter_is_generic_on_methods_not_hardcoded_to_win_r():

117
tests/unit/test_competence_verdicts.py Normal file
View File

@@ -0,0 +1,117 @@
from datetime import datetime, timezone
import pytest
from core.competences.verdicts import (
CompetenceVerdictError,
iter_competence_verdicts,
store_competence_verdict,
)
VERDICT_ID = "123e4567-e89b-42d3-a456-426614174000"
def _payload(**overrides):
payload = {
"verdict_id": VERDICT_ID,
"verdict_kind": "valid",
"verdict_by": "human:dom",
"context_signature": {
"machine_id": "DESKTOP-58D5CAC_windows",
"screen_state_initial": "before_hash",
"screen_state_after_action": "after_hash",
},
"evidence": {
"screenshot_before": "evidence/before.png",
"screenshot_after": "evidence/after.png",
"wait_state_matched_evidence": {
"window_title": "Executer",
"process_name": "explorer.exe",
},
},
"comments": "Supervised replay ok",
}
payload.update(overrides)
return payload
def test_store_competence_verdict_appends_jsonl(tmp_path):
log_path = tmp_path / "verdicts.jsonl"
record = store_competence_verdict(
"key_win_r_wait_explorer_exe",
_payload(),
log_path=log_path,
now=datetime(2026, 5, 29, 16, 30, tzinfo=timezone.utc),
)
assert record["schema_version"] == "lea_competence_verdict.v1"
assert record["competence_id"] == "key_win_r_wait_explorer_exe"
assert record["verdict_kind"] == "valid"
assert record["write_back_enabled"] is False
assert record["yaml_write"] is False
assert record["duplicate"] is False
assert record["context_signature"]["machine_id"] == "DESKTOP-58D5CAC_windows"
records = iter_competence_verdicts(log_path=log_path)
assert len(records) == 1
assert records[0]["verdict_id"] == VERDICT_ID
def test_store_competence_verdict_is_idempotent(tmp_path):
log_path = tmp_path / "verdicts.jsonl"
first = store_competence_verdict(
"key_win_r_wait_explorer_exe",
_payload(),
log_path=log_path,
)
second = store_competence_verdict(
"key_win_r_wait_explorer_exe",
_payload(comments="second click"),
log_path=log_path,
)
assert first["duplicate"] is False
assert second["duplicate"] is True
assert len(log_path.read_text(encoding="utf-8").splitlines()) == 1
assert second["comments"] == "Supervised replay ok"
@pytest.mark.parametrize("kind", ["valid", "invalid", "inconclusive"])
def test_store_competence_verdict_accepts_three_kinds(tmp_path, kind):
record = store_competence_verdict(
"key_win_r_wait_explorer_exe",
_payload(
verdict_id={
"valid": "123e4567-e89b-42d3-a456-426614174000",
"invalid": "123e4567-e89b-42d3-a456-426614174001",
"inconclusive": "123e4567-e89b-42d3-a456-426614174002",
}[kind],
verdict_kind=kind,
),
log_path=tmp_path / "verdicts.jsonl",
)
assert record["verdict_kind"] == kind
def test_store_competence_verdict_requires_context_machine(tmp_path):
with pytest.raises(CompetenceVerdictError, match="machine_id"):
store_competence_verdict(
"key_win_r_wait_explorer_exe",
_payload(context_signature={}),
log_path=tmp_path / "verdicts.jsonl",
)
def test_store_competence_verdict_rejects_yaml_write_attempt(tmp_path):
record = store_competence_verdict(
"key_win_r_wait_explorer_exe",
_payload(write_back_enabled=True, yaml_write=True),
log_path=tmp_path / "verdicts.jsonl",
)
assert record["write_back_enabled"] is False
assert record["yaml_write"] is False

109
tests/unit/test_lea_competence_verdict_api.py Normal file
View File

@@ -0,0 +1,109 @@
from flask import Flask
import importlib.util
from pathlib import Path
MODULE_PATH = (
Path(__file__).resolve().parents[2]
/ "visual_workflow_builder"
/ "backend"
/ "api"
/ "lea_competences.py"
)
SPEC = importlib.util.spec_from_file_location("lea_competences_test_module", MODULE_PATH)
lea_competences_module = importlib.util.module_from_spec(SPEC)
assert SPEC and SPEC.loader
SPEC.loader.exec_module(lea_competences_module)
lea_competences_bp = lea_competences_module.lea_competences_bp
def _app():
app = Flask(__name__)
app.register_blueprint(lea_competences_bp)
return app
def test_submit_competence_verdict_endpoint(monkeypatch):
def fake_store(competence_id, payload):
assert competence_id == "key_win_r_wait_explorer_exe"
assert payload["verdict_kind"] == "valid"
return {
"verdict_id": payload["verdict_id"],
"competence_id": competence_id,
"verdict_kind": "valid",
"duplicate": False,
"write_back_enabled": False,
"yaml_write": False,
}
monkeypatch.setattr(
lea_competences_module,
"store_competence_verdict",
fake_store,
)
with _app().test_client() as client:
response = client.post(
"/api/v1/lea/competences/key_win_r_wait_explorer_exe/verdict",
json={
"verdict_id": "123e4567-e89b-42d3-a456-426614174000",
"verdict_kind": "valid",
},
)
assert response.status_code == 201
data = response.get_json()
assert data["success"] is True
assert data["write_back_enabled"] is False
assert data["yaml_write"] is False
assert data["verdict"]["duplicate"] is False
def test_submit_competence_verdict_endpoint_returns_duplicate_200(monkeypatch):
def fake_store(_competence_id, payload):
return {
"verdict_id": payload["verdict_id"],
"competence_id": "key_win_r_wait_explorer_exe",
"verdict_kind": "valid",
"duplicate": True,
"write_back_enabled": False,
"yaml_write": False,
}
monkeypatch.setattr(
lea_competences_module,
"store_competence_verdict",
fake_store,
)
with _app().test_client() as client:
response = client.post(
"/api/v1/lea/competences/key_win_r_wait_explorer_exe/verdict",
json={
"verdict_id": "123e4567-e89b-42d3-a456-426614174000",
"verdict_kind": "valid",
},
)
assert response.status_code == 200
assert response.get_json()["duplicate"] is True
def test_list_competence_verdicts_endpoint(monkeypatch):
monkeypatch.setattr(
lea_competences_module,
"iter_competence_verdicts",
lambda competence_id: [{"competence_id": competence_id, "verdict_kind": "valid"}],
)
with _app().test_client() as client:
response = client.get(
"/api/v1/lea/competences/key_win_r_wait_explorer_exe/verdicts"
)
data = response.get_json()
assert response.status_code == 200
assert data["success"] is True
assert data["verdicts"] == [
{"competence_id": "key_win_r_wait_explorer_exe", "verdict_kind": "valid"}
]

38
tests/unit/test_supervised_popup_guard.py Normal file
View File

@@ -0,0 +1,38 @@
from visual_workflow_builder.backend.services.supervised_popup_guard import (
build_unexpected_popup_pause,
)
def test_unexpected_popup_builds_pause_without_auto_resolution():
pause = build_unexpected_popup_pause(
{"pattern": "confirm_save_overwrite", "title": "Enregistrer sous"},
expected_state={
"window_title_in": ["Executer"],
"process_active": "explorer.exe",
},
competence_id="key_win_r_wait_explorer_exe",
source_method_id="step_2_wait_state",
)
assert pause is not None
assert pause["needs_human"] is True
assert pause["pause_reason"] == "unexpected_popup"
assert pause["auto_resolution"] is False
assert pause["write_back_enabled"] is False
assert pause["detected_popup"]["title"] == "Enregistrer sous"
def test_expected_popup_title_does_not_build_pause():
pause = build_unexpected_popup_pause(
{"pattern": "run_dialog", "title": "Executer"},
expected_state={"window_title_in": ["Executer"]},
)
assert pause is None
def test_missing_popup_does_not_build_pause():
assert build_unexpected_popup_pause(
None,
expected_state={"window_title_in": ["Executer"]},
) is None

81
tests/unit/test_vwb_supervised_pause_runtime.py Normal file
View File

@@ -0,0 +1,81 @@
from flask import Flask
from visual_workflow_builder.backend.catalog_routes_v2_vlm import catalog_bp
class _WaitResult:
matched = False
def to_dict(self):
return {
"matched": False,
"timed_out": True,
"match": {
"expected_state": {"window_title_in": ["Executer"]},
"observed_state": {"window_title": "Enregistrer sous"},
},
}
def test_catalog_pause_for_human_returns_paused_status():
app = Flask(__name__)
app.register_blueprint(catalog_bp)
with app.test_client() as client:
response = client.post(
"/api/vwb/catalog/execute",
json={
"type": "pause_for_human",
"step_id": "pause_1",
"parameters": {
"message": "Valider le resultat",
"phase": "after",
"verdict_required": True,
"competence_id": "key_win_r_wait_explorer_exe",
},
},
)
data = response.get_json()
assert response.status_code == 200
assert data["result"]["status"] == "paused"
assert data["result"]["output_data"]["needs_human"] is True
assert data["result"]["output_data"]["write_back_enabled"] is False
def test_catalog_wait_for_state_popup_pauses_without_auto_resolution(monkeypatch):
monkeypatch.setattr(
"visual_workflow_builder.backend.services.wait_for_state.wait_for_expected_state",
lambda **_kwargs: _WaitResult(),
)
monkeypatch.setattr(
"core.execution.input_handler.check_screen_for_patterns",
lambda: {"pattern": "save_as", "title": "Enregistrer sous"},
)
app = Flask(__name__)
app.register_blueprint(catalog_bp)
with app.test_client() as client:
response = client.post(
"/api/vwb/catalog/execute",
json={
"type": "wait_for_state",
"step_id": "wait_1",
"parameters": {
"expected_state": {"window_title_in": ["Executer"]},
"timeout_ms": 1,
"poll_interval_ms": 1,
"supervised_popup_detection": True,
"competence_id": "key_win_r_wait_explorer_exe",
"source_method_id": "step_2_wait_state",
},
},
)
data = response.get_json()
assert response.status_code == 200
assert data["result"]["status"] == "paused"
pause = data["result"]["output_data"]["human_pause"]
assert pause["pause_reason"] == "unexpected_popup"
assert pause["auto_resolution"] is False

61
visual_workflow_builder/backend/api/lea_competences.py Normal file
View File

@@ -0,0 +1,61 @@
"""Lea competence supervision endpoints exposed by the VWB backend."""
from __future__ import annotations
from flask import Blueprint, jsonify, request
from core.competences.verdicts import (
CompetenceVerdictError,
iter_competence_verdicts,
store_competence_verdict,
)
lea_competences_bp = Blueprint(
"lea_competences",
__name__,
url_prefix="/api/v1/lea/competences",
)
@lea_competences_bp.route("/<competence_id>/verdict", methods=["POST"])
def submit_competence_verdict(competence_id: str):
"""Persist one supervised human verdict without touching YAML."""
payload = request.get_json(silent=True) or {}
try:
verdict = store_competence_verdict(competence_id, payload)
except KeyError:
return jsonify({
"success": False,
"error": f"Competence '{competence_id}' introuvable",
}), 404
except CompetenceVerdictError as exc:
return jsonify({
"success": False,
"error": str(exc),
"write_back_enabled": False,
"yaml_write": False,
}), 400
return jsonify({
"success": True,
"competence_id": competence_id,
"verdict": verdict,
"duplicate": verdict["duplicate"],
"write_back_enabled": False,
"yaml_write": False,
}), 200 if verdict["duplicate"] else 201
@lea_competences_bp.route("/<competence_id>/verdicts", methods=["GET"])
def list_competence_verdicts(competence_id: str):
"""Return logged supervised verdicts for one competence."""
return jsonify({
"success": True,
"competence_id": competence_id,
"verdicts": iter_competence_verdicts(competence_id=competence_id),
"write_back_enabled": False,
"yaml_write": False,
})

91
visual_workflow_builder/backend/api_v3/execute.py
View File

@@ -103,6 +103,33 @@ def _execute_wait_for_state(params: Dict[str, Any]) -> dict:
output = result.to_dict()
if result.matched:
return {"success": True, "output": output}
if params.get("supervised_popup_detection", False):
detected_popup = None
try:
detected_popup = _check_screen_for_patterns()
except Exception as exc:
print(f"⚠️ [wait_for_state] Detection popup indisponible: {exc}")
from visual_workflow_builder.backend.services.supervised_popup_guard import (
build_unexpected_popup_pause,
)
pause_payload = build_unexpected_popup_pause(
detected_popup,
expected_state=expected_state,
competence_id=str(params.get("competence_id") or ""),
source_method_id=str(params.get("source_method_id") or ""),
)
if pause_payload:
return {
"success": True,
"needs_human_pause": True,
"output": {
"wait_for_state": output,
"human_pause": pause_payload,
"write_back_enabled": False,
},
}
return {
"success": False,
"error": "Etat attendu non observe avant timeout",
@@ -110,6 +137,23 @@ def _execute_wait_for_state(params: Dict[str, Any]) -> dict:
}
def _execute_pause_for_human(params: Dict[str, Any]) -> dict:
return {
"success": True,
"needs_human_pause": True,
"output": {
"needs_human": True,
"pause_reason": "supervised_pause",
"message": params.get("message", "Validation humaine requise"),
"phase": params.get("phase"),
"verdict_required": bool(params.get("verdict_required", False)),
"verdict_endpoint": params.get("verdict_endpoint"),
"competence_id": params.get("competence_id"),
"write_back_enabled": False,
},
}
def minimize_active_window():
"""Minimise le navigateur VWB et active la fenêtre suivante (VM, app cible)."""
try:
@@ -168,7 +212,8 @@ _execution_state = {
'pending_action': None, # Action en attente de choix utilisateur
'candidates': [], # Candidats proposés
'user_choice': None, # Choix de l'utilisateur (coordonnées ou 'skip' ou 'static')
'current_step_info': None # Info sur l'étape en cours pour affichage
'current_step_info': None, # Info sur l'étape en cours pour affichage
'human_pause': None,
}
@@ -307,6 +352,35 @@ def execute_workflow_thread(execution_id: str, workflow_id: str, app):
params['_step_label'] = step.label
result = execute_action(step.action_type, params)
if result.get('needs_human_pause'):
pause_payload = result.get('output', {})
print(f"⏸️ [Supervision] Pause humaine étape {index + 1}: {pause_payload}")
with _execution_lock:
_execution_state['is_paused'] = True
_execution_state['human_pause'] = pause_payload
_execution_state['current_step_info'] = {
'index': index,
'total': len(steps),
'step_id': step.id,
'action_type': step.action_type,
'label': step.label,
'human_pause': pause_payload,
}
execution.status = 'paused'
db.session.commit()
while _execution_state['is_paused'] and not _execution_state['should_stop']:
time.sleep(0.1)
if _execution_state['should_stop']:
execution.status = 'cancelled'
break
with _execution_lock:
_execution_state['human_pause'] = None
execution.status = 'running'
db.session.commit()
# === SELF-HEALING INTERACTIF ===
# Si l'action nécessite un choix utilisateur, attendre
if result.get('needs_user_choice'):
@@ -456,6 +530,7 @@ def execute_workflow_thread(execution_id: str, workflow_id: str, app):
with _execution_lock:
_execution_state['is_running'] = False
_execution_state['current_execution_id'] = None
_execution_state['human_pause'] = None
def execute_ai_analyze(params: dict) -> dict:
@@ -1198,6 +1273,9 @@ def execute_action(action_type: str, params: dict) -> dict:
elif action_type == 'wait_for_state':
return _execute_wait_for_state(params)
elif action_type == 'pause_for_human':
return _execute_pause_for_human(params)
elif action_type == 'keyboard_shortcut':
keys = params.get('keys', [])
if not keys:
@@ -1631,6 +1709,7 @@ def start_execution():
_execution_state['current_execution_id'] = execution.id
_execution_state['execution_mode'] = execution_mode
_execution_state['variables'] = {} # Reset des variables
_execution_state['human_pause'] = None
print(f"🎯 [API v3] Mode d'exécution: {execution_mode}")
@@ -1790,7 +1869,15 @@ def get_execution_status():
# Self-healing interactif
'waiting_for_choice': _execution_state.get('waiting_for_choice', False),
'candidates': _execution_state.get('candidates', []) if _execution_state.get('waiting_for_choice') else [],
'current_step_info': _execution_state.get('current_step_info') if _execution_state.get('waiting_for_choice') else None
'current_step_info': (
_execution_state.get('current_step_info')
if (
_execution_state.get('waiting_for_choice')
or _execution_state.get('human_pause')
)
else None
),
'human_pause': _execution_state.get('human_pause'),
})

7
visual_workflow_builder/backend/app.py
View File

@@ -170,6 +170,13 @@ try:
except ImportError as e:
print(f"⚠️ Blueprint correction_packs désactivé: {e}")
try:
from api.lea_competences import lea_competences_bp
app.register_blueprint(lea_competences_bp)
print("✅ Blueprint lea_competences enregistré")
except ImportError as e:
print(f"⚠️ Blueprint lea_competences désactivé: {e}")
try:
from api.coaching_sessions import coaching_sessions_bp
app.register_blueprint(coaching_sessions_bp, url_prefix='/api/coaching-sessions')

165
visual_workflow_builder/backend/catalog_routes_v2_vlm.py
View File

@@ -1253,6 +1253,108 @@ def preview_lea_competence_workflow(competence_id: str):
}), 500
def _catalog_paused_result(
*,
action_type: str,
step_id: str,
output_data: Dict[str, Any],
start_time: datetime,
) -> Any:
end_time = datetime.now()
execution_time_ms = (end_time - start_time).total_seconds() * 1000
return jsonify({
"success": True,
"result": {
"action_id": f"paused_{action_type}_{step_id}",
"step_id": step_id,
"status": "paused",
"execution_time_ms": execution_time_ms,
"output_data": output_data,
"evidence_list": [],
"error": None,
},
})
def _execute_catalog_wait_for_state(
*,
parameters: Dict[str, Any],
action_type: str,
step_id: str,
start_time: datetime,
) -> Any:
from visual_workflow_builder.backend.services.wait_for_state import (
wait_for_expected_state,
)
expected_state = parameters.get('expected_state') or {}
timeout_ms = int(parameters.get('timeout_ms', 5000))
poll_interval_ms = int(parameters.get('poll_interval_ms', 250))
evidence_required = parameters.get('evidence_required', 'window_or_process')
wait_result = wait_for_expected_state(
expected_state=expected_state,
timeout_ms=timeout_ms,
poll_interval_ms=poll_interval_ms,
evidence_required=evidence_required,
)
output_data = wait_result.to_dict()
if (
not wait_result.matched
and parameters.get('supervised_popup_detection', False)
):
detected_popup = None
try:
from core.execution.input_handler import check_screen_for_patterns
detected_popup = check_screen_for_patterns()
except Exception as popup_error:
print(f"⚠️ [wait_for_state] Detection popup indisponible: {popup_error}")
from visual_workflow_builder.backend.services.supervised_popup_guard import (
build_unexpected_popup_pause,
)
pause_payload = build_unexpected_popup_pause(
detected_popup,
expected_state=expected_state,
competence_id=str(parameters.get('competence_id') or ''),
source_method_id=str(parameters.get('source_method_id') or ''),
)
if pause_payload:
return _catalog_paused_result(
action_type=action_type,
step_id=step_id,
start_time=start_time,
output_data={
"needs_human": True,
"wait_for_state": output_data,
"human_pause": pause_payload,
"write_back_enabled": False,
},
)
end_time = datetime.now()
execution_time_ms = (end_time - start_time).total_seconds() * 1000
result_message = (
'Etat attendu observe'
if wait_result.matched
else 'Etat attendu non observe avant timeout'
)
return jsonify({
"success": True,
"result": {
"action_id": f"direct_{action_type}_{step_id}",
"step_id": step_id,
"status": "success" if wait_result.matched else "failed",
"execution_time_ms": execution_time_ms,
"output_data": output_data,
"evidence_list": [],
"error": None if wait_result.matched else {"message": result_message},
},
})
def get_screen_capturer():
"""
Obtient l'instance du ScreenCapturer (initialisation paresseuse).
@@ -2096,6 +2198,33 @@ def execute_action():
if action_type == "test_competence" or action_type.startswith("lea_competence_"):
return _execute_lea_competence_action(data, action_type, step_id, parameters)
request_start_time = datetime.now()
if action_type == "pause_for_human":
return _catalog_paused_result(
action_type=action_type,
step_id=step_id,
start_time=request_start_time,
output_data={
"needs_human": True,
"pause_reason": "supervised_pause",
"message": parameters.get("message", "Validation humaine requise"),
"phase": parameters.get("phase"),
"verdict_required": bool(parameters.get("verdict_required", False)),
"verdict_endpoint": parameters.get("verdict_endpoint"),
"competence_id": parameters.get("competence_id"),
"write_back_enabled": False,
},
)
if action_type == "wait_for_state":
return _execute_catalog_wait_for_state(
parameters=parameters,
action_type=action_type,
step_id=step_id,
start_time=request_start_time,
)
# LOG DEBUG - Voir ce qui arrive du frontend
print(f"\n{'='*60}")
print(f"🔥 REQUÊTE EXECUTE REÇUE:")
@@ -2768,6 +2897,40 @@ def execute_action():
if wait_result.matched
else 'Etat attendu non observe avant timeout'
)
if (
not wait_result.matched
and parameters.get('supervised_popup_detection', False)
):
detected_popup = None
try:
from core.execution.input_handler import check_screen_for_patterns
detected_popup = check_screen_for_patterns()
except Exception as popup_error:
print(f"⚠️ [wait_for_state] Detection popup indisponible: {popup_error}")
from visual_workflow_builder.backend.services.supervised_popup_guard import (
build_unexpected_popup_pause,
)
pause_payload = build_unexpected_popup_pause(
detected_popup,
expected_state=expected_state,
competence_id=str(parameters.get('competence_id') or ''),
source_method_id=str(parameters.get('source_method_id') or ''),
)
if pause_payload:
return _catalog_paused_result(
action_type=action_type,
step_id=step_id,
start_time=start_time,
output_data={
"needs_human": True,
"wait_for_state": direct_output_data,
"human_pause": pause_payload,
"write_back_enabled": False,
},
)
elif action_type in ['focus', 'focus_anchor', 'focaliser']:
# focus_anchor: chercher l'ancre et cliquer dessus pour donner le focus
@@ -2845,7 +3008,7 @@ def execute_action():
result_message = 'Coordonnées invalides - ancre non trouvée'
execution_success = False
elif action_type in ['hotkey', 'raccourci']:
elif action_type in ['hotkey', 'raccourci', 'keyboard_shortcut']:
keys = parameters.get('keys', parameters.get('touches', []))
if keys:
print(f"⌨️ [Direct] Raccourci: {'+'.join(keys)}")

31
visual_workflow_builder/backend/contracts/action_contracts.py
View File

@@ -202,7 +202,17 @@ VWB_ACTION_CONTRACTS: Dict[str, ActionContract] = {
action_type="wait_for_state",
description="Attendre qu'un etat semantique d'ecran soit observe",
required_params=["expected_state"],
optional_params=["timeout_ms", "poll_interval_ms", "evidence_required"],
optional_params=[
"timeout_ms",
"poll_interval_ms",
"evidence_required",
"supervised_popup_detection",
"popup_policy",
"competence_id",
"source",
"source_method_id",
"write_back_enabled",
],
param_validators={
"expected_state": lambda p: isinstance(p, dict) and any(
p.get(key)
@@ -214,6 +224,25 @@ VWB_ACTION_CONTRACTS: Dict[str, ActionContract] = {
)
}
),
"pause_for_human": ActionContract(
action_type="pause_for_human",
description="Suspendre l'execution pour une validation humaine",
required_params=["message"],
optional_params=[
"phase",
"verdict_required",
"verdict_endpoint",
"competence_id",
"source",
"write_back_enabled",
"intention",
"attendu",
"demande",
"safety_level",
"safety_checks",
],
param_validators={"message": lambda p: bool(p and isinstance(p, str))}
),
# --- ACTIONS DE SCROLL ---
"scroll_to_anchor": ActionContract(

4
visual_workflow_builder/backend/services/competence_vwb_preview.py
View File

@@ -49,6 +49,7 @@ def competence_yaml_to_vwb_preview(
"review_status": "preview",
"tags": ["lea", "competence", competence.learning_state],
"competence_id": competence.id,
"verdict_endpoint": f"/api/v1/lea/competences/{competence.id}/verdict",
"learning_state": competence.learning_state,
"source_path": competence.source_path,
"readonly": True,
@@ -120,6 +121,8 @@ def _method_to_vwb_step(
"timeout_ms": int(params.get("timeout_ms", 5000)),
"poll_interval_ms": int(params.get("poll_interval_ms", 250)),
"evidence_required": params.get("evidence_required", "window_or_process"),
"supervised_popup_detection": True,
"popup_policy": "pause_only",
**source_params,
}
return _step(
@@ -185,6 +188,7 @@ def _pause_step(
"message": message,
"phase": phase,
"verdict_required": verdict_required,
"verdict_endpoint": f"/api/v1/lea/competences/{competence.id}/verdict",
"competence_id": competence.id,
"source": f"lea_competence:{competence.id}",
"write_back_enabled": False,

69
visual_workflow_builder/backend/services/supervised_popup_guard.py Normal file
View File

@@ -0,0 +1,69 @@
"""Helpers for supervised popup detection without auto-resolution."""
from __future__ import annotations
from typing import Any, Dict, Mapping, Optional
def build_unexpected_popup_pause(
detected_popup: Optional[Mapping[str, Any]],
*,
expected_state: Mapping[str, Any],
competence_id: str = "",
source_method_id: str = "",
) -> Optional[Dict[str, Any]]:
"""Return a human-pause payload when a detected popup is not expected."""
if not detected_popup:
return None
popup = dict(detected_popup)
popup_title = _popup_title(popup)
if popup_title and _title_is_expected(popup_title, expected_state):
return None
competence_label = f" pour {competence_id}" if competence_id else ""
title_label = popup_title or str(popup.get("pattern") or "popup inconnue")
return {
"needs_human": True,
"pause_reason": "unexpected_popup",
"message": (
f"Popup inattendue detectee{competence_label}: '{title_label}'. "
"Mode supervise: aucune resolution automatique n'est lancee."
),
"detected_popup": popup,
"expected_state": dict(expected_state or {}),
"competence_id": competence_id,
"source_method_id": source_method_id,
"auto_resolution": False,
"write_back_enabled": False,
}
def _popup_title(popup: Mapping[str, Any]) -> str:
for key in ("title", "window_title", "target", "pattern", "name"):
value = popup.get(key)
if isinstance(value, str) and value.strip():
return value.strip()
return ""
def _title_is_expected(title: str, expected_state: Mapping[str, Any]) -> bool:
normalized = _norm(title)
exact = expected_state.get("window_title_in")
if isinstance(exact, str):
exact = [exact]
if isinstance(exact, list) and any(_norm(candidate) == normalized for candidate in exact):
return True
contains = expected_state.get("window_title_contains")
if isinstance(contains, str):
contains = [contains]
if isinstance(contains, list) and any(_norm(candidate) in normalized for candidate in contains):
return True
return False
def _norm(value: Any) -> str:
return str(value or "").strip().casefold()

27
visual_workflow_builder/frontend_v4/src/App.tsx
View File

@@ -106,6 +106,24 @@ function App() {
setRuntimeVariables(status.variables as Record<string, unknown>);
}
const localExecution = status.execution;
if (status.human_pause && localExecution) {
const rawPause = status.human_pause as any;
const pause = rawPause.human_pause || rawPause;
setAppState((prev) => prev ? ({
...prev,
execution: {
...localExecution,
pause_message: pause.message || 'Validation humaine requise',
pause_reason: pause.pause_reason || 'supervised_pause',
safety_checks: [],
verdict_required: Boolean(pause.verdict_required),
verdict_endpoint: pause.verdict_endpoint,
competence_id: pause.competence_id,
},
}) : prev);
}
// Self-healing interactif: detecter si on attend un choix utilisateur
if (status.waiting_for_choice && status.candidates) {
setHealingCandidates(status.candidates);
@@ -640,12 +658,19 @@ function App() {
safety_checks vide, PauseDialog rend la bulle simple legacy. */}
{(appState?.execution?.status === 'paused_need_help' ||
(appState?.execution?.status === 'paused' &&
(appState?.execution?.safety_checks?.length ?? 0) > 0)) && (
(
(appState?.execution?.safety_checks?.length ?? 0) > 0 ||
Boolean(appState?.execution?.pause_message)
))) && (
<div className="pause-dialog-overlay">
<PauseDialog
pauseMessage={appState.execution.pause_message || 'Validation requise'}
pauseReason={appState.execution.pause_reason}
safetyChecks={appState.execution.safety_checks || []}
verdictRequired={appState.execution.verdict_required}
verdictEndpoint={appState.execution.verdict_endpoint}
competenceId={appState.execution.competence_id}
executionId={appState.execution.id}
onResume={async (ackIds) => {
const replayId = appState.execution?.replay_id || appState.execution?.id;
if (replayId) {

85
visual_workflow_builder/frontend_v4/src/components/PauseDialog.tsx
View File

@@ -14,6 +14,10 @@ interface Props {
pauseMessage: string;
pauseReason?: string;
safetyChecks: SafetyCheck[];
verdictRequired?: boolean;
verdictEndpoint?: string;
competenceId?: string;
executionId?: string;
onResume: (acknowledgedIds: string[]) => Promise<void>;
onCancel: () => void;
}
@@ -22,6 +26,10 @@ export default function PauseDialog({
pauseMessage,
pauseReason,
safetyChecks,
verdictRequired = false,
verdictEndpoint,
competenceId,
executionId,
onResume,
onCancel,
}: Props) {
@@ -54,6 +62,57 @@ export default function PauseDialog({
}
};
const newVerdictId = (): string => {
if (window.crypto?.randomUUID) {
return window.crypto.randomUUID();
}
return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, (char) => {
const value = Math.floor(Math.random() * 16);
const resolved = char === 'x' ? value : (value & 0x3) | 0x8;
return resolved.toString(16);
});
};
const submitVerdict = async (verdictKind: 'valid' | 'invalid' | 'inconclusive') => {
if (!verdictEndpoint || !competenceId) return;
setSubmitting(true);
setError(null);
try {
const response = await fetch(verdictEndpoint, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
verdict_id: newVerdictId(),
verdict_kind: verdictKind,
verdict_by: 'human:dom',
context_signature: {
machine_id: `browser:${window.navigator.platform || 'unknown'}`,
screen_state_initial: '',
screen_state_after_action: '',
},
evidence: {
execution_id: executionId || '',
pause_reason: pauseReason || '',
},
source: {
frontend: 'vwb_v4',
execution_id: executionId || '',
},
comments: `Verdict humain VWB: ${verdictKind}`,
}),
});
if (!response.ok) {
const data = await response.json().catch(() => ({}));
throw new Error(data.error || response.statusText);
}
await onResume([]);
} catch (e: any) {
setError(e?.message || 'Erreur lors du verdict');
} finally {
setSubmitting(false);
}
};
// Backward compat : pas de checks -> bulle simple legacy
if (safetyChecks.length === 0) {
return (
@@ -61,6 +120,19 @@ export default function PauseDialog({
<p>{pauseMessage}</p>
{pauseReason && <small className="pause-reason">Raison : {pauseReason}</small>}
<div className="pause-actions">
{verdictRequired && verdictEndpoint && competenceId && (
<>
<button onClick={() => submitVerdict('valid')} disabled={submitting}>
Valide
</button>
<button onClick={() => submitVerdict('invalid')} disabled={submitting}>
Invalide
</button>
<button onClick={() => submitVerdict('inconclusive')} disabled={submitting}>
Incertain
</button>
</>
)}
<button onClick={() => onResume([])} disabled={submitting}>
Continuer
</button>
@@ -110,6 +182,19 @@ export default function PauseDialog({
{error && <div className="pause-error">{error}</div>}
<div className="pause-actions">
{verdictRequired && verdictEndpoint && competenceId && (
<>
<button onClick={() => submitVerdict('valid')} disabled={submitting}>
Valide
</button>
<button onClick={() => submitVerdict('invalid')} disabled={submitting}>
Invalide
</button>
<button onClick={() => submitVerdict('inconclusive')} disabled={submitting}>
Incertain
</button>
</>
)}
<button
onClick={handleResume}
disabled={!allRequiredOK || submitting}

2
visual_workflow_builder/frontend_v4/src/services/api.ts
View File

@@ -191,7 +191,9 @@ export async function getExecutionStatus(): Promise<{
total: number;
original_bbox?: { x: number; y: number; width: number; height: number };
error?: string;
human_pause?: Record<string, unknown>;
};
human_pause?: Record<string, unknown> | null;
}> {
return request('GET', '/execute/status');
}

3
visual_workflow_builder/frontend_v4/src/types.ts
View File

@@ -350,6 +350,9 @@ export interface Execution {
pause_reason?: string;
pause_message?: string;
safety_checks?: SafetyCheck[];
verdict_required?: boolean;
verdict_endpoint?: string;
competence_id?: string;
// ID du replay (utile pour appeler /replay/resume avec acknowledged_check_ids)
replay_id?: string;
}