From aba849324a1746513f7887679a7ca712c061e4e6 Mon Sep 17 00:00:00 2001 From: Dom Date: Fri, 29 May 2026 18:36:06 +0200 Subject: [PATCH] feat(vwb): log supervised competence verdicts --- core/competences/__init__.py | 8 + core/competences/verdicts.py | 181 ++++++++++++++++++ tests/unit/test_competence_to_vwb_preview.py | 8 + tests/unit/test_competence_verdicts.py | 117 +++++++++++ tests/unit/test_lea_competence_verdict_api.py | 109 +++++++++++ tests/unit/test_supervised_popup_guard.py | 38 ++++ .../unit/test_vwb_supervised_pause_runtime.py | 81 ++++++++ .../backend/api/lea_competences.py | 61 ++++++ .../backend/api_v3/execute.py | 91 ++++++++- visual_workflow_builder/backend/app.py | 7 + .../backend/catalog_routes_v2_vlm.py | 165 +++++++++++++++- .../backend/contracts/action_contracts.py | 31 ++- .../services/competence_vwb_preview.py | 4 + .../services/supervised_popup_guard.py | 69 +++++++ .../frontend_v4/src/App.tsx | 27 ++- .../src/components/PauseDialog.tsx | 85 ++++++++ .../frontend_v4/src/services/api.ts | 2 + .../frontend_v4/src/types.ts | 3 + 18 files changed, 1082 insertions(+), 5 deletions(-) create mode 100644 core/competences/verdicts.py create mode 100644 tests/unit/test_competence_verdicts.py create mode 100644 tests/unit/test_lea_competence_verdict_api.py create mode 100644 tests/unit/test_supervised_popup_guard.py create mode 100644 tests/unit/test_vwb_supervised_pause_runtime.py create mode 100644 visual_workflow_builder/backend/api/lea_competences.py create mode 100644 visual_workflow_builder/backend/services/supervised_popup_guard.py diff --git a/core/competences/__init__.py b/core/competences/__init__.py index dd9c87d51..0c114b719 100644 --- a/core/competences/__init__.py +++ b/core/competences/__init__.py @@ -10,12 +10,20 @@ from .replay import ( build_competence_replay_payload, find_competence, ) +from .verdicts import ( + CompetenceVerdictError, + iter_competence_verdicts, + store_competence_verdict, +) __all__ = [ "CompetenceSummary", + "CompetenceVerdictError", "build_competence_replay_actions", "build_competence_replay_payload", "find_competence", + "iter_competence_verdicts", "load_competence_catalog_actions", "load_competences", + "store_competence_verdict", ] diff --git a/core/competences/verdicts.py b/core/competences/verdicts.py new file mode 100644 index 000000000..620f55195 --- /dev/null +++ b/core/competences/verdicts.py @@ -0,0 +1,181 @@ +"""Persist supervised human verdicts for Lea competences.""" + +from __future__ import annotations + +import json +import uuid +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Dict, Iterable, Optional + +from .catalog import DEFAULT_COMPETENCE_ROOT, REPO_ROOT +from .replay import find_competence + + +DEFAULT_VERDICT_LOG = REPO_ROOT / "data" / "competence_verdicts" / "verdicts.jsonl" +VALID_VERDICT_KINDS = {"valid", "invalid", "inconclusive"} +SCHEMA_VERSION = "lea_competence_verdict.v1" + + +class CompetenceVerdictError(ValueError): + """Raised when a supervised verdict payload is invalid.""" + + +def store_competence_verdict( + competence_id: str, + payload: Dict[str, Any], + *, + log_path: Path | str = DEFAULT_VERDICT_LOG, + competence_root: Path | str = DEFAULT_COMPETENCE_ROOT, + states: Optional[Iterable[str]] = None, + now: Optional[datetime] = None, +) -> Dict[str, Any]: + """Validate and append one supervised verdict. + + The function is idempotent on ``verdict_id``. If the same verdict was + already logged for the same competence, the stored record is returned with + ``duplicate=True`` and the log is left untouched. + """ + + if not isinstance(payload, dict): + raise CompetenceVerdictError("Payload verdict invalide") + + competence = find_competence(competence_id, root=competence_root, states=states) + log = Path(log_path) + verdict_id = _required_text(payload, "verdict_id") + _validate_uuid(verdict_id) + + for existing in iter_competence_verdicts(log_path=log): + if existing.get("verdict_id") != verdict_id: + continue + if existing.get("competence_id") != competence_id: + raise CompetenceVerdictError( + f"verdict_id deja utilise pour {existing.get('competence_id')}" + ) + duplicate = dict(existing) + duplicate["duplicate"] = True + return duplicate + + verdict_kind = _required_text(payload, "verdict_kind") + if verdict_kind not in VALID_VERDICT_KINDS: + raise CompetenceVerdictError( + "verdict_kind doit etre valid, invalid ou inconclusive" + ) + + verdict_at = _timestamp(payload.get("verdict_at"), now=now) + context_signature = _context_signature(payload.get("context_signature")) + evidence = _mapping(payload.get("evidence"), field="evidence") + source = _mapping(payload.get("source"), field="source") + + record = { + "schema_version": SCHEMA_VERSION, + "verdict_id": verdict_id, + "competence_id": competence.id, + "competence_source_path": competence.source_path, + "learning_state": competence.learning_state, + "verdict_kind": verdict_kind, + "verdict_at": verdict_at, + "verdict_by": str(payload.get("verdict_by") or "human:dom"), + "context_signature": context_signature, + "evidence": evidence, + "comments": str(payload.get("comments") or ""), + "source": source, + "write_back_enabled": False, + "yaml_write": False, + "duplicate": False, + } + + _append_jsonl(log, record) + return record + + +def iter_competence_verdicts( + *, + log_path: Path | str = DEFAULT_VERDICT_LOG, + competence_id: Optional[str] = None, +) -> list[Dict[str, Any]]: + """Load logged verdict records, skipping malformed historical lines.""" + + log = Path(log_path) + if not log.exists(): + return [] + + records: list[Dict[str, Any]] = [] + with log.open("r", encoding="utf-8") as handle: + for line in handle: + line = line.strip() + if not line: + continue + try: + record = json.loads(line) + except json.JSONDecodeError: + continue + if not isinstance(record, dict): + continue + if competence_id and record.get("competence_id") != competence_id: + continue + records.append(record) + return records + + +def _required_text(payload: Dict[str, Any], key: str) -> str: + value = payload.get(key) + if not isinstance(value, str) or not value.strip(): + raise CompetenceVerdictError(f"{key} requis") + return value.strip() + + +def _validate_uuid(value: str) -> None: + try: + parsed = uuid.UUID(value, version=4) + except ValueError as exc: + raise CompetenceVerdictError("verdict_id doit etre un UUID v4") from exc + if str(parsed) != value.lower(): + raise CompetenceVerdictError("verdict_id UUID v4 invalide") + + +def _timestamp(value: Any, *, now: Optional[datetime]) -> str: + if value is None: + timestamp = now or datetime.now(timezone.utc) + elif isinstance(value, datetime): + timestamp = value + elif isinstance(value, str) and value.strip(): + text = value.strip() + try: + parsed = datetime.fromisoformat(text.replace("Z", "+00:00")) + except ValueError as exc: + raise CompetenceVerdictError("verdict_at doit etre ISO 8601") from exc + timestamp = parsed + else: + raise CompetenceVerdictError("verdict_at doit etre ISO 8601") + + if timestamp.tzinfo is None: + timestamp = timestamp.replace(tzinfo=timezone.utc) + return timestamp.astimezone(timezone.utc).isoformat() + + +def _context_signature(value: Any) -> Dict[str, Any]: + context = _mapping(value, field="context_signature") + machine_id = context.get("machine_id") + if not isinstance(machine_id, str) or not machine_id.strip(): + raise CompetenceVerdictError("context_signature.machine_id requis") + normalized = dict(context) + normalized["machine_id"] = machine_id.strip() + normalized.setdefault("screen_state_initial", "") + normalized.setdefault("screen_state_after_action", "") + return normalized + + +def _mapping(value: Any, *, field: str) -> Dict[str, Any]: + if value is None: + return {} + if not isinstance(value, dict): + raise CompetenceVerdictError(f"{field} doit etre un objet") + return dict(value) + + +def _append_jsonl(log_path: Path, record: Dict[str, Any]) -> None: + log_path.parent.mkdir(parents=True, exist_ok=True) + with log_path.open("a", encoding="utf-8") as handle: + handle.write(json.dumps(record, ensure_ascii=False, sort_keys=True)) + handle.write("\n") diff --git a/tests/unit/test_competence_to_vwb_preview.py b/tests/unit/test_competence_to_vwb_preview.py index a0f8ba0f1..ebf9552af 100644 --- a/tests/unit/test_competence_to_vwb_preview.py +++ b/tests/unit/test_competence_to_vwb_preview.py @@ -16,6 +16,9 @@ def test_competence_to_vwb_preview_key_win_r_steps(): assert preview["readonly"] is True assert preview["write_back_enabled"] is False assert preview["workflow"]["competence_id"] == "key_win_r_wait_explorer_exe" + assert preview["workflow"]["verdict_endpoint"] == ( + "/api/v1/lea/competences/key_win_r_wait_explorer_exe/verdict" + ) assert _by_type(preview) == [ "pause_for_human", "keyboard_shortcut", @@ -47,6 +50,8 @@ def test_wait_for_state_expected_state_is_preserved(): assert params["timeout_ms"] == 5000 assert params["poll_interval_ms"] == 250 assert params["evidence_required"] == "window_or_process" + assert params["supervised_popup_detection"] is True + assert params["popup_policy"] == "pause_only" def test_pause_for_human_before_and_after_are_supervision_only(): @@ -59,6 +64,9 @@ def test_pause_for_human_before_and_after_are_supervision_only(): assert after["parameters"]["phase"] == "after" assert after["parameters"]["verdict_required"] is True assert after["parameters"]["write_back_enabled"] is False + assert after["parameters"]["verdict_endpoint"] == ( + "/api/v1/lea/competences/key_win_r_wait_explorer_exe/verdict" + ) def test_adapter_is_generic_on_methods_not_hardcoded_to_win_r(): diff --git a/tests/unit/test_competence_verdicts.py b/tests/unit/test_competence_verdicts.py new file mode 100644 index 000000000..98bcee40b --- /dev/null +++ b/tests/unit/test_competence_verdicts.py @@ -0,0 +1,117 @@ +from datetime import datetime, timezone + +import pytest + +from core.competences.verdicts import ( + CompetenceVerdictError, + iter_competence_verdicts, + store_competence_verdict, +) + + +VERDICT_ID = "123e4567-e89b-42d3-a456-426614174000" + + +def _payload(**overrides): + payload = { + "verdict_id": VERDICT_ID, + "verdict_kind": "valid", + "verdict_by": "human:dom", + "context_signature": { + "machine_id": "DESKTOP-58D5CAC_windows", + "screen_state_initial": "before_hash", + "screen_state_after_action": "after_hash", + }, + "evidence": { + "screenshot_before": "evidence/before.png", + "screenshot_after": "evidence/after.png", + "wait_state_matched_evidence": { + "window_title": "Executer", + "process_name": "explorer.exe", + }, + }, + "comments": "Supervised replay ok", + } + payload.update(overrides) + return payload + + +def test_store_competence_verdict_appends_jsonl(tmp_path): + log_path = tmp_path / "verdicts.jsonl" + + record = store_competence_verdict( + "key_win_r_wait_explorer_exe", + _payload(), + log_path=log_path, + now=datetime(2026, 5, 29, 16, 30, tzinfo=timezone.utc), + ) + + assert record["schema_version"] == "lea_competence_verdict.v1" + assert record["competence_id"] == "key_win_r_wait_explorer_exe" + assert record["verdict_kind"] == "valid" + assert record["write_back_enabled"] is False + assert record["yaml_write"] is False + assert record["duplicate"] is False + assert record["context_signature"]["machine_id"] == "DESKTOP-58D5CAC_windows" + + records = iter_competence_verdicts(log_path=log_path) + assert len(records) == 1 + assert records[0]["verdict_id"] == VERDICT_ID + + +def test_store_competence_verdict_is_idempotent(tmp_path): + log_path = tmp_path / "verdicts.jsonl" + + first = store_competence_verdict( + "key_win_r_wait_explorer_exe", + _payload(), + log_path=log_path, + ) + second = store_competence_verdict( + "key_win_r_wait_explorer_exe", + _payload(comments="second click"), + log_path=log_path, + ) + + assert first["duplicate"] is False + assert second["duplicate"] is True + assert len(log_path.read_text(encoding="utf-8").splitlines()) == 1 + assert second["comments"] == "Supervised replay ok" + + +@pytest.mark.parametrize("kind", ["valid", "invalid", "inconclusive"]) +def test_store_competence_verdict_accepts_three_kinds(tmp_path, kind): + record = store_competence_verdict( + "key_win_r_wait_explorer_exe", + _payload( + verdict_id={ + "valid": "123e4567-e89b-42d3-a456-426614174000", + "invalid": "123e4567-e89b-42d3-a456-426614174001", + "inconclusive": "123e4567-e89b-42d3-a456-426614174002", + }[kind], + verdict_kind=kind, + ), + log_path=tmp_path / "verdicts.jsonl", + ) + + assert record["verdict_kind"] == kind + + +def test_store_competence_verdict_requires_context_machine(tmp_path): + with pytest.raises(CompetenceVerdictError, match="machine_id"): + store_competence_verdict( + "key_win_r_wait_explorer_exe", + _payload(context_signature={}), + log_path=tmp_path / "verdicts.jsonl", + ) + + +def test_store_competence_verdict_rejects_yaml_write_attempt(tmp_path): + record = store_competence_verdict( + "key_win_r_wait_explorer_exe", + _payload(write_back_enabled=True, yaml_write=True), + log_path=tmp_path / "verdicts.jsonl", + ) + + assert record["write_back_enabled"] is False + assert record["yaml_write"] is False diff --git a/tests/unit/test_lea_competence_verdict_api.py b/tests/unit/test_lea_competence_verdict_api.py new file mode 100644 index 000000000..f7cbd8870 --- /dev/null +++ b/tests/unit/test_lea_competence_verdict_api.py @@ -0,0 +1,109 @@ +from flask import Flask +import importlib.util +from pathlib import Path + + +MODULE_PATH = ( + Path(__file__).resolve().parents[2] + / "visual_workflow_builder" + / "backend" + / "api" + / "lea_competences.py" +) +SPEC = importlib.util.spec_from_file_location("lea_competences_test_module", MODULE_PATH) +lea_competences_module = importlib.util.module_from_spec(SPEC) +assert SPEC and SPEC.loader +SPEC.loader.exec_module(lea_competences_module) +lea_competences_bp = lea_competences_module.lea_competences_bp + + +def _app(): + app = Flask(__name__) + app.register_blueprint(lea_competences_bp) + return app + + +def test_submit_competence_verdict_endpoint(monkeypatch): + def fake_store(competence_id, payload): + assert competence_id == "key_win_r_wait_explorer_exe" + assert payload["verdict_kind"] == "valid" + return { + "verdict_id": payload["verdict_id"], + "competence_id": competence_id, + "verdict_kind": "valid", + "duplicate": False, + "write_back_enabled": False, + "yaml_write": False, + } + + monkeypatch.setattr( + lea_competences_module, + "store_competence_verdict", + fake_store, + ) + + with _app().test_client() as client: + response = client.post( + "/api/v1/lea/competences/key_win_r_wait_explorer_exe/verdict", + json={ + "verdict_id": "123e4567-e89b-42d3-a456-426614174000", + "verdict_kind": "valid", + }, + ) + + assert response.status_code == 201 + data = response.get_json() + assert data["success"] is True + assert data["write_back_enabled"] is False + assert data["yaml_write"] is False + assert data["verdict"]["duplicate"] is False + + +def test_submit_competence_verdict_endpoint_returns_duplicate_200(monkeypatch): + def fake_store(_competence_id, payload): + return { + "verdict_id": payload["verdict_id"], + "competence_id": "key_win_r_wait_explorer_exe", + "verdict_kind": "valid", + "duplicate": True, + "write_back_enabled": False, + "yaml_write": False, + } + + monkeypatch.setattr( + lea_competences_module, + "store_competence_verdict", + fake_store, + ) + + with _app().test_client() as client: + response = client.post( + "/api/v1/lea/competences/key_win_r_wait_explorer_exe/verdict", + json={ + "verdict_id": "123e4567-e89b-42d3-a456-426614174000", + "verdict_kind": "valid", + }, + ) + + assert response.status_code == 200 + assert response.get_json()["duplicate"] is True + + +def test_list_competence_verdicts_endpoint(monkeypatch): + monkeypatch.setattr( + lea_competences_module, + "iter_competence_verdicts", + lambda competence_id: [{"competence_id": competence_id, "verdict_kind": "valid"}], + ) + + with _app().test_client() as client: + response = client.get( + "/api/v1/lea/competences/key_win_r_wait_explorer_exe/verdicts" + ) + + data = response.get_json() + assert response.status_code == 200 + assert data["success"] is True + assert data["verdicts"] == [ + {"competence_id": "key_win_r_wait_explorer_exe", "verdict_kind": "valid"} + ] diff --git a/tests/unit/test_supervised_popup_guard.py b/tests/unit/test_supervised_popup_guard.py new file mode 100644 index 000000000..58ebfbb9a --- /dev/null +++ b/tests/unit/test_supervised_popup_guard.py @@ -0,0 +1,38 @@ +from visual_workflow_builder.backend.services.supervised_popup_guard import ( + build_unexpected_popup_pause, +) + + +def test_unexpected_popup_builds_pause_without_auto_resolution(): + pause = build_unexpected_popup_pause( + {"pattern": "confirm_save_overwrite", "title": "Enregistrer sous"}, + expected_state={ + "window_title_in": ["Executer"], + "process_active": "explorer.exe", + }, + competence_id="key_win_r_wait_explorer_exe", + source_method_id="step_2_wait_state", + ) + + assert pause is not None + assert pause["needs_human"] is True + assert pause["pause_reason"] == "unexpected_popup" + assert pause["auto_resolution"] is False + assert pause["write_back_enabled"] is False + assert pause["detected_popup"]["title"] == "Enregistrer sous" + + +def test_expected_popup_title_does_not_build_pause(): + pause = build_unexpected_popup_pause( + {"pattern": "run_dialog", "title": "Executer"}, + expected_state={"window_title_in": ["Executer"]}, + ) + + assert pause is None + + +def test_missing_popup_does_not_build_pause(): + assert build_unexpected_popup_pause( + None, + expected_state={"window_title_in": ["Executer"]}, + ) is None diff --git a/tests/unit/test_vwb_supervised_pause_runtime.py b/tests/unit/test_vwb_supervised_pause_runtime.py new file mode 100644 index 000000000..4c7d26cb3 --- /dev/null +++ b/tests/unit/test_vwb_supervised_pause_runtime.py @@ -0,0 +1,81 @@ +from flask import Flask + +from visual_workflow_builder.backend.catalog_routes_v2_vlm import catalog_bp + + +class _WaitResult: + matched = False + + def to_dict(self): + return { + "matched": False, + "timed_out": True, + "match": { + "expected_state": {"window_title_in": ["Executer"]}, + "observed_state": {"window_title": "Enregistrer sous"}, + }, + } + + +def test_catalog_pause_for_human_returns_paused_status(): + app = Flask(__name__) + app.register_blueprint(catalog_bp) + + with app.test_client() as client: + response = client.post( + "/api/vwb/catalog/execute", + json={ + "type": "pause_for_human", + "step_id": "pause_1", + "parameters": { + "message": "Valider le resultat", + "phase": "after", + "verdict_required": True, + "competence_id": "key_win_r_wait_explorer_exe", + }, + }, + ) + + data = response.get_json() + assert response.status_code == 200 + assert data["result"]["status"] == "paused" + assert data["result"]["output_data"]["needs_human"] is True + assert data["result"]["output_data"]["write_back_enabled"] is False + + +def test_catalog_wait_for_state_popup_pauses_without_auto_resolution(monkeypatch): + monkeypatch.setattr( + "visual_workflow_builder.backend.services.wait_for_state.wait_for_expected_state", + lambda **_kwargs: _WaitResult(), + ) + monkeypatch.setattr( + "core.execution.input_handler.check_screen_for_patterns", + lambda: {"pattern": "save_as", "title": "Enregistrer sous"}, + ) + + app = Flask(__name__) + app.register_blueprint(catalog_bp) + + with app.test_client() as client: + response = client.post( + "/api/vwb/catalog/execute", + json={ + "type": "wait_for_state", + "step_id": "wait_1", + "parameters": { + "expected_state": {"window_title_in": ["Executer"]}, + "timeout_ms": 1, + "poll_interval_ms": 1, + "supervised_popup_detection": True, + "competence_id": "key_win_r_wait_explorer_exe", + "source_method_id": "step_2_wait_state", + }, + }, + ) + + data = response.get_json() + assert response.status_code == 200 + assert data["result"]["status"] == "paused" + pause = data["result"]["output_data"]["human_pause"] + assert pause["pause_reason"] == "unexpected_popup" + assert pause["auto_resolution"] is False diff --git a/visual_workflow_builder/backend/api/lea_competences.py b/visual_workflow_builder/backend/api/lea_competences.py new file mode 100644 index 000000000..952ad9151 --- /dev/null +++ b/visual_workflow_builder/backend/api/lea_competences.py @@ -0,0 +1,61 @@ +"""Lea competence supervision endpoints exposed by the VWB backend.""" + +from __future__ import annotations + +from flask import Blueprint, jsonify, request + +from core.competences.verdicts import ( + CompetenceVerdictError, + iter_competence_verdicts, + store_competence_verdict, +) + + +lea_competences_bp = Blueprint( + "lea_competences", + __name__, + url_prefix="/api/v1/lea/competences", +) + + +@lea_competences_bp.route("//verdict", methods=["POST"]) +def submit_competence_verdict(competence_id: str): + """Persist one supervised human verdict without touching YAML.""" + + payload = request.get_json(silent=True) or {} + try: + verdict = store_competence_verdict(competence_id, payload) + except KeyError: + return jsonify({ + "success": False, + "error": f"Competence '{competence_id}' introuvable", + }), 404 + except CompetenceVerdictError as exc: + return jsonify({ + "success": False, + "error": str(exc), + "write_back_enabled": False, + "yaml_write": False, + }), 400 + + return jsonify({ + "success": True, + "competence_id": competence_id, + "verdict": verdict, + "duplicate": verdict["duplicate"], + "write_back_enabled": False, + "yaml_write": False, + }), 200 if verdict["duplicate"] else 201 + + +@lea_competences_bp.route("//verdicts", methods=["GET"]) +def list_competence_verdicts(competence_id: str): + """Return logged supervised verdicts for one competence.""" + + return jsonify({ + "success": True, + "competence_id": competence_id, + "verdicts": iter_competence_verdicts(competence_id=competence_id), + "write_back_enabled": False, + "yaml_write": False, + }) diff --git a/visual_workflow_builder/backend/api_v3/execute.py b/visual_workflow_builder/backend/api_v3/execute.py index af511f879..df2ec735d 100644 --- a/visual_workflow_builder/backend/api_v3/execute.py +++ b/visual_workflow_builder/backend/api_v3/execute.py @@ -103,6 +103,33 @@ def _execute_wait_for_state(params: Dict[str, Any]) -> dict: output = result.to_dict() if result.matched: return {"success": True, "output": output} + if params.get("supervised_popup_detection", False): + detected_popup = None + try: + detected_popup = _check_screen_for_patterns() + except Exception as exc: + print(f"⚠️ [wait_for_state] Detection popup indisponible: {exc}") + + from visual_workflow_builder.backend.services.supervised_popup_guard import ( + build_unexpected_popup_pause, + ) + + pause_payload = build_unexpected_popup_pause( + detected_popup, + expected_state=expected_state, + competence_id=str(params.get("competence_id") or ""), + source_method_id=str(params.get("source_method_id") or ""), + ) + if pause_payload: + return { + "success": True, + "needs_human_pause": True, + "output": { + "wait_for_state": output, + "human_pause": pause_payload, + "write_back_enabled": False, + }, + } return { "success": False, "error": "Etat attendu non observe avant timeout", @@ -110,6 +137,23 @@ def _execute_wait_for_state(params: Dict[str, Any]) -> dict: } +def _execute_pause_for_human(params: Dict[str, Any]) -> dict: + return { + "success": True, + "needs_human_pause": True, + "output": { + "needs_human": True, + "pause_reason": "supervised_pause", + "message": params.get("message", "Validation humaine requise"), + "phase": params.get("phase"), + "verdict_required": bool(params.get("verdict_required", False)), + "verdict_endpoint": params.get("verdict_endpoint"), + "competence_id": params.get("competence_id"), + "write_back_enabled": False, + }, + } + + def minimize_active_window(): """Minimise le navigateur VWB et active la fenêtre suivante (VM, app cible).""" try: @@ -168,7 +212,8 @@ _execution_state = { 'pending_action': None, # Action en attente de choix utilisateur 'candidates': [], # Candidats proposés 'user_choice': None, # Choix de l'utilisateur (coordonnées ou 'skip' ou 'static') - 'current_step_info': None # Info sur l'étape en cours pour affichage + 'current_step_info': None, # Info sur l'étape en cours pour affichage + 'human_pause': None, } @@ -307,6 +352,35 @@ def execute_workflow_thread(execution_id: str, workflow_id: str, app): params['_step_label'] = step.label result = execute_action(step.action_type, params) + if result.get('needs_human_pause'): + pause_payload = result.get('output', {}) + print(f"⏸️ [Supervision] Pause humaine étape {index + 1}: {pause_payload}") + with _execution_lock: + _execution_state['is_paused'] = True + _execution_state['human_pause'] = pause_payload + _execution_state['current_step_info'] = { + 'index': index, + 'total': len(steps), + 'step_id': step.id, + 'action_type': step.action_type, + 'label': step.label, + 'human_pause': pause_payload, + } + execution.status = 'paused' + db.session.commit() + + while _execution_state['is_paused'] and not _execution_state['should_stop']: + time.sleep(0.1) + + if _execution_state['should_stop']: + execution.status = 'cancelled' + break + + with _execution_lock: + _execution_state['human_pause'] = None + execution.status = 'running' + db.session.commit() + # === SELF-HEALING INTERACTIF === # Si l'action nécessite un choix utilisateur, attendre if result.get('needs_user_choice'): @@ -456,6 +530,7 @@ def execute_workflow_thread(execution_id: str, workflow_id: str, app): with _execution_lock: _execution_state['is_running'] = False _execution_state['current_execution_id'] = None + _execution_state['human_pause'] = None def execute_ai_analyze(params: dict) -> dict: @@ -1198,6 +1273,9 @@ def execute_action(action_type: str, params: dict) -> dict: elif action_type == 'wait_for_state': return _execute_wait_for_state(params) + elif action_type == 'pause_for_human': + return _execute_pause_for_human(params) + elif action_type == 'keyboard_shortcut': keys = params.get('keys', []) if not keys: @@ -1631,6 +1709,7 @@ def start_execution(): _execution_state['current_execution_id'] = execution.id _execution_state['execution_mode'] = execution_mode _execution_state['variables'] = {} # Reset des variables + _execution_state['human_pause'] = None print(f"🎯 [API v3] Mode d'exécution: {execution_mode}") @@ -1790,7 +1869,15 @@ def get_execution_status(): # Self-healing interactif 'waiting_for_choice': _execution_state.get('waiting_for_choice', False), 'candidates': _execution_state.get('candidates', []) if _execution_state.get('waiting_for_choice') else [], - 'current_step_info': _execution_state.get('current_step_info') if _execution_state.get('waiting_for_choice') else None + 'current_step_info': ( + _execution_state.get('current_step_info') + if ( + _execution_state.get('waiting_for_choice') + or _execution_state.get('human_pause') + ) + else None + ), + 'human_pause': _execution_state.get('human_pause'), }) diff --git a/visual_workflow_builder/backend/app.py b/visual_workflow_builder/backend/app.py index 5f0f048a2..0f874b4c3 100644 --- a/visual_workflow_builder/backend/app.py +++ b/visual_workflow_builder/backend/app.py @@ -170,6 +170,13 @@ try: except ImportError as e: print(f"⚠️ Blueprint correction_packs désactivé: {e}") +try: + from api.lea_competences import lea_competences_bp + app.register_blueprint(lea_competences_bp) + print("✅ Blueprint lea_competences enregistré") +except ImportError as e: + print(f"⚠️ Blueprint lea_competences désactivé: {e}") + try: from api.coaching_sessions import coaching_sessions_bp app.register_blueprint(coaching_sessions_bp, url_prefix='/api/coaching-sessions') diff --git a/visual_workflow_builder/backend/catalog_routes_v2_vlm.py b/visual_workflow_builder/backend/catalog_routes_v2_vlm.py index cc5c583cf..abc92f794 100644 --- a/visual_workflow_builder/backend/catalog_routes_v2_vlm.py +++ b/visual_workflow_builder/backend/catalog_routes_v2_vlm.py @@ -1253,6 +1253,108 @@ def preview_lea_competence_workflow(competence_id: str): }), 500 +def _catalog_paused_result( + *, + action_type: str, + step_id: str, + output_data: Dict[str, Any], + start_time: datetime, +) -> Any: + end_time = datetime.now() + execution_time_ms = (end_time - start_time).total_seconds() * 1000 + return jsonify({ + "success": True, + "result": { + "action_id": f"paused_{action_type}_{step_id}", + "step_id": step_id, + "status": "paused", + "execution_time_ms": execution_time_ms, + "output_data": output_data, + "evidence_list": [], + "error": None, + }, + }) + + +def _execute_catalog_wait_for_state( + *, + parameters: Dict[str, Any], + action_type: str, + step_id: str, + start_time: datetime, +) -> Any: + from visual_workflow_builder.backend.services.wait_for_state import ( + wait_for_expected_state, + ) + + expected_state = parameters.get('expected_state') or {} + timeout_ms = int(parameters.get('timeout_ms', 5000)) + poll_interval_ms = int(parameters.get('poll_interval_ms', 250)) + evidence_required = parameters.get('evidence_required', 'window_or_process') + wait_result = wait_for_expected_state( + expected_state=expected_state, + timeout_ms=timeout_ms, + poll_interval_ms=poll_interval_ms, + evidence_required=evidence_required, + ) + output_data = wait_result.to_dict() + + if ( + not wait_result.matched + and parameters.get('supervised_popup_detection', False) + ): + detected_popup = None + try: + from core.execution.input_handler import check_screen_for_patterns + + detected_popup = check_screen_for_patterns() + except Exception as popup_error: + print(f"⚠️ [wait_for_state] Detection popup indisponible: {popup_error}") + + from visual_workflow_builder.backend.services.supervised_popup_guard import ( + build_unexpected_popup_pause, + ) + + pause_payload = build_unexpected_popup_pause( + detected_popup, + expected_state=expected_state, + competence_id=str(parameters.get('competence_id') or ''), + source_method_id=str(parameters.get('source_method_id') or ''), + ) + if pause_payload: + return _catalog_paused_result( + action_type=action_type, + step_id=step_id, + start_time=start_time, + output_data={ + "needs_human": True, + "wait_for_state": output_data, + "human_pause": pause_payload, + "write_back_enabled": False, + }, + ) + + end_time = datetime.now() + execution_time_ms = (end_time - start_time).total_seconds() * 1000 + result_message = ( + 'Etat attendu observe' + if wait_result.matched + else 'Etat attendu non observe avant timeout' + ) + return jsonify({ + "success": True, + "result": { + "action_id": f"direct_{action_type}_{step_id}", + "step_id": step_id, + "status": "success" if wait_result.matched else "failed", + "execution_time_ms": execution_time_ms, + "output_data": output_data, + "evidence_list": [], + "error": None if wait_result.matched else {"message": result_message}, + }, + }) + + def get_screen_capturer(): """ Obtient l'instance du ScreenCapturer (initialisation paresseuse). @@ -2096,6 +2198,33 @@ def execute_action(): if action_type == "test_competence" or action_type.startswith("lea_competence_"): return _execute_lea_competence_action(data, action_type, step_id, parameters) + request_start_time = datetime.now() + + if action_type == "pause_for_human": + return _catalog_paused_result( + action_type=action_type, + step_id=step_id, + start_time=request_start_time, + output_data={ + "needs_human": True, + "pause_reason": "supervised_pause", + "message": parameters.get("message", "Validation humaine requise"), + "phase": parameters.get("phase"), + "verdict_required": bool(parameters.get("verdict_required", False)), + "verdict_endpoint": parameters.get("verdict_endpoint"), + "competence_id": parameters.get("competence_id"), + "write_back_enabled": False, + }, + ) + + if action_type == "wait_for_state": + return _execute_catalog_wait_for_state( + parameters=parameters, + action_type=action_type, + step_id=step_id, + start_time=request_start_time, + ) + # LOG DEBUG - Voir ce qui arrive du frontend print(f"\n{'='*60}") print(f"🔥 REQUÊTE EXECUTE REÇUE:") @@ -2768,6 +2897,40 @@ def execute_action(): if wait_result.matched else 'Etat attendu non observe avant timeout' ) + if ( + not wait_result.matched + and parameters.get('supervised_popup_detection', False) + ): + detected_popup = None + try: + from core.execution.input_handler import check_screen_for_patterns + + detected_popup = check_screen_for_patterns() + except Exception as popup_error: + print(f"⚠️ [wait_for_state] Detection popup indisponible: {popup_error}") + + from visual_workflow_builder.backend.services.supervised_popup_guard import ( + build_unexpected_popup_pause, + ) + + pause_payload = build_unexpected_popup_pause( + detected_popup, + expected_state=expected_state, + competence_id=str(parameters.get('competence_id') or ''), + source_method_id=str(parameters.get('source_method_id') or ''), + ) + if pause_payload: + return _catalog_paused_result( + action_type=action_type, + step_id=step_id, + start_time=start_time, + output_data={ + "needs_human": True, + "wait_for_state": direct_output_data, + "human_pause": pause_payload, + "write_back_enabled": False, + }, + ) elif action_type in ['focus', 'focus_anchor', 'focaliser']: # focus_anchor: chercher l'ancre et cliquer dessus pour donner le focus @@ -2845,7 +3008,7 @@ def execute_action(): result_message = 'Coordonnées invalides - ancre non trouvée' execution_success = False - elif action_type in ['hotkey', 'raccourci']: + elif action_type in ['hotkey', 'raccourci', 'keyboard_shortcut']: keys = parameters.get('keys', parameters.get('touches', [])) if keys: print(f"⌨️ [Direct] Raccourci: {'+'.join(keys)}") diff --git a/visual_workflow_builder/backend/contracts/action_contracts.py b/visual_workflow_builder/backend/contracts/action_contracts.py index 591ac113c..193ab81b7 100644 --- a/visual_workflow_builder/backend/contracts/action_contracts.py +++ b/visual_workflow_builder/backend/contracts/action_contracts.py @@ -202,7 +202,17 @@ VWB_ACTION_CONTRACTS: Dict[str, ActionContract] = { action_type="wait_for_state", description="Attendre qu'un etat semantique d'ecran soit observe", required_params=["expected_state"], - optional_params=["timeout_ms", "poll_interval_ms", "evidence_required"], + optional_params=[ + "timeout_ms", + "poll_interval_ms", + "evidence_required", + "supervised_popup_detection", + "popup_policy", + "competence_id", + "source", + "source_method_id", + "write_back_enabled", + ], param_validators={ "expected_state": lambda p: isinstance(p, dict) and any( p.get(key) @@ -214,6 +224,25 @@ VWB_ACTION_CONTRACTS: Dict[str, ActionContract] = { ) } ), + "pause_for_human": ActionContract( + action_type="pause_for_human", + description="Suspendre l'execution pour une validation humaine", + required_params=["message"], + optional_params=[ + "phase", + "verdict_required", + "verdict_endpoint", + "competence_id", + "source", + "write_back_enabled", + "intention", + "attendu", + "demande", + "safety_level", + "safety_checks", + ], + param_validators={"message": lambda p: bool(p and isinstance(p, str))} + ), # --- ACTIONS DE SCROLL --- "scroll_to_anchor": ActionContract( diff --git a/visual_workflow_builder/backend/services/competence_vwb_preview.py b/visual_workflow_builder/backend/services/competence_vwb_preview.py index 4c97da2e2..1688072cf 100644 --- a/visual_workflow_builder/backend/services/competence_vwb_preview.py +++ b/visual_workflow_builder/backend/services/competence_vwb_preview.py @@ -49,6 +49,7 @@ def competence_yaml_to_vwb_preview( "review_status": "preview", "tags": ["lea", "competence", competence.learning_state], "competence_id": competence.id, + "verdict_endpoint": f"/api/v1/lea/competences/{competence.id}/verdict", "learning_state": competence.learning_state, "source_path": competence.source_path, "readonly": True, @@ -120,6 +121,8 @@ def _method_to_vwb_step( "timeout_ms": int(params.get("timeout_ms", 5000)), "poll_interval_ms": int(params.get("poll_interval_ms", 250)), "evidence_required": params.get("evidence_required", "window_or_process"), + "supervised_popup_detection": True, + "popup_policy": "pause_only", **source_params, } return _step( @@ -185,6 +188,7 @@ def _pause_step( "message": message, "phase": phase, "verdict_required": verdict_required, + "verdict_endpoint": f"/api/v1/lea/competences/{competence.id}/verdict", "competence_id": competence.id, "source": f"lea_competence:{competence.id}", "write_back_enabled": False, diff --git a/visual_workflow_builder/backend/services/supervised_popup_guard.py b/visual_workflow_builder/backend/services/supervised_popup_guard.py new file mode 100644 index 000000000..a8dc7aa95 --- /dev/null +++ b/visual_workflow_builder/backend/services/supervised_popup_guard.py @@ -0,0 +1,69 @@ +"""Helpers for supervised popup detection without auto-resolution.""" + +from __future__ import annotations + +from typing import Any, Dict, Mapping, Optional + + +def build_unexpected_popup_pause( + detected_popup: Optional[Mapping[str, Any]], + *, + expected_state: Mapping[str, Any], + competence_id: str = "", + source_method_id: str = "", +) -> Optional[Dict[str, Any]]: + """Return a human-pause payload when a detected popup is not expected.""" + + if not detected_popup: + return None + + popup = dict(detected_popup) + popup_title = _popup_title(popup) + if popup_title and _title_is_expected(popup_title, expected_state): + return None + + competence_label = f" pour {competence_id}" if competence_id else "" + title_label = popup_title or str(popup.get("pattern") or "popup inconnue") + return { + "needs_human": True, + "pause_reason": "unexpected_popup", + "message": ( + f"Popup inattendue detectee{competence_label}: '{title_label}'. " + "Mode supervise: aucune resolution automatique n'est lancee." + ), + "detected_popup": popup, + "expected_state": dict(expected_state or {}), + "competence_id": competence_id, + "source_method_id": source_method_id, + "auto_resolution": False, + "write_back_enabled": False, + } + + +def _popup_title(popup: Mapping[str, Any]) -> str: + for key in ("title", "window_title", "target", "pattern", "name"): + value = popup.get(key) + if isinstance(value, str) and value.strip(): + return value.strip() + return "" + + +def _title_is_expected(title: str, expected_state: Mapping[str, Any]) -> bool: + normalized = _norm(title) + exact = expected_state.get("window_title_in") + if isinstance(exact, str): + exact = [exact] + if isinstance(exact, list) and any(_norm(candidate) == normalized for candidate in exact): + return True + + contains = expected_state.get("window_title_contains") + if isinstance(contains, str): + contains = [contains] + if isinstance(contains, list) and any(_norm(candidate) in normalized for candidate in contains): + return True + + return False + + +def _norm(value: Any) -> str: + return str(value or "").strip().casefold() diff --git a/visual_workflow_builder/frontend_v4/src/App.tsx b/visual_workflow_builder/frontend_v4/src/App.tsx index 2e41e6549..13a012593 100644 --- a/visual_workflow_builder/frontend_v4/src/App.tsx +++ b/visual_workflow_builder/frontend_v4/src/App.tsx @@ -106,6 +106,24 @@ function App() { setRuntimeVariables(status.variables as Record); } + const localExecution = status.execution; + if (status.human_pause && localExecution) { + const rawPause = status.human_pause as any; + const pause = rawPause.human_pause || rawPause; + setAppState((prev) => prev ? ({ + ...prev, + execution: { + ...localExecution, + pause_message: pause.message || 'Validation humaine requise', + pause_reason: pause.pause_reason || 'supervised_pause', + safety_checks: [], + verdict_required: Boolean(pause.verdict_required), + verdict_endpoint: pause.verdict_endpoint, + competence_id: pause.competence_id, + }, + }) : prev); + } + // Self-healing interactif: detecter si on attend un choix utilisateur if (status.waiting_for_choice && status.candidates) { setHealingCandidates(status.candidates); @@ -640,12 +658,19 @@ function App() { safety_checks vide, PauseDialog rend la bulle simple legacy. */} {(appState?.execution?.status === 'paused_need_help' || (appState?.execution?.status === 'paused' && - (appState?.execution?.safety_checks?.length ?? 0) > 0)) && ( + ( + (appState?.execution?.safety_checks?.length ?? 0) > 0 || + Boolean(appState?.execution?.pause_message) + ))) && (
{ const replayId = appState.execution?.replay_id || appState.execution?.id; if (replayId) { diff --git a/visual_workflow_builder/frontend_v4/src/components/PauseDialog.tsx b/visual_workflow_builder/frontend_v4/src/components/PauseDialog.tsx index 73b34a91b..4622b2c0c 100644 --- a/visual_workflow_builder/frontend_v4/src/components/PauseDialog.tsx +++ b/visual_workflow_builder/frontend_v4/src/components/PauseDialog.tsx @@ -14,6 +14,10 @@ interface Props { pauseMessage: string; pauseReason?: string; safetyChecks: SafetyCheck[]; + verdictRequired?: boolean; + verdictEndpoint?: string; + competenceId?: string; + executionId?: string; onResume: (acknowledgedIds: string[]) => Promise; onCancel: () => void; } @@ -22,6 +26,10 @@ export default function PauseDialog({ pauseMessage, pauseReason, safetyChecks, + verdictRequired = false, + verdictEndpoint, + competenceId, + executionId, onResume, onCancel, }: Props) { @@ -54,6 +62,57 @@ export default function PauseDialog({ } }; + const newVerdictId = (): string => { + if (window.crypto?.randomUUID) { + return window.crypto.randomUUID(); + } + return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, (char) => { + const value = Math.floor(Math.random() * 16); + const resolved = char === 'x' ? value : (value & 0x3) | 0x8; + return resolved.toString(16); + }); + }; + + const submitVerdict = async (verdictKind: 'valid' | 'invalid' | 'inconclusive') => { + if (!verdictEndpoint || !competenceId) return; + setSubmitting(true); + setError(null); + try { + const response = await fetch(verdictEndpoint, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + verdict_id: newVerdictId(), + verdict_kind: verdictKind, + verdict_by: 'human:dom', + context_signature: { + machine_id: `browser:${window.navigator.platform || 'unknown'}`, + screen_state_initial: '', + screen_state_after_action: '', + }, + evidence: { + execution_id: executionId || '', + pause_reason: pauseReason || '', + }, + source: { + frontend: 'vwb_v4', + execution_id: executionId || '', + }, + comments: `Verdict humain VWB: ${verdictKind}`, + }), + }); + if (!response.ok) { + const data = await response.json().catch(() => ({})); + throw new Error(data.error || response.statusText); + } + await onResume([]); + } catch (e: any) { + setError(e?.message || 'Erreur lors du verdict'); + } finally { + setSubmitting(false); + } + }; + // Backward compat : pas de checks -> bulle simple legacy if (safetyChecks.length === 0) { return ( @@ -61,6 +120,19 @@ export default function PauseDialog({

{pauseMessage}

{pauseReason && Raison : {pauseReason}}
+ {verdictRequired && verdictEndpoint && competenceId && ( + <> + + + + + )} @@ -110,6 +182,19 @@ export default function PauseDialog({ {error &&
{error}
}
+ {verdictRequired && verdictEndpoint && competenceId && ( + <> + + + + + )}