From 45b6da5e3fca8079639e974ed46559dda0e33891 Mon Sep 17 00:00:00 2001 From: Dom Date: Fri, 29 May 2026 17:09:47 +0200 Subject: [PATCH] feat(vwb): load palette from catalog --- tests/unit/test_competence_catalog_loader.py | 35 ++++ .../backend/catalog_routes_v2_vlm.py | 161 +++++++++++++++++- .../src/components/ConfidenceDashboard.tsx | 3 +- .../src/components/PropertiesPanel.tsx | 92 ++++++++++ .../src/components/ToolPalette.tsx | 89 +++++++++- .../frontend_v4/src/types.ts | 6 + 6 files changed, 379 insertions(+), 7 deletions(-) diff --git a/tests/unit/test_competence_catalog_loader.py b/tests/unit/test_competence_catalog_loader.py index 097a81a4b..5ae7660f0 100644 --- a/tests/unit/test_competence_catalog_loader.py +++ b/tests/unit/test_competence_catalog_loader.py @@ -90,3 +90,38 @@ def test_vwb_catalog_execute_plans_competence_replay(): "verify_screen", "pause_for_human", ] + + +def test_vwb_catalog_exposes_lea_mapping_actions_without_competence_noise(): + from visual_workflow_builder.backend.catalog_routes_v2_vlm import catalog_bp + + app = Flask(__name__) + app.register_blueprint(catalog_bp) + + with app.test_client() as client: + response = client.get("/api/vwb/catalog/actions") + + assert response.status_code == 200 + data = response.get_json() + assert data["success"] is True + + ids = {action["id"] for action in data["actions"]} + assert "keyboard_shortcut" in ids + assert "pause_for_human" in ids + assert "wait_for_state" in ids + assert not any(action_id.startswith("lea_competence_") for action_id in ids) + + +def test_vwb_catalog_lea_competences_are_explicit_opt_in(): + from visual_workflow_builder.backend.catalog_routes_v2_vlm import catalog_bp + + app = Flask(__name__) + app.register_blueprint(catalog_bp) + + with app.test_client() as client: + response = client.get("/api/vwb/catalog/actions?show_competences=1") + + assert response.status_code == 200 + data = response.get_json() + ids = {action["id"] for action in data["actions"]} + assert "lea_competence_key_win_r_wait_explorer_exe" in ids diff --git a/visual_workflow_builder/backend/catalog_routes_v2_vlm.py b/visual_workflow_builder/backend/catalog_routes_v2_vlm.py index e7ae67a7f..5a1083883 100644 --- a/visual_workflow_builder/backend/catalog_routes_v2_vlm.py +++ b/visual_workflow_builder/backend/catalog_routes_v2_vlm.py @@ -1090,6 +1090,13 @@ def _load_lea_competence_actions() -> List[Dict[str, Any]]: return [] +def _truthy_query_arg(name: str) -> bool: + """Return True for explicit opt-in query flags.""" + return str(request.args.get(name, "")).strip().lower() in ( + "1", "true", "yes", "on", + ) + + def _extract_competence_id(action_type: str, parameters: Dict[str, Any]) -> str: competence_id = str(parameters.get("competence_id") or "").strip() if competence_id: @@ -1334,6 +1341,7 @@ def list_actions(): # Paramètres de requête category_filter = request.args.get('category') search_query = request.args.get('search', '').lower() + show_competences = _truthy_query_arg('show_competences') # Définir les actions disponibles available_actions = [ @@ -1443,6 +1451,36 @@ def list_actions(): } ] }, + { + "id": "keyboard_shortcut", + "name": "Raccourci Clavier", + "description": "Exécute une combinaison de touches normalisée", + "category": "keyboard", + "icon": "⌘", + "parameters": { + "keys": { + "type": "array", + "required": True, + "description": "Touches à envoyer dans l'ordre, ex: ['ctrl', 's']" + }, + "hold_duration_ms": { + "type": "number", + "required": False, + "default": 50, + "min": 0, + "description": "Durée de maintien optionnelle en millisecondes" + } + }, + "examples": [ + { + "name": "Ouvrir Exécuter", + "description": "Envoie Win+R pour ouvrir la boîte Exécuter", + "parameters": { + "keys": ["win", "r"] + } + } + ] + }, { "id": "wait_for_anchor", "name": "Attente d'Ancre Visuelle", @@ -1488,6 +1526,54 @@ def list_actions(): } ] }, + { + "id": "wait_for_state", + "name": "Attendre un État", + "description": "Attend un état sémantique d'écran, comme une fenêtre active ou un processus cible", + "category": "wait", + "icon": "⏱️", + "parameters": { + "expected_state": { + "type": "object", + "required": True, + "description": "Critères d'état attendus: window_title_in, window_title_contains, process_active" + }, + "timeout_ms": { + "type": "number", + "required": False, + "default": 5000, + "min": 100, + "description": "Délai d'attente maximum en millisecondes" + }, + "poll_interval_ms": { + "type": "number", + "required": False, + "default": 250, + "min": 50, + "description": "Intervalle de vérification en millisecondes" + }, + "evidence_required": { + "type": "string", + "required": False, + "default": "window_or_process", + "options": ["window_or_process", "uia", "ocr", "screenshot_diff"], + "description": "Niveau de preuve requis" + } + }, + "examples": [ + { + "name": "Attendre Exécuter", + "description": "Attend que la fenêtre Exécuter soit au premier plan", + "parameters": { + "expected_state": { + "window_title_in": ["Exécuter"], + "process_active": "explorer.exe" + }, + "timeout_ms": 5000 + } + } + ] + }, { "id": "focus_anchor", "name": "Donner le Focus à un Élément", @@ -1533,6 +1619,42 @@ def list_actions(): } ] }, + { + "id": "pause_for_human", + "name": "Pause Supervisée", + "description": "Suspend l'exécution et demande une validation humaine avant de continuer", + "category": "logic", + "icon": "⏸️", + "parameters": { + "message": { + "type": "string", + "required": True, + "description": "Message affiché à l'opérateur" + }, + "safety_level": { + "type": "string", + "required": False, + "default": "standard", + "options": ["standard", "medical_critical"], + "description": "Niveau de sécurité de la pause" + }, + "safety_checks": { + "type": "array", + "required": False, + "default": [], + "description": "Liste de contrôles à acquitter avant reprise" + } + }, + "examples": [ + { + "name": "Validation humaine", + "description": "Demande à l'opérateur de valider l'état visible", + "parameters": { + "message": "Validez-vous que l'état attendu est visible ?" + } + } + ] + }, { "id": "type_secret", "name": "Saisie Sécurisée", @@ -1688,8 +1810,11 @@ def list_actions(): } ] - # Ajouter les compétences YAML Léa comme actions testables. - available_actions.extend(_load_lea_competence_actions()) + # Les compétences Léa sont masquées par défaut : elles ne doivent pas + # ressembler à des actions ordinaires tant que le replay supervisé + # complet (verdict + write-back YAML) n'est pas branché. + if show_competences: + available_actions.extend(_load_lea_competence_actions()) # Filtrer par catégorie if category_filter: @@ -2879,7 +3004,8 @@ def list_categories(): } """ try: - competence_action_count = len(_load_lea_competence_actions()) + show_competences = _truthy_query_arg('show_competences') + competence_action_count = len(_load_lea_competence_actions()) if show_competences else 0 # Définir les catégories disponibles avec métadonnées available_categories = [ @@ -2892,6 +3018,15 @@ def list_categories(): "color": "#2196f3", "isEnabled": True }, + { + "id": "keyboard", + "name": "Clavier", + "description": "Raccourcis clavier et actions de saisie", + "icon": "⌨️", + "actionCount": 1, + "color": "#607d8b", + "isEnabled": True + }, { "id": "control", "name": "Contrôle de Flux", @@ -2901,6 +3036,24 @@ def list_categories(): "color": "#ff9800", "isEnabled": True }, + { + "id": "wait", + "name": "Attente d'État", + "description": "Attentes sémantiques d'écran et de processus", + "icon": "⏱️", + "actionCount": 1, + "color": "#795548", + "isEnabled": True + }, + { + "id": "logic", + "name": "Logique", + "description": "Pauses supervisées et contrôle humain", + "icon": "⏸️", + "actionCount": 1, + "color": "#9c27b0", + "isEnabled": True + }, { "id": "data", "name": "Données", @@ -2962,7 +3115,7 @@ def catalog_health(): services_status = { "screen_capturer": screen_capturer is not None, - "actions": 7 + competence_action_count, + "actions": 10, "lea_competences": competence_action_count, "screen_capturer_method": getattr(screen_capturer, 'method', 'unavailable') if screen_capturer else 'unavailable' } diff --git a/visual_workflow_builder/frontend_v4/src/components/ConfidenceDashboard.tsx b/visual_workflow_builder/frontend_v4/src/components/ConfidenceDashboard.tsx index 7bab752fc..5490172b7 100644 --- a/visual_workflow_builder/frontend_v4/src/components/ConfidenceDashboard.tsx +++ b/visual_workflow_builder/frontend_v4/src/components/ConfidenceDashboard.tsx @@ -6,6 +6,7 @@ */ import { useState, useEffect, useRef } from 'react'; +import type { ExecutionMode } from '../types'; interface StepScore { stepIndex: number; @@ -21,7 +22,7 @@ interface StepScore { interface Props { isExecutionRunning: boolean; - executionMode: 'basic' | 'intelligent' | 'debug'; + executionMode: ExecutionMode; } export default function ConfidenceDashboard({ isExecutionRunning, executionMode }: Props) { diff --git a/visual_workflow_builder/frontend_v4/src/components/PropertiesPanel.tsx b/visual_workflow_builder/frontend_v4/src/components/PropertiesPanel.tsx index 92acec574..484da2b3a 100644 --- a/visual_workflow_builder/frontend_v4/src/components/PropertiesPanel.tsx +++ b/visual_workflow_builder/frontend_v4/src/components/PropertiesPanel.tsx @@ -343,6 +343,98 @@ export default function PropertiesPanel({ step, onUpdateParams, onDelete }: Prop ); + case 'wait_for_state': { + const expectedState = ( + params.expected_state && + typeof params.expected_state === 'object' && + !Array.isArray(params.expected_state) + ) ? params.expected_state as Record : {}; + + const updateExpectedState = (key: string, value: unknown) => { + updateParam('expected_state', { + ...expectedState, + [key]: value, + }); + }; + + const titleIn = Array.isArray(expectedState.window_title_in) + ? expectedState.window_title_in.join(', ') + : ''; + const titleContains = Array.isArray(expectedState.window_title_contains) + ? expectedState.window_title_contains.join(', ') + : ''; + + return ( + <> +
+ + updateExpectedState( + 'window_title_in', + e.target.value.split(',').map(v => v.trim()).filter(Boolean), + )} + placeholder="Exécuter, Enregistrer sous" + /> +
+
+ + updateExpectedState( + 'window_title_contains', + e.target.value.split(',').map(v => v.trim()).filter(Boolean), + )} + placeholder="Bloc-notes" + /> +
+
+ + updateExpectedState('process_active', e.target.value)} + placeholder="explorer.exe" + /> +
+
+ + updateParam('timeout_ms', Number(e.target.value))} + min="100" + step="250" + /> +
+
+ + updateParam('poll_interval_ms', Number(e.target.value))} + min="50" + step="50" + /> +
+
+ + +
+ + ); + } + // === DONNÉES === case 'extract_text': return ( diff --git a/visual_workflow_builder/frontend_v4/src/components/ToolPalette.tsx b/visual_workflow_builder/frontend_v4/src/components/ToolPalette.tsx index c4622e6c1..b19cc1187 100644 --- a/visual_workflow_builder/frontend_v4/src/components/ToolPalette.tsx +++ b/visual_workflow_builder/frontend_v4/src/components/ToolPalette.tsx @@ -1,9 +1,90 @@ -import { useState } from 'react'; +import { useEffect, useMemo, useState } from 'react'; import { ACTIONS, ACTION_CATEGORIES } from '../types'; +import type { ActionDefinition, ActionType } from '../types'; + +interface CatalogActionParameter { + type?: string; + description?: string; +} + +interface CatalogAction { + id: string; + name?: string; + description?: string; + category?: string; + icon?: string; + parameters?: Record; +} + +const ACTION_TYPE_SET = new Set(ACTIONS.map(action => action.type)); + +function mapCatalogCategory(action: CatalogAction): ActionDefinition['category'] { + if (action.category === 'keyboard') return 'keyboard'; + if (action.category === 'wait' || action.category === 'control') return 'wait'; + if (action.category === 'logic') return 'logic'; + if (action.category === 'data') return 'data'; + if (action.id === 'type_text' || action.id === 'type_secret') return 'keyboard'; + return 'mouse'; +} + +function needsAnchor(action: CatalogAction): boolean { + const params = action.parameters || {}; + return Boolean(params.visual_anchor); +} + +function mapCatalogAction(action: CatalogAction): ActionDefinition | null { + if (!ACTION_TYPE_SET.has(action.id as ActionType)) { + return null; + } + const staticAction = ACTIONS.find(item => item.type === action.id); + const catalogParams = Object.entries(action.parameters || {}).map(([name, param]) => ({ + name, + type: String(param.type || 'string'), + description: String(param.description || ''), + })); + return { + type: action.id as ActionType, + label: action.name || staticAction?.label || action.id, + icon: action.icon || staticAction?.icon || '⚙️', + description: action.description || staticAction?.description || '', + category: staticAction?.category || mapCatalogCategory(action), + needsAnchor: staticAction?.needsAnchor ?? needsAnchor(action), + params: staticAction?.params.length ? staticAction.params : catalogParams, + }; +} export default function ToolPalette() { // État des catégories dépliées (toutes fermées par défaut sauf 'mouse') const [expandedCategories, setExpandedCategories] = useState(['mouse']); + const [catalogActions, setCatalogActions] = useState(ACTIONS); + + useEffect(() => { + let cancelled = false; + + async function loadCatalog() { + try { + const response = await fetch('/api/vwb/catalog/actions'); + if (!response.ok) throw new Error(`HTTP ${response.status}`); + const data = await response.json(); + if (!data.success || !Array.isArray(data.actions)) { + throw new Error(data.error || 'Catalogue VWB invalide'); + } + const mapped = data.actions + .map((action: CatalogAction) => mapCatalogAction(action)) + .filter((action: ActionDefinition | null): action is ActionDefinition => action !== null); + if (!cancelled && mapped.length > 0) { + setCatalogActions(mapped); + } + } catch (error) { + console.warn('Catalogue VWB indisponible, palette statique utilisée:', error); + } + } + + loadCatalog(); + return () => { + cancelled = true; + }; + }, []); const onDragStart = (event: React.DragEvent, actionType: string) => { event.dataTransfer.setData('actionType', actionType); @@ -20,6 +101,10 @@ export default function ToolPalette() { // Grouper par catégorie const categories = Object.keys(ACTION_CATEGORIES) as Array; + const visibleActions = useMemo( + () => catalogActions.filter(action => !action.hidden), + [catalogActions], + ); return (
@@ -27,7 +112,7 @@ export default function ToolPalette() {
{categories.map((catKey) => { const cat = ACTION_CATEGORIES[catKey]; - const tools = ACTIONS.filter(a => a.category === catKey && !a.hidden); + const tools = visibleActions.filter(a => a.category === catKey); const isExpanded = expandedCategories.includes(catKey); if (tools.length === 0) return null; diff --git a/visual_workflow_builder/frontend_v4/src/types.ts b/visual_workflow_builder/frontend_v4/src/types.ts index 3c69cfc0f..6c6363e72 100644 --- a/visual_workflow_builder/frontend_v4/src/types.ts +++ b/visual_workflow_builder/frontend_v4/src/types.ts @@ -46,6 +46,7 @@ export type ActionType = | 'type_secret' | 'focus_anchor' | 'wait_for_anchor' + | 'wait_for_state' | 'scroll_to_anchor' | 'drag_drop_anchor' | 'keyboard_shortcut' @@ -120,6 +121,11 @@ export const ACTIONS: ActionDefinition[] = [ { type: 'wait_for_anchor', label: 'Attendre élément', icon: '⏳', description: 'Attend qu\'un élément visuel apparaisse à l\'écran.', category: 'wait', needsAnchor: true, params: [ { name: 'timeout_ms', type: 'number', description: 'Délai max d\'attente en millisecondes' } ] }, + { type: 'wait_for_state', label: 'Attendre état', icon: '⏱️', description: 'Attend un état d\'écran attendu, comme une fenêtre active ou un processus cible.', category: 'wait', needsAnchor: false, params: [ + { name: 'expected_state', type: 'object', description: 'Critères attendus (window_title_in, window_title_contains, process_active)' }, + { name: 'timeout_ms', type: 'number', description: 'Délai max d\'attente en millisecondes' }, + { name: 'poll_interval_ms', type: 'number', description: 'Intervalle de vérification en millisecondes' } + ] }, // === EXTRACTION DE DONNÉES === { type: 'extract_text', label: 'Extraire texte (OCR écran)', icon: '📋', description: 'OCR EasyOCR fr+en sur le dernier screenshot. Stocke le texte dans une variable réutilisable plus loin via {{output_var}}. Pas d\'ancre nécessaire — extrait toute la page visible.', category: 'data', needsAnchor: false, params: [