Dom/rpa_vision_v3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(vwb): load palette from catalog

Browse Source
This commit is contained in:
Dom
2026-05-29 17:09:47 +02:00
parent 02211fddf2
commit 45b6da5e3f
6 changed files with 379 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

161
visual_workflow_builder/backend/catalog_routes_v2_vlm.py
View File

@@ -1090,6 +1090,13 @@ def _load_lea_competence_actions() -> List[Dict[str, Any]]:
return []
def _truthy_query_arg(name: str) -> bool:
"""Return True for explicit opt-in query flags."""
return str(request.args.get(name, "")).strip().lower() in (
"1", "true", "yes", "on",
)
def _extract_competence_id(action_type: str, parameters: Dict[str, Any]) -> str:
competence_id = str(parameters.get("competence_id") or "").strip()
if competence_id:
@@ -1334,6 +1341,7 @@ def list_actions():
# Paramètres de requête
category_filter = request.args.get('category')
search_query = request.args.get('search', '').lower()
show_competences = _truthy_query_arg('show_competences')
# Définir les actions disponibles
available_actions = [
@@ -1443,6 +1451,36 @@ def list_actions():
}
]
},
{
"id": "keyboard_shortcut",
"name": "Raccourci Clavier",
"description": "Exécute une combinaison de touches normalisée",
"category": "keyboard",
"icon": "⌘",
"parameters": {
"keys": {
"type": "array",
"required": True,
"description": "Touches à envoyer dans l'ordre, ex: ['ctrl', 's']"
},
"hold_duration_ms": {
"type": "number",
"required": False,
"default": 50,
"min": 0,
"description": "Durée de maintien optionnelle en millisecondes"
}
},
"examples": [
{
"name": "Ouvrir Exécuter",
"description": "Envoie Win+R pour ouvrir la boîte Exécuter",
"parameters": {
"keys": ["win", "r"]
}
}
]
},
{
"id": "wait_for_anchor",
"name": "Attente d'Ancre Visuelle",
@@ -1488,6 +1526,54 @@ def list_actions():
}
]
},
{
"id": "wait_for_state",
"name": "Attendre un État",
"description": "Attend un état sémantique d'écran, comme une fenêtre active ou un processus cible",
"category": "wait",
"icon": "⏱️",
"parameters": {
"expected_state": {
"type": "object",
"required": True,
"description": "Critères d'état attendus: window_title_in, window_title_contains, process_active"
},
"timeout_ms": {
"type": "number",
"required": False,
"default": 5000,
"min": 100,
"description": "Délai d'attente maximum en millisecondes"
},
"poll_interval_ms": {
"type": "number",
"required": False,
"default": 250,
"min": 50,
"description": "Intervalle de vérification en millisecondes"
},
"evidence_required": {
"type": "string",
"required": False,
"default": "window_or_process",
"options": ["window_or_process", "uia", "ocr", "screenshot_diff"],
"description": "Niveau de preuve requis"
}
},
"examples": [
{
"name": "Attendre Exécuter",
"description": "Attend que la fenêtre Exécuter soit au premier plan",
"parameters": {
"expected_state": {
"window_title_in": ["Exécuter"],
"process_active": "explorer.exe"
},
"timeout_ms": 5000
}
}
]
},
{
"id": "focus_anchor",
"name": "Donner le Focus à un Élément",
@@ -1533,6 +1619,42 @@ def list_actions():
}
]
},
{
"id": "pause_for_human",
"name": "Pause Supervisée",
"description": "Suspend l'exécution et demande une validation humaine avant de continuer",
"category": "logic",
"icon": "⏸️",
"parameters": {
"message": {
"type": "string",
"required": True,
"description": "Message affiché à l'opérateur"
},
"safety_level": {
"type": "string",
"required": False,
"default": "standard",
"options": ["standard", "medical_critical"],
"description": "Niveau de sécurité de la pause"
},
"safety_checks": {
"type": "array",
"required": False,
"default": [],
"description": "Liste de contrôles à acquitter avant reprise"
}
},
"examples": [
{
"name": "Validation humaine",
"description": "Demande à l'opérateur de valider l'état visible",
"parameters": {
"message": "Validez-vous que l'état attendu est visible ?"
}
}
]
},
{
"id": "type_secret",
"name": "Saisie Sécurisée",
@@ -1688,8 +1810,11 @@ def list_actions():
}
]
# Ajouter les compétences YAML Léa comme actions testables.
available_actions.extend(_load_lea_competence_actions())
# Les compétences Léa sont masquées par défaut : elles ne doivent pas
# ressembler à des actions ordinaires tant que le replay supervisé
# complet (verdict + write-back YAML) n'est pas branché.
if show_competences:
available_actions.extend(_load_lea_competence_actions())
# Filtrer par catégorie
if category_filter:
@@ -2879,7 +3004,8 @@ def list_categories():
}
"""
try:
competence_action_count = len(_load_lea_competence_actions())
show_competences = _truthy_query_arg('show_competences')
competence_action_count = len(_load_lea_competence_actions()) if show_competences else 0
# Définir les catégories disponibles avec métadonnées
available_categories = [
@@ -2892,6 +3018,15 @@ def list_categories():
"color": "#2196f3",
"isEnabled": True
},
{
"id": "keyboard",
"name": "Clavier",
"description": "Raccourcis clavier et actions de saisie",
"icon": "⌨️",
"actionCount": 1,
"color": "#607d8b",
"isEnabled": True
},
{
"id": "control",
"name": "Contrôle de Flux",
@@ -2901,6 +3036,24 @@ def list_categories():
"color": "#ff9800",
"isEnabled": True
},
{
"id": "wait",
"name": "Attente d'État",
"description": "Attentes sémantiques d'écran et de processus",
"icon": "⏱️",
"actionCount": 1,
"color": "#795548",
"isEnabled": True
},
{
"id": "logic",
"name": "Logique",
"description": "Pauses supervisées et contrôle humain",
"icon": "⏸️",
"actionCount": 1,
"color": "#9c27b0",
"isEnabled": True
},
{
"id": "data",
"name": "Données",
@@ -2962,7 +3115,7 @@ def catalog_health():
services_status = {
"screen_capturer": screen_capturer is not None,
"actions": 7 + competence_action_count,
"actions": 10,
"lea_competences": competence_action_count,
"screen_capturer_method": getattr(screen_capturer, 'method', 'unavailable') if screen_capturer else 'unavailable'
}

3
visual_workflow_builder/frontend_v4/src/components/ConfidenceDashboard.tsx
View File

@@ -6,6 +6,7 @@
*/
import { useState, useEffect, useRef } from 'react';
import type { ExecutionMode } from '../types';
interface StepScore {
stepIndex: number;
@@ -21,7 +22,7 @@ interface StepScore {
interface Props {
isExecutionRunning: boolean;
executionMode: 'basic' | 'intelligent' | 'debug';
executionMode: ExecutionMode;
}
export default function ConfidenceDashboard({ isExecutionRunning, executionMode }: Props) {

92
visual_workflow_builder/frontend_v4/src/components/PropertiesPanel.tsx
View File

@@ -343,6 +343,98 @@ export default function PropertiesPanel({ step, onUpdateParams, onDelete }: Prop
</>
);
case 'wait_for_state': {
const expectedState = (
params.expected_state &&
typeof params.expected_state === 'object' &&
!Array.isArray(params.expected_state)
) ? params.expected_state as Record<string, unknown> : {};
const updateExpectedState = (key: string, value: unknown) => {
updateParam('expected_state', {
...expectedState,
[key]: value,
});
};
const titleIn = Array.isArray(expectedState.window_title_in)
? expectedState.window_title_in.join(', ')
: '';
const titleContains = Array.isArray(expectedState.window_title_contains)
? expectedState.window_title_contains.join(', ')
: '';
return (
<>
<div className="prop-field">
<label>Titres exacts acceptés</label>
<input
type="text"
value={titleIn}
onChange={(e) => updateExpectedState(
'window_title_in',
e.target.value.split(',').map(v => v.trim()).filter(Boolean),
)}
placeholder="Exécuter, Enregistrer sous"
/>
</div>
<div className="prop-field">
<label>Titre contient</label>
<input
type="text"
value={titleContains}
onChange={(e) => updateExpectedState(
'window_title_contains',
e.target.value.split(',').map(v => v.trim()).filter(Boolean),
)}
placeholder="Bloc-notes"
/>
</div>
<div className="prop-field">
<label>Processus actif</label>
<input
type="text"
value={String(expectedState.process_active || '')}
onChange={(e) => updateExpectedState('process_active', e.target.value)}
placeholder="explorer.exe"
/>
</div>
<div className="prop-field">
<label>Timeout (ms)</label>
<input
type="number"
value={Number(params.timeout_ms || 5000)}
onChange={(e) => updateParam('timeout_ms', Number(e.target.value))}
min="100"
step="250"
/>
</div>
<div className="prop-field">
<label>Intervalle de vérification (ms)</label>
<input
type="number"
value={Number(params.poll_interval_ms || 250)}
onChange={(e) => updateParam('poll_interval_ms', Number(e.target.value))}
min="50"
step="50"
/>
</div>
<div className="prop-field">
<label>Preuve requise</label>
<select
value={String(params.evidence_required || 'window_or_process')}
onChange={(e) => updateParam('evidence_required', e.target.value)}
>
<option value="window_or_process">Fenêtre ou process</option>
<option value="uia">UIA</option>
<option value="ocr">OCR</option>
<option value="screenshot_diff">Différence écran</option>
</select>
</div>
</>
);
}
// === DONNÉES ===
case 'extract_text':
return (

89
visual_workflow_builder/frontend_v4/src/components/ToolPalette.tsx
View File

@@ -1,9 +1,90 @@
import { useState } from 'react';
import { useEffect, useMemo, useState } from 'react';
import { ACTIONS, ACTION_CATEGORIES } from '../types';
import type { ActionDefinition, ActionType } from '../types';
interface CatalogActionParameter {
type?: string;
description?: string;
}
interface CatalogAction {
id: string;
name?: string;
description?: string;
category?: string;
icon?: string;
parameters?: Record<string, CatalogActionParameter>;
}
const ACTION_TYPE_SET = new Set(ACTIONS.map(action => action.type));
function mapCatalogCategory(action: CatalogAction): ActionDefinition['category'] {
if (action.category === 'keyboard') return 'keyboard';
if (action.category === 'wait' || action.category === 'control') return 'wait';
if (action.category === 'logic') return 'logic';
if (action.category === 'data') return 'data';
if (action.id === 'type_text' || action.id === 'type_secret') return 'keyboard';
return 'mouse';
}
function needsAnchor(action: CatalogAction): boolean {
const params = action.parameters || {};
return Boolean(params.visual_anchor);
}
function mapCatalogAction(action: CatalogAction): ActionDefinition | null {
if (!ACTION_TYPE_SET.has(action.id as ActionType)) {
return null;
}
const staticAction = ACTIONS.find(item => item.type === action.id);
const catalogParams = Object.entries(action.parameters || {}).map(([name, param]) => ({
name,
type: String(param.type || 'string'),
description: String(param.description || ''),
}));
return {
type: action.id as ActionType,
label: action.name || staticAction?.label || action.id,
icon: action.icon || staticAction?.icon || '⚙️',
description: action.description || staticAction?.description || '',
category: staticAction?.category || mapCatalogCategory(action),
needsAnchor: staticAction?.needsAnchor ?? needsAnchor(action),
params: staticAction?.params.length ? staticAction.params : catalogParams,
};
}
export default function ToolPalette() {
// État des catégories dépliées (toutes fermées par défaut sauf 'mouse')
const [expandedCategories, setExpandedCategories] = useState<string[]>(['mouse']);
const [catalogActions, setCatalogActions] = useState<ActionDefinition[]>(ACTIONS);
useEffect(() => {
let cancelled = false;
async function loadCatalog() {
try {
const response = await fetch('/api/vwb/catalog/actions');
if (!response.ok) throw new Error(`HTTP ${response.status}`);
const data = await response.json();
if (!data.success || !Array.isArray(data.actions)) {
throw new Error(data.error || 'Catalogue VWB invalide');
}
const mapped = data.actions
.map((action: CatalogAction) => mapCatalogAction(action))
.filter((action: ActionDefinition | null): action is ActionDefinition => action !== null);
if (!cancelled && mapped.length > 0) {
setCatalogActions(mapped);
}
} catch (error) {
console.warn('Catalogue VWB indisponible, palette statique utilisée:', error);
}
}
loadCatalog();
return () => {
cancelled = true;
};
}, []);
const onDragStart = (event: React.DragEvent, actionType: string) => {
event.dataTransfer.setData('actionType', actionType);
@@ -20,6 +101,10 @@ export default function ToolPalette() {
// Grouper par catégorie
const categories = Object.keys(ACTION_CATEGORIES) as Array<keyof typeof ACTION_CATEGORIES>;
const visibleActions = useMemo(
() => catalogActions.filter(action => !action.hidden),
[catalogActions],
);
return (
<div className="tool-palette">
@@ -27,7 +112,7 @@ export default function ToolPalette() {
<div className="tool-categories">
{categories.map((catKey) => {
const cat = ACTION_CATEGORIES[catKey];
const tools = ACTIONS.filter(a => a.category === catKey && !a.hidden);
const tools = visibleActions.filter(a => a.category === catKey);
const isExpanded = expandedCategories.includes(catKey);
if (tools.length === 0) return null;

6
visual_workflow_builder/frontend_v4/src/types.ts
View File

@@ -46,6 +46,7 @@ export type ActionType =
| 'type_secret'
| 'focus_anchor'
| 'wait_for_anchor'
| 'wait_for_state'
| 'scroll_to_anchor'
| 'drag_drop_anchor'
| 'keyboard_shortcut'
@@ -120,6 +121,11 @@ export const ACTIONS: ActionDefinition[] = [
{ type: 'wait_for_anchor', label: 'Attendre élément', icon: '⏳', description: 'Attend qu\'un élément visuel apparaisse à l\'écran.', category: 'wait', needsAnchor: true, params: [
{ name: 'timeout_ms', type: 'number', description: 'Délai max d\'attente en millisecondes' }
] },
{ type: 'wait_for_state', label: 'Attendre état', icon: '⏱️', description: 'Attend un état d\'écran attendu, comme une fenêtre active ou un processus cible.', category: 'wait', needsAnchor: false, params: [
{ name: 'expected_state', type: 'object', description: 'Critères attendus (window_title_in, window_title_contains, process_active)' },
{ name: 'timeout_ms', type: 'number', description: 'Délai max d\'attente en millisecondes' },
{ name: 'poll_interval_ms', type: 'number', description: 'Intervalle de vérification en millisecondes' }
] },
// === EXTRACTION DE DONNÉES ===
{ type: 'extract_text', label: 'Extraire texte (OCR écran)', icon: '📋', description: 'OCR EasyOCR fr+en sur le dernier screenshot. Stocke le texte dans une variable réutilisable plus loin via {{output_var}}. Pas d\'ancre nécessaire — extrait toute la page visible.', category: 'data', needsAnchor: false, params: [