feat: replay visuel VLM-first, worker séparé, package Léa, AZERTY, sécurité HTTPS
Pipeline replay visuel : - VLM-first : l'agent appelle Ollama directement pour trouver les éléments - Template matching en fallback (seuil strict 0.90) - Stop immédiat si élément non trouvé (pas de clic blind) - Replay depuis session brute (/replay-session) sans attendre le VLM - Vérification post-action (screenshot hash avant/après) - Gestion des popups (Enter/Escape/Tab+Enter) Worker VLM séparé : - run_worker.py : process distinct du serveur HTTP - Communication par fichiers (_worker_queue.txt + _replay_active.lock) - Le serveur HTTP ne fait plus jamais de VLM → toujours réactif - Service systemd rpa-worker.service Capture clavier : - raw_keys (vk + press/release) pour replay exact indépendant du layout - Fix AZERTY : ToUnicodeEx + AltGr detection - Enter capturé comme \n, Tab comme \t - Filtrage modificateurs seuls (Ctrl/Alt/Shift parasites) - Fusion text_input consécutifs, dédup key_combo Sécurité & Internet : - HTTPS Let's Encrypt (lea.labs + vwb.labs.laurinebazin.design) - Token API fixe dans .env.local - HTTP Basic Auth sur VWB - Security headers (HSTS, CSP, nosniff) - CORS domaines publics, plus de wildcard Infrastructure : - DPI awareness (SetProcessDpiAwareness) Python + Rust - Métadonnées système (dpi_scale, window_bounds, monitors, os_theme) - Template matching multi-scale [0.5, 2.0] - Résolution dynamique (plus de hardcode 1920x1080) - VLM prefill fix (47x speedup, 3.5s au lieu de 180s) Modules : - core/auth/ : credential vault (Fernet AES), TOTP (RFC 6238), auth handler - core/federation/ : LearningPack export/import anonymisé, FAISS global - deploy/ : package Léa (config.txt, Lea.bat, install.bat, LISEZMOI.txt) UX : - Filtrage OS (VWB + Chat montrent que les workflows de l'OS courant) - Bibliothèque persistante (cache local + SQLite) - Clustering hybride (titre fenêtre + DBSCAN) - EdgeConstraints + PostConditions peuplés - GraphBuilder compound actions (toutes les frappes) Agent Rust : - Token Bearer auth (network.rs) - sysinfo.rs (DPI, résolution, window bounds via Win32 API) - config.txt lu automatiquement - Support Chrome/Brave/Firefox (pas que Edge) Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Dom
@@ -248,6 +248,16 @@ function App() {
|
||||
}
|
||||
};
|
||||
|
||||
// Écouter les custom events de suppression (depuis StepNode via memo)
|
||||
useEffect(() => {
|
||||
const handler = (e: Event) => {
|
||||
const stepId = (e as CustomEvent).detail;
|
||||
if (stepId) handleDeleteStep(stepId);
|
||||
};
|
||||
window.addEventListener('rpa-delete-step', handler);
|
||||
return () => window.removeEventListener('rpa-delete-step', handler);
|
||||
});
|
||||
|
||||
const handleUpdateStepParams = async (id: string, params: Record<string, unknown>) => {
|
||||
if (!appState?.session.active_workflow_id) return;
|
||||
try {
|
||||
|
||||
@@ -2,6 +2,9 @@
|
||||
import { useState } from 'react';
|
||||
import type { Execution } from '../types';
|
||||
|
||||
// Détecter l'OS du navigateur pour n'afficher que le bouton d'exécution pertinent
|
||||
const userOS: 'windows' | 'linux' = navigator.platform.includes('Win') ? 'windows' : 'linux';
|
||||
|
||||
interface Props {
|
||||
execution: Execution | null;
|
||||
onStart: () => void;
|
||||
@@ -71,25 +74,28 @@ export default function ExecutionControls({ execution, onStart, onStop }: Props)
|
||||
<div className="execution-controls">
|
||||
{!isRunning ? (
|
||||
<div style={{ display: 'flex', gap: '4px', alignItems: 'center' }}>
|
||||
<button className="btn-start" onClick={onStart} title="Exécuter sur cet écran (Linux)">
|
||||
🐧 Linux
|
||||
</button>
|
||||
<button
|
||||
className="btn-start"
|
||||
onClick={handleExecuteWindows}
|
||||
disabled={windowsStatus === 'sending'}
|
||||
style={{
|
||||
background: windowsStatus === 'sent' ? '#22c55e' : windowsStatus === 'error' ? '#ef4444' : '#0078d4',
|
||||
fontSize: '12px',
|
||||
opacity: windowsStatus === 'sending' ? 0.6 : 1,
|
||||
}}
|
||||
title="Exécuter sur le PC Windows distant"
|
||||
>
|
||||
{windowsStatus === 'sending' ? '⏳ Envoi...' :
|
||||
windowsStatus === 'sent' ? '✅ Lancé !' :
|
||||
windowsStatus === 'error' ? '❌ Erreur' :
|
||||
'🖥️ Windows'}
|
||||
</button>
|
||||
{userOS === 'linux' ? (
|
||||
<button className="btn-start" onClick={onStart} title="Exécuter sur cet écran">
|
||||
Exécuter
|
||||
</button>
|
||||
) : (
|
||||
<button
|
||||
className="btn-start"
|
||||
onClick={handleExecuteWindows}
|
||||
disabled={windowsStatus === 'sending'}
|
||||
style={{
|
||||
background: windowsStatus === 'sent' ? '#22c55e' : windowsStatus === 'error' ? '#ef4444' : '#0078d4',
|
||||
fontSize: '12px',
|
||||
opacity: windowsStatus === 'sending' ? 0.6 : 1,
|
||||
}}
|
||||
title="Exécuter sur ce PC"
|
||||
>
|
||||
{windowsStatus === 'sending' ? 'Envoi...' :
|
||||
windowsStatus === 'sent' ? 'Lancé !' :
|
||||
windowsStatus === 'error' ? 'Erreur' :
|
||||
'Exécuter'}
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
) : (
|
||||
<>
|
||||
|
||||
@@ -41,7 +41,14 @@ export default function WorkflowList({ workflows, activeId, onSelect, onCreate,
|
||||
onClick={() => onSelect(wf.id)}
|
||||
>
|
||||
<div className="wf-main-row">
|
||||
<span className="wf-name">{wf.name}</span>
|
||||
<span className="wf-name">
|
||||
{wf.name}
|
||||
{wf.source === 'learned_import' && (
|
||||
<span className="imported-badge" title="Importé depuis Léa">
|
||||
appris
|
||||
</span>
|
||||
)}
|
||||
</span>
|
||||
|
||||
{/* Indicateurs compacts des métadonnées */}
|
||||
<span className="wf-meta-indicators">
|
||||
|
||||
@@ -12,6 +12,9 @@ interface Props {
|
||||
onRename: (id: string, newName: string) => void;
|
||||
}
|
||||
|
||||
// Détecter l'OS du navigateur (pour ne montrer que les workflows compatibles)
|
||||
const userOS: 'windows' | 'linux' = navigator.platform.includes('Win') ? 'windows' : 'linux';
|
||||
|
||||
export default function WorkflowSelector({
|
||||
workflows,
|
||||
activeWorkflow,
|
||||
@@ -50,11 +53,11 @@ export default function WorkflowSelector({
|
||||
}
|
||||
}, [editingId]);
|
||||
|
||||
// Charger les workflows appris quand le dropdown s'ouvre
|
||||
// Charger les workflows appris quand le dropdown s'ouvre (filtrés par OS)
|
||||
const loadLearnedWorkflows = useCallback(async () => {
|
||||
setLearnedLoading(true);
|
||||
try {
|
||||
const data = await api.getLearnedWorkflows();
|
||||
const data = await api.getLearnedWorkflows(undefined, userOS);
|
||||
// Ne garder que ceux qui ne sont pas encore importés
|
||||
setLearnedWorkflows(data.workflows.filter(w => !w.already_imported));
|
||||
} catch {
|
||||
@@ -77,10 +80,11 @@ export default function WorkflowSelector({
|
||||
(wf.tags || []).some(tag => tag.toLowerCase().includes(search.toLowerCase()))
|
||||
);
|
||||
|
||||
// Filtrer les workflows appris
|
||||
// Filtrer les workflows appris (par recherche + OS)
|
||||
const filteredLearned = learnedWorkflows.filter(wf =>
|
||||
wf.name.toLowerCase().includes(search.toLowerCase()) ||
|
||||
wf.workflow_id.toLowerCase().includes(search.toLowerCase())
|
||||
(wf.name.toLowerCase().includes(search.toLowerCase()) ||
|
||||
wf.workflow_id.toLowerCase().includes(search.toLowerCase())) &&
|
||||
(wf.machine_id || '').toLowerCase().includes(userOS)
|
||||
);
|
||||
|
||||
// Workflows récents (les 8 premiers)
|
||||
@@ -202,6 +206,11 @@ export default function WorkflowSelector({
|
||||
<>
|
||||
<span className="item-name">
|
||||
{wf.name}
|
||||
{wf.source === 'learned_import' && (
|
||||
<span className="imported-badge" title="Importé depuis Léa">
|
||||
appris
|
||||
</span>
|
||||
)}
|
||||
{wf.review_status === 'pending_review' && (
|
||||
<span className="review-badge-inline pending_review" title="En attente de validation">
|
||||
● A valider
|
||||
|
||||
@@ -322,12 +322,15 @@ export interface LearnedWorkflow {
|
||||
created_at?: string;
|
||||
}
|
||||
|
||||
export async function getLearnedWorkflows(machineId?: string): Promise<{
|
||||
export async function getLearnedWorkflows(machineId?: string, os?: string): Promise<{
|
||||
workflows: LearnedWorkflow[];
|
||||
streaming_server_available: boolean;
|
||||
}> {
|
||||
const params = machineId ? `?machine_id=${encodeURIComponent(machineId)}` : '';
|
||||
return request('GET', `/learned-workflows${params}`);
|
||||
const searchParams = new URLSearchParams();
|
||||
if (machineId) searchParams.set('machine_id', machineId);
|
||||
if (os) searchParams.set('os', os);
|
||||
const qs = searchParams.toString();
|
||||
return request('GET', `/learned-workflows${qs ? '?' + qs : ''}`);
|
||||
}
|
||||
|
||||
export async function importLearnedWorkflow(
|
||||
|
||||
@@ -2544,6 +2544,17 @@ body {
|
||||
gap: 0.3rem;
|
||||
}
|
||||
|
||||
.learned-list .learned-item .item-name {
|
||||
color: #1e293b;
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
.learned-list .learned-item .item-meta {
|
||||
color: #64748b;
|
||||
font-size: 0.8rem;
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
.learned-badge {
|
||||
display: inline-block;
|
||||
font-size: 0.65rem;
|
||||
@@ -2555,6 +2566,18 @@ body {
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
.imported-badge {
|
||||
display: inline-block;
|
||||
font-size: 0.6rem;
|
||||
padding: 0.1rem 0.35rem;
|
||||
border-radius: 3px;
|
||||
background: #e0f2fe;
|
||||
color: #0369a1;
|
||||
margin-left: 0.3rem;
|
||||
font-weight: 500;
|
||||
vertical-align: middle;
|
||||
}
|
||||
|
||||
.import-btn {
|
||||
margin-left: auto;
|
||||
padding: 0.2rem 0.6rem;
|
||||
|
||||
@@ -4,10 +4,18 @@ import react from '@vitejs/plugin-react'
|
||||
export default defineConfig({
|
||||
plugins: [react()],
|
||||
server: {
|
||||
port: 3002,
|
||||
allowedHosts: [
|
||||
'vwb.labs.laurinebazin.design',
|
||||
],
|
||||
proxy: {
|
||||
'/api': {
|
||||
target: 'http://localhost:5001',
|
||||
target: 'http://localhost:5002',
|
||||
changeOrigin: true
|
||||
},
|
||||
'/socket.io': {
|
||||
target: 'http://localhost:5002',
|
||||
ws: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user