feat: replay visuel Windows opérationnel — template matching + VWB complet
- Bouton "Windows" dans VWB pour exécuter sur le PC distant - Template matching OpenCV multi-scale pour localiser les ancres visuelles - Proxy VWB→streaming server avec chargement ancre (thumb, pas full) - Fix executor Windows : mss lazy, result reporting, debug prints - Fix poll replay permanent (sans session active) - Mapping types VWB→executor (click_anchor→click, type_text→type) - CORS streaming server, capture Windows dans VWB - Dédup heartbeats côté client (hash perceptuel) - Mode cloud VLM configurable via RPA_VLM_MODEL - Fix resolve_target : pas de ScreenAnalyzer fallback (trop lent) Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -150,6 +150,29 @@ export default function CapturePanel({
|
||||
<button onClick={onCapture} disabled={countdown !== null}>
|
||||
Capturer
|
||||
</button>
|
||||
<button
|
||||
onClick={async () => {
|
||||
try {
|
||||
const resp = await fetch('/api/screen-capture/capture-windows', { method: 'POST' });
|
||||
const data = await resp.json();
|
||||
if (data.image) {
|
||||
const fakeCapture = {
|
||||
screenshot_base64: data.image,
|
||||
width: data.width,
|
||||
height: data.height,
|
||||
source: 'windows',
|
||||
};
|
||||
setCurrentCapture(fakeCapture as any);
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('Capture Windows échouée:', err);
|
||||
}
|
||||
}}
|
||||
title="Capture le dernier écran du PC Windows"
|
||||
style={{ fontSize: '12px' }}
|
||||
>
|
||||
🖥️ Windows
|
||||
</button>
|
||||
<select value={timerSeconds} onChange={(e) => setTimerSeconds(Number(e.target.value))}>
|
||||
<option value="0">Immédiat</option>
|
||||
<option value="3">3 sec</option>
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
// @ts-nocheck
|
||||
import type { Execution } from '../types';
|
||||
|
||||
interface Props {
|
||||
@@ -9,12 +10,73 @@ interface Props {
|
||||
export default function ExecutionControls({ execution, onStart, onStop }: Props) {
|
||||
const isRunning = execution?.status === 'running' || execution?.status === 'paused';
|
||||
|
||||
const handleExecuteWindows = async () => {
|
||||
try {
|
||||
// Récupérer le workflow actif depuis l'état de la session
|
||||
const stateResp = await fetch('/api/v3/session/state');
|
||||
const state = await stateResp.json();
|
||||
let workflowId = state?.session?.active_workflow_id;
|
||||
let steps = state?.workflow?.steps || [];
|
||||
|
||||
// Si pas de workflow actif, essayer de charger le premier disponible
|
||||
if (!steps.length && state?.workflows_list?.length) {
|
||||
const firstWf = state.workflows_list[0];
|
||||
workflowId = firstWf.id;
|
||||
// Charger les étapes du workflow
|
||||
const wfResp = await fetch(`/api/v3/workflow/${firstWf.id}`);
|
||||
const wfData = await wfResp.json();
|
||||
steps = wfData?.steps || wfData?.workflow?.steps || [];
|
||||
}
|
||||
|
||||
if (!steps.length) {
|
||||
alert('Aucune étape dans le workflow. Sélectionnez un workflow d\'abord.');
|
||||
return;
|
||||
}
|
||||
|
||||
// Via le proxy Vite (/api → port 5002)
|
||||
const resp = await fetch('/api/v3/execute-windows', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
workflow_id: workflowId,
|
||||
session_id: `replay_${Date.now()}`,
|
||||
actions: steps.map((step: any, i: number) => ({
|
||||
action_id: step.id || `action_${i}`,
|
||||
type: step.action_type,
|
||||
parameters: step.parameters || {},
|
||||
anchor_id: step.anchor_id || null,
|
||||
order: i,
|
||||
})),
|
||||
}),
|
||||
});
|
||||
|
||||
const result = await resp.json();
|
||||
if (result.replay_id) {
|
||||
alert(`Replay lancé sur Windows ! ID: ${result.replay_id}`);
|
||||
} else {
|
||||
alert(`Erreur: ${result.error || JSON.stringify(result)}`);
|
||||
}
|
||||
} catch (err) {
|
||||
alert(`Erreur connexion streaming server: ${err}`);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="execution-controls">
|
||||
{!isRunning ? (
|
||||
<button className="btn-start" onClick={onStart}>
|
||||
▶️ Exécuter
|
||||
</button>
|
||||
<div style={{ display: 'flex', gap: '4px' }}>
|
||||
<button className="btn-start" onClick={onStart}>
|
||||
▶️ Exécuter
|
||||
</button>
|
||||
<button
|
||||
className="btn-start"
|
||||
onClick={handleExecuteWindows}
|
||||
style={{ background: '#0078d4', fontSize: '12px' }}
|
||||
title="Envoyer les actions au PC Windows via le streaming server"
|
||||
>
|
||||
🖥️ Windows
|
||||
</button>
|
||||
</div>
|
||||
) : (
|
||||
<>
|
||||
<div className="exec-status">
|
||||
|
||||
@@ -2,8 +2,8 @@
|
||||
* Service de détection UI (UI-DETR-1)
|
||||
*/
|
||||
|
||||
// VWB backend (port 5002) — contient le screen capturer et la détection UI
|
||||
const API_BASE = `http://${window.location.hostname}:5002`;
|
||||
// Via le proxy Vite (/api → port 5002) — fonctionne depuis n'importe quel navigateur
|
||||
const API_BASE = '';
|
||||
|
||||
export interface UIElement {
|
||||
id: number;
|
||||
@@ -52,7 +52,8 @@ export async function detectUIElements(
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
image_base64: imageBase64,
|
||||
// Enlever le préfixe data:image/...;base64, si présent
|
||||
image_base64: imageBase64.replace(/^data:image\/[^;]+;base64,/, ''),
|
||||
threshold: options.threshold ?? 0.35,
|
||||
annotate: options.annotate ?? false,
|
||||
show_confidence: options.showConfidence ?? false,
|
||||
|
||||
Reference in New Issue
Block a user