feat: Léa chat + IRBuilder enrichi (stratégies V4 complètes)

Aspect 2/4 Léa : interface conversationnelle
- chat_interface.py : ChatSession thread-safe, états idle/planning/awaiting/executing/done
- 5 endpoints REST : /api/v1/chat/* (session, message, history, confirm, sessions)
- web_dashboard/chat.html + chat.js : UI minimaliste, polling 2s, pas de framework
- Proxy Flask /api/chat/* → serveur streaming
- 34 tests (happy path, abandon, refus, erreurs, gemma4 down)

IRBuilder enrichi pour plans V4 complets
- _event_to_action() appelle enrich_click_from_screenshot() quand session_dir dispo
- Chaque clic porte _enrichment (by_text OCR, anchor_image_base64, vlm_description)
- ExecutionCompiler consomme l'enrichissement pour produire 3 stratégies par clic
  Avant : [ocr] uniquement, target="unknown_window"
  Après : [ocr, template, vlm] avec vrai texte OCR ("Rechercher", "Ouvrir")

Validé sur session réelle : 10/10 clics enrichis (by_text + anchor + vlm_description)

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Dom
2026-04-10 09:01:13 +02:00
parent a6eb4c168f
commit f541bb8ce4
8 changed files with 2241 additions and 26 deletions

View File

@@ -1876,7 +1876,7 @@ def load_system_config():
"version": "1.0.0",
"services": {},
"llm": {"provider": "ollama", "base_url": "http://localhost:11434", "model": "qwen2.5:7b"},
"vlm": {"provider": "ollama", "base_url": "http://localhost:11434", "model": "qwen2.5vl:7b"},
"vlm": {"provider": "ollama", "base_url": "http://localhost:11434", "model": "gemma4:e4b"},
"detection": {"owl_model": "google/owlv2-base-patch16-ensemble", "confidence_threshold": 0.3},
"database": {"type": "sqlite", "path": "data/training/workflows.db"},
"security": {"enable_encryption": True, "require_authentication": False}
@@ -2371,6 +2371,93 @@ def proxy_streaming(endpoint):
return jsonify({'error': str(e)}), 500
# =============================================================================
# Chat conversationnel — Léa
# =============================================================================
CHAT_BASE_URL = 'http://localhost:5005/api/v1/chat'
@app.route('/chat')
def chat_page():
"""Page de chat conversationnel avec Léa."""
return render_template('chat.html')
@app.route('/api/chat/session', methods=['POST'])
def proxy_chat_session():
"""Proxy : créer une session de chat côté serveur streaming."""
return _proxy_chat(
method='POST',
path='/session',
payload=request.get_json(silent=True) or {},
)
@app.route('/api/chat/<session_id>/message', methods=['POST'])
def proxy_chat_message(session_id):
"""Proxy : envoyer un message dans une session."""
return _proxy_chat(
method='POST',
path=f'/{session_id}/message',
payload=request.get_json(silent=True) or {},
)
@app.route('/api/chat/<session_id>/history', methods=['GET'])
def proxy_chat_history(session_id):
"""Proxy : récupérer l'historique."""
return _proxy_chat(method='GET', path=f'/{session_id}/history')
@app.route('/api/chat/<session_id>/confirm', methods=['POST'])
def proxy_chat_confirm(session_id):
"""Proxy : confirmer l'exécution d'un plan."""
return _proxy_chat(
method='POST',
path=f'/{session_id}/confirm',
payload=request.get_json(silent=True) or {},
)
def _proxy_chat(method, path, payload=None):
"""Helper pour proxyfier les requêtes vers le serveur streaming (:5005)."""
import urllib.request
import urllib.error
url = f'{CHAT_BASE_URL}{path}'
headers = {
'Accept': 'application/json',
'Content-Type': 'application/json',
}
# Token Bearer (lu depuis l'env — même token que le serveur streaming)
token = os.environ.get('RPA_API_TOKEN', '')
if token:
headers['Authorization'] = f'Bearer {token}'
try:
data_bytes = None
if payload is not None and method != 'GET':
data_bytes = json.dumps(payload).encode('utf-8')
req = urllib.request.Request(url, data=data_bytes, headers=headers, method=method)
with urllib.request.urlopen(req, timeout=15) as response:
body = response.read().decode('utf-8')
try:
return jsonify(json.loads(body))
except json.JSONDecodeError:
return body, response.status, {'Content-Type': 'application/json'}
except urllib.error.HTTPError as e:
try:
detail = json.loads(e.read().decode('utf-8'))
except Exception:
detail = {'error': str(e)}
return jsonify(detail), e.code
except urllib.error.URLError as e:
return jsonify({'error': f'Serveur chat inaccessible : {e}'}), 502
except Exception as e:
return jsonify({'error': str(e)}), 500
# =============================================================================
# Main
# =============================================================================

View File

@@ -0,0 +1,240 @@
// chat.js — Client Léa conversationnelle
// Logique minimaliste : pas de framework, fetch + polling.
const API_BASE = "/api/chat"; // Proxyfié par le dashboard Flask vers :5005
let sessionId = null;
let pollTimer = null;
let lastMessageCount = 0;
let currentState = "idle";
const STATE_LABELS = {
idle: "En attente",
planning: "Léa réfléchit…",
awaiting_confirmation: "En attente de confirmation",
executing: "Léa exécute le workflow…",
done: "Terminé",
error: "Erreur",
};
// -----------------------------------------------------------------------------
// Initialisation
// -----------------------------------------------------------------------------
async function initChat() {
try {
const resp = await fetch(`${API_BASE}/session`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ machine_id: "default" }),
});
if (!resp.ok) throw new Error(`HTTP ${resp.status}`);
const data = await resp.json();
sessionId = data.session_id;
currentState = data.state || "idle";
updateStatus(currentState);
renderMessages(data.history || []);
document.getElementById("sessionInfo").textContent = `Session ${sessionId}`;
startPolling();
} catch (err) {
console.error("Impossible de créer la session chat :", err);
showSystemMessage(`Impossible de créer la session chat : ${err.message}. Vérifiez que le serveur streaming (5005) est démarré.`);
}
}
// -----------------------------------------------------------------------------
// Envoi de messages
// -----------------------------------------------------------------------------
async function sendMessage() {
const input = document.getElementById("composerInput");
const text = (input.value || "").trim();
if (!text || !sessionId) return;
const sendBtn = document.getElementById("sendBtn");
sendBtn.disabled = true;
input.value = "";
autosizeTextarea();
// Affichage optimiste
appendMessage({
role: "user",
content: text,
timestamp: Date.now() / 1000,
});
try {
updateStatus("planning");
const resp = await fetch(`${API_BASE}/${sessionId}/message`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ message: text }),
});
if (!resp.ok) throw new Error(`HTTP ${resp.status}`);
const data = await resp.json();
currentState = data.state || "idle";
updateStatus(currentState);
renderMessages(data.history || []);
} catch (err) {
console.error("Erreur envoi message :", err);
showSystemMessage(`Erreur : ${err.message}`);
updateStatus("error");
} finally {
sendBtn.disabled = false;
input.focus();
}
}
async function confirmPlan(confirmed) {
if (!sessionId) return;
const confirmBar = document.getElementById("confirmBar");
confirmBar.classList.remove("visible");
try {
const resp = await fetch(`${API_BASE}/${sessionId}/confirm`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ confirmed }),
});
if (!resp.ok) throw new Error(`HTTP ${resp.status}`);
const data = await resp.json();
currentState = data.state || "idle";
updateStatus(currentState);
renderMessages(data.history || []);
} catch (err) {
console.error("Erreur confirmation :", err);
showSystemMessage(`Erreur confirmation : ${err.message}`);
}
}
// -----------------------------------------------------------------------------
// Polling
// -----------------------------------------------------------------------------
function startPolling() {
if (pollTimer) clearInterval(pollTimer);
pollTimer = setInterval(pollHistory, 2000);
}
async function pollHistory() {
if (!sessionId) return;
try {
const resp = await fetch(`${API_BASE}/${sessionId}/history`);
if (!resp.ok) return;
const data = await resp.json();
const snap = data.snapshot || {};
currentState = snap.state || "idle";
updateStatus(currentState, snap.progress || {});
const messages = snap.messages || [];
if (messages.length !== lastMessageCount) {
renderMessages(messages);
}
} catch (err) {
// Silencieux — on réessayera au prochain tick
}
}
// -----------------------------------------------------------------------------
// Rendu
// -----------------------------------------------------------------------------
function renderMessages(messages) {
const container = document.getElementById("messages");
container.innerHTML = "";
messages.forEach(msg => appendMessage(msg, false));
lastMessageCount = messages.length;
container.scrollTop = container.scrollHeight;
// Afficher/masquer la barre de confirmation
const confirmBar = document.getElementById("confirmBar");
if (currentState === "awaiting_confirmation") {
confirmBar.classList.add("visible");
} else {
confirmBar.classList.remove("visible");
}
}
function appendMessage(msg, autoscroll = true) {
const container = document.getElementById("messages");
const div = document.createElement("div");
div.className = `message ${msg.role}`;
const avatar = document.createElement("div");
avatar.className = "avatar";
if (msg.role === "user") avatar.textContent = "Vous";
else if (msg.role === "lea") avatar.textContent = "L";
else avatar.textContent = "i";
const bubbleWrap = document.createElement("div");
const bubble = document.createElement("div");
bubble.className = "bubble";
bubble.textContent = msg.content || "";
bubbleWrap.appendChild(bubble);
const ts = document.createElement("div");
ts.className = "timestamp";
try {
const d = new Date((msg.timestamp || 0) * 1000);
ts.textContent = d.toLocaleTimeString("fr-FR");
} catch (e) { ts.textContent = ""; }
bubbleWrap.appendChild(ts);
div.appendChild(avatar);
div.appendChild(bubbleWrap);
container.appendChild(div);
if (autoscroll) container.scrollTop = container.scrollHeight;
}
function showSystemMessage(text) {
appendMessage({
role: "system",
content: text,
timestamp: Date.now() / 1000,
});
}
function updateStatus(state, progress = {}) {
const dot = document.getElementById("statusDot");
const txt = document.getElementById("statusText");
dot.className = `status-dot ${state}`;
let label = STATE_LABELS[state] || state;
if (state === "executing" && progress && progress.total_actions) {
const done = progress.completed_actions || 0;
const total = progress.total_actions || 0;
label = `Léa exécute… ${done}/${total}`;
}
txt.textContent = label;
// Bloquer la saisie pendant planning/executing
const input = document.getElementById("composerInput");
const sendBtn = document.getElementById("sendBtn");
const blocked = (state === "planning" || state === "executing");
input.disabled = blocked;
sendBtn.disabled = blocked;
}
// -----------------------------------------------------------------------------
// UX composer
// -----------------------------------------------------------------------------
function handleKeydown(event) {
if (event.key === "Enter" && !event.shiftKey) {
event.preventDefault();
sendMessage();
}
}
function autosizeTextarea() {
const input = document.getElementById("composerInput");
input.style.height = "auto";
input.style.height = Math.min(input.scrollHeight, 120) + "px";
}
document.addEventListener("DOMContentLoaded", () => {
const input = document.getElementById("composerInput");
input.addEventListener("input", autosizeTextarea);
initChat();
});

View File

@@ -0,0 +1,309 @@
<!DOCTYPE html>
<html lang="fr">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Léa — Chat RPA Vision V3</title>
<style>
* { margin: 0; padding: 0; box-sizing: border-box; }
body {
font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
background: #0f172a;
color: #e2e8f0;
min-height: 100vh;
display: flex;
flex-direction: column;
}
.header {
background: linear-gradient(135deg, #3b82f6 0%, #8b5cf6 100%);
color: white;
padding: 16px 24px;
display: flex;
justify-content: space-between;
align-items: center;
box-shadow: 0 2px 8px rgba(0,0,0,0.3);
}
.header h1 {
font-size: 20px;
display: flex;
align-items: center;
gap: 10px;
}
.header .home-link {
color: rgba(255,255,255,0.85);
text-decoration: none;
font-size: 13px;
padding: 6px 14px;
border-radius: 6px;
background: rgba(255,255,255,0.1);
transition: background 0.2s;
}
.header .home-link:hover { background: rgba(255,255,255,0.2); }
.chat-wrapper {
flex: 1;
display: flex;
flex-direction: column;
max-width: 900px;
width: 100%;
margin: 0 auto;
padding: 20px;
gap: 16px;
}
.status-bar {
background: #1e293b;
border: 1px solid #334155;
border-radius: 12px;
padding: 12px 18px;
display: flex;
align-items: center;
justify-content: space-between;
gap: 12px;
font-size: 14px;
}
.status-bar .status-label {
color: #94a3b8;
}
.status-bar .status-value {
font-weight: 600;
display: flex;
align-items: center;
gap: 8px;
}
.status-dot {
width: 10px;
height: 10px;
border-radius: 50%;
background: #64748b;
}
.status-dot.idle { background: #64748b; }
.status-dot.planning { background: #f59e0b; animation: pulse 1.2s infinite; }
.status-dot.awaiting_confirmation { background: #3b82f6; animation: pulse 1.8s infinite; }
.status-dot.executing { background: #22c55e; animation: pulse 1s infinite; }
.status-dot.done { background: #22c55e; }
.status-dot.error { background: #ef4444; }
@keyframes pulse {
0%, 100% { opacity: 1; transform: scale(1); }
50% { opacity: 0.5; transform: scale(1.15); }
}
.messages {
flex: 1;
background: #1e293b;
border: 1px solid #334155;
border-radius: 12px;
padding: 20px;
overflow-y: auto;
display: flex;
flex-direction: column;
gap: 14px;
min-height: 400px;
max-height: calc(100vh - 320px);
}
.messages::-webkit-scrollbar { width: 8px; }
.messages::-webkit-scrollbar-thumb { background: #334155; border-radius: 4px; }
.messages::-webkit-scrollbar-track { background: transparent; }
.message {
display: flex;
gap: 12px;
max-width: 85%;
animation: fadeIn 0.25s ease-out;
}
@keyframes fadeIn {
from { opacity: 0; transform: translateY(6px); }
to { opacity: 1; transform: translateY(0); }
}
.message.user {
align-self: flex-end;
flex-direction: row-reverse;
}
.message .avatar {
width: 34px;
height: 34px;
border-radius: 50%;
display: flex;
align-items: center;
justify-content: center;
font-size: 16px;
flex-shrink: 0;
font-weight: 600;
}
.message.user .avatar {
background: #3b82f6;
color: white;
}
.message.lea .avatar {
background: linear-gradient(135deg, #8b5cf6, #ec4899);
color: white;
}
.message.system .avatar {
background: #475569;
color: #cbd5e1;
}
.message .bubble {
background: #334155;
padding: 12px 16px;
border-radius: 14px;
line-height: 1.5;
font-size: 14px;
white-space: pre-wrap;
word-wrap: break-word;
box-shadow: 0 1px 3px rgba(0,0,0,0.2);
}
.message.user .bubble {
background: #1d4ed8;
color: white;
}
.message.lea .bubble {
background: #334155;
}
.message.system .bubble {
background: transparent;
border: 1px dashed #475569;
color: #94a3b8;
font-style: italic;
}
.message .timestamp {
font-size: 11px;
color: #64748b;
margin-top: 4px;
}
.confirm-bar {
background: #1e293b;
border: 1px solid #3b82f6;
border-radius: 12px;
padding: 14px 18px;
display: none;
align-items: center;
justify-content: space-between;
gap: 12px;
}
.confirm-bar.visible { display: flex; }
.confirm-bar .label {
font-size: 14px;
color: #93c5fd;
font-weight: 500;
}
.confirm-bar .actions { display: flex; gap: 10px; }
.btn {
padding: 9px 20px;
border-radius: 8px;
border: none;
font-size: 14px;
font-weight: 600;
cursor: pointer;
transition: all 0.15s;
}
.btn-confirm {
background: #22c55e;
color: white;
}
.btn-confirm:hover { background: #16a34a; }
.btn-cancel {
background: #475569;
color: #e2e8f0;
}
.btn-cancel:hover { background: #64748b; }
.btn:disabled {
opacity: 0.5;
cursor: not-allowed;
}
.composer {
background: #1e293b;
border: 1px solid #334155;
border-radius: 12px;
padding: 14px;
display: flex;
gap: 10px;
align-items: flex-end;
}
.composer textarea {
flex: 1;
background: #0f172a;
border: 1px solid #334155;
border-radius: 8px;
padding: 10px 14px;
color: #e2e8f0;
font-family: inherit;
font-size: 14px;
resize: none;
min-height: 42px;
max-height: 120px;
line-height: 1.5;
}
.composer textarea:focus {
outline: none;
border-color: #3b82f6;
}
.composer textarea:disabled {
opacity: 0.6;
cursor: not-allowed;
}
.composer .btn-send {
background: #3b82f6;
color: white;
padding: 10px 22px;
}
.composer .btn-send:hover { background: #2563eb; }
.progress-bar {
margin-top: 8px;
height: 6px;
background: #0f172a;
border-radius: 3px;
overflow: hidden;
}
.progress-bar .fill {
height: 100%;
background: linear-gradient(90deg, #3b82f6, #22c55e);
width: 0%;
transition: width 0.4s ease;
}
</style>
</head>
<body>
<div class="header">
<h1>Léa &mdash; Assistant RPA Vision V3</h1>
<a href="/" class="home-link">Retour au dashboard</a>
</div>
<div class="chat-wrapper">
<div class="status-bar">
<div>
<span class="status-label">État :</span>
<span class="status-value">
<span class="status-dot idle" id="statusDot"></span>
<span id="statusText">En attente</span>
</span>
</div>
<div id="sessionInfo" style="color:#64748b;font-size:12px;">Aucune session</div>
</div>
<div class="messages" id="messages"></div>
<div class="confirm-bar" id="confirmBar">
<div class="label">Léa propose un plan. Confirmer l'exécution ?</div>
<div class="actions">
<button class="btn btn-cancel" onclick="confirmPlan(false)">Non, annuler</button>
<button class="btn btn-confirm" onclick="confirmPlan(true)">Oui, y aller</button>
</div>
</div>
<div class="composer">
<textarea
id="composerInput"
placeholder="Dites à Léa ce que vous voulez faire (ex. « Ouvre le Bloc-notes et écris bonjour »)…"
rows="1"
onkeydown="handleKeydown(event)"
></textarea>
<button class="btn btn-send" id="sendBtn" onclick="sendMessage()">Envoyer</button>
</div>
</div>
<script src="/static/js/chat.js"></script>
</body>
</html>