Files
rpa_vision_v3/agent_v0/agent_v1/ui/smart_tray.py
Dom 647aa610fd feat: popup VLM double-appel, auth Bearer partout, texte AZERTY corrigé
- Popup handling via double appel VLM (détection + localisation précise du bouton)
- Reconstruction texte depuis raw_keys (numpad /, @ AltGr fusionné)
- Clipboard paste pour texte riche, raw_keys pour commandes simples (Win+R)
- Skip des release orphelins dans raw_keys (fix menu Démarrer parasite)
- Auth Bearer sur toutes les requêtes agent → streaming server
- Endpoints /replay/next et /stream/image publics (agent Rust legacy)
- alt_gr ajouté dans _MODIFIER_ONLY_KEYS
- _key_combo_printable_char détecte ctrl+@ comme caractère imprimable
- start.bat tue les anciens process (python + rpa-agent) au démarrage
- Heartbeat avec token Bearer dans main.py et deploy/

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-03-30 16:45:09 +02:00

782 lines
28 KiB
Python

# agent_v1/ui/smart_tray.py
"""
Tray intelligent pour Agent V1 — remplace tray.py (plus de PyQt5).
Utilise pystray pour l'icone systray et tkinter (stdlib) pour les dialogues.
Communication serveur via LeaServerClient (chat:5004, streaming:5005).
Notifications via NotificationManager (module parallele).
Fenetre de chat Lea integree via ChatWindow (pywebview).
Architecture de threads :
- Thread principal : boucle pystray (icon.run)
- Thread daemon : verification connexion serveur (toutes les 30s)
- Thread daemon : rafraichissement cache workflows (toutes les 5 min)
- Thread daemon : pywebview (fenetre de chat Lea)
- Thread daemon : hotkey global Ctrl+Shift+L (si keyboard disponible)
- Threads ephemeres : dialogues tkinter (chaque dialogue cree son propre Tk())
"""
from __future__ import annotations
import logging
import os
import threading
import time
from typing import Any, Callable, Dict, List, Optional
from PIL import Image, ImageDraw
import pystray
from pystray import MenuItem as item
from .notifications import NotificationManager
from .shared_state import AgentState
logger = logging.getLogger(__name__)
# Intervalles (secondes)
_CONNECTION_CHECK_INTERVAL = 30
_WORKFLOW_CACHE_TTL = 300 # 5 minutes
# ---------------------------------------------------------------------------
# Helpers tkinter (sans PyQt5)
# ---------------------------------------------------------------------------
def _ask_string(title: str, prompt: str, default: str = "") -> Optional[str]:
"""Dialogue de saisie texte via tkinter (sans PyQt5).
Cree une instance Tk() ephemere, affiche le dialogue, puis la detruit.
Compatible avec la boucle pystray (pas de mainloop persistant).
"""
import tkinter as tk
from tkinter import simpledialog
root = tk.Tk()
root.withdraw()
root.attributes('-topmost', True)
result = simpledialog.askstring(title, prompt, initialvalue=default, parent=root)
root.destroy()
return result
def _show_info(title: str, message: str) -> None:
"""Affiche une boite d'information via tkinter (sans PyQt5)."""
import tkinter as tk
from tkinter import messagebox
root = tk.Tk()
root.withdraw()
root.attributes('-topmost', True)
messagebox.showinfo(title, message, parent=root)
root.destroy()
def _ask_consent(title: str, message: str) -> bool:
"""Dialogue de consentement Oui/Non via tkinter (sans PyQt5).
Utilise pour la notification prealable obligatoire (Articles 13/14,
Reglement IA) avant tout enregistrement.
"""
import tkinter as tk
from tkinter import messagebox
root = tk.Tk()
root.withdraw()
root.attributes('-topmost', True)
result = messagebox.askyesno(title, message, parent=root)
root.destroy()
return result
# ---------------------------------------------------------------------------
# SmartTrayV1
# ---------------------------------------------------------------------------
class SmartTrayV1:
"""Tray systeme intelligent pour Agent V1.
Remplace TrayAppV1 (PyQt5) par pystray + tkinter.
Meme interface constructeur pour compatibilite avec main.py.
"""
def __init__(
self,
on_start_callback: Callable[[str], None],
on_stop_callback: Callable[[], None],
server_client: Optional[Any] = None,
chat_window: Optional[Any] = None,
machine_id: str = "default",
shared_state: Optional[AgentState] = None,
) -> None:
self.on_start = on_start_callback
self.on_stop = on_stop_callback
self.server_client = server_client
self.machine_id = machine_id # Identifiant machine (multi-machine)
# Fenetre de chat Lea (pywebview)
self._chat_window = chat_window
# Etat partage avec le chat (source de verite unique)
self._shared_state = shared_state
# Etat interne (synchronise avec shared_state si disponible)
self.icon: Optional[pystray.Icon] = None
self.is_recording = False
self.actions_count = 0
# Etat connexion serveur
self._connected = False
self._replay_active = False
# Cache workflows
self._workflows: List[Dict[str, Any]] = []
self._workflows_lock = threading.Lock()
self._workflows_last_fetch: float = 0.0
# Verrous
self._state_lock = threading.Lock()
self._stop_event = threading.Event()
# Notifications
self._notifier = NotificationManager()
# Icones d'etat (cercles colores)
self.icons = {
"idle": self._create_circle_icon("gray"),
"recording": self._create_circle_icon("red"),
"connected": self._create_circle_icon("green"),
"disconnected": self._create_circle_icon("orange"),
"replay": self._create_circle_icon("blue"),
}
# Enregistrer le callback de changement de connexion sur le client
if self.server_client is not None:
self.server_client.set_on_connection_change(self._on_connection_change)
# S'abonner aux changements de l'etat partage
if self._shared_state is not None:
self._shared_state.on_change(self._on_shared_state_change)
logger.info("SmartTrayV1 initialise")
# ------------------------------------------------------------------
# Icones
# ------------------------------------------------------------------
@staticmethod
def _create_circle_icon(color: str) -> Image.Image:
"""Genere une icone circulaire simple mais propre."""
img = Image.new("RGBA", (64, 64), (0, 0, 0, 0))
draw = ImageDraw.Draw(img)
draw.ellipse((4, 4, 60, 60), fill=color, outline="white", width=2)
return img
def _current_icon(self) -> Image.Image:
"""Retourne l'icone correspondant a l'etat courant."""
if self._replay_active:
return self.icons["replay"]
if self.is_recording:
return self.icons["recording"]
if self._connected:
return self.icons["connected"]
if self.server_client is not None:
return self.icons["disconnected"]
return self.icons["idle"]
def _update_icon(self) -> None:
"""Met a jour l'icone et le menu du tray."""
if self.icon is not None:
self.icon.icon = self._current_icon()
self.icon.update_menu()
# ------------------------------------------------------------------
# Menu dynamique
# ------------------------------------------------------------------
def _get_menu_items(self):
"""Retourne les items du menu (appele a chaque ouverture du menu)."""
# Ligne de statut (féminin : Léa est connectée/déconnectée)
if self.is_recording:
status_text = "\U0001f534 Apprentissage en cours..."
elif self._connected:
status_text = "\U0001f7e2 Connect\u00e9e"
else:
status_text = "\U0001f534 D\u00e9connect\u00e9e"
# Compteur d'actions (visible uniquement en enregistrement)
actions_text = f"\U0001f4ca {self.actions_count} \u00e9tapes m\u00e9moris\u00e9es"
# Sous-menu workflows
workflow_items = self._build_workflow_submenu()
# Ligne d'identification machine (toujours visible)
machine_text = f"\U0001f4bb {self.machine_id}"
items = [
# --- Identite machine ---
item(machine_text, lambda: None, enabled=False),
# --- Statut ---
item(status_text, lambda: None, enabled=False),
item(
actions_text,
lambda: None,
enabled=False,
visible=lambda _i: self.is_recording,
),
pystray.Menu.SEPARATOR,
# --- Actions session ---
item(
"\U0001f393 Apprenez-moi une t\u00e2che",
self._on_start_session,
visible=lambda _i: not self.is_recording,
),
item(
"\u23f9\ufe0f C'est termin\u00e9",
self._on_stop_session,
visible=lambda _i: self.is_recording,
),
pystray.Menu.SEPARATOR,
# --- Workflows ---
item(
"\U0001f4cb Mes t\u00e2ches",
pystray.Menu(*workflow_items) if workflow_items else pystray.Menu(
item("(aucune t\u00e2che apprise)", lambda: None, enabled=False),
),
visible=lambda _i: self.server_client is not None,
),
item(
"\U0001f504 Actualiser",
self._on_refresh_workflows,
visible=lambda _i: self.server_client is not None,
),
pystray.Menu.SEPARATOR,
# --- Chat ---
item(
"\U0001f4ac Discuter avec L\u00e9a",
self._on_toggle_chat,
visible=lambda _i: self._chat_window is not None,
),
pystray.Menu.SEPARATOR,
# --- Arret d'urgence (Article 14, Reglement IA — controle humain) ---
# Toujours visible, quel que soit l'etat de l'agent
item(
"\u26d4 ARR\u00caT D'URGENCE",
self._on_emergency_stop,
),
pystray.Menu.SEPARATOR,
# --- Utilitaires ---
item("\U0001f4c2 Mes fichiers", self._on_open_folder),
item("\u274c Quitter L\u00e9a", self._on_quit),
]
return items
@staticmethod
def _human_workflow_name(wf: Dict[str, Any]) -> str:
"""Retourne un nom lisible pour un workflow.
Priorite :
1. Champ 'display_name' (nom humain saisi par l'utilisateur)
2. Champ 'name' ou 'workflow_name'
3. Fallback : "Tache du <date>"
"""
# Nom humain explicite (nouveau champ)
display = wf.get("display_name", "").strip()
if display:
return display
# Nom technique existant
name = wf.get("name", wf.get("workflow_name", "")).strip()
if name:
return name
# Fallback avec date de creation
created = wf.get("created_at", wf.get("timestamp", ""))
if created:
# Extraire juste la date (format ISO ou timestamp)
try:
from datetime import datetime
if isinstance(created, (int, float)):
dt = datetime.fromtimestamp(created)
else:
dt = datetime.fromisoformat(str(created).replace("Z", "+00:00"))
return f"T\u00e2che du {dt.strftime('%d %B')}"
except Exception:
pass
return "T\u00e2che sans nom"
def _build_workflow_submenu(self) -> List[pystray.MenuItem]:
"""Construit la liste des workflows comme items de sous-menu."""
with self._workflows_lock:
workflows = list(self._workflows)
if not workflows:
return [item("(aucune t\u00e2che apprise)", lambda: None, enabled=False)]
items = []
for wf in workflows:
wf_name = self._human_workflow_name(wf)
wf_id = wf.get("id", wf.get("workflow_id", ""))
# Creer une closure avec les bonnes valeurs
items.append(
item(wf_name, self._make_replay_callback(wf_id, wf_name))
)
return items
def _make_replay_callback(
self, workflow_id: str, workflow_name: str
) -> Callable:
"""Cree un callback de lancement de replay pour un workflow donne."""
def _callback(_icon=None, _item=None):
self._launch_replay(workflow_id, workflow_name)
return _callback
# ------------------------------------------------------------------
# Actions utilisateur
# ------------------------------------------------------------------
def _on_shared_state_change(self, state: AgentState) -> None:
"""Callback appele quand l'etat partage change (depuis le chat ou ailleurs).
Met a jour l'etat local du systray pour refleter le changement.
"""
with self._state_lock:
self.is_recording = state.is_recording
self.actions_count = state.actions_count
self._replay_active = state.is_replay_active
self._update_icon()
def _on_start_session(self, _icon=None, _item=None) -> None:
"""Demande le consentement puis le nom de la tache et demarre la session.
Notification prealable obligatoire (Articles 13/14, Reglement IA) :
l'utilisateur doit etre informe de ce qui sera capture AVANT le demarrage.
"""
# Dialogue tkinter dans un thread dedie
def _dialog():
# --- Consentement prealable (Articles 13/14, Reglement IA) ---
if not _ask_consent(
"Enregistrement — Information",
"\u26a0\ufe0f L'enregistrement va capturer votre \u00e9cran, "
"vos clics et vos frappes clavier pour apprendre cette t\u00e2che.\n\n"
"Les donn\u00e9es sensibles seront automatiquement flout\u00e9es.\n\n"
"Voulez-vous continuer ?",
):
return
name = _ask_string(
"Nouvelle t\u00e2che",
"D\u00e9crivez la t\u00e2che \u00e0 apprendre :",
default="",
)
if name and name.strip():
name = name.strip()
# Utiliser l'etat partage si disponible
if self._shared_state is not None:
try:
self._shared_state.start_recording(name)
except Exception as e:
self._notifier.notify("L\u00e9a", f"Oups : {e}")
return
else:
# Fallback sans etat partage
with self._state_lock:
self.is_recording = True
self.actions_count = 0
self._update_icon()
self.on_start(name)
self._notifier.notify(
"L\u00e9a",
"C'est parti ! Montrez-moi comment faire.",
)
threading.Thread(target=_dialog, daemon=True).start()
def _on_stop_session(self, _icon=None, _item=None) -> None:
"""Termine la session en cours et envoie les donnees."""
count = self.actions_count
# Utiliser l'etat partage si disponible
if self._shared_state is not None:
self._shared_state.stop_recording()
else:
with self._state_lock:
self.is_recording = False
self._update_icon()
self.on_stop()
self._notifier.notify(
"L\u00e9a",
f"Merci ! J'ai bien m\u00e9moris\u00e9 vos {count} actions.",
)
def _on_refresh_workflows(self, _icon=None, _item=None) -> None:
"""Rafraichit la liste des workflows depuis le serveur."""
threading.Thread(target=self._fetch_workflows, daemon=True).start()
def _on_ask_server(self, _icon=None, _item=None) -> None:
"""Envoie 'Que dois-je faire ?' au serveur et affiche la reponse."""
def _ask():
if self.server_client is None:
return
response = self.server_client.send_chat_message(
"Que dois-je faire maintenant ?"
)
if response:
# L'API renvoie {"response": {"message": "..."}} ou {"response": "..."}
resp = response.get("response", {})
if isinstance(resp, dict):
text = resp.get("message", str(resp))
else:
text = str(resp)
self._notifier.notify("Léa", text)
else:
self._notifier.notify(
"Erreur",
"Impossible de contacter le serveur.",
)
threading.Thread(target=_ask, daemon=True).start()
def _on_toggle_chat(self, _icon=None, _item=None) -> None:
"""Affiche ou masque la fenetre de chat Lea (pywebview)."""
if self._chat_window is None:
return
def _toggle():
try:
self._chat_window.toggle()
except Exception as e:
logger.error("Erreur toggle chat : %s", e)
self._notifier.notify(
"Erreur Chat",
f"Impossible d'ouvrir le chat : {e}",
)
threading.Thread(target=_toggle, daemon=True).start()
def _launch_replay(self, workflow_id: str, workflow_name: str) -> None:
"""Lance le replay d'un workflow."""
def _replay():
if self.server_client is None:
return
with self._state_lock:
self._replay_active = True
self._update_icon()
# Transparence mode autonome (Article 50, Reglement IA)
self._notifier.notify(
"L\u00e9a",
f"Le syst\u00e8me d'intelligence artificielle ex\u00e9cute la "
f"t\u00e2che '{workflow_name}' sur votre \u00e9cran.",
)
try:
import requests
# Auth headers pour le streaming server (port 5005)
auth_headers = {}
if self.server_client is not None:
auth_headers = self.server_client._auth_headers()
resp = requests.post(
f"{self.server_client._stream_base}/api/v1/traces/stream/replay/start",
json={"workflow_id": workflow_id},
headers=auth_headers,
timeout=10,
)
if resp.ok:
logger.info("Replay demarre pour workflow %s", workflow_id)
else:
self._notifier.notify(
"L\u00e9a",
"Hmm, le serveur a refus\u00e9. R\u00e9essayons plus tard.",
)
except Exception as e:
logger.error("Erreur lancement replay : %s", e)
self._notifier.notify(
"L\u00e9a",
f"Oups, un probl\u00e8me : {e}",
)
finally:
with self._state_lock:
self._replay_active = False
self._update_icon()
threading.Thread(target=_replay, daemon=True).start()
def _on_emergency_stop(self, _icon=None, _item=None) -> None:
"""Arret d'urgence — stoppe TOUTES les activites de l'agent immediatement.
Controle humain obligatoire (Article 14, Reglement IA).
Arrete l'enregistrement, le replay ET le heartbeat d'un seul clic.
Toujours accessible dans le menu, quel que soit l'etat de l'agent.
"""
logger.warning("ARRET D'URGENCE declenche par l'utilisateur")
# Arreter l'enregistrement si en cours
if self._shared_state is not None:
if self._shared_state.is_recording:
try:
self._shared_state.stop_recording()
except Exception as e:
logger.error("Erreur arret enregistrement d'urgence : %s", e)
# Arreter le replay si en cours
if self._shared_state.is_replay_active:
self._shared_state.set_replay_active(False)
else:
# Fallback sans etat partage
if self.is_recording:
try:
self.on_stop()
except Exception as e:
logger.error("Erreur arret session d'urgence : %s", e)
# Forcer l'etat local a l'arret
with self._state_lock:
self.is_recording = False
self.actions_count = 0
self._replay_active = False
self._update_icon()
# Notification
self._notifier.notify(
"\u26d4 Arr\u00eat d'urgence",
"Toutes les activit\u00e9s ont \u00e9t\u00e9 arr\u00eat\u00e9es.",
timeout=10,
)
def _on_open_folder(self, _icon=None, _item=None) -> None:
"""Ouvre le dossier des sessions dans l'explorateur de fichiers."""
from ..config import SESSIONS_ROOT
sessions_path = str(SESSIONS_ROOT)
if os.name == "nt":
os.startfile(sessions_path)
else:
os.system(f'xdg-open "{sessions_path}"')
def _on_quit(self, _icon=None, _item=None) -> None:
"""Arrete proprement l'agent et quitte."""
logger.info("Arret demande par l'utilisateur")
# Arreter la session si en cours
if self.is_recording:
self.on_stop()
# Signaler l'arret aux threads de fond
self._stop_event.set()
# Fermer la fenetre de chat si ouverte
if self._chat_window is not None:
try:
self._chat_window.destroy()
except Exception as e:
logger.debug("Erreur fermeture chat : %s", e)
# Arreter le hotkey global si actif
self._stop_hotkey()
# Arreter le client serveur si present
if self.server_client is not None:
self.server_client.shutdown()
# Arreter l'icone pystray
if self.icon is not None:
self.icon.stop()
# ------------------------------------------------------------------
# Verification connexion serveur (thread daemon)
# ------------------------------------------------------------------
def _connection_checker_loop(self) -> None:
"""Verifie la connexion au serveur toutes les 30 secondes."""
logger.info("Thread de verification connexion demarre")
while not self._stop_event.is_set():
if self.server_client is not None:
try:
was_connected = self._connected
self._connected = self.server_client.check_connection()
if self._connected != was_connected:
self._update_icon()
# La notification est geree par _on_connection_change
except Exception as e:
logger.error("Erreur verification connexion : %s", e)
self._stop_event.wait(timeout=_CONNECTION_CHECK_INTERVAL)
logger.info("Thread de verification connexion arrete")
def _on_connection_change(self, connected: bool) -> None:
"""Callback appelee par LeaServerClient quand l'etat de connexion change."""
with self._state_lock:
self._connected = connected
self._update_icon()
if connected:
self._notifier.notify(
"L\u00e9a",
"Connect\u00e9e au serveur.",
)
# Rafraichir les taches a la connexion
threading.Thread(target=self._fetch_workflows, daemon=True).start()
else:
self._notifier.notify(
"L\u00e9a",
"J'ai perdu la connexion avec le serveur.",
)
# ------------------------------------------------------------------
# Cache workflows (thread daemon)
# ------------------------------------------------------------------
def _workflow_cache_loop(self) -> None:
"""Rafraichit le cache des workflows toutes les 5 minutes."""
logger.info("Thread de cache workflows demarre")
while not self._stop_event.is_set():
if self.server_client is not None and self._connected:
self._fetch_workflows()
self._stop_event.wait(timeout=_WORKFLOW_CACHE_TTL)
logger.info("Thread de cache workflows arrete")
def _fetch_workflows(self) -> None:
"""Recupere la liste des workflows depuis le serveur."""
if self.server_client is None:
return
try:
workflows = self.server_client.list_workflows()
with self._workflows_lock:
self._workflows = workflows
self._workflows_last_fetch = time.time()
logger.debug(
"Cache workflows mis a jour : %d workflows", len(workflows)
)
# Forcer la reconstruction du menu
self._update_icon()
except Exception as e:
logger.error("Erreur recuperation workflows : %s", e)
# ------------------------------------------------------------------
# Mise a jour du compteur (compatibilite main.py)
# ------------------------------------------------------------------
def update_stats(self, count: int) -> None:
"""Met a jour le compteur d'actions en temps reel dans le menu."""
with self._state_lock:
self.actions_count = count
if self.icon is not None:
self.icon.update_menu()
def set_replay_active(self, active: bool) -> None:
"""Signale qu'un replay est en cours (appele depuis main.py)."""
with self._state_lock:
self._replay_active = active
self._update_icon()
if active:
# Transparence mode autonome (Article 50, Reglement IA)
self._notifier.notify(
"L\u00e9a",
"Le syst\u00e8me d'intelligence artificielle ex\u00e9cute "
"une t\u00e2che sur votre \u00e9cran.",
)
else:
self._notifier.notify("L\u00e9a", "C'est fait !")
# ------------------------------------------------------------------
# Hotkey global Ctrl+Shift+L (toggle chat)
# ------------------------------------------------------------------
_hotkey_hook = None # reference pour pouvoir le retirer
def _start_hotkey(self) -> None:
"""Enregistre le raccourci global Ctrl+Shift+L pour ouvrir le chat.
Utilise la librairie 'keyboard' si disponible.
Silencieux si elle n'est pas installee (pas critique).
"""
if self._chat_window is None:
return
try:
import keyboard
self._hotkey_hook = keyboard.add_hotkey(
"ctrl+shift+l",
self._on_toggle_chat,
suppress=False,
)
logger.info("Hotkey Ctrl+Shift+L enregistre pour le chat Lea")
except ImportError:
logger.debug(
"keyboard non installe — hotkey Ctrl+Shift+L desactive. "
"Installer avec : pip install keyboard"
)
except Exception as e:
logger.warning("Impossible d'enregistrer le hotkey : %s", e)
def _stop_hotkey(self) -> None:
"""Retire le raccourci global."""
if self._hotkey_hook is not None:
try:
import keyboard
keyboard.remove_hotkey(self._hotkey_hook)
self._hotkey_hook = None
logger.debug("Hotkey Ctrl+Shift+L retire")
except Exception:
pass
# ------------------------------------------------------------------
# Point d'entree
# ------------------------------------------------------------------
def run(self) -> None:
"""Demarre le tray, les threads de fond, et entre dans la boucle principale."""
# Notification d'accueil — divulgation IA (Article 50, Reglement IA)
self._notifier.greet()
# Enregistrer le hotkey global Ctrl+Shift+L (toggle chat)
self._start_hotkey()
# Tooltip avec identifiant machine pour le multi-machine
tray_title = f"Agent V1 - {self.machine_id}"
# Menu statique — reconstruit via _update_icon() quand l'état change
self.icon = pystray.Icon(
"AgentV1",
self._current_icon(),
tray_title,
menu=pystray.Menu(*self._get_menu_items()),
)
# Demarrer le thread de verification connexion
if self.server_client is not None:
conn_thread = threading.Thread(
target=self._connection_checker_loop,
daemon=True,
name="smart-tray-conn-check",
)
conn_thread.start()
# Demarrer le thread de cache workflows
wf_thread = threading.Thread(
target=self._workflow_cache_loop,
daemon=True,
name="smart-tray-wf-cache",
)
wf_thread.start()
# Premiere verification immediate
threading.Thread(
target=self._fetch_workflows, daemon=True
).start()
# Boucle principale pystray (bloquante)
logger.info("SmartTrayV1 demarre — entree dans la boucle pystray")
self.icon.run()