chore: ajouter agent_v0/ au tracking git (était un repo embarqué)
Suppression du .git embarqué dans agent_v0/ — le code est maintenant tracké normalement dans le repo principal. Inclut : agent_v1 (client), server_v1 (streaming), lea_ui (chat client) Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
17
agent_v0/deploy/windows_client/LISEZMOI.txt
Normal file
17
agent_v0/deploy/windows_client/LISEZMOI.txt
Normal file
@@ -0,0 +1,17 @@
|
||||
=== Agent V1 — RPA Vision — Client Windows ===
|
||||
|
||||
Installation :
|
||||
1. Double-cliquer sur setup.bat
|
||||
2. Configurer le serveur : éditer agent_config.json
|
||||
ou définir la variable RPA_SERVER_HOST=192.168.1.x
|
||||
3. Lancer : python run_agent_v1.py
|
||||
|
||||
L'agent apparaît dans la zone de notification (systray).
|
||||
Clic droit pour accéder au menu : démarrer une session,
|
||||
lancer un replay, voir les workflows appris, etc.
|
||||
|
||||
Léa communique par des notifications toast sur votre écran.
|
||||
|
||||
Prérequis :
|
||||
- Python 3.10 ou plus récent
|
||||
- Connexion réseau vers le serveur Linux
|
||||
1
agent_v0/deploy/windows_client/__init__.py
Normal file
1
agent_v0/deploy/windows_client/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
# agent_v0 — Agent RPA Vision V3
|
||||
15
agent_v0/deploy/windows_client/agent_config.json
Normal file
15
agent_v0/deploy/windows_client/agent_config.json
Normal file
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"user_id": "demo_user",
|
||||
"user_label": "Démo agent_v0",
|
||||
"customer": "Clinique Demo",
|
||||
"training_label": "Facturation_T2A_demo",
|
||||
"notes": "Session réelle avec clics + screenshots + key combos.",
|
||||
"mode": "enriched",
|
||||
"screenshot_mode": "crop",
|
||||
"screenshot_crop_width": 900,
|
||||
"screenshot_crop_height": 700,
|
||||
"capture_hover": true,
|
||||
"hover_min_idle_ms": 700,
|
||||
"capture_scroll": true,
|
||||
"network_save_path": ""
|
||||
}
|
||||
0
agent_v0/deploy/windows_client/agent_v1/__init__.py
Normal file
0
agent_v0/deploy/windows_client/agent_v1/__init__.py
Normal file
43
agent_v0/deploy/windows_client/agent_v1/config.py
Normal file
43
agent_v0/deploy/windows_client/agent_v1/config.py
Normal file
@@ -0,0 +1,43 @@
|
||||
# agent_v1/config.py
|
||||
"""
|
||||
Configuration avancée pour Agent V1.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import os
|
||||
import platform
|
||||
import socket
|
||||
from pathlib import Path
|
||||
|
||||
AGENT_VERSION = "1.0.0"
|
||||
|
||||
# Identifiant unique de la machine (utilisé pour le multi-machine)
|
||||
# Configurable via variable d'environnement, sinon auto-généré depuis hostname + OS
|
||||
MACHINE_ID = os.environ.get(
|
||||
"RPA_MACHINE_ID",
|
||||
f"{socket.gethostname()}_{platform.system().lower()}",
|
||||
)
|
||||
|
||||
# Dossier racine de l'agent
|
||||
BASE_DIR = Path(__file__).resolve().parent
|
||||
|
||||
# Endpoint du serveur Streaming (port 5005)
|
||||
SERVER_URL = os.getenv("RPA_SERVER_URL", "http://localhost:5005/api/v1")
|
||||
UPLOAD_ENDPOINT = f"{SERVER_URL}/traces/upload"
|
||||
STREAMING_ENDPOINT = f"{SERVER_URL}/traces/stream"
|
||||
|
||||
# Paramètres de session
|
||||
MAX_SESSION_DURATION_S = 60 * 60 # 1 heure
|
||||
SESSIONS_ROOT = BASE_DIR / "sessions"
|
||||
|
||||
# Paramètres Vision (Crops pour qwen3-vl)
|
||||
TARGETED_CROP_SIZE = (400, 400)
|
||||
SCREENSHOT_QUALITY = 85
|
||||
|
||||
# Monitoring
|
||||
PERF_MONITOR_INTERVAL_S = 30
|
||||
LOGS_DIR = BASE_DIR / "logs"
|
||||
LOG_FILE = LOGS_DIR / "agent_v1.log"
|
||||
|
||||
# Création des dossiers
|
||||
os.makedirs(SESSIONS_ROOT, exist_ok=True)
|
||||
os.makedirs(LOGS_DIR, exist_ok=True)
|
||||
319
agent_v0/deploy/windows_client/agent_v1/core/captor.py
Normal file
319
agent_v0/deploy/windows_client/agent_v1/core/captor.py
Normal file
@@ -0,0 +1,319 @@
|
||||
# agent_v1/core/captor.py
|
||||
"""
|
||||
Moteur de capture d'événements Agent V1.
|
||||
Capture enrichie avec focus sur le contexte UI pour le stagiaire.
|
||||
|
||||
Fonctionnalités :
|
||||
- Capture clics souris (simple et double-clic)
|
||||
- Capture scroll souris
|
||||
- Capture combos clavier (Ctrl+C, Alt+Tab, etc.)
|
||||
- Buffer de saisie texte : accumule les frappes et émet un événement
|
||||
text_input après 500ms d'inactivité clavier
|
||||
- Surveillance du focus fenêtre
|
||||
"""
|
||||
|
||||
import threading
|
||||
import time
|
||||
import logging
|
||||
from typing import Callable, Optional, List, Dict, Any, Tuple
|
||||
from pynput import mouse, keyboard
|
||||
from pynput.mouse import Button
|
||||
from pynput.keyboard import Key, KeyCode
|
||||
|
||||
# Importation relative pour rester dans le module v1
|
||||
from ..vision.capturer import VisionCapturer
|
||||
# from ..monitoring.system import SystemMonitor
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Délai d'inactivité avant flush du buffer texte (en secondes)
|
||||
TEXT_FLUSH_DELAY = 0.5
|
||||
# Délai max entre deux clics pour un double-clic (en secondes)
|
||||
DOUBLE_CLICK_DELAY = 0.3
|
||||
# Tolérance en pixels pour considérer deux clics au même endroit
|
||||
DOUBLE_CLICK_TOLERANCE = 10
|
||||
|
||||
|
||||
class EventCaptorV1:
|
||||
def __init__(self, on_event_callback: Callable[[Dict[str, Any]], None]):
|
||||
self.on_event = on_event_callback
|
||||
self.mouse_listener = None
|
||||
self.keyboard_listener = None
|
||||
self.running = False
|
||||
|
||||
# État des touches modificatrices
|
||||
self.modifiers = set()
|
||||
|
||||
# Tracking du focus fenêtre
|
||||
self.last_window = None
|
||||
self._focus_thread = None
|
||||
|
||||
# --- Buffer de saisie texte ---
|
||||
# Lock pour accès thread-safe au buffer (le listener pynput
|
||||
# tourne dans un thread séparé)
|
||||
self._text_lock = threading.Lock()
|
||||
self._text_buffer: list[str] = []
|
||||
# Position de la souris au moment de la première frappe du buffer
|
||||
self._text_start_pos: Optional[Tuple[int, int]] = None
|
||||
# Timer pour le flush après inactivité
|
||||
self._text_flush_timer: Optional[threading.Timer] = None
|
||||
# Dernière position connue de la souris (pour associer le texte
|
||||
# au champ dans lequel l'utilisateur tape)
|
||||
self._last_mouse_pos: Tuple[int, int] = (0, 0)
|
||||
|
||||
# --- Détection double-clic ---
|
||||
# Dernier clic : (x, y, timestamp, button)
|
||||
self._last_click: Optional[Tuple[int, int, float, str]] = None
|
||||
|
||||
def start(self):
|
||||
self.running = True
|
||||
self.mouse_listener = mouse.Listener(
|
||||
on_click=self._on_click,
|
||||
on_scroll=self._on_scroll,
|
||||
on_move=self._on_move
|
||||
)
|
||||
self.keyboard_listener = keyboard.Listener(
|
||||
on_press=self._on_press,
|
||||
on_release=self._on_release
|
||||
)
|
||||
|
||||
self.mouse_listener.start()
|
||||
self.keyboard_listener.start()
|
||||
|
||||
# Thread de surveillance du focus fenêtre (Proactif)
|
||||
self._focus_thread = threading.Thread(target=self._watch_window_focus, daemon=True)
|
||||
self._focus_thread.start()
|
||||
|
||||
logger.info("Agent V1 Captor démarré")
|
||||
|
||||
def stop(self):
|
||||
self.running = False
|
||||
# Flush du buffer texte restant avant arrêt
|
||||
self._flush_text_buffer()
|
||||
# Annuler le timer s'il est en cours
|
||||
with self._text_lock:
|
||||
if self._text_flush_timer is not None:
|
||||
self._text_flush_timer.cancel()
|
||||
self._text_flush_timer = None
|
||||
if self.mouse_listener: self.mouse_listener.stop()
|
||||
if self.keyboard_listener: self.keyboard_listener.stop()
|
||||
logger.info("Agent V1 Captor arrêté")
|
||||
|
||||
# ----------------------------------------------------------------
|
||||
# Souris
|
||||
# ----------------------------------------------------------------
|
||||
|
||||
def _on_move(self, x, y):
|
||||
"""Mémorise la position souris pour l'associer aux événements texte."""
|
||||
self._last_mouse_pos = (x, y)
|
||||
|
||||
def _on_click(self, x, y, button, pressed):
|
||||
if not pressed:
|
||||
return
|
||||
|
||||
now = time.time()
|
||||
|
||||
# --- Flush du buffer texte : l'utilisateur a cliqué, donc
|
||||
# il change probablement de champ ---
|
||||
self._flush_text_buffer()
|
||||
|
||||
# --- Détection double-clic ---
|
||||
if self._last_click is not None:
|
||||
lx, ly, lt, lb = self._last_click
|
||||
# Même bouton, même zone, délai court → double-clic
|
||||
if (button.name == lb
|
||||
and abs(x - lx) <= DOUBLE_CLICK_TOLERANCE
|
||||
and abs(y - ly) <= DOUBLE_CLICK_TOLERANCE
|
||||
and (now - lt) <= DOUBLE_CLICK_DELAY):
|
||||
event = {
|
||||
"type": "double_click",
|
||||
"button": button.name,
|
||||
"pos": (x, y),
|
||||
"timestamp": now,
|
||||
}
|
||||
self.on_event(event)
|
||||
# Réinitialiser pour éviter un triple-clic = 2 double-clics
|
||||
self._last_click = None
|
||||
return
|
||||
|
||||
# Clic simple — on le mémorise pour comparer au prochain
|
||||
self._last_click = (x, y, now, button.name)
|
||||
event = {
|
||||
"type": "mouse_click",
|
||||
"button": button.name,
|
||||
"pos": (x, y),
|
||||
"timestamp": now,
|
||||
}
|
||||
self.on_event(event)
|
||||
|
||||
def _on_scroll(self, x, y, dx, dy):
|
||||
event = {
|
||||
"type": "mouse_scroll",
|
||||
"pos": (x, y),
|
||||
"delta": (dx, dy),
|
||||
"timestamp": time.time(),
|
||||
}
|
||||
self.on_event(event)
|
||||
|
||||
# ----------------------------------------------------------------
|
||||
# Clavier
|
||||
# ----------------------------------------------------------------
|
||||
|
||||
def _on_press(self, key):
|
||||
# Gestion des touches modificatrices
|
||||
if key in (Key.ctrl, Key.ctrl_l, Key.ctrl_r):
|
||||
self.modifiers.add("ctrl")
|
||||
elif key in (Key.alt, Key.alt_l, Key.alt_r):
|
||||
self.modifiers.add("alt")
|
||||
elif key in (Key.shift, Key.shift_l, Key.shift_r):
|
||||
self.modifiers.add("shift")
|
||||
|
||||
# --- Combos avec modificateur (sauf Shift seul) ---
|
||||
# Shift seul n'est pas un « vrai » modificateur pour les combos :
|
||||
# Shift+a = 'A' = saisie texte, pas un raccourci.
|
||||
# On considère un combo seulement si Ctrl ou Alt est enfoncé.
|
||||
has_real_modifier = self.modifiers & {"ctrl", "alt"}
|
||||
if has_real_modifier:
|
||||
key_name = self._get_key_name(key)
|
||||
if key_name and key_name not in ("ctrl", "alt", "shift"):
|
||||
# Un combo interrompt la saisie texte en cours
|
||||
self._flush_text_buffer()
|
||||
event = {
|
||||
"type": "key_combo",
|
||||
"keys": list(self.modifiers) + [key_name],
|
||||
"timestamp": time.time(),
|
||||
}
|
||||
self.on_event(event)
|
||||
return
|
||||
|
||||
# --- Saisie texte (pas de Ctrl/Alt enfoncé) ---
|
||||
self._handle_text_key(key)
|
||||
|
||||
def _handle_text_key(self, key):
|
||||
"""Gère l'accumulation des frappes texte dans le buffer.
|
||||
|
||||
Touches spéciales :
|
||||
- Backspace : supprime le dernier caractère du buffer
|
||||
- Enter / Tab : flush immédiat + émission de l'événement
|
||||
- Escape : vide le buffer sans émettre
|
||||
"""
|
||||
with self._text_lock:
|
||||
# --- Touches spéciales ---
|
||||
if key == Key.backspace:
|
||||
if self._text_buffer:
|
||||
self._text_buffer.pop()
|
||||
self._reset_flush_timer()
|
||||
return
|
||||
|
||||
if key == Key.escape:
|
||||
# Annuler la saisie en cours
|
||||
self._text_buffer.clear()
|
||||
self._text_start_pos = None
|
||||
self._cancel_flush_timer()
|
||||
return
|
||||
|
||||
if key in (Key.enter, Key.tab):
|
||||
# Flush immédiat — on relâche le lock avant d'appeler
|
||||
# _flush_text_buffer (qui prend aussi le lock)
|
||||
pass # on sort du with et on flush après
|
||||
|
||||
elif key == Key.space:
|
||||
# Espace = caractère normal
|
||||
if not self._text_buffer:
|
||||
self._text_start_pos = self._last_mouse_pos
|
||||
self._text_buffer.append(" ")
|
||||
self._reset_flush_timer()
|
||||
return
|
||||
|
||||
elif isinstance(key, KeyCode) and key.char is not None:
|
||||
# Caractère alphanumérique / ponctuation
|
||||
# pynput renvoie déjà le bon caractère selon le layout
|
||||
# (AZERTY inclus) — on ne convertit rien.
|
||||
if not self._text_buffer:
|
||||
self._text_start_pos = self._last_mouse_pos
|
||||
self._text_buffer.append(key.char)
|
||||
self._reset_flush_timer()
|
||||
return
|
||||
else:
|
||||
# Touche spéciale non gérée (F1, Insert, etc.) — on ignore
|
||||
return
|
||||
|
||||
# Si on arrive ici, c'est Enter ou Tab → flush immédiat
|
||||
self._flush_text_buffer()
|
||||
|
||||
def _reset_flush_timer(self):
|
||||
"""Réarme le timer de flush après chaque frappe.
|
||||
|
||||
Doit être appelé avec self._text_lock déjà acquis.
|
||||
"""
|
||||
if self._text_flush_timer is not None:
|
||||
self._text_flush_timer.cancel()
|
||||
self._text_flush_timer = threading.Timer(
|
||||
TEXT_FLUSH_DELAY, self._flush_text_buffer
|
||||
)
|
||||
self._text_flush_timer.daemon = True
|
||||
self._text_flush_timer.start()
|
||||
|
||||
def _cancel_flush_timer(self):
|
||||
"""Annule le timer de flush sans émettre.
|
||||
|
||||
Doit être appelé avec self._text_lock déjà acquis.
|
||||
"""
|
||||
if self._text_flush_timer is not None:
|
||||
self._text_flush_timer.cancel()
|
||||
self._text_flush_timer = None
|
||||
|
||||
def _flush_text_buffer(self):
|
||||
"""Émet un événement text_input avec le contenu du buffer, puis
|
||||
le vide. Thread-safe — peut être appelé depuis le timer, le
|
||||
listener souris ou le listener clavier."""
|
||||
with self._text_lock:
|
||||
if not self._text_buffer:
|
||||
# Rien à émettre
|
||||
self._cancel_flush_timer()
|
||||
return
|
||||
text = "".join(self._text_buffer)
|
||||
pos = self._text_start_pos or self._last_mouse_pos
|
||||
self._text_buffer.clear()
|
||||
self._text_start_pos = None
|
||||
self._cancel_flush_timer()
|
||||
|
||||
# Émission hors du lock pour éviter un deadlock si le callback
|
||||
# est lent ou prend d'autres locks
|
||||
event = {
|
||||
"type": "text_input",
|
||||
"text": text,
|
||||
"pos": pos,
|
||||
"timestamp": time.time(),
|
||||
}
|
||||
logger.debug(f"text_input émis : {len(text)} caractères")
|
||||
self.on_event(event)
|
||||
|
||||
def _on_release(self, key):
|
||||
if key in (Key.ctrl, Key.ctrl_l, Key.ctrl_r):
|
||||
self.modifiers.discard("ctrl")
|
||||
elif key in (Key.alt, Key.alt_l, Key.alt_r):
|
||||
self.modifiers.discard("alt")
|
||||
elif key in (Key.shift, Key.shift_l, Key.shift_r):
|
||||
self.modifiers.discard("shift")
|
||||
|
||||
def _watch_window_focus(self):
|
||||
"""Surveille proactivement le changement de fenêtre pour le stagiaire."""
|
||||
# Importation relative simple
|
||||
from ..window_info_crossplatform import get_active_window_info
|
||||
|
||||
while self.running:
|
||||
try:
|
||||
info = get_active_window_info()
|
||||
if info and info != self.last_window:
|
||||
event = {
|
||||
"type": "window_focus_change",
|
||||
"from": self.last_window,
|
||||
"to": info,
|
||||
"timestamp": time.time()
|
||||
}
|
||||
self.last_window = info
|
||||
self.on_event(event)
|
||||
except Exception as e:
|
||||
logger.error(f"Erreur focus window: {e}")
|
||||
time.sleep(0.5)
|
||||
523
agent_v0/deploy/windows_client/agent_v1/core/executor.py
Normal file
523
agent_v0/deploy/windows_client/agent_v1/core/executor.py
Normal file
@@ -0,0 +1,523 @@
|
||||
# agent_v1/core/executor.py
|
||||
"""
|
||||
Executeur d'actions visuelles pour Agent V1.
|
||||
Opere par coordonnees normalisees (proportions) pour le rejeu en univers ferme (VM).
|
||||
|
||||
Supporte deux modes :
|
||||
- Watchdog fichier (command.json) — legacy
|
||||
- Polling serveur (GET /replay/next) — mode replay P0-5
|
||||
"""
|
||||
|
||||
import base64
|
||||
import io
|
||||
import time
|
||||
import logging
|
||||
|
||||
import mss
|
||||
from pynput.mouse import Button, Controller as MouseController
|
||||
from pynput.keyboard import Controller as KeyboardController, Key
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Mapping des noms de touches spéciales vers pynput.Key
|
||||
_SPECIAL_KEYS = {
|
||||
"enter": Key.enter,
|
||||
"return": Key.enter,
|
||||
"tab": Key.tab,
|
||||
"escape": Key.esc,
|
||||
"esc": Key.esc,
|
||||
"backspace": Key.backspace,
|
||||
"delete": Key.delete,
|
||||
"space": Key.space,
|
||||
"up": Key.up,
|
||||
"down": Key.down,
|
||||
"left": Key.left,
|
||||
"right": Key.right,
|
||||
"home": Key.home,
|
||||
"end": Key.end,
|
||||
"page_up": Key.page_up,
|
||||
"page_down": Key.page_down,
|
||||
"f1": Key.f1, "f2": Key.f2, "f3": Key.f3, "f4": Key.f4,
|
||||
"f5": Key.f5, "f6": Key.f6, "f7": Key.f7, "f8": Key.f8,
|
||||
"f9": Key.f9, "f10": Key.f10, "f11": Key.f11, "f12": Key.f12,
|
||||
"ctrl": Key.ctrl, "ctrl_l": Key.ctrl_l, "ctrl_r": Key.ctrl_r,
|
||||
"alt": Key.alt, "alt_l": Key.alt_l, "alt_r": Key.alt_r,
|
||||
"shift": Key.shift, "shift_l": Key.shift_l, "shift_r": Key.shift_r,
|
||||
"cmd": Key.cmd, "win": Key.cmd,
|
||||
"super": Key.cmd, "super_l": Key.cmd, "super_r": Key.cmd,
|
||||
"windows": Key.cmd, "meta": Key.cmd,
|
||||
"insert": Key.insert, "print_screen": Key.print_screen,
|
||||
"caps_lock": Key.caps_lock, "num_lock": Key.num_lock,
|
||||
}
|
||||
|
||||
|
||||
class ActionExecutorV1:
|
||||
def __init__(self):
|
||||
self.mouse = MouseController()
|
||||
self.keyboard = KeyboardController()
|
||||
# NB: mss est initialise paresseusement pour eviter les problemes
|
||||
# de thread-safety (le constructeur peut etre appele dans un thread
|
||||
# different de celui qui utilise l'instance).
|
||||
self._sct = None
|
||||
self.running = True
|
||||
# Backoff exponentiel pour le polling replay (evite de marteler le serveur)
|
||||
self._poll_backoff = 1.0 # Delai actuel (secondes)
|
||||
self._poll_backoff_min = 1.0 # Delai minimal (reset apres succes)
|
||||
self._poll_backoff_max = 30.0 # Delai maximal
|
||||
self._poll_backoff_factor = 1.5 # Multiplicateur en cas d'echec
|
||||
|
||||
@property
|
||||
def sct(self):
|
||||
"""Instance mss paresseuse, creee dans le thread appelant."""
|
||||
if self._sct is None:
|
||||
self._sct = mss.mss()
|
||||
return self._sct
|
||||
|
||||
# =========================================================================
|
||||
# Execution legacy (watchdog command.json)
|
||||
# =========================================================================
|
||||
|
||||
def execute_normalized_order(self, order: dict):
|
||||
"""
|
||||
Execute un ordre base sur des proportions (0.0 a 1.0).
|
||||
Ex: {"action": "mouse_click", "x_pct": 0.5, "y_pct": 0.5} (Clic au centre)
|
||||
"""
|
||||
action = order.get("action")
|
||||
try:
|
||||
# Recuperation de la resolution actuelle de la VM
|
||||
monitor = self.sct.monitors[1]
|
||||
width, height = monitor["width"], monitor["height"]
|
||||
|
||||
if action == "mouse_click":
|
||||
# Traduction Proportions -> Pixels reels de la VM
|
||||
real_x = int(order.get("x_pct", 0) * width)
|
||||
real_y = int(order.get("y_pct", 0) * height)
|
||||
self._click((real_x, real_y), order.get("button", "left"))
|
||||
|
||||
elif action == "text_input":
|
||||
self.keyboard.type(order.get("text", ""))
|
||||
|
||||
logger.info(f"Ordre Visuel execute : {action} sur ({width}x{height})")
|
||||
except Exception as e:
|
||||
logger.error(f"Echec de l'ordre {action} : {e}")
|
||||
|
||||
# =========================================================================
|
||||
# Execution replay (polling serveur)
|
||||
# =========================================================================
|
||||
|
||||
def execute_replay_action(self, action: dict, server_url: str = "") -> dict:
|
||||
"""
|
||||
Execute une action normalisee recue du serveur de replay.
|
||||
|
||||
Supporte deux modes :
|
||||
- Visual mode (visual_mode=True + target_spec) : capture un screenshot,
|
||||
l'envoie au serveur pour resolution visuelle, puis execute a la position trouvee.
|
||||
- Blind mode (defaut) : utilise les coordonnees statiques x_pct/y_pct.
|
||||
|
||||
Format d'entree :
|
||||
{
|
||||
"action_id": "act_xxxx",
|
||||
"type": "click|type|key_combo|scroll|wait",
|
||||
"x_pct": 0.5,
|
||||
"y_pct": 0.3,
|
||||
"text": "...",
|
||||
"keys": [...],
|
||||
"button": "left",
|
||||
"duration_ms": 500,
|
||||
"visual_mode": true,
|
||||
"target_spec": {"by_role": "button", "by_text": "Submit"}
|
||||
}
|
||||
|
||||
Retourne :
|
||||
{
|
||||
"action_id": "act_xxxx",
|
||||
"success": True/False,
|
||||
"error": None ou message,
|
||||
"screenshot": base64 du screenshot post-action,
|
||||
"visual_resolved": True/False
|
||||
}
|
||||
"""
|
||||
action_id = action.get("action_id", "unknown")
|
||||
action_type = action.get("type", "unknown")
|
||||
visual_mode = action.get("visual_mode", False)
|
||||
target_spec = action.get("target_spec", {})
|
||||
result = {
|
||||
"action_id": action_id,
|
||||
"success": False,
|
||||
"error": None,
|
||||
"screenshot": None,
|
||||
"visual_resolved": False,
|
||||
}
|
||||
|
||||
try:
|
||||
monitor = self.sct.monitors[1]
|
||||
width, height = monitor["width"], monitor["height"]
|
||||
|
||||
# Resolution visuelle des coordonnees si demande
|
||||
x_pct = action.get("x_pct", 0.0)
|
||||
y_pct = action.get("y_pct", 0.0)
|
||||
|
||||
if visual_mode and target_spec and server_url:
|
||||
resolved = self._resolve_target_visual(
|
||||
server_url, target_spec, x_pct, y_pct, width, height
|
||||
)
|
||||
if resolved:
|
||||
x_pct = resolved["x_pct"]
|
||||
y_pct = resolved["y_pct"]
|
||||
result["visual_resolved"] = resolved.get("resolved", False)
|
||||
if resolved.get("resolved"):
|
||||
logger.info(
|
||||
f"Visual resolve OK: {resolved.get('matched_element', {}).get('label', '?')} "
|
||||
f"-> ({x_pct:.4f}, {y_pct:.4f})"
|
||||
)
|
||||
|
||||
if action_type == "click":
|
||||
real_x = int(x_pct * width)
|
||||
real_y = int(y_pct * height)
|
||||
button = action.get("button", "left")
|
||||
mode = "VISUAL" if result["visual_resolved"] else "BLIND"
|
||||
print(
|
||||
f" [CLICK] [{mode}] ({x_pct:.3f}, {y_pct:.3f}) -> "
|
||||
f"({real_x}, {real_y}) sur ({width}x{height}), bouton={button}"
|
||||
)
|
||||
self._click((real_x, real_y), button)
|
||||
print(f" [CLICK] Termine.")
|
||||
logger.info(
|
||||
f"Replay click [{mode}] : ({x_pct:.3f}, {y_pct:.3f}) -> "
|
||||
f"({real_x}, {real_y}) sur ({width}x{height})"
|
||||
)
|
||||
|
||||
elif action_type == "type":
|
||||
text = action.get("text", "")
|
||||
print(f" [TYPE] Texte: '{text[:50]}' ({len(text)} chars)")
|
||||
# Cliquer sur le champ avant de taper (si coordonnees disponibles)
|
||||
if x_pct > 0 and y_pct > 0:
|
||||
real_x = int(x_pct * width)
|
||||
real_y = int(y_pct * height)
|
||||
print(f" [TYPE] Clic prealable sur ({real_x}, {real_y})")
|
||||
self._click((real_x, real_y), "left")
|
||||
time.sleep(0.3)
|
||||
self.keyboard.type(text)
|
||||
print(f" [TYPE] Termine.")
|
||||
logger.info(f"Replay type : '{text[:30]}...' ({len(text)} chars)")
|
||||
|
||||
elif action_type == "key_combo":
|
||||
keys = action.get("keys", [])
|
||||
print(f" [KEY_COMBO] Touches: {keys}")
|
||||
self._execute_key_combo(keys)
|
||||
print(f" [KEY_COMBO] Termine.")
|
||||
logger.info(f"Replay key_combo : {keys}")
|
||||
|
||||
elif action_type == "scroll":
|
||||
real_x = int(x_pct * width) if x_pct > 0 else int(0.5 * width)
|
||||
real_y = int(y_pct * height) if y_pct > 0 else int(0.5 * height)
|
||||
delta = action.get("delta", -3)
|
||||
print(f" [SCROLL] delta={delta} a ({real_x}, {real_y})")
|
||||
self.mouse.position = (real_x, real_y)
|
||||
time.sleep(0.05)
|
||||
self.mouse.scroll(0, delta)
|
||||
print(f" [SCROLL] Termine.")
|
||||
logger.info(f"Replay scroll : delta={delta} a ({real_x}, {real_y})")
|
||||
|
||||
elif action_type == "wait":
|
||||
duration_ms = action.get("duration_ms", 500)
|
||||
print(f" [WAIT] {duration_ms}ms...")
|
||||
time.sleep(duration_ms / 1000.0)
|
||||
print(f" [WAIT] Termine.")
|
||||
logger.info(f"Replay wait : {duration_ms}ms")
|
||||
|
||||
else:
|
||||
result["error"] = f"Type d'action inconnu : {action_type}"
|
||||
logger.warning(result["error"])
|
||||
return result
|
||||
|
||||
result["success"] = True
|
||||
|
||||
# Capturer un screenshot post-action
|
||||
time.sleep(0.5)
|
||||
result["screenshot"] = self._capture_screenshot_b64()
|
||||
|
||||
except Exception as e:
|
||||
result["error"] = str(e)
|
||||
logger.error(f"Echec replay action {action_id} ({action_type}) : {e}")
|
||||
|
||||
return result
|
||||
|
||||
def _resolve_target_visual(
|
||||
self, server_url: str, target_spec: dict,
|
||||
fallback_x: float, fallback_y: float,
|
||||
screen_width: int, screen_height: int,
|
||||
) -> dict:
|
||||
"""
|
||||
Envoyer un screenshot au serveur pour resolution visuelle de la cible.
|
||||
|
||||
Capture l'ecran en haute resolution (pas de downscale pour le template
|
||||
matching), l'encode en base64 JPEG, et POST au endpoint
|
||||
/replay/resolve_target. Retourne les coordonnees resolues.
|
||||
"""
|
||||
import requests
|
||||
|
||||
try:
|
||||
# Capturer à 1280px max — assez pour le template matching
|
||||
# et raisonnable pour le transfert réseau (~200-400Ko)
|
||||
screenshot_b64 = self._capture_screenshot_b64(
|
||||
max_width=1280,
|
||||
quality=75,
|
||||
)
|
||||
if not screenshot_b64:
|
||||
logger.warning("Capture screenshot echouee pour visual resolve")
|
||||
return None
|
||||
|
||||
print(
|
||||
f" [VISUAL] Envoi screenshot ({len(screenshot_b64) // 1024} Ko) "
|
||||
f"au serveur pour resolution..."
|
||||
)
|
||||
|
||||
# Appel au serveur
|
||||
resolve_url = f"{server_url}/traces/stream/replay/resolve_target"
|
||||
payload = {
|
||||
"session_id": "", # Pas critique pour la resolution
|
||||
"screenshot_b64": screenshot_b64,
|
||||
"target_spec": target_spec,
|
||||
"fallback_x_pct": fallback_x,
|
||||
"fallback_y_pct": fallback_y,
|
||||
"screen_width": screen_width,
|
||||
"screen_height": screen_height,
|
||||
}
|
||||
|
||||
resp = requests.post(resolve_url, json=payload, timeout=60)
|
||||
if resp.ok:
|
||||
data = resp.json()
|
||||
method = data.get("method", "?")
|
||||
resolved = data.get("resolved", False)
|
||||
print(
|
||||
f" [VISUAL] Reponse serveur : resolved={resolved}, "
|
||||
f"method={method}, score={data.get('score', 'N/A')}"
|
||||
)
|
||||
return data
|
||||
else:
|
||||
logger.warning(f"Visual resolve HTTP {resp.status_code}: {resp.text[:200]}")
|
||||
return None
|
||||
|
||||
except requests.exceptions.Timeout:
|
||||
logger.warning("Visual resolve timeout (30s)")
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.warning(f"Visual resolve echoue: {e}")
|
||||
return None
|
||||
|
||||
def poll_and_execute(self, session_id: str, server_url: str, machine_id: str = "default") -> bool:
|
||||
"""
|
||||
Poll le serveur pour recuperer et executer la prochaine action.
|
||||
|
||||
1. GET /replay/next pour recuperer l'action
|
||||
2. Execute l'action (clic, texte, etc.)
|
||||
3. POST /replay/result avec le resultat + screenshot
|
||||
|
||||
Args:
|
||||
session_id: Identifiant de la session courante
|
||||
server_url: URL de base du serveur streaming
|
||||
machine_id: Identifiant de la machine (pour le replay multi-machine)
|
||||
|
||||
Retourne True si une action a ete executee, False sinon.
|
||||
IMPORTANT: Si une action est recue, le resultat est TOUJOURS rapporte
|
||||
au serveur (meme en cas d'erreur d'execution).
|
||||
"""
|
||||
import requests
|
||||
|
||||
replay_next_url = f"{server_url}/traces/stream/replay/next"
|
||||
replay_result_url = f"{server_url}/traces/stream/replay/result"
|
||||
|
||||
# Phase 1 : Recuperer la prochaine action (filtree par machine_id)
|
||||
try:
|
||||
resp = requests.get(
|
||||
replay_next_url,
|
||||
params={"session_id": session_id, "machine_id": machine_id},
|
||||
timeout=5,
|
||||
)
|
||||
if not resp.ok:
|
||||
logger.debug(f"Poll replay echoue : HTTP {resp.status_code}")
|
||||
return False
|
||||
|
||||
data = resp.json()
|
||||
action = data.get("action")
|
||||
if action is None:
|
||||
return False
|
||||
|
||||
except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as e:
|
||||
# Backoff exponentiel : augmenter le delai de polling
|
||||
self._poll_backoff = min(
|
||||
self._poll_backoff * self._poll_backoff_factor,
|
||||
self._poll_backoff_max,
|
||||
)
|
||||
if not hasattr(self, '_last_conn_error_logged'):
|
||||
self._last_conn_error_logged = True
|
||||
print(f"[REPLAY] Serveur non disponible (backoff={self._poll_backoff:.1f}s) : {e}")
|
||||
logger.warning(f"Serveur non disponible pour replay (backoff={self._poll_backoff:.1f}s): {e}")
|
||||
return False
|
||||
except Exception as e:
|
||||
self._poll_backoff = min(
|
||||
self._poll_backoff * self._poll_backoff_factor,
|
||||
self._poll_backoff_max,
|
||||
)
|
||||
print(f"[REPLAY] ERREUR poll (GET) : {e}")
|
||||
logger.error(f"Erreur poll GET : {e}")
|
||||
return False
|
||||
|
||||
# Reset du flag d'erreur connexion et du backoff (on a reussi le GET)
|
||||
self._last_conn_error_logged = False
|
||||
self._poll_backoff = self._poll_backoff_min
|
||||
|
||||
# Phase 2 : Executer l'action et rapporter le resultat
|
||||
# TOUJOURS rapporter au serveur, meme en cas d'erreur d'execution
|
||||
action_type = action.get('type', '?')
|
||||
action_id = action.get('action_id', '?')
|
||||
print(f"\n>>> REPLAY ACTION RECUE : {action_type} (id={action_id})")
|
||||
print(f" Contenu: {action}")
|
||||
logger.info(f"Action de replay recue : {action_type} (id={action_id})")
|
||||
|
||||
result = None
|
||||
try:
|
||||
print(f">>> Execution de l'action {action_type}...")
|
||||
result = self.execute_replay_action(action, server_url=server_url)
|
||||
print(
|
||||
f">>> Resultat execution : success={result['success']}, "
|
||||
f"error={result.get('error')}"
|
||||
)
|
||||
except Exception as e:
|
||||
print(f">>> ERREUR EXECUTION : {e}")
|
||||
logger.error(f"Erreur execute_replay_action: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
result = {
|
||||
"action_id": action_id,
|
||||
"success": False,
|
||||
"error": f"Exception executor: {e}",
|
||||
"screenshot": None,
|
||||
}
|
||||
|
||||
# Phase 3 : Rapporter le resultat au serveur (TOUJOURS)
|
||||
report = {
|
||||
"session_id": session_id,
|
||||
"action_id": result["action_id"],
|
||||
"success": result["success"],
|
||||
"error": result.get("error"),
|
||||
"screenshot": result.get("screenshot"),
|
||||
}
|
||||
try:
|
||||
resp2 = requests.post(
|
||||
replay_result_url,
|
||||
json=report,
|
||||
timeout=10,
|
||||
)
|
||||
if resp2.ok:
|
||||
server_resp = resp2.json()
|
||||
msg = (
|
||||
f"Resultat rapporte : replay_status={server_resp.get('replay_status')}, "
|
||||
f"restant={server_resp.get('remaining_actions')}"
|
||||
)
|
||||
print(f">>> {msg}")
|
||||
logger.info(msg)
|
||||
else:
|
||||
print(f">>> Rapport resultat echoue : HTTP {resp2.status_code}")
|
||||
logger.warning(f"Rapport resultat echoue : HTTP {resp2.status_code}")
|
||||
except Exception as e:
|
||||
print(f">>> Impossible de rapporter le resultat : {e}")
|
||||
logger.warning(f"Impossible de rapporter le resultat : {e}")
|
||||
|
||||
return True
|
||||
|
||||
# =========================================================================
|
||||
# Helpers
|
||||
# =========================================================================
|
||||
|
||||
def _click(self, pos, button_name):
|
||||
"""Deplacer la souris et cliquer.
|
||||
|
||||
Supporte les boutons : left, right, double (double-clic gauche).
|
||||
"""
|
||||
self.mouse.position = pos
|
||||
time.sleep(0.1) # Delai pour simuler le temps de reaction humain
|
||||
|
||||
if button_name == "double":
|
||||
self.mouse.click(Button.left, 2)
|
||||
elif button_name == "right":
|
||||
self.mouse.click(Button.right)
|
||||
else:
|
||||
self.mouse.click(Button.left)
|
||||
|
||||
def _execute_key_combo(self, keys: list):
|
||||
"""
|
||||
Executer une combinaison de touches.
|
||||
Ex: ["ctrl", "a"] -> Ctrl+A
|
||||
Ex: ["enter"] -> Enter
|
||||
"""
|
||||
if not keys:
|
||||
return
|
||||
|
||||
# Resoudre les noms de touches vers les objets pynput
|
||||
resolved = []
|
||||
for key_name in keys:
|
||||
key_lower = key_name.lower()
|
||||
if key_lower in _SPECIAL_KEYS:
|
||||
resolved.append(_SPECIAL_KEYS[key_lower])
|
||||
elif len(key_name) == 1:
|
||||
resolved.append(key_name)
|
||||
else:
|
||||
logger.warning(f"Touche inconnue : '{key_name}', ignoree")
|
||||
|
||||
if not resolved:
|
||||
return
|
||||
|
||||
# Si une seule touche, simple press
|
||||
if len(resolved) == 1:
|
||||
self.keyboard.press(resolved[0])
|
||||
self.keyboard.release(resolved[0])
|
||||
return
|
||||
|
||||
# Combo : maintenir les modificateurs, taper la derniere touche
|
||||
modifiers = resolved[:-1]
|
||||
final_key = resolved[-1]
|
||||
|
||||
for mod in modifiers:
|
||||
self.keyboard.press(mod)
|
||||
time.sleep(0.05)
|
||||
|
||||
self.keyboard.press(final_key)
|
||||
self.keyboard.release(final_key)
|
||||
|
||||
for mod in reversed(modifiers):
|
||||
self.keyboard.release(mod)
|
||||
|
||||
def _capture_screenshot_b64(self, max_width: int = 800, quality: int = 60) -> str:
|
||||
"""
|
||||
Capturer l'ecran et retourner le screenshot en base64.
|
||||
|
||||
Args:
|
||||
max_width: Largeur maximale en pixels (0 = pas de redimensionnement,
|
||||
utile pour le template matching qui a besoin de la resolution native)
|
||||
quality: Qualite JPEG (1-100, 60 pour preview, 85+ pour template matching)
|
||||
"""
|
||||
try:
|
||||
from PIL import Image
|
||||
|
||||
monitor = self.sct.monitors[1]
|
||||
raw = self.sct.grab(monitor)
|
||||
img = Image.frombytes("RGB", raw.size, raw.bgra, "raw", "BGRX")
|
||||
|
||||
# Redimensionner si max_width > 0
|
||||
if max_width > 0 and img.width > max_width:
|
||||
ratio = max_width / img.width
|
||||
new_h = int(img.height * ratio)
|
||||
img = img.resize((max_width, new_h), Image.LANCZOS)
|
||||
|
||||
buffer = io.BytesIO()
|
||||
img.save(buffer, format="JPEG", quality=quality)
|
||||
return base64.b64encode(buffer.getvalue()).decode("utf-8")
|
||||
except ImportError:
|
||||
# PIL non disponible — retourner None
|
||||
logger.debug("PIL non disponible, pas de screenshot base64")
|
||||
return ""
|
||||
except Exception as e:
|
||||
logger.debug(f"Capture screenshot base64 echouee : {e}")
|
||||
return ""
|
||||
55
agent_v0/deploy/windows_client/agent_v1/core/window_info.py
Normal file
55
agent_v0/deploy/windows_client/agent_v1/core/window_info.py
Normal file
@@ -0,0 +1,55 @@
|
||||
# window_info.py
|
||||
"""
|
||||
Récupération des informations sur la fenêtre active (X11).
|
||||
|
||||
v0 :
|
||||
- utilise xdotool pour obtenir :
|
||||
- le titre de la fenêtre active
|
||||
- le PID de la fenêtre active, puis le nom du process via ps
|
||||
|
||||
Si quelque chose ne fonctionne pas, on renvoie des valeurs "unknown".
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
from typing import Dict, Optional
|
||||
|
||||
|
||||
def _run_cmd(cmd: list[str]) -> Optional[str]:
|
||||
"""Exécute une commande et renvoie la sortie texte (strippée), ou None en cas d'erreur."""
|
||||
try:
|
||||
out = subprocess.check_output(cmd, stderr=subprocess.DEVNULL)
|
||||
return out.decode("utf-8", errors="ignore").strip()
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def get_active_window_info() -> Dict[str, str]:
|
||||
"""
|
||||
Renvoie un dict :
|
||||
{
|
||||
"title": "...",
|
||||
"app_name": "..."
|
||||
}
|
||||
|
||||
Nécessite xdotool installé sur le système.
|
||||
"""
|
||||
title = _run_cmd(["xdotool", "getactivewindow", "getwindowname"])
|
||||
pid_str = _run_cmd(["xdotool", "getactivewindow", "getwindowpid"])
|
||||
|
||||
app_name: Optional[str] = None
|
||||
if pid_str:
|
||||
pid_str = pid_str.strip()
|
||||
# On récupère le nom du binaire via ps
|
||||
app_name = _run_cmd(["ps", "-p", pid_str, "-o", "comm="])
|
||||
|
||||
if not title:
|
||||
title = "unknown_window"
|
||||
if not app_name:
|
||||
app_name = "unknown_app"
|
||||
|
||||
return {
|
||||
"title": title,
|
||||
"app_name": app_name,
|
||||
}
|
||||
@@ -0,0 +1,192 @@
|
||||
# window_info_crossplatform.py
|
||||
"""
|
||||
Récupération des informations sur la fenêtre active - CROSS-PLATFORM
|
||||
|
||||
Supporte:
|
||||
- Linux (X11 via xdotool)
|
||||
- Windows (via pywin32)
|
||||
- macOS (via pyobjc)
|
||||
|
||||
Installation des dépendances:
|
||||
pip install pywin32 # Windows
|
||||
pip install pyobjc-framework-Cocoa # macOS
|
||||
pip install psutil # Tous OS
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import platform
|
||||
import subprocess
|
||||
from typing import Dict, Optional
|
||||
|
||||
|
||||
def _run_cmd(cmd: list[str]) -> Optional[str]:
|
||||
"""Exécute une commande et renvoie la sortie texte (strippée), ou None en cas d'erreur."""
|
||||
try:
|
||||
out = subprocess.check_output(cmd, stderr=subprocess.DEVNULL)
|
||||
return out.decode("utf-8", errors="ignore").strip()
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def get_active_window_info() -> Dict[str, str]:
|
||||
"""
|
||||
Renvoie un dict :
|
||||
{
|
||||
"title": "...",
|
||||
"app_name": "..."
|
||||
}
|
||||
|
||||
Détecte automatiquement l'OS et utilise la méthode appropriée.
|
||||
"""
|
||||
system = platform.system()
|
||||
|
||||
if system == "Linux":
|
||||
return _get_window_info_linux()
|
||||
elif system == "Windows":
|
||||
return _get_window_info_windows()
|
||||
elif system == "Darwin": # macOS
|
||||
return _get_window_info_macos()
|
||||
else:
|
||||
return {"title": "unknown_window", "app_name": "unknown_app"}
|
||||
|
||||
|
||||
def _get_window_info_linux() -> Dict[str, str]:
|
||||
"""
|
||||
Linux: utilise xdotool (X11)
|
||||
|
||||
Nécessite: sudo apt-get install xdotool
|
||||
"""
|
||||
title = _run_cmd(["xdotool", "getactivewindow", "getwindowname"])
|
||||
pid_str = _run_cmd(["xdotool", "getactivewindow", "getwindowpid"])
|
||||
|
||||
app_name: Optional[str] = None
|
||||
if pid_str:
|
||||
pid_str = pid_str.strip()
|
||||
# On récupère le nom du binaire via ps
|
||||
app_name = _run_cmd(["ps", "-p", pid_str, "-o", "comm="])
|
||||
|
||||
if not title:
|
||||
title = "unknown_window"
|
||||
if not app_name:
|
||||
app_name = "unknown_app"
|
||||
|
||||
return {
|
||||
"title": title,
|
||||
"app_name": app_name,
|
||||
}
|
||||
|
||||
|
||||
def _get_window_info_windows() -> Dict[str, str]:
|
||||
"""
|
||||
Windows: utilise pywin32 + psutil
|
||||
|
||||
Nécessite: pip install pywin32 psutil
|
||||
"""
|
||||
try:
|
||||
import win32gui
|
||||
import win32process
|
||||
import psutil
|
||||
|
||||
# Fenêtre au premier plan
|
||||
hwnd = win32gui.GetForegroundWindow()
|
||||
|
||||
# Titre de la fenêtre
|
||||
title = win32gui.GetWindowText(hwnd)
|
||||
if not title:
|
||||
title = "unknown_window"
|
||||
|
||||
# PID du processus
|
||||
_, pid = win32process.GetWindowThreadProcessId(hwnd)
|
||||
|
||||
# Nom du processus
|
||||
try:
|
||||
process = psutil.Process(pid)
|
||||
app_name = process.name()
|
||||
except (psutil.NoSuchProcess, psutil.AccessDenied):
|
||||
app_name = "unknown_app"
|
||||
|
||||
return {
|
||||
"title": title,
|
||||
"app_name": app_name,
|
||||
}
|
||||
|
||||
except ImportError:
|
||||
# pywin32 ou psutil non installé
|
||||
return {
|
||||
"title": "unknown_window (pywin32 missing)",
|
||||
"app_name": "unknown_app (pywin32 missing)",
|
||||
}
|
||||
except Exception as e:
|
||||
return {
|
||||
"title": f"error: {e}",
|
||||
"app_name": "unknown_app",
|
||||
}
|
||||
|
||||
|
||||
def _get_window_info_macos() -> Dict[str, str]:
|
||||
"""
|
||||
macOS: utilise pyobjc (AppKit)
|
||||
|
||||
Nécessite: pip install pyobjc-framework-Cocoa
|
||||
|
||||
Note: Nécessite les permissions "Accessibility" dans System Preferences
|
||||
"""
|
||||
try:
|
||||
from AppKit import NSWorkspace
|
||||
from Quartz import (
|
||||
CGWindowListCopyWindowInfo,
|
||||
kCGWindowListOptionOnScreenOnly,
|
||||
kCGNullWindowID
|
||||
)
|
||||
|
||||
# Application active
|
||||
active_app = NSWorkspace.sharedWorkspace().activeApplication()
|
||||
app_name = active_app.get('NSApplicationName', 'unknown_app')
|
||||
|
||||
# Titre de la fenêtre (via Quartz)
|
||||
# On cherche la fenêtre de l'app active qui est au premier plan
|
||||
window_list = CGWindowListCopyWindowInfo(
|
||||
kCGWindowListOptionOnScreenOnly,
|
||||
kCGNullWindowID
|
||||
)
|
||||
|
||||
title = "unknown_window"
|
||||
for window in window_list:
|
||||
owner_name = window.get('kCGWindowOwnerName', '')
|
||||
if owner_name == app_name:
|
||||
window_title = window.get('kCGWindowName', '')
|
||||
if window_title:
|
||||
title = window_title
|
||||
break
|
||||
|
||||
return {
|
||||
"title": title,
|
||||
"app_name": app_name,
|
||||
}
|
||||
|
||||
except ImportError:
|
||||
# pyobjc non installé
|
||||
return {
|
||||
"title": "unknown_window (pyobjc missing)",
|
||||
"app_name": "unknown_app (pyobjc missing)",
|
||||
}
|
||||
except Exception as e:
|
||||
return {
|
||||
"title": f"error: {e}",
|
||||
"app_name": "unknown_app",
|
||||
}
|
||||
|
||||
|
||||
# Test rapide
|
||||
if __name__ == "__main__":
|
||||
import time
|
||||
|
||||
print(f"OS détecté: {platform.system()}")
|
||||
print("\nTest de capture fenêtre active (5 secondes)...")
|
||||
print("Changez de fenêtre pour tester!\n")
|
||||
|
||||
for i in range(5):
|
||||
info = get_active_window_info()
|
||||
print(f"[{i+1}] App: {info['app_name']:20s} | Title: {info['title']}")
|
||||
time.sleep(1)
|
||||
325
agent_v0/deploy/windows_client/agent_v1/main.py
Normal file
325
agent_v0/deploy/windows_client/agent_v1/main.py
Normal file
@@ -0,0 +1,325 @@
|
||||
# agent_v1/main.py
|
||||
"""
|
||||
Point d'entree Agent V1 - Enrichi avec Intelligence de Contexte, Heartbeat et Replay.
|
||||
|
||||
Boucles paralleles (threads daemon) :
|
||||
- _heartbeat_loop : capture periodique toutes les 5s
|
||||
- _command_watchdog_loop : surveillance du fichier command.json (legacy)
|
||||
- _replay_poll_loop : polling du serveur pour les actions de replay (P0-5)
|
||||
"""
|
||||
|
||||
import sys
|
||||
import os
|
||||
import uuid
|
||||
import time
|
||||
import logging
|
||||
import threading
|
||||
from .config import SESSIONS_ROOT, AGENT_VERSION, SERVER_URL, MACHINE_ID
|
||||
from .core.captor import EventCaptorV1
|
||||
from .core.executor import ActionExecutorV1
|
||||
from .network.streamer import TraceStreamer
|
||||
from .ui.smart_tray import SmartTrayV1
|
||||
from .ui.chat_window import ChatWindow
|
||||
from .session.storage import SessionStorage
|
||||
from .vision.capturer import VisionCapturer
|
||||
|
||||
# Import optionnel du client serveur (pour le chat et les workflows)
|
||||
# Deux chemins : relatif (depuis agent_v0.agent_v1) ou absolu (depuis C:\rpa_vision\agent_v1)
|
||||
try:
|
||||
from ..lea_ui.server_client import LeaServerClient
|
||||
except (ImportError, ValueError):
|
||||
try:
|
||||
from lea_ui.server_client import LeaServerClient
|
||||
except ImportError:
|
||||
LeaServerClient = None
|
||||
|
||||
# Configuration du logging
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Intervalle de polling replay (secondes)
|
||||
REPLAY_POLL_INTERVAL = 1.0
|
||||
|
||||
|
||||
class AgentV1:
|
||||
def __init__(self, user_id="demo_user"):
|
||||
self.user_id = user_id
|
||||
self.machine_id = MACHINE_ID
|
||||
self.session_id = None
|
||||
self.session_dir = None
|
||||
|
||||
# Gestion du stockage local et nettoyage
|
||||
self.storage = SessionStorage(SESSIONS_ROOT)
|
||||
threading.Thread(target=self._delayed_cleanup, daemon=True).start()
|
||||
|
||||
self.vision = None
|
||||
self.streamer = None
|
||||
self.captor = None
|
||||
self.shot_counter = 0
|
||||
self.running = False
|
||||
|
||||
# Executeur partage entre watchdog et replay
|
||||
self._executor = None
|
||||
# Flag pour indiquer qu'un replay est en cours (eviter les conflits)
|
||||
self._replay_active = False
|
||||
|
||||
# Client serveur pour le chat et les workflows
|
||||
self._server_client = None
|
||||
if LeaServerClient is not None:
|
||||
self._server_client = LeaServerClient()
|
||||
|
||||
# Fenetre de chat Lea (tkinter natif)
|
||||
server_host = (
|
||||
self._server_client.server_host
|
||||
if self._server_client is not None
|
||||
else os.getenv("RPA_SERVER_HOST", "localhost")
|
||||
)
|
||||
self._chat_window = ChatWindow(
|
||||
server_client=self._server_client,
|
||||
on_start_callback=self.start_session,
|
||||
server_host=server_host,
|
||||
chat_port=5004,
|
||||
)
|
||||
|
||||
# Executeur pour le replay (doit exister avant le poll)
|
||||
self._executor = ActionExecutorV1()
|
||||
|
||||
# Boucle de polling replay PERMANENTE (pas besoin de session active)
|
||||
self.running = True
|
||||
threading.Thread(target=self._replay_poll_loop, daemon=True).start()
|
||||
|
||||
# UI Tray intelligent (remplace TrayAppV1, plus de PyQt5)
|
||||
self.ui = SmartTrayV1(
|
||||
self.start_session,
|
||||
self.stop_session,
|
||||
server_client=self._server_client,
|
||||
chat_window=self._chat_window,
|
||||
machine_id=self.machine_id,
|
||||
)
|
||||
|
||||
def _delayed_cleanup(self):
|
||||
"""Nettoyage en arrière-plan après 30s pour ne pas bloquer le démarrage."""
|
||||
time.sleep(30)
|
||||
self.storage.run_auto_cleanup()
|
||||
|
||||
def start_session(self, workflow_name):
|
||||
self.session_id = f"sess_{time.strftime('%Y%m%dT%H%M%S')}_{uuid.uuid4().hex[:6]}"
|
||||
self.session_dir = self.storage.get_session_dir(self.session_id)
|
||||
|
||||
self.vision = VisionCapturer(str(self.session_dir))
|
||||
|
||||
self.streamer = TraceStreamer(self.session_id, machine_id=self.machine_id)
|
||||
self.captor = EventCaptorV1(self._on_event_bridge)
|
||||
|
||||
# Initialiser l'executeur partage
|
||||
self._executor = ActionExecutorV1()
|
||||
|
||||
self.shot_counter = 0
|
||||
self.running = True
|
||||
self._replay_active = False
|
||||
self.streamer.start()
|
||||
self.captor.start()
|
||||
|
||||
# Heartbeat Contextuel (Toutes les 5s par defaut)
|
||||
threading.Thread(target=self._heartbeat_loop, daemon=True).start()
|
||||
|
||||
# Watchdog de Commandes (GHOST Replay — legacy fichier)
|
||||
threading.Thread(target=self._command_watchdog_loop, daemon=True).start()
|
||||
|
||||
# Boucle de polling replay (P0-5 — pull depuis le serveur)
|
||||
threading.Thread(target=self._replay_poll_loop, daemon=True).start()
|
||||
|
||||
logger.info(f"Session {self.session_id} ({workflow_name}) sur machine {self.machine_id} en cours...")
|
||||
|
||||
def _command_watchdog_loop(self):
|
||||
"""Surveille un fichier de commande pour executer des ordres visuels (legacy)."""
|
||||
import json
|
||||
import platform
|
||||
from .config import BASE_DIR
|
||||
|
||||
# Chemin du fichier de commande selon l'OS
|
||||
if platform.system() == "Windows":
|
||||
cmd_path = "C:\\rpa_vision\\command.json"
|
||||
else:
|
||||
cmd_path = str(BASE_DIR / "command.json")
|
||||
|
||||
while self.running:
|
||||
# Ne pas traiter les commandes fichier pendant un replay serveur
|
||||
if self._replay_active:
|
||||
time.sleep(1)
|
||||
continue
|
||||
|
||||
if os.path.exists(cmd_path):
|
||||
try:
|
||||
with open(cmd_path, "r") as f:
|
||||
order = json.load(f)
|
||||
os.remove(cmd_path) # On consomme l'ordre
|
||||
if self._executor:
|
||||
self._executor.execute_normalized_order(order)
|
||||
except Exception as e:
|
||||
logger.error(f"Erreur Watchdog: {e}")
|
||||
time.sleep(1)
|
||||
|
||||
def _replay_poll_loop(self):
|
||||
"""
|
||||
Boucle de polling pour les actions de replay depuis le serveur (P0-5).
|
||||
|
||||
Tourne en parallele du heartbeat et du watchdog.
|
||||
Poll GET /replay/next toutes les REPLAY_POLL_INTERVAL secondes.
|
||||
Quand une action est recue, l'execute via l'executor et rapporte le resultat.
|
||||
"""
|
||||
msg = (
|
||||
f"[REPLAY] Boucle replay demarree — poll toutes les "
|
||||
f"{REPLAY_POLL_INTERVAL}s sur {SERVER_URL}"
|
||||
)
|
||||
print(msg)
|
||||
logger.info(msg)
|
||||
|
||||
poll_count = 0
|
||||
while self.running:
|
||||
if not self._executor:
|
||||
time.sleep(REPLAY_POLL_INTERVAL)
|
||||
continue
|
||||
|
||||
# Utiliser la session active ou un ID par défaut pour le replay
|
||||
poll_session = self.session_id or f"agent_{self.user_id}"
|
||||
|
||||
# Log periodique pour confirmer que la boucle tourne (toutes les 60s)
|
||||
poll_count += 1
|
||||
if poll_count % int(60 / REPLAY_POLL_INTERVAL) == 0:
|
||||
print(
|
||||
f"[REPLAY] Poll #{poll_count} — session={poll_session} "
|
||||
f"— serveur={SERVER_URL}"
|
||||
)
|
||||
|
||||
try:
|
||||
# Tenter de recuperer et executer une action
|
||||
had_action = self._executor.poll_and_execute(
|
||||
session_id=poll_session,
|
||||
server_url=SERVER_URL,
|
||||
machine_id=self.machine_id,
|
||||
)
|
||||
|
||||
if had_action:
|
||||
if not self._replay_active:
|
||||
self._replay_active = True
|
||||
self.ui.set_replay_active(True)
|
||||
# Si une action a ete executee, poll plus rapidement
|
||||
# pour enchainer les actions du workflow
|
||||
time.sleep(0.2)
|
||||
else:
|
||||
# Pas d'action en attente — utiliser le backoff de l'executor
|
||||
# (augmente si le serveur est indisponible, reset a 1s sinon)
|
||||
if self._replay_active:
|
||||
print("[REPLAY] Replay termine — retour en mode capture")
|
||||
logger.info("Replay termine — retour en mode capture")
|
||||
self._replay_active = False
|
||||
self.ui.set_replay_active(False)
|
||||
poll_delay = getattr(self._executor, '_poll_backoff', REPLAY_POLL_INTERVAL)
|
||||
time.sleep(max(poll_delay, REPLAY_POLL_INTERVAL))
|
||||
|
||||
except Exception as e:
|
||||
print(f"[REPLAY] ERREUR boucle replay : {e}")
|
||||
logger.error(f"Erreur replay poll loop : {e}")
|
||||
self._replay_active = False
|
||||
poll_delay = getattr(self._executor, '_poll_backoff', REPLAY_POLL_INTERVAL)
|
||||
time.sleep(max(poll_delay, REPLAY_POLL_INTERVAL))
|
||||
|
||||
def stop_session(self):
|
||||
self.running = False
|
||||
if self.captor: self.captor.stop()
|
||||
if self.streamer: self.streamer.stop()
|
||||
logger.info(f"Session {self.session_id} terminée.")
|
||||
|
||||
_last_heartbeat_hash: str = ""
|
||||
|
||||
def _heartbeat_loop(self):
|
||||
"""Capture périodique pour donner du contexte au stagiaire.
|
||||
Déduplication : n'envoie que si l'écran a changé.
|
||||
"""
|
||||
while self.running:
|
||||
try:
|
||||
full_path = self.vision.capture_full_context("heartbeat")
|
||||
if full_path:
|
||||
# Hash rapide pour détecter les changements d'écran
|
||||
img_hash = self._quick_hash(full_path)
|
||||
if img_hash != self._last_heartbeat_hash:
|
||||
self._last_heartbeat_hash = img_hash
|
||||
self.streamer.push_image(full_path, f"heartbeat_{int(time.time())}")
|
||||
self.streamer.push_event({"type": "heartbeat", "image": full_path, "timestamp": time.time(), "machine_id": self.machine_id})
|
||||
except Exception as e:
|
||||
logger.error(f"Heartbeat error: {e}")
|
||||
time.sleep(5)
|
||||
|
||||
@staticmethod
|
||||
def _quick_hash(image_path: str) -> str:
|
||||
"""Hash perceptuel rapide (16x16 niveaux de gris)."""
|
||||
try:
|
||||
from PIL import Image
|
||||
import hashlib
|
||||
img = Image.open(image_path).resize((16, 16)).convert('L')
|
||||
return hashlib.md5(img.tobytes()).hexdigest()
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
def _on_event_bridge(self, event):
|
||||
"""Pont intelligent avec capture duale et post-action monitoring."""
|
||||
if not self.session_id:
|
||||
return
|
||||
|
||||
# Injecter l'identifiant machine dans chaque événement (multi-machine)
|
||||
event["machine_id"] = self.machine_id
|
||||
|
||||
# Injecter le contexte fenêtre dans chaque événement (nécessaire
|
||||
# pour que le serveur maintienne last_window_info)
|
||||
if self.captor and self.captor.last_window:
|
||||
event["window"] = self.captor.last_window
|
||||
|
||||
# Capture Proactive sur changement de fenêtre
|
||||
if event["type"] == "window_focus_change":
|
||||
full_path = self.vision.capture_full_context("focus_change")
|
||||
event["screenshot_context"] = full_path
|
||||
self.streamer.push_image(full_path, f"focus_{int(time.time())}")
|
||||
|
||||
# 🔴 Capture Interactive (Dual)
|
||||
if event["type"] in ["mouse_click", "key_combo"]:
|
||||
self.shot_counter += 1
|
||||
shot_id = f"shot_{self.shot_counter:04d}"
|
||||
|
||||
pos = event.get("pos", (0, 0))
|
||||
capture_info = self.vision.capture_dual(pos[0], pos[1], shot_id)
|
||||
|
||||
event["screenshot_id"] = shot_id
|
||||
event["vision_info"] = capture_info
|
||||
|
||||
self._stream_capture_info(capture_info, shot_id)
|
||||
|
||||
# 🕒 POST-ACTION : Capture du résultat après 1s (pour voir le résultat du clic)
|
||||
threading.Timer(1.0, self._capture_result, args=(shot_id,)).start()
|
||||
|
||||
self.ui.update_stats(self.shot_counter)
|
||||
print(f"📸 Action capturée : {event['type']}")
|
||||
self.streamer.push_event(event)
|
||||
|
||||
def _capture_result(self, base_shot_id: str):
|
||||
"""Capture l'état de l'écran 1s après l'action pour voir l'effet."""
|
||||
if not self.running: return
|
||||
res_path = self.vision.capture_full_context(f"result_of_{base_shot_id}")
|
||||
self.streamer.push_image(res_path, f"res_{base_shot_id}")
|
||||
self.streamer.push_event({"type": "action_result", "base_shot_id": base_shot_id, "image": res_path})
|
||||
|
||||
def _stream_capture_info(self, capture_info, shot_id):
|
||||
if "full" in capture_info:
|
||||
self.streamer.push_image(capture_info["full"], f"{shot_id}_full")
|
||||
if "crop" in capture_info:
|
||||
self.streamer.push_image(capture_info["crop"], f"{shot_id}_crop")
|
||||
|
||||
def run(self):
|
||||
self.ui.run()
|
||||
|
||||
def main():
|
||||
agent = AgentV1()
|
||||
agent.run()
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
398
agent_v0/deploy/windows_client/agent_v1/network/streamer.py
Normal file
398
agent_v0/deploy/windows_client/agent_v1/network/streamer.py
Normal file
@@ -0,0 +1,398 @@
|
||||
# agent_v1/network/streamer.py
|
||||
"""
|
||||
Streaming temps réel pour Agent V1.
|
||||
Exploite la fibre pour envoyer les événements au fur et à mesure.
|
||||
|
||||
Endpoints serveur (api_stream.py, port 5005) :
|
||||
POST /api/v1/traces/stream/register — enregistrer la session
|
||||
POST /api/v1/traces/stream/event — événement temps réel
|
||||
POST /api/v1/traces/stream/image — screenshot (full ou crop)
|
||||
POST /api/v1/traces/stream/finalize — clôturer et construire le workflow
|
||||
|
||||
Robustesse (P0-2) :
|
||||
- Retry avec backoff exponentiel (1s/2s/4s, max 3 tentatives)
|
||||
- Health-check périodique (30s) pour recovery du flag _server_available
|
||||
- Compression JPEG qualité 85 pour les images (réduction ~5-10x)
|
||||
- Backpressure : queue bornée (maxsize=100), drop des heartbeat si pleine
|
||||
"""
|
||||
|
||||
import io
|
||||
import logging
|
||||
import queue
|
||||
import threading
|
||||
import time
|
||||
|
||||
import requests
|
||||
from PIL import Image
|
||||
|
||||
from ..config import STREAMING_ENDPOINT
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Paramètres de retry
|
||||
MAX_RETRIES = 3
|
||||
RETRY_DELAYS = [1.0, 2.0, 4.0] # Backoff exponentiel
|
||||
|
||||
# Paramètres de health-check
|
||||
HEALTH_CHECK_INTERVAL_S = 30
|
||||
|
||||
# Paramètres de compression
|
||||
JPEG_QUALITY = 85
|
||||
|
||||
# Taille max de la queue (backpressure)
|
||||
QUEUE_MAX_SIZE = 100
|
||||
|
||||
# Types d'événements à ne jamais dropper
|
||||
PRIORITY_EVENT_TYPES = {"click", "key", "scroll", "action", "screenshot"}
|
||||
|
||||
|
||||
class TraceStreamer:
|
||||
def __init__(self, session_id: str, machine_id: str = "default"):
|
||||
self.session_id = session_id
|
||||
self.machine_id = machine_id # Identifiant machine pour le multi-machine
|
||||
self.queue: queue.Queue = queue.Queue(maxsize=QUEUE_MAX_SIZE)
|
||||
self.running = False
|
||||
self._thread = None
|
||||
self._health_thread = None
|
||||
self._server_available = True # Désactivé après trop d'échecs
|
||||
|
||||
def start(self):
|
||||
"""Démarrer le streaming et enregistrer la session côté serveur."""
|
||||
self.running = True
|
||||
self._register_session()
|
||||
# Thread principal d'envoi
|
||||
self._thread = threading.Thread(target=self._stream_loop, daemon=True)
|
||||
self._thread.start()
|
||||
# Thread de health-check pour recovery
|
||||
self._health_thread = threading.Thread(
|
||||
target=self._health_check_loop, daemon=True
|
||||
)
|
||||
self._health_thread.start()
|
||||
logger.info(f"Streamer pour {self.session_id} démarré")
|
||||
|
||||
def stop(self):
|
||||
"""Arrêter le streaming et finaliser la session côté serveur.
|
||||
|
||||
Attend que la queue se vide (max 30s) avant de finaliser,
|
||||
pour que toutes les images soient envoyées au serveur.
|
||||
"""
|
||||
self.running = False
|
||||
|
||||
# Attendre que la queue se vide (les images doivent être envoyées)
|
||||
if self._thread:
|
||||
drain_start = time.time()
|
||||
while not self.queue.empty() and (time.time() - drain_start) < 30:
|
||||
time.sleep(0.5)
|
||||
if not self.queue.empty():
|
||||
logger.warning(
|
||||
f"Queue non vide après 30s ({self.queue.qsize()} items restants)"
|
||||
)
|
||||
self._thread.join(timeout=5.0)
|
||||
|
||||
if self._health_thread:
|
||||
self._health_thread.join(timeout=2.0)
|
||||
|
||||
self._finalize_session()
|
||||
logger.info(f"Streamer pour {self.session_id} arrêté")
|
||||
|
||||
def push_event(self, event_data: dict):
|
||||
"""Enfile un événement pour envoi immédiat.
|
||||
|
||||
Si la queue est pleine (backpressure), les heartbeat sont droppés
|
||||
tandis que les événements utilisateur (click, key, scroll, action)
|
||||
et screenshots sont toujours conservés.
|
||||
"""
|
||||
self._enqueue_with_backpressure("event", event_data)
|
||||
|
||||
def push_image(self, image_path: str, screenshot_id: str):
|
||||
"""Enfile une image pour envoi asynchrone."""
|
||||
if not image_path:
|
||||
return # Ignorer les chemins vides (heartbeat sans changement)
|
||||
self._enqueue_with_backpressure("image", (image_path, screenshot_id))
|
||||
|
||||
# =========================================================================
|
||||
# Backpressure — gestion de la queue bornée
|
||||
# =========================================================================
|
||||
|
||||
def _enqueue_with_backpressure(self, item_type: str, data):
|
||||
"""Ajouter un item à la queue avec gestion du backpressure.
|
||||
|
||||
Quand la queue est pleine :
|
||||
- Les événements prioritaires (click, key, action, screenshot) sont
|
||||
ajoutés en bloquant brièvement (0.5s)
|
||||
- Les heartbeat sont silencieusement droppés
|
||||
"""
|
||||
is_priority = self._is_priority_item(item_type, data)
|
||||
|
||||
try:
|
||||
self.queue.put_nowait((item_type, data))
|
||||
except queue.Full:
|
||||
if is_priority:
|
||||
# Événement prioritaire : on attend un peu pour l'ajouter
|
||||
try:
|
||||
self.queue.put((item_type, data), timeout=0.5)
|
||||
except queue.Full:
|
||||
logger.warning(
|
||||
f"Queue pleine — événement prioritaire droppé "
|
||||
f"(type={item_type})"
|
||||
)
|
||||
else:
|
||||
# Heartbeat ou événement non-critique : on drop silencieusement
|
||||
logger.debug(
|
||||
f"Queue pleine — heartbeat/non-prioritaire droppé "
|
||||
f"(type={item_type})"
|
||||
)
|
||||
|
||||
def _is_priority_item(self, item_type: str, data) -> bool:
|
||||
"""Vérifie si un item est prioritaire (ne doit pas être droppé).
|
||||
|
||||
Les images sont toujours prioritaires. Pour les événements,
|
||||
on regarde le type d'événement (click, key, scroll, action).
|
||||
"""
|
||||
if item_type == "image":
|
||||
return True
|
||||
if item_type == "event" and isinstance(data, dict):
|
||||
event_type = data.get("type", "").lower()
|
||||
return event_type in PRIORITY_EVENT_TYPES
|
||||
return False
|
||||
|
||||
# =========================================================================
|
||||
# Boucle d'envoi
|
||||
# =========================================================================
|
||||
|
||||
def _stream_loop(self):
|
||||
"""Boucle d'envoi asynchrone (thread daemon)."""
|
||||
consecutive_failures = 0
|
||||
while self.running or not self.queue.empty():
|
||||
try:
|
||||
item_type, data = self.queue.get(timeout=0.5)
|
||||
success = False
|
||||
if item_type == "event":
|
||||
success = self._send_with_retry(self._send_event, data)
|
||||
elif item_type == "image":
|
||||
success = self._send_with_retry(self._send_image, *data)
|
||||
self.queue.task_done()
|
||||
|
||||
if success:
|
||||
consecutive_failures = 0
|
||||
else:
|
||||
consecutive_failures += 1
|
||||
if consecutive_failures >= 10:
|
||||
logger.warning(
|
||||
"10 échecs consécutifs — serveur marqué indisponible"
|
||||
)
|
||||
self._server_available = False
|
||||
consecutive_failures = 0
|
||||
|
||||
except queue.Empty:
|
||||
continue
|
||||
except Exception as e:
|
||||
logger.error(f"Erreur Streaming Loop: {e}")
|
||||
|
||||
# =========================================================================
|
||||
# Retry avec backoff exponentiel
|
||||
# =========================================================================
|
||||
|
||||
def _send_with_retry(self, send_fn, *args) -> bool:
|
||||
"""Tente l'envoi avec retry et backoff exponentiel.
|
||||
|
||||
3 tentatives max avec délais de 1s, 2s, 4s entre chaque.
|
||||
Retourne True si l'envoi a réussi, False sinon.
|
||||
"""
|
||||
# Première tentative (sans délai)
|
||||
if send_fn(*args):
|
||||
return True
|
||||
|
||||
# Retries avec backoff
|
||||
for attempt, delay in enumerate(RETRY_DELAYS, start=1):
|
||||
if not self.running:
|
||||
# On arrête les retries si le streamer est en cours d'arrêt
|
||||
break
|
||||
logger.debug(
|
||||
f"Retry {attempt}/{MAX_RETRIES} dans {delay}s..."
|
||||
)
|
||||
time.sleep(delay)
|
||||
if send_fn(*args):
|
||||
logger.debug(f"Retry {attempt} réussi")
|
||||
return True
|
||||
|
||||
logger.debug(f"Envoi échoué après {MAX_RETRIES} retries")
|
||||
return False
|
||||
|
||||
# =========================================================================
|
||||
# Health-check périodique pour recovery
|
||||
# =========================================================================
|
||||
|
||||
def _health_check_loop(self):
|
||||
"""Vérifie périodiquement si le serveur est redevenu disponible.
|
||||
|
||||
Toutes les 30s, tente un GET /stats. Si le serveur répond,
|
||||
remet _server_available = True et ré-enregistre la session.
|
||||
"""
|
||||
while self.running:
|
||||
time.sleep(HEALTH_CHECK_INTERVAL_S)
|
||||
if not self.running:
|
||||
break
|
||||
if self._server_available:
|
||||
# Serveur déjà disponible, rien à faire
|
||||
continue
|
||||
# Tenter un health-check
|
||||
try:
|
||||
resp = requests.get(
|
||||
f"{STREAMING_ENDPOINT}/stats",
|
||||
timeout=3,
|
||||
)
|
||||
if resp.ok:
|
||||
logger.info(
|
||||
"Health-check OK — serveur redevenu disponible, "
|
||||
"ré-enregistrement de la session"
|
||||
)
|
||||
self._server_available = True
|
||||
self._register_session()
|
||||
except Exception:
|
||||
logger.debug("Health-check échoué — serveur toujours indisponible")
|
||||
|
||||
# =========================================================================
|
||||
# Compression JPEG
|
||||
# =========================================================================
|
||||
|
||||
def _compress_image_to_jpeg(self, path: str) -> tuple:
|
||||
"""Compresse une image (PNG ou autre) en JPEG qualité 85 en mémoire.
|
||||
|
||||
Retourne un tuple (bytes_io, content_type, filename_suffix).
|
||||
Si la compression échoue, renvoie le fichier original en PNG.
|
||||
"""
|
||||
try:
|
||||
img = Image.open(path)
|
||||
# Convertir en RGB si nécessaire (JPEG ne supporte pas l'alpha)
|
||||
if img.mode in ("RGBA", "LA", "P"):
|
||||
img = img.convert("RGB")
|
||||
buf = io.BytesIO()
|
||||
img.save(buf, format="JPEG", quality=JPEG_QUALITY, optimize=True)
|
||||
buf.seek(0)
|
||||
return buf, "image/jpeg", ".jpg"
|
||||
except FileNotFoundError:
|
||||
# Fichier introuvable — propager l'erreur (pas de fallback possible)
|
||||
logger.warning(f"Fichier image introuvable pour compression : {path}")
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.warning(f"Compression JPEG échouée, envoi PNG brut: {e}")
|
||||
return None, None, None
|
||||
|
||||
# =========================================================================
|
||||
# Envois HTTP
|
||||
# =========================================================================
|
||||
|
||||
def _register_session(self):
|
||||
"""Enregistrer la session auprès du serveur (avec identifiant machine)."""
|
||||
try:
|
||||
resp = requests.post(
|
||||
f"{STREAMING_ENDPOINT}/register",
|
||||
params={
|
||||
"session_id": self.session_id,
|
||||
"machine_id": self.machine_id,
|
||||
},
|
||||
timeout=3,
|
||||
)
|
||||
if resp.ok:
|
||||
logger.info(
|
||||
f"Session {self.session_id} enregistrée sur le serveur "
|
||||
f"(machine={self.machine_id})"
|
||||
)
|
||||
self._server_available = True
|
||||
else:
|
||||
logger.warning(f"Enregistrement session échoué: {resp.status_code}")
|
||||
except Exception as e:
|
||||
logger.debug(f"Serveur indisponible pour register: {e}")
|
||||
self._server_available = False
|
||||
|
||||
def _finalize_session(self):
|
||||
"""Finaliser la session (construction du workflow côté serveur).
|
||||
|
||||
IMPORTANT : tente TOUJOURS l'envoi, indépendamment de _server_available.
|
||||
C'est la dernière chance de sauver les données de la session.
|
||||
"""
|
||||
try:
|
||||
resp = requests.post(
|
||||
f"{STREAMING_ENDPOINT}/finalize",
|
||||
params={
|
||||
"session_id": self.session_id,
|
||||
"machine_id": self.machine_id,
|
||||
},
|
||||
timeout=30, # Le build workflow peut prendre du temps
|
||||
)
|
||||
if resp.ok:
|
||||
result = resp.json()
|
||||
logger.info(f"Session finalisée: {result}")
|
||||
else:
|
||||
logger.warning(f"Finalisation échouée: {resp.status_code}")
|
||||
except Exception as e:
|
||||
logger.debug(f"Finalisation échouée: {e}")
|
||||
|
||||
def _send_event(self, event: dict) -> bool:
|
||||
"""Envoyer un événement au serveur (avec identifiant machine)."""
|
||||
if not self._server_available:
|
||||
return False
|
||||
try:
|
||||
payload = {
|
||||
"session_id": self.session_id,
|
||||
"timestamp": time.time(),
|
||||
"event": event,
|
||||
"machine_id": self.machine_id,
|
||||
}
|
||||
resp = requests.post(
|
||||
f"{STREAMING_ENDPOINT}/event",
|
||||
json=payload,
|
||||
timeout=2,
|
||||
)
|
||||
return resp.ok
|
||||
except Exception as e:
|
||||
logger.debug(f"Streaming Event échoué: {e}")
|
||||
return False
|
||||
|
||||
def _send_image(self, path: str, shot_id: str) -> bool:
|
||||
"""Envoyer un screenshot au serveur, compressé en JPEG.
|
||||
|
||||
Utilise un context manager pour le fallback PNG afin d'éviter
|
||||
les fuites de descripteurs de fichier.
|
||||
"""
|
||||
if not self._server_available:
|
||||
return False
|
||||
try:
|
||||
# Tenter la compression JPEG (réduction ~5-10x vs PNG)
|
||||
jpeg_buf, content_type, suffix = self._compress_image_to_jpeg(path)
|
||||
|
||||
params = {
|
||||
"session_id": self.session_id,
|
||||
"shot_id": shot_id,
|
||||
"machine_id": self.machine_id,
|
||||
}
|
||||
|
||||
if jpeg_buf is not None:
|
||||
# Envoi du JPEG compressé (BytesIO, pas de fuite possible)
|
||||
files = {
|
||||
"file": (f"{shot_id}{suffix}", jpeg_buf, content_type)
|
||||
}
|
||||
resp = requests.post(
|
||||
f"{STREAMING_ENDPOINT}/image",
|
||||
files=files,
|
||||
params=params,
|
||||
timeout=5,
|
||||
)
|
||||
return resp.ok
|
||||
else:
|
||||
# Fallback : envoi PNG original avec context manager
|
||||
with open(path, "rb") as f:
|
||||
files = {
|
||||
"file": (f"{shot_id}.png", f, "image/png")
|
||||
}
|
||||
resp = requests.post(
|
||||
f"{STREAMING_ENDPOINT}/image",
|
||||
files=files,
|
||||
params=params,
|
||||
timeout=5,
|
||||
)
|
||||
return resp.ok
|
||||
except Exception as e:
|
||||
logger.debug(f"Streaming Image échoué: {e}")
|
||||
return False
|
||||
65
agent_v0/deploy/windows_client/agent_v1/session/storage.py
Normal file
65
agent_v0/deploy/windows_client/agent_v1/session/storage.py
Normal file
@@ -0,0 +1,65 @@
|
||||
# agent_v1/session/storage.py
|
||||
"""
|
||||
Gestionnaire de stockage local robuste pour Agent V1.
|
||||
Gère le chiffrement des données au repos et l'auto-nettoyage du disque.
|
||||
"""
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import time
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
logger = logging.getLogger("session_storage")
|
||||
|
||||
class SessionStorage:
|
||||
def __init__(self, base_dir: Path, max_size_gb: int = 5, retention_days: int = 1):
|
||||
self.base_dir = base_dir
|
||||
self.max_size_bytes = max_size_gb * 1024 * 1024 * 1024
|
||||
self.retention_days = retention_days
|
||||
self.base_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def get_session_dir(self, session_id: str) -> Path:
|
||||
"""Retourne et crée le dossier pour une session."""
|
||||
session_path = self.base_dir / session_id
|
||||
session_path.mkdir(exist_ok=True)
|
||||
(session_path / "shots").mkdir(exist_ok=True)
|
||||
return session_path
|
||||
|
||||
def run_auto_cleanup(self):
|
||||
"""Lance le nettoyage automatique basé sur l'âge et la taille."""
|
||||
logger.info("🧹 Lancement du nettoyage automatique du stockage local...")
|
||||
self._cleanup_by_age()
|
||||
self._cleanup_by_size()
|
||||
|
||||
def _cleanup_by_age(self):
|
||||
"""Supprime les sessions plus vieilles que retention_days."""
|
||||
threshold = datetime.now() - timedelta(days=self.retention_days)
|
||||
for session_path in self.base_dir.iterdir():
|
||||
if session_path.is_dir():
|
||||
mtime = datetime.fromtimestamp(session_path.stat().st_mtime)
|
||||
if mtime < threshold:
|
||||
logger.info(f"🗑️ Purge session ancienne : {session_path.name}")
|
||||
shutil.rmtree(session_path)
|
||||
|
||||
def _cleanup_by_size(self):
|
||||
"""Supprime les sessions les plus anciennes si la taille totale dépasse max_size_bytes."""
|
||||
sessions = []
|
||||
total_size = 0
|
||||
for session_path in self.base_dir.iterdir():
|
||||
if session_path.is_dir():
|
||||
size = sum(f.stat().st_size for f in session_path.rglob('*') if f.is_file())
|
||||
sessions.append((session_path, session_path.stat().st_mtime, size))
|
||||
total_size += size
|
||||
|
||||
if total_size > self.max_size_bytes:
|
||||
logger.warning(f"⚠️ Stockage saturé ({total_size/1e9:.2f} GB). Purge nécessaire.")
|
||||
# Trier par date de modif (plus ancien d'abord)
|
||||
sessions.sort(key=lambda x: x[1])
|
||||
for path, _, size in sessions:
|
||||
if total_size <= self.max_size_bytes * 0.8: # On libère jusqu'à 80% du max
|
||||
break
|
||||
logger.info(f"🗑️ Purge session pour libérer de l'espace : {path.name} ({size/1e6:.1f} MB)")
|
||||
shutil.rmtree(path)
|
||||
total_size -= size
|
||||
201
agent_v0/deploy/windows_client/agent_v1/ui/notifications.py
Normal file
201
agent_v0/deploy/windows_client/agent_v1/ui/notifications.py
Normal file
@@ -0,0 +1,201 @@
|
||||
# agent_v1/ui/notifications.py
|
||||
"""
|
||||
Gestionnaire de notifications toast natives (Windows/Linux/macOS).
|
||||
Utilise plyer pour les notifications système, sans dépendance PyQt5.
|
||||
|
||||
Remplace les dialogues Qt par des toasts non-bloquants.
|
||||
Thread-safe avec rate limiting (1 notification / 2 secondes max).
|
||||
"""
|
||||
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from typing import Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Import conditionnel de plyer — fallback silencieux si absent
|
||||
try:
|
||||
from plyer import notification as _plyer_notification
|
||||
_PLYER_AVAILABLE = True
|
||||
except ImportError:
|
||||
_plyer_notification = None
|
||||
_PLYER_AVAILABLE = False
|
||||
logger.warning(
|
||||
"plyer non installé — les notifications toast sont désactivées. "
|
||||
"Installer avec : pip install plyer"
|
||||
)
|
||||
|
||||
# Nom de l'application affiché dans les toasts
|
||||
APP_NAME = "Léa - RPA Vision"
|
||||
|
||||
# Intervalle minimum entre deux notifications (secondes)
|
||||
RATE_LIMIT_SECONDS = 2
|
||||
|
||||
|
||||
class NotificationManager:
|
||||
"""
|
||||
Gestionnaire centralisé de notifications toast.
|
||||
|
||||
Thread-safe : peut être appelé depuis n'importe quel thread.
|
||||
Rate limiting : une seule notification toutes les 2 secondes,
|
||||
les notifications excédentaires sont ignorées (pas de file d'attente
|
||||
pour éviter un flood différé).
|
||||
"""
|
||||
|
||||
def __init__(self, icon_path: Optional[str] = None):
|
||||
"""
|
||||
Initialise le gestionnaire.
|
||||
|
||||
Args:
|
||||
icon_path: Chemin vers l'icône (.ico/.png) pour les toasts.
|
||||
None = icône par défaut du système.
|
||||
"""
|
||||
self._icon_path = icon_path
|
||||
self._lock = threading.Lock()
|
||||
self._last_notification_time: float = 0.0
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# Méthode générique
|
||||
# ------------------------------------------------------------------ #
|
||||
|
||||
def notify(self, title: str, message: str, timeout: int = 5) -> bool:
|
||||
"""
|
||||
Affiche une notification toast.
|
||||
|
||||
Args:
|
||||
title: Titre de la notification.
|
||||
message: Corps du message.
|
||||
timeout: Durée d'affichage en secondes.
|
||||
|
||||
Returns:
|
||||
True si la notification a été envoyée, False sinon
|
||||
(plyer absent ou rate limit atteint).
|
||||
"""
|
||||
if not _PLYER_AVAILABLE:
|
||||
logger.debug("Notification ignorée (plyer absent) : %s", title)
|
||||
return False
|
||||
|
||||
with self._lock:
|
||||
now = time.monotonic()
|
||||
elapsed = now - self._last_notification_time
|
||||
if elapsed < RATE_LIMIT_SECONDS:
|
||||
logger.debug(
|
||||
"Notification ignorée (rate limit, %.1fs restantes) : %s",
|
||||
RATE_LIMIT_SECONDS - elapsed,
|
||||
title,
|
||||
)
|
||||
return False
|
||||
self._last_notification_time = now
|
||||
|
||||
# Envoi dans un thread dédié pour ne jamais bloquer l'appelant
|
||||
thread = threading.Thread(
|
||||
target=self._send,
|
||||
args=(title, message, timeout),
|
||||
daemon=True,
|
||||
)
|
||||
thread.start()
|
||||
return True
|
||||
|
||||
def _send(self, title: str, message: str, timeout: int) -> None:
|
||||
"""Envoi effectif de la notification (exécuté dans un thread dédié)."""
|
||||
try:
|
||||
_plyer_notification.notify(
|
||||
title=title,
|
||||
message=message,
|
||||
app_name=APP_NAME,
|
||||
app_icon=self._icon_path,
|
||||
timeout=timeout,
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("Erreur lors de l'envoi de la notification toast")
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# Méthodes métier
|
||||
# ------------------------------------------------------------------ #
|
||||
|
||||
def greet(self) -> bool:
|
||||
"""Notification de bienvenue au démarrage."""
|
||||
return self.notify(
|
||||
title=APP_NAME,
|
||||
message="Bonjour ! Léa est prête à travailler.",
|
||||
timeout=5,
|
||||
)
|
||||
|
||||
def session_started(self, workflow_name: str) -> bool:
|
||||
"""Notification de début de session."""
|
||||
return self.notify(
|
||||
title="Session démarrée",
|
||||
message=f"Session démarrée : {workflow_name}",
|
||||
timeout=5,
|
||||
)
|
||||
|
||||
def session_ended(self, action_count: int) -> bool:
|
||||
"""Notification de fin de session avec le nombre d'actions."""
|
||||
return self.notify(
|
||||
title="Session terminée",
|
||||
message=f"Session terminée : {action_count} actions capturées.",
|
||||
timeout=5,
|
||||
)
|
||||
|
||||
def workflow_learned(self, name: str) -> bool:
|
||||
"""Notification quand un workflow a été appris."""
|
||||
return self.notify(
|
||||
title="Nouveau workflow appris",
|
||||
message=f"J'ai appris '{name}' ! Je peux essayer quand vous voulez.",
|
||||
timeout=7,
|
||||
)
|
||||
|
||||
def replay_started(self, workflow_name: str, step_count: int) -> bool:
|
||||
"""Notification de début de replay."""
|
||||
return self.notify(
|
||||
title="Replay en cours",
|
||||
message=f"Replay de '{workflow_name}' ({step_count} étapes)",
|
||||
timeout=5,
|
||||
)
|
||||
|
||||
def replay_step(self, current: int, total: int, description: str) -> bool:
|
||||
"""Notification de progression d'une étape de replay."""
|
||||
return self.notify(
|
||||
title=f"Étape {current}/{total}",
|
||||
message=f"Étape {current}/{total} : {description}",
|
||||
timeout=3,
|
||||
)
|
||||
|
||||
def replay_finished(self, success: bool, workflow_name: str) -> bool:
|
||||
"""Notification de fin de replay (succès ou échec)."""
|
||||
if success:
|
||||
return self.notify(
|
||||
title="Replay terminé !",
|
||||
message=f"Replay de '{workflow_name}' terminé avec succès.",
|
||||
timeout=5,
|
||||
)
|
||||
else:
|
||||
return self.notify(
|
||||
title="Replay échoué",
|
||||
message=f"Le replay de '{workflow_name}' a échoué.",
|
||||
timeout=7,
|
||||
)
|
||||
|
||||
def connection_changed(self, connected: bool, server_host: str) -> bool:
|
||||
"""Notification de changement d'état de la connexion serveur."""
|
||||
if connected:
|
||||
return self.notify(
|
||||
title="Connexion établie",
|
||||
message=f"Connecté au serveur {server_host}",
|
||||
timeout=5,
|
||||
)
|
||||
else:
|
||||
return self.notify(
|
||||
title="Connexion perdue",
|
||||
message=f"Connexion perdue avec le serveur {server_host}",
|
||||
timeout=7,
|
||||
)
|
||||
|
||||
def error(self, message: str) -> bool:
|
||||
"""Notification d'erreur."""
|
||||
return self.notify(
|
||||
title="Erreur - Léa",
|
||||
message=message,
|
||||
timeout=10,
|
||||
)
|
||||
625
agent_v0/deploy/windows_client/agent_v1/ui/smart_tray.py
Normal file
625
agent_v0/deploy/windows_client/agent_v1/ui/smart_tray.py
Normal file
@@ -0,0 +1,625 @@
|
||||
# agent_v1/ui/smart_tray.py
|
||||
"""
|
||||
Tray intelligent pour Agent V1 — remplace tray.py (plus de PyQt5).
|
||||
|
||||
Utilise pystray pour l'icone systray et tkinter (stdlib) pour les dialogues.
|
||||
Communication serveur via LeaServerClient (chat:5004, streaming:5005).
|
||||
Notifications via NotificationManager (module parallele).
|
||||
Fenetre de chat Lea integree via ChatWindow (pywebview).
|
||||
|
||||
Architecture de threads :
|
||||
- Thread principal : boucle pystray (icon.run)
|
||||
- Thread daemon : verification connexion serveur (toutes les 30s)
|
||||
- Thread daemon : rafraichissement cache workflows (toutes les 5 min)
|
||||
- Thread daemon : pywebview (fenetre de chat Lea)
|
||||
- Thread daemon : hotkey global Ctrl+Shift+L (si keyboard disponible)
|
||||
- Threads ephemeres : dialogues tkinter (chaque dialogue cree son propre Tk())
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from typing import Any, Callable, Dict, List, Optional
|
||||
|
||||
from PIL import Image, ImageDraw
|
||||
import pystray
|
||||
from pystray import MenuItem as item
|
||||
|
||||
from .notifications import NotificationManager
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Intervalles (secondes)
|
||||
_CONNECTION_CHECK_INTERVAL = 30
|
||||
_WORKFLOW_CACHE_TTL = 300 # 5 minutes
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers tkinter (sans PyQt5)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _ask_string(title: str, prompt: str, default: str = "") -> Optional[str]:
|
||||
"""Dialogue de saisie texte via tkinter (sans PyQt5).
|
||||
|
||||
Cree une instance Tk() ephemere, affiche le dialogue, puis la detruit.
|
||||
Compatible avec la boucle pystray (pas de mainloop persistant).
|
||||
"""
|
||||
import tkinter as tk
|
||||
from tkinter import simpledialog
|
||||
|
||||
root = tk.Tk()
|
||||
root.withdraw()
|
||||
root.attributes('-topmost', True)
|
||||
result = simpledialog.askstring(title, prompt, initialvalue=default, parent=root)
|
||||
root.destroy()
|
||||
return result
|
||||
|
||||
|
||||
def _show_info(title: str, message: str) -> None:
|
||||
"""Affiche une boite d'information via tkinter (sans PyQt5)."""
|
||||
import tkinter as tk
|
||||
from tkinter import messagebox
|
||||
|
||||
root = tk.Tk()
|
||||
root.withdraw()
|
||||
root.attributes('-topmost', True)
|
||||
messagebox.showinfo(title, message, parent=root)
|
||||
root.destroy()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# SmartTrayV1
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class SmartTrayV1:
|
||||
"""Tray systeme intelligent pour Agent V1.
|
||||
|
||||
Remplace TrayAppV1 (PyQt5) par pystray + tkinter.
|
||||
Meme interface constructeur pour compatibilite avec main.py.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
on_start_callback: Callable[[str], None],
|
||||
on_stop_callback: Callable[[], None],
|
||||
server_client: Optional[Any] = None,
|
||||
chat_window: Optional[Any] = None,
|
||||
machine_id: str = "default",
|
||||
) -> None:
|
||||
self.on_start = on_start_callback
|
||||
self.on_stop = on_stop_callback
|
||||
self.server_client = server_client
|
||||
self.machine_id = machine_id # Identifiant machine (multi-machine)
|
||||
|
||||
# Fenetre de chat Lea (pywebview)
|
||||
self._chat_window = chat_window
|
||||
|
||||
# Etat interne
|
||||
self.icon: Optional[pystray.Icon] = None
|
||||
self.is_recording = False
|
||||
self.actions_count = 0
|
||||
|
||||
# Etat connexion serveur
|
||||
self._connected = False
|
||||
self._replay_active = False
|
||||
|
||||
# Cache workflows
|
||||
self._workflows: List[Dict[str, Any]] = []
|
||||
self._workflows_lock = threading.Lock()
|
||||
self._workflows_last_fetch: float = 0.0
|
||||
|
||||
# Verrous
|
||||
self._state_lock = threading.Lock()
|
||||
self._stop_event = threading.Event()
|
||||
|
||||
# Notifications
|
||||
self._notifier = NotificationManager()
|
||||
|
||||
# Icones d'etat (cercles colores)
|
||||
self.icons = {
|
||||
"idle": self._create_circle_icon("gray"),
|
||||
"recording": self._create_circle_icon("red"),
|
||||
"connected": self._create_circle_icon("green"),
|
||||
"disconnected": self._create_circle_icon("orange"),
|
||||
"replay": self._create_circle_icon("blue"),
|
||||
}
|
||||
|
||||
# Enregistrer le callback de changement de connexion sur le client
|
||||
if self.server_client is not None:
|
||||
self.server_client.set_on_connection_change(self._on_connection_change)
|
||||
|
||||
logger.info("SmartTrayV1 initialise")
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Icones
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@staticmethod
|
||||
def _create_circle_icon(color: str) -> Image.Image:
|
||||
"""Genere une icone circulaire simple mais propre."""
|
||||
img = Image.new("RGBA", (64, 64), (0, 0, 0, 0))
|
||||
draw = ImageDraw.Draw(img)
|
||||
draw.ellipse((4, 4, 60, 60), fill=color, outline="white", width=2)
|
||||
return img
|
||||
|
||||
def _current_icon(self) -> Image.Image:
|
||||
"""Retourne l'icone correspondant a l'etat courant."""
|
||||
if self._replay_active:
|
||||
return self.icons["replay"]
|
||||
if self.is_recording:
|
||||
return self.icons["recording"]
|
||||
if self._connected:
|
||||
return self.icons["connected"]
|
||||
if self.server_client is not None:
|
||||
return self.icons["disconnected"]
|
||||
return self.icons["idle"]
|
||||
|
||||
def _update_icon(self) -> None:
|
||||
"""Met a jour l'icone et le menu du tray."""
|
||||
if self.icon is not None:
|
||||
self.icon.icon = self._current_icon()
|
||||
self.icon.update_menu()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Menu dynamique
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _get_menu_items(self):
|
||||
"""Retourne les items du menu (appele a chaque ouverture du menu)."""
|
||||
# Ligne de statut
|
||||
if self.is_recording:
|
||||
status_text = "\U0001f534 Enregistrement..."
|
||||
elif self._connected:
|
||||
status_text = "\U0001f7e2 Connect\u00e9"
|
||||
else:
|
||||
status_text = "\U0001f534 D\u00e9connect\u00e9"
|
||||
|
||||
# Compteur d'actions (visible uniquement en enregistrement)
|
||||
actions_text = f"\U0001f4ca {self.actions_count} actions captur\u00e9es"
|
||||
|
||||
# Sous-menu workflows
|
||||
workflow_items = self._build_workflow_submenu()
|
||||
|
||||
# Ligne d'identification machine (toujours visible)
|
||||
machine_text = f"\U0001f4bb {self.machine_id}"
|
||||
|
||||
items = [
|
||||
# --- Identite machine ---
|
||||
item(machine_text, lambda: None, enabled=False),
|
||||
# --- Statut ---
|
||||
item(status_text, lambda: None, enabled=False),
|
||||
item(
|
||||
actions_text,
|
||||
lambda: None,
|
||||
enabled=False,
|
||||
visible=lambda _i: self.is_recording,
|
||||
),
|
||||
pystray.Menu.SEPARATOR,
|
||||
# --- Actions session ---
|
||||
item(
|
||||
"\U0001f680 D\u00e9marrer une session",
|
||||
self._on_start_session,
|
||||
visible=lambda _i: not self.is_recording,
|
||||
),
|
||||
item(
|
||||
"\u23f9\ufe0f Terminer et Envoyer",
|
||||
self._on_stop_session,
|
||||
visible=lambda _i: self.is_recording,
|
||||
),
|
||||
pystray.Menu.SEPARATOR,
|
||||
# --- Workflows ---
|
||||
item(
|
||||
"\U0001f4cb Workflows connus",
|
||||
pystray.Menu(*workflow_items) if workflow_items else pystray.Menu(
|
||||
item("(aucun workflow)", lambda: None, enabled=False),
|
||||
),
|
||||
visible=lambda _i: self.server_client is not None,
|
||||
),
|
||||
item(
|
||||
"\U0001f504 Rafra\u00eechir les workflows",
|
||||
self._on_refresh_workflows,
|
||||
visible=lambda _i: self.server_client is not None,
|
||||
),
|
||||
pystray.Menu.SEPARATOR,
|
||||
# --- Chat ---
|
||||
item(
|
||||
"\U0001f4ac Que dois-je faire ?",
|
||||
self._on_ask_server,
|
||||
visible=lambda _i: self.server_client is not None and self._connected,
|
||||
),
|
||||
item(
|
||||
"\U0001f4ac Discuter avec L\u00e9a",
|
||||
self._on_toggle_chat,
|
||||
visible=lambda _i: self._chat_window is not None,
|
||||
),
|
||||
pystray.Menu.SEPARATOR,
|
||||
# --- Utilitaires ---
|
||||
item("\U0001f4c2 Ouvrir le dossier sessions", self._on_open_folder),
|
||||
item("\u274c Quitter", self._on_quit),
|
||||
]
|
||||
return items
|
||||
|
||||
def _build_workflow_submenu(self) -> List[pystray.MenuItem]:
|
||||
"""Construit la liste des workflows comme items de sous-menu."""
|
||||
with self._workflows_lock:
|
||||
workflows = list(self._workflows)
|
||||
|
||||
if not workflows:
|
||||
return [item("(aucun workflow)", lambda: None, enabled=False)]
|
||||
|
||||
items = []
|
||||
for wf in workflows:
|
||||
wf_name = wf.get("name", wf.get("workflow_name", "Sans nom"))
|
||||
wf_id = wf.get("id", wf.get("workflow_id", ""))
|
||||
# Creer une closure avec les bonnes valeurs
|
||||
items.append(
|
||||
item(wf_name, self._make_replay_callback(wf_id, wf_name))
|
||||
)
|
||||
return items
|
||||
|
||||
def _make_replay_callback(
|
||||
self, workflow_id: str, workflow_name: str
|
||||
) -> Callable:
|
||||
"""Cree un callback de lancement de replay pour un workflow donne."""
|
||||
def _callback(_icon=None, _item=None):
|
||||
self._launch_replay(workflow_id, workflow_name)
|
||||
return _callback
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Actions utilisateur
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _on_start_session(self, _icon=None, _item=None) -> None:
|
||||
"""Demande le nom du workflow et demarre la session."""
|
||||
# Dialogue tkinter dans un thread dedie
|
||||
def _dialog():
|
||||
name = _ask_string(
|
||||
"Nouvelle Session",
|
||||
"Quel workflow allons-nous apprendre aujourd'hui ?",
|
||||
default="Ma_Tache_Quotidienne",
|
||||
)
|
||||
if name:
|
||||
with self._state_lock:
|
||||
self.is_recording = True
|
||||
self.actions_count = 0
|
||||
self._update_icon()
|
||||
self._notifier.notify(
|
||||
"Session d\u00e9marr\u00e9e",
|
||||
f"Enregistrement du workflow \u00ab {name} \u00bb en cours.",
|
||||
)
|
||||
self.on_start(name)
|
||||
|
||||
threading.Thread(target=_dialog, daemon=True).start()
|
||||
|
||||
def _on_stop_session(self, _icon=None, _item=None) -> None:
|
||||
"""Termine la session en cours et envoie les donnees."""
|
||||
count = self.actions_count
|
||||
with self._state_lock:
|
||||
self.is_recording = False
|
||||
self._update_icon()
|
||||
self.on_stop()
|
||||
self._notifier.notify(
|
||||
"Session termin\u00e9e",
|
||||
f"Bravo ! {count} actions enregistr\u00e9es et envoy\u00e9es au serveur.",
|
||||
)
|
||||
|
||||
def _on_refresh_workflows(self, _icon=None, _item=None) -> None:
|
||||
"""Rafraichit la liste des workflows depuis le serveur."""
|
||||
threading.Thread(target=self._fetch_workflows, daemon=True).start()
|
||||
|
||||
def _on_ask_server(self, _icon=None, _item=None) -> None:
|
||||
"""Envoie 'Que dois-je faire ?' au serveur et affiche la reponse."""
|
||||
def _ask():
|
||||
if self.server_client is None:
|
||||
return
|
||||
response = self.server_client.send_chat_message(
|
||||
"Que dois-je faire maintenant ?"
|
||||
)
|
||||
if response:
|
||||
# L'API renvoie {"response": {"message": "..."}} ou {"response": "..."}
|
||||
resp = response.get("response", {})
|
||||
if isinstance(resp, dict):
|
||||
text = resp.get("message", str(resp))
|
||||
else:
|
||||
text = str(resp)
|
||||
self._notifier.notify("Léa", text)
|
||||
else:
|
||||
self._notifier.notify(
|
||||
"Erreur",
|
||||
"Impossible de contacter le serveur.",
|
||||
)
|
||||
|
||||
threading.Thread(target=_ask, daemon=True).start()
|
||||
|
||||
def _on_toggle_chat(self, _icon=None, _item=None) -> None:
|
||||
"""Affiche ou masque la fenetre de chat Lea (pywebview)."""
|
||||
if self._chat_window is None:
|
||||
return
|
||||
|
||||
def _toggle():
|
||||
try:
|
||||
self._chat_window.toggle()
|
||||
except Exception as e:
|
||||
logger.error("Erreur toggle chat : %s", e)
|
||||
self._notifier.notify(
|
||||
"Erreur Chat",
|
||||
f"Impossible d'ouvrir le chat : {e}",
|
||||
)
|
||||
|
||||
threading.Thread(target=_toggle, daemon=True).start()
|
||||
|
||||
def _launch_replay(self, workflow_id: str, workflow_name: str) -> None:
|
||||
"""Lance le replay d'un workflow."""
|
||||
def _replay():
|
||||
if self.server_client is None:
|
||||
return
|
||||
|
||||
with self._state_lock:
|
||||
self._replay_active = True
|
||||
self._update_icon()
|
||||
self._notifier.notify(
|
||||
"Replay",
|
||||
f"Lancement du workflow \u00ab {workflow_name} \u00bb...",
|
||||
)
|
||||
|
||||
try:
|
||||
import requests
|
||||
resp = requests.post(
|
||||
f"{self.server_client._stream_base}/api/v1/traces/stream/replay/start",
|
||||
json={"workflow_id": workflow_id},
|
||||
timeout=10,
|
||||
)
|
||||
if resp.ok:
|
||||
logger.info("Replay demarre pour workflow %s", workflow_id)
|
||||
else:
|
||||
self._notifier.notify(
|
||||
"Erreur Replay",
|
||||
f"Le serveur a refus\u00e9 : HTTP {resp.status_code}",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error("Erreur lancement replay : %s", e)
|
||||
self._notifier.notify(
|
||||
"Erreur Replay",
|
||||
f"Impossible de lancer le replay : {e}",
|
||||
)
|
||||
finally:
|
||||
with self._state_lock:
|
||||
self._replay_active = False
|
||||
self._update_icon()
|
||||
|
||||
threading.Thread(target=_replay, daemon=True).start()
|
||||
|
||||
def _on_open_folder(self, _icon=None, _item=None) -> None:
|
||||
"""Ouvre le dossier des sessions dans l'explorateur de fichiers."""
|
||||
from ..config import SESSIONS_ROOT
|
||||
|
||||
sessions_path = str(SESSIONS_ROOT)
|
||||
if os.name == "nt":
|
||||
os.startfile(sessions_path)
|
||||
else:
|
||||
os.system(f'xdg-open "{sessions_path}"')
|
||||
|
||||
def _on_quit(self, _icon=None, _item=None) -> None:
|
||||
"""Arrete proprement l'agent et quitte."""
|
||||
logger.info("Arret demande par l'utilisateur")
|
||||
|
||||
# Arreter la session si en cours
|
||||
if self.is_recording:
|
||||
self.on_stop()
|
||||
|
||||
# Signaler l'arret aux threads de fond
|
||||
self._stop_event.set()
|
||||
|
||||
# Fermer la fenetre de chat si ouverte
|
||||
if self._chat_window is not None:
|
||||
try:
|
||||
self._chat_window.destroy()
|
||||
except Exception as e:
|
||||
logger.debug("Erreur fermeture chat : %s", e)
|
||||
|
||||
# Arreter le hotkey global si actif
|
||||
self._stop_hotkey()
|
||||
|
||||
# Arreter le client serveur si present
|
||||
if self.server_client is not None:
|
||||
self.server_client.shutdown()
|
||||
|
||||
# Arreter l'icone pystray
|
||||
if self.icon is not None:
|
||||
self.icon.stop()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Verification connexion serveur (thread daemon)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _connection_checker_loop(self) -> None:
|
||||
"""Verifie la connexion au serveur toutes les 30 secondes."""
|
||||
logger.info("Thread de verification connexion demarre")
|
||||
|
||||
while not self._stop_event.is_set():
|
||||
if self.server_client is not None:
|
||||
try:
|
||||
was_connected = self._connected
|
||||
self._connected = self.server_client.check_connection()
|
||||
|
||||
if self._connected != was_connected:
|
||||
self._update_icon()
|
||||
# La notification est geree par _on_connection_change
|
||||
except Exception as e:
|
||||
logger.error("Erreur verification connexion : %s", e)
|
||||
|
||||
self._stop_event.wait(timeout=_CONNECTION_CHECK_INTERVAL)
|
||||
|
||||
logger.info("Thread de verification connexion arrete")
|
||||
|
||||
def _on_connection_change(self, connected: bool) -> None:
|
||||
"""Callback appelee par LeaServerClient quand l'etat de connexion change."""
|
||||
with self._state_lock:
|
||||
self._connected = connected
|
||||
self._update_icon()
|
||||
|
||||
if connected:
|
||||
self._notifier.notify(
|
||||
"Connexion \u00e9tablie",
|
||||
f"Connect\u00e9 au serveur {self.server_client.server_host}.",
|
||||
)
|
||||
# Rafraichir les workflows a la connexion
|
||||
threading.Thread(target=self._fetch_workflows, daemon=True).start()
|
||||
else:
|
||||
self._notifier.notify(
|
||||
"Connexion perdue",
|
||||
"Le serveur n'est plus accessible.",
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Cache workflows (thread daemon)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _workflow_cache_loop(self) -> None:
|
||||
"""Rafraichit le cache des workflows toutes les 5 minutes."""
|
||||
logger.info("Thread de cache workflows demarre")
|
||||
|
||||
while not self._stop_event.is_set():
|
||||
if self.server_client is not None and self._connected:
|
||||
self._fetch_workflows()
|
||||
|
||||
self._stop_event.wait(timeout=_WORKFLOW_CACHE_TTL)
|
||||
|
||||
logger.info("Thread de cache workflows arrete")
|
||||
|
||||
def _fetch_workflows(self) -> None:
|
||||
"""Recupere la liste des workflows depuis le serveur."""
|
||||
if self.server_client is None:
|
||||
return
|
||||
|
||||
try:
|
||||
workflows = self.server_client.list_workflows()
|
||||
with self._workflows_lock:
|
||||
self._workflows = workflows
|
||||
self._workflows_last_fetch = time.time()
|
||||
logger.debug(
|
||||
"Cache workflows mis a jour : %d workflows", len(workflows)
|
||||
)
|
||||
# Forcer la reconstruction du menu
|
||||
self._update_icon()
|
||||
except Exception as e:
|
||||
logger.error("Erreur recuperation workflows : %s", e)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Mise a jour du compteur (compatibilite main.py)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def update_stats(self, count: int) -> None:
|
||||
"""Met a jour le compteur d'actions en temps reel dans le menu."""
|
||||
with self._state_lock:
|
||||
self.actions_count = count
|
||||
if self.icon is not None:
|
||||
self.icon.update_menu()
|
||||
|
||||
def set_replay_active(self, active: bool) -> None:
|
||||
"""Signale qu'un replay est en cours (appele depuis main.py)."""
|
||||
with self._state_lock:
|
||||
self._replay_active = active
|
||||
self._update_icon()
|
||||
|
||||
if active:
|
||||
self._notifier.notify("Replay", "Execution du replay en cours...")
|
||||
else:
|
||||
self._notifier.notify("Replay termin\u00e9", "Le replay est termin\u00e9.")
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Hotkey global Ctrl+Shift+L (toggle chat)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
_hotkey_hook = None # reference pour pouvoir le retirer
|
||||
|
||||
def _start_hotkey(self) -> None:
|
||||
"""Enregistre le raccourci global Ctrl+Shift+L pour ouvrir le chat.
|
||||
|
||||
Utilise la librairie 'keyboard' si disponible.
|
||||
Silencieux si elle n'est pas installee (pas critique).
|
||||
"""
|
||||
if self._chat_window is None:
|
||||
return
|
||||
|
||||
try:
|
||||
import keyboard
|
||||
self._hotkey_hook = keyboard.add_hotkey(
|
||||
"ctrl+shift+l",
|
||||
self._on_toggle_chat,
|
||||
suppress=False,
|
||||
)
|
||||
logger.info("Hotkey Ctrl+Shift+L enregistre pour le chat Lea")
|
||||
except ImportError:
|
||||
logger.debug(
|
||||
"keyboard non installe — hotkey Ctrl+Shift+L desactive. "
|
||||
"Installer avec : pip install keyboard"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("Impossible d'enregistrer le hotkey : %s", e)
|
||||
|
||||
def _stop_hotkey(self) -> None:
|
||||
"""Retire le raccourci global."""
|
||||
if self._hotkey_hook is not None:
|
||||
try:
|
||||
import keyboard
|
||||
keyboard.remove_hotkey(self._hotkey_hook)
|
||||
self._hotkey_hook = None
|
||||
logger.debug("Hotkey Ctrl+Shift+L retire")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Point d'entree
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def run(self) -> None:
|
||||
"""Demarre le tray, les threads de fond, et entre dans la boucle principale."""
|
||||
# Notification d'accueil (avec identifiant machine)
|
||||
self._notifier.notify(
|
||||
"Agent V1",
|
||||
f"Bonjour ! Agent RPA Vision pr\u00eat.\nMachine : {self.machine_id}",
|
||||
)
|
||||
|
||||
# Enregistrer le hotkey global Ctrl+Shift+L (toggle chat)
|
||||
self._start_hotkey()
|
||||
|
||||
# Tooltip avec identifiant machine pour le multi-machine
|
||||
tray_title = f"Agent V1 - {self.machine_id}"
|
||||
|
||||
# Menu statique — reconstruit via _update_icon() quand l'état change
|
||||
self.icon = pystray.Icon(
|
||||
"AgentV1",
|
||||
self._current_icon(),
|
||||
tray_title,
|
||||
menu=pystray.Menu(*self._get_menu_items()),
|
||||
)
|
||||
|
||||
# Demarrer le thread de verification connexion
|
||||
if self.server_client is not None:
|
||||
conn_thread = threading.Thread(
|
||||
target=self._connection_checker_loop,
|
||||
daemon=True,
|
||||
name="smart-tray-conn-check",
|
||||
)
|
||||
conn_thread.start()
|
||||
|
||||
# Demarrer le thread de cache workflows
|
||||
wf_thread = threading.Thread(
|
||||
target=self._workflow_cache_loop,
|
||||
daemon=True,
|
||||
name="smart-tray-wf-cache",
|
||||
)
|
||||
wf_thread.start()
|
||||
|
||||
# Premiere verification immediate
|
||||
threading.Thread(
|
||||
target=self._fetch_workflows, daemon=True
|
||||
).start()
|
||||
|
||||
# Boucle principale pystray (bloquante)
|
||||
logger.info("SmartTrayV1 demarre — entree dans la boucle pystray")
|
||||
self.icon.run()
|
||||
84
agent_v0/deploy/windows_client/agent_v1/vision/capturer.py
Normal file
84
agent_v0/deploy/windows_client/agent_v1/vision/capturer.py
Normal file
@@ -0,0 +1,84 @@
|
||||
# agent_v1/vision/capturer.py
|
||||
"""
|
||||
Gestionnaire de vision avancé pour Agent V1.
|
||||
Optimisé pour le streaming fibre avec détection de changement.
|
||||
"""
|
||||
|
||||
import os
|
||||
import time
|
||||
import logging
|
||||
import hashlib
|
||||
from PIL import Image, ImageFilter, ImageStat
|
||||
import mss
|
||||
from ..config import TARGETED_CROP_SIZE, SCREENSHOT_QUALITY
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
class VisionCapturer:
|
||||
def __init__(self, session_dir: str):
|
||||
self.session_dir = session_dir
|
||||
self.shots_dir = os.path.join(session_dir, "shots")
|
||||
os.makedirs(self.shots_dir, exist_ok=True)
|
||||
# On ne crée plus self.sct ici car mss n'est pas thread-safe sous Windows
|
||||
self.last_img_hash = None
|
||||
|
||||
def capture_full_context(self, name_suffix: str, force=False) -> str:
|
||||
"""
|
||||
Capture l'écran complet.
|
||||
Si force=False, vérifie d'abord si l'écran a changé.
|
||||
"""
|
||||
try:
|
||||
with mss.mss() as sct:
|
||||
monitor = sct.monitors[1]
|
||||
sct_img = sct.grab(monitor)
|
||||
img = Image.frombytes("RGB", sct_img.size, sct_img.bgra, "raw", "BGRX")
|
||||
|
||||
# Détection de changement (pour Heartbeat)
|
||||
if not force:
|
||||
current_hash = self._compute_quick_hash(img)
|
||||
if current_hash == self.last_img_hash:
|
||||
return "" # Pas de changement, on économise la fibre
|
||||
self.last_img_hash = current_hash
|
||||
|
||||
path = os.path.join(self.shots_dir, f"context_{int(time.time())}_{name_suffix}.png")
|
||||
img.save(path, "PNG", quality=SCREENSHOT_QUALITY)
|
||||
return path
|
||||
except Exception as e:
|
||||
logger.error(f"Erreur Context Capture: {e}")
|
||||
return ""
|
||||
|
||||
def capture_dual(self, x: int, y: int, screenshot_id: str, anonymize=False) -> dict:
|
||||
"""Capture duale (Full + Crop) systématique (forcée car liée à une action)."""
|
||||
try:
|
||||
with mss.mss() as sct:
|
||||
full_path = os.path.join(self.shots_dir, f"{screenshot_id}_full.png")
|
||||
monitor = sct.monitors[1]
|
||||
sct_img = sct.grab(monitor)
|
||||
img = Image.frombytes("RGB", sct_img.size, sct_img.bgra, "raw", "BGRX")
|
||||
|
||||
# Capture du Crop (Cœur de l'apprentissage qwen3-vl)
|
||||
crop_path = os.path.join(self.shots_dir, f"{screenshot_id}_crop.png")
|
||||
w, h = TARGETED_CROP_SIZE
|
||||
left = max(0, x - w // 2)
|
||||
top = max(0, y - h // 2)
|
||||
crop_img = img.crop((left, top, left + w, top + h))
|
||||
|
||||
if anonymize:
|
||||
crop_img = crop_img.filter(ImageFilter.GaussianBlur(radius=4))
|
||||
|
||||
img.save(full_path, "PNG", quality=SCREENSHOT_QUALITY)
|
||||
crop_img.save(crop_path, "PNG", quality=SCREENSHOT_QUALITY)
|
||||
|
||||
# Mise à jour du hash pour le prochain heartbeat
|
||||
self.last_img_hash = self._compute_quick_hash(img)
|
||||
|
||||
return {"full": full_path, "crop": crop_path}
|
||||
except Exception as e:
|
||||
logger.error(f"Erreur Dual Capture: {e}")
|
||||
return {}
|
||||
|
||||
def _compute_quick_hash(self, img: Image) -> str:
|
||||
"""Calcule un hash rapide basé sur une vignette réduite pour détecter les changements."""
|
||||
# On réduit l'image à 64x64 pour comparer les masses de couleurs (très rapide)
|
||||
small_img = img.resize((64, 64), Image.NEAREST).convert("L")
|
||||
return hashlib.md5(small_img.tobytes()).hexdigest()
|
||||
55
agent_v0/deploy/windows_client/agent_v1/window_info.py
Normal file
55
agent_v0/deploy/windows_client/agent_v1/window_info.py
Normal file
@@ -0,0 +1,55 @@
|
||||
# window_info.py
|
||||
"""
|
||||
Récupération des informations sur la fenêtre active (X11).
|
||||
|
||||
v0 :
|
||||
- utilise xdotool pour obtenir :
|
||||
- le titre de la fenêtre active
|
||||
- le PID de la fenêtre active, puis le nom du process via ps
|
||||
|
||||
Si quelque chose ne fonctionne pas, on renvoie des valeurs "unknown".
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
from typing import Dict, Optional
|
||||
|
||||
|
||||
def _run_cmd(cmd: list[str]) -> Optional[str]:
|
||||
"""Exécute une commande et renvoie la sortie texte (strippée), ou None en cas d'erreur."""
|
||||
try:
|
||||
out = subprocess.check_output(cmd, stderr=subprocess.DEVNULL)
|
||||
return out.decode("utf-8", errors="ignore").strip()
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def get_active_window_info() -> Dict[str, str]:
|
||||
"""
|
||||
Renvoie un dict :
|
||||
{
|
||||
"title": "...",
|
||||
"app_name": "..."
|
||||
}
|
||||
|
||||
Nécessite xdotool installé sur le système.
|
||||
"""
|
||||
title = _run_cmd(["xdotool", "getactivewindow", "getwindowname"])
|
||||
pid_str = _run_cmd(["xdotool", "getactivewindow", "getwindowpid"])
|
||||
|
||||
app_name: Optional[str] = None
|
||||
if pid_str:
|
||||
pid_str = pid_str.strip()
|
||||
# On récupère le nom du binaire via ps
|
||||
app_name = _run_cmd(["ps", "-p", pid_str, "-o", "comm="])
|
||||
|
||||
if not title:
|
||||
title = "unknown_window"
|
||||
if not app_name:
|
||||
app_name = "unknown_app"
|
||||
|
||||
return {
|
||||
"title": title,
|
||||
"app_name": app_name,
|
||||
}
|
||||
@@ -0,0 +1,192 @@
|
||||
# window_info_crossplatform.py
|
||||
"""
|
||||
Récupération des informations sur la fenêtre active - CROSS-PLATFORM
|
||||
|
||||
Supporte:
|
||||
- Linux (X11 via xdotool)
|
||||
- Windows (via pywin32)
|
||||
- macOS (via pyobjc)
|
||||
|
||||
Installation des dépendances:
|
||||
pip install pywin32 # Windows
|
||||
pip install pyobjc-framework-Cocoa # macOS
|
||||
pip install psutil # Tous OS
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import platform
|
||||
import subprocess
|
||||
from typing import Dict, Optional
|
||||
|
||||
|
||||
def _run_cmd(cmd: list[str]) -> Optional[str]:
|
||||
"""Exécute une commande et renvoie la sortie texte (strippée), ou None en cas d'erreur."""
|
||||
try:
|
||||
out = subprocess.check_output(cmd, stderr=subprocess.DEVNULL)
|
||||
return out.decode("utf-8", errors="ignore").strip()
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def get_active_window_info() -> Dict[str, str]:
|
||||
"""
|
||||
Renvoie un dict :
|
||||
{
|
||||
"title": "...",
|
||||
"app_name": "..."
|
||||
}
|
||||
|
||||
Détecte automatiquement l'OS et utilise la méthode appropriée.
|
||||
"""
|
||||
system = platform.system()
|
||||
|
||||
if system == "Linux":
|
||||
return _get_window_info_linux()
|
||||
elif system == "Windows":
|
||||
return _get_window_info_windows()
|
||||
elif system == "Darwin": # macOS
|
||||
return _get_window_info_macos()
|
||||
else:
|
||||
return {"title": "unknown_window", "app_name": "unknown_app"}
|
||||
|
||||
|
||||
def _get_window_info_linux() -> Dict[str, str]:
|
||||
"""
|
||||
Linux: utilise xdotool (X11)
|
||||
|
||||
Nécessite: sudo apt-get install xdotool
|
||||
"""
|
||||
title = _run_cmd(["xdotool", "getactivewindow", "getwindowname"])
|
||||
pid_str = _run_cmd(["xdotool", "getactivewindow", "getwindowpid"])
|
||||
|
||||
app_name: Optional[str] = None
|
||||
if pid_str:
|
||||
pid_str = pid_str.strip()
|
||||
# On récupère le nom du binaire via ps
|
||||
app_name = _run_cmd(["ps", "-p", pid_str, "-o", "comm="])
|
||||
|
||||
if not title:
|
||||
title = "unknown_window"
|
||||
if not app_name:
|
||||
app_name = "unknown_app"
|
||||
|
||||
return {
|
||||
"title": title,
|
||||
"app_name": app_name,
|
||||
}
|
||||
|
||||
|
||||
def _get_window_info_windows() -> Dict[str, str]:
|
||||
"""
|
||||
Windows: utilise pywin32 + psutil
|
||||
|
||||
Nécessite: pip install pywin32 psutil
|
||||
"""
|
||||
try:
|
||||
import win32gui
|
||||
import win32process
|
||||
import psutil
|
||||
|
||||
# Fenêtre au premier plan
|
||||
hwnd = win32gui.GetForegroundWindow()
|
||||
|
||||
# Titre de la fenêtre
|
||||
title = win32gui.GetWindowText(hwnd)
|
||||
if not title:
|
||||
title = "unknown_window"
|
||||
|
||||
# PID du processus
|
||||
_, pid = win32process.GetWindowThreadProcessId(hwnd)
|
||||
|
||||
# Nom du processus
|
||||
try:
|
||||
process = psutil.Process(pid)
|
||||
app_name = process.name()
|
||||
except (psutil.NoSuchProcess, psutil.AccessDenied):
|
||||
app_name = "unknown_app"
|
||||
|
||||
return {
|
||||
"title": title,
|
||||
"app_name": app_name,
|
||||
}
|
||||
|
||||
except ImportError:
|
||||
# pywin32 ou psutil non installé
|
||||
return {
|
||||
"title": "unknown_window (pywin32 missing)",
|
||||
"app_name": "unknown_app (pywin32 missing)",
|
||||
}
|
||||
except Exception as e:
|
||||
return {
|
||||
"title": f"error: {e}",
|
||||
"app_name": "unknown_app",
|
||||
}
|
||||
|
||||
|
||||
def _get_window_info_macos() -> Dict[str, str]:
|
||||
"""
|
||||
macOS: utilise pyobjc (AppKit)
|
||||
|
||||
Nécessite: pip install pyobjc-framework-Cocoa
|
||||
|
||||
Note: Nécessite les permissions "Accessibility" dans System Preferences
|
||||
"""
|
||||
try:
|
||||
from AppKit import NSWorkspace
|
||||
from Quartz import (
|
||||
CGWindowListCopyWindowInfo,
|
||||
kCGWindowListOptionOnScreenOnly,
|
||||
kCGNullWindowID
|
||||
)
|
||||
|
||||
# Application active
|
||||
active_app = NSWorkspace.sharedWorkspace().activeApplication()
|
||||
app_name = active_app.get('NSApplicationName', 'unknown_app')
|
||||
|
||||
# Titre de la fenêtre (via Quartz)
|
||||
# On cherche la fenêtre de l'app active qui est au premier plan
|
||||
window_list = CGWindowListCopyWindowInfo(
|
||||
kCGWindowListOptionOnScreenOnly,
|
||||
kCGNullWindowID
|
||||
)
|
||||
|
||||
title = "unknown_window"
|
||||
for window in window_list:
|
||||
owner_name = window.get('kCGWindowOwnerName', '')
|
||||
if owner_name == app_name:
|
||||
window_title = window.get('kCGWindowName', '')
|
||||
if window_title:
|
||||
title = window_title
|
||||
break
|
||||
|
||||
return {
|
||||
"title": title,
|
||||
"app_name": app_name,
|
||||
}
|
||||
|
||||
except ImportError:
|
||||
# pyobjc non installé
|
||||
return {
|
||||
"title": "unknown_window (pyobjc missing)",
|
||||
"app_name": "unknown_app (pyobjc missing)",
|
||||
}
|
||||
except Exception as e:
|
||||
return {
|
||||
"title": f"error: {e}",
|
||||
"app_name": "unknown_app",
|
||||
}
|
||||
|
||||
|
||||
# Test rapide
|
||||
if __name__ == "__main__":
|
||||
import time
|
||||
|
||||
print(f"OS détecté: {platform.system()}")
|
||||
print("\nTest de capture fenêtre active (5 secondes)...")
|
||||
print("Changez de fenêtre pour tester!\n")
|
||||
|
||||
for i in range(5):
|
||||
info = get_active_window_info()
|
||||
print(f"[{i+1}] App: {info['app_name']:20s} | Title: {info['title']}")
|
||||
time.sleep(1)
|
||||
58
agent_v0/deploy/windows_client/config.py
Normal file
58
agent_v0/deploy/windows_client/config.py
Normal file
@@ -0,0 +1,58 @@
|
||||
# config.py
|
||||
"""
|
||||
Configuration de base pour agent_v0.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
AGENT_VERSION = "0.1.0"
|
||||
|
||||
# Dossier racine du projet (là où se trouve ce fichier)
|
||||
BASE_DIR = Path(__file__).resolve().parent
|
||||
|
||||
# Chargement automatique de .env.local depuis le répertoire parent
|
||||
def load_env_file(env_path):
|
||||
"""Charge un fichier .env dans les variables d'environnement"""
|
||||
if not env_path.exists():
|
||||
return False
|
||||
|
||||
with open(env_path, 'r') as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if line and not line.startswith('#') and '=' in line:
|
||||
key, value = line.split('=', 1)
|
||||
os.environ[key.strip()] = value.strip()
|
||||
return True
|
||||
|
||||
# Charger .env.local depuis le répertoire parent (racine du projet)
|
||||
env_local_path = BASE_DIR.parent / ".env.local"
|
||||
if load_env_file(env_local_path):
|
||||
print(f"[agent_v0] Variables d'environnement chargées depuis {env_local_path}")
|
||||
|
||||
|
||||
|
||||
# Endpoint du serveur RPA Vision V3
|
||||
# En développement local : http://localhost:8000/api/traces/upload
|
||||
# En production : configurer via variable d'environnement
|
||||
import os
|
||||
SERVER_URL = os.getenv("RPA_SERVER_URL", "http://localhost:8000/api/traces/upload")
|
||||
|
||||
# Durée max d'une session en secondes (ex: 30 minutes)
|
||||
MAX_SESSION_DURATION_S = 30 * 60
|
||||
|
||||
# Dossier racine local où stocker les sessions (chemin ABSOLU)
|
||||
SESSIONS_ROOT = str(BASE_DIR / "sessions")
|
||||
|
||||
# Dossier et fichier de logs
|
||||
LOGS_DIR = BASE_DIR / "logs"
|
||||
LOG_FILE = LOGS_DIR / "agent_v0.log"
|
||||
|
||||
# Faut-il quitter l'application après un Stop session ?
|
||||
EXIT_AFTER_SESSION = True
|
||||
|
||||
# Création des dossiers si besoin
|
||||
os.makedirs(SESSIONS_ROOT, exist_ok=True)
|
||||
os.makedirs(LOGS_DIR, exist_ok=True)
|
||||
13
agent_v0/deploy/windows_client/lea_ui/__init__.py
Normal file
13
agent_v0/deploy/windows_client/lea_ui/__init__.py
Normal file
@@ -0,0 +1,13 @@
|
||||
# agent_v0.lea_ui — Interface utilisateur "Lea"
|
||||
#
|
||||
# Panneau PyQt5 integre qui remplace le system tray + navigateur web
|
||||
# par une interface unifiee pour piloter l'Agent RPA Vision V3.
|
||||
#
|
||||
# Composants :
|
||||
# - LeaMainWindow : fenetre principale ancree a droite
|
||||
# - ChatWidget : zone de conversation avec le serveur
|
||||
# - OverlayWidget : feedback visuel pendant le replay
|
||||
# - LeaServerClient : client API vers le serveur Linux
|
||||
# - styles : theme et couleurs
|
||||
|
||||
__version__ = "0.1.0"
|
||||
350
agent_v0/deploy/windows_client/lea_ui/server_client.py
Normal file
350
agent_v0/deploy/windows_client/lea_ui/server_client.py
Normal file
@@ -0,0 +1,350 @@
|
||||
# agent_v0/lea_ui/server_client.py
|
||||
"""
|
||||
Client API pour communiquer avec le serveur Linux RPA Vision V3.
|
||||
|
||||
Endpoints cibles :
|
||||
- Agent Chat (port 5004) : /api/chat, /api/workflows
|
||||
- Streaming Server (port 5005) : /api/v1/traces/stream/replay/next, etc.
|
||||
|
||||
Le polling tourne dans un thread separe pour ne pas bloquer la UI Qt.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from typing import Any, Callable, Dict, List, Optional
|
||||
|
||||
logger = logging.getLogger("lea_ui.server_client")
|
||||
|
||||
|
||||
def _get_server_host() -> str:
|
||||
"""Recuperer l'adresse du serveur Linux.
|
||||
|
||||
Ordre de resolution :
|
||||
1. Variable d'environnement RPA_SERVER_HOST
|
||||
2. Fichier de config agent_config.json (cle "server_host")
|
||||
3. Fallback localhost
|
||||
"""
|
||||
# 1. Variable d'environnement
|
||||
host = os.environ.get("RPA_SERVER_HOST", "").strip()
|
||||
if host:
|
||||
return host
|
||||
|
||||
# 2. Fichier de config
|
||||
config_paths = [
|
||||
os.path.join(os.path.dirname(__file__), "..", "agent_config.json"),
|
||||
os.path.join(os.path.dirname(__file__), "..", "..", "agent_config.json"),
|
||||
]
|
||||
for config_path in config_paths:
|
||||
try:
|
||||
with open(config_path, "r", encoding="utf-8") as f:
|
||||
cfg = json.load(f)
|
||||
host = cfg.get("server_host", "").strip()
|
||||
if host:
|
||||
return host
|
||||
except (OSError, json.JSONDecodeError):
|
||||
continue
|
||||
|
||||
# 3. Fallback
|
||||
return "localhost"
|
||||
|
||||
|
||||
class LeaServerClient:
|
||||
"""Client API thread-safe vers le serveur RPA Vision V3.
|
||||
|
||||
Gere la communication HTTP avec le serveur chat (port 5004)
|
||||
et le serveur de streaming (port 5005).
|
||||
Le polling replay tourne dans un thread daemon separe.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
server_host: Optional[str] = None,
|
||||
chat_port: int = 5004,
|
||||
stream_port: int = 5005,
|
||||
) -> None:
|
||||
self._host = server_host or _get_server_host()
|
||||
self._chat_port = chat_port
|
||||
self._stream_port = stream_port
|
||||
|
||||
self._chat_base = f"http://{self._host}:{self._chat_port}"
|
||||
self._stream_base = f"http://{self._host}:{self._stream_port}"
|
||||
|
||||
# Etat de connexion
|
||||
self._connected = False
|
||||
self._last_error: Optional[str] = None
|
||||
|
||||
# Callbacks UI (appelees depuis le thread de polling)
|
||||
self._on_connection_change: Optional[Callable[[bool], None]] = None
|
||||
self._on_replay_action: Optional[Callable[[Dict[str, Any]], None]] = None
|
||||
self._on_chat_response: Optional[Callable[[Dict[str, Any]], None]] = None
|
||||
|
||||
# Thread de polling
|
||||
self._polling = False
|
||||
self._poll_thread: Optional[threading.Thread] = None
|
||||
self._poll_interval = 1.0 # secondes
|
||||
|
||||
# Session de chat
|
||||
self._chat_session_id: Optional[str] = None
|
||||
|
||||
logger.info(
|
||||
"LeaServerClient initialise : chat=%s, stream=%s",
|
||||
self._chat_base, self._stream_base,
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Proprietes
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@property
|
||||
def connected(self) -> bool:
|
||||
return self._connected
|
||||
|
||||
@property
|
||||
def server_host(self) -> str:
|
||||
return self._host
|
||||
|
||||
@property
|
||||
def last_error(self) -> Optional[str]:
|
||||
return self._last_error
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Callbacks
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def set_on_connection_change(self, callback: Callable[[bool], None]) -> None:
|
||||
"""Callback appelee quand l'etat de connexion change."""
|
||||
self._on_connection_change = callback
|
||||
|
||||
def set_on_replay_action(self, callback: Callable[[Dict[str, Any]], None]) -> None:
|
||||
"""Callback appelee quand une action de replay est recue."""
|
||||
self._on_replay_action = callback
|
||||
|
||||
def set_on_chat_response(self, callback: Callable[[Dict[str, Any]], None]) -> None:
|
||||
"""Callback appelee quand une reponse chat est recue."""
|
||||
self._on_chat_response = callback
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Connexion
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def check_connection(self) -> bool:
|
||||
"""Tester la connexion au serveur chat."""
|
||||
try:
|
||||
import requests
|
||||
resp = requests.get(
|
||||
f"{self._chat_base}/api/status",
|
||||
timeout=5,
|
||||
)
|
||||
was_connected = self._connected
|
||||
self._connected = resp.ok
|
||||
self._last_error = None
|
||||
|
||||
if self._connected != was_connected and self._on_connection_change:
|
||||
self._on_connection_change(self._connected)
|
||||
|
||||
return self._connected
|
||||
except Exception as e:
|
||||
was_connected = self._connected
|
||||
self._connected = False
|
||||
self._last_error = str(e)
|
||||
|
||||
if was_connected and self._on_connection_change:
|
||||
self._on_connection_change(False)
|
||||
|
||||
return False
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Chat API (port 5004)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def send_chat_message(self, message: str) -> Optional[Dict[str, Any]]:
|
||||
"""Envoyer un message au chat et retourner la reponse.
|
||||
|
||||
Retourne None en cas d'erreur reseau.
|
||||
"""
|
||||
try:
|
||||
import requests
|
||||
payload = {
|
||||
"message": message,
|
||||
}
|
||||
if self._chat_session_id:
|
||||
payload["session_id"] = self._chat_session_id
|
||||
|
||||
resp = requests.post(
|
||||
f"{self._chat_base}/api/chat",
|
||||
json=payload,
|
||||
timeout=30,
|
||||
)
|
||||
|
||||
if resp.ok:
|
||||
data = resp.json()
|
||||
# Sauvegarder le session_id pour le contexte multi-tour
|
||||
if "session_id" in data:
|
||||
self._chat_session_id = data["session_id"]
|
||||
self._connected = True
|
||||
return data
|
||||
else:
|
||||
self._last_error = f"HTTP {resp.status_code}"
|
||||
logger.warning("Chat API erreur : %s", self._last_error)
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
self._last_error = str(e)
|
||||
self._connected = False
|
||||
logger.error("Chat API exception : %s", e)
|
||||
return None
|
||||
|
||||
def list_workflows(self) -> List[Dict[str, Any]]:
|
||||
"""Recuperer la liste des workflows depuis le serveur chat."""
|
||||
try:
|
||||
import requests
|
||||
resp = requests.get(
|
||||
f"{self._chat_base}/api/workflows",
|
||||
timeout=10,
|
||||
)
|
||||
if resp.ok:
|
||||
data = resp.json()
|
||||
self._connected = True
|
||||
return data.get("workflows", [])
|
||||
return []
|
||||
except Exception as e:
|
||||
self._last_error = str(e)
|
||||
logger.error("List workflows erreur : %s", e)
|
||||
return []
|
||||
|
||||
def list_gestures(self) -> List[Dict[str, Any]]:
|
||||
"""Recuperer la liste des gestes depuis le serveur chat."""
|
||||
try:
|
||||
import requests
|
||||
resp = requests.get(
|
||||
f"{self._chat_base}/api/workflows",
|
||||
timeout=10,
|
||||
)
|
||||
if resp.ok:
|
||||
data = resp.json()
|
||||
return data.get("workflows", [])
|
||||
return []
|
||||
except Exception as e:
|
||||
logger.error("List gestures erreur : %s", e)
|
||||
return []
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Replay Polling (port 5005)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def start_polling(self, session_id: str) -> None:
|
||||
"""Demarrer le polling des actions de replay dans un thread daemon."""
|
||||
if self._polling:
|
||||
return
|
||||
|
||||
self._polling = True
|
||||
self._poll_session_id = session_id
|
||||
self._poll_thread = threading.Thread(
|
||||
target=self._poll_loop,
|
||||
daemon=True,
|
||||
name="lea-replay-poll",
|
||||
)
|
||||
self._poll_thread.start()
|
||||
logger.info("Polling replay demarre pour session %s", session_id)
|
||||
|
||||
def stop_polling(self) -> None:
|
||||
"""Arreter le polling."""
|
||||
self._polling = False
|
||||
if self._poll_thread:
|
||||
self._poll_thread.join(timeout=3)
|
||||
self._poll_thread = None
|
||||
logger.info("Polling replay arrete")
|
||||
|
||||
def _poll_loop(self) -> None:
|
||||
"""Boucle de polling dans un thread separe."""
|
||||
import requests as req_lib
|
||||
|
||||
while self._polling:
|
||||
try:
|
||||
resp = req_lib.get(
|
||||
f"{self._stream_base}/api/v1/traces/stream/replay/next",
|
||||
params={"session_id": self._poll_session_id},
|
||||
timeout=5,
|
||||
)
|
||||
|
||||
if resp.ok:
|
||||
data = resp.json()
|
||||
action = data.get("action")
|
||||
if action and self._on_replay_action:
|
||||
self._on_replay_action(action)
|
||||
# Apres une action, poll plus rapidement
|
||||
time.sleep(0.2)
|
||||
continue
|
||||
|
||||
except req_lib.exceptions.ConnectionError:
|
||||
# Serveur non disponible — silencieux
|
||||
pass
|
||||
except req_lib.exceptions.Timeout:
|
||||
pass
|
||||
except Exception as e:
|
||||
logger.error("Erreur poll replay : %s", e)
|
||||
|
||||
time.sleep(self._poll_interval)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Replay Status
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def get_replay_status(self) -> Optional[Dict[str, Any]]:
|
||||
"""Recuperer l'etat des replays en cours."""
|
||||
try:
|
||||
import requests
|
||||
resp = requests.get(
|
||||
f"{self._stream_base}/api/v1/traces/stream/replays",
|
||||
timeout=5,
|
||||
)
|
||||
if resp.ok:
|
||||
data = resp.json()
|
||||
replays = data.get("replays", [])
|
||||
# Retourner le premier replay actif
|
||||
for r in replays:
|
||||
if r.get("status") == "running":
|
||||
return r
|
||||
return None
|
||||
return None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def report_action_result(
|
||||
self,
|
||||
session_id: str,
|
||||
action_id: str,
|
||||
success: bool,
|
||||
error: Optional[str] = None,
|
||||
screenshot: Optional[str] = None,
|
||||
) -> None:
|
||||
"""Rapporter le resultat d'execution d'une action au serveur."""
|
||||
try:
|
||||
import requests
|
||||
requests.post(
|
||||
f"{self._stream_base}/api/v1/traces/stream/replay/result",
|
||||
json={
|
||||
"session_id": session_id,
|
||||
"action_id": action_id,
|
||||
"success": success,
|
||||
"error": error,
|
||||
"screenshot": screenshot,
|
||||
},
|
||||
timeout=5,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error("Report action result erreur : %s", e)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Lifecycle
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def shutdown(self) -> None:
|
||||
"""Arreter proprement le client."""
|
||||
self.stop_polling()
|
||||
logger.info("LeaServerClient arrete")
|
||||
15
agent_v0/deploy/windows_client/requirements.txt
Normal file
15
agent_v0/deploy/windows_client/requirements.txt
Normal file
@@ -0,0 +1,15 @@
|
||||
# agent_v1/requirements.txt
|
||||
mss>=9.0.1 # Capture d'écran haute performance
|
||||
pynput>=1.7.7 # Clavier/Souris Cross-plateforme
|
||||
Pillow>=10.0.0 # Crops et processing image
|
||||
requests>=2.31.0 # Streaming réseau
|
||||
psutil>=5.9.0 # Monitoring CPU/RAM
|
||||
pystray>=0.19.5 # Icône Tray UI
|
||||
plyer>=2.1.0 # Notifications toast natives (remplace PyQt5)
|
||||
|
||||
# Windows spécifique
|
||||
pywin32>=306 ; sys_platform == 'win32'
|
||||
|
||||
# macOS spécifique
|
||||
pyobjc-framework-Cocoa>=10.0 ; sys_platform == 'darwin'
|
||||
pyobjc-framework-Quartz>=10.0 ; sys_platform == 'darwin'
|
||||
16
agent_v0/deploy/windows_client/run_agent_v1.py
Normal file
16
agent_v0/deploy/windows_client/run_agent_v1.py
Normal file
@@ -0,0 +1,16 @@
|
||||
# run_agent_v1.py
|
||||
import sys
|
||||
import os
|
||||
|
||||
# Ajout du répertoire courant au PYTHONPATH pour permettre les imports de modules
|
||||
current_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
if current_dir not in sys.path:
|
||||
sys.path.append(current_dir)
|
||||
|
||||
try:
|
||||
from agent_v1.main import main
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
except ImportError as e:
|
||||
print(f"Erreur d'importation : {e}")
|
||||
print("Assurez-vous d'être dans le répertoire racine du projet et que agent_v1 est bien un package Python.")
|
||||
64
agent_v0/deploy/windows_client/setup.bat
Normal file
64
agent_v0/deploy/windows_client/setup.bat
Normal file
@@ -0,0 +1,64 @@
|
||||
@echo off
|
||||
:: setup_v1.bat - Installation conviviale pour Windows
|
||||
|
||||
echo ==================================================
|
||||
echo Agent V1 - RPA Vision - Installation Windows
|
||||
echo ==================================================
|
||||
echo.
|
||||
|
||||
:: 0. Verifier que Python est installe
|
||||
python --version >nul 2>&1
|
||||
if errorlevel 1 (
|
||||
echo [ERREUR] Python n'est pas installe ou pas dans le PATH.
|
||||
echo Telecharger Python 3.10+ depuis https://python.org
|
||||
pause
|
||||
exit /b 1
|
||||
)
|
||||
|
||||
:: 1. Creation de l'environnement virtuel
|
||||
if not exist ".venv_v1_win" (
|
||||
echo [1/4] Creation de l'environnement virtuel...
|
||||
python -m venv .venv_v1_win
|
||||
) else (
|
||||
echo [1/4] Environnement virtuel existant detecte.
|
||||
)
|
||||
|
||||
:: 2. Activation
|
||||
call .venv_v1_win\Scripts\activate.bat
|
||||
|
||||
:: 3. Mise a jour pip et installation des dependances
|
||||
echo [2/4] Installation des dependances...
|
||||
python -m pip install --upgrade pip --quiet
|
||||
pip install -r agent_v1\requirements.txt --quiet
|
||||
|
||||
:: 4. Post-installation Windows (pywin32)
|
||||
echo [3/4] Configuration Windows...
|
||||
python -c "import win32api" >nul 2>&1
|
||||
if errorlevel 1 (
|
||||
echo pywin32 post-install...
|
||||
python .venv_v1_win\Scripts\pywin32_postinstall.py -install >nul 2>&1
|
||||
)
|
||||
|
||||
:: 5. Verification rapide
|
||||
echo [4/4] Verification...
|
||||
python -c "import pystray; import plyer; import mss; import pynput; print(' Toutes les dependances OK')"
|
||||
if errorlevel 1 (
|
||||
echo [ERREUR] Certaines dependances sont manquantes.
|
||||
echo Relancer : pip install -r agent_v1\requirements.txt
|
||||
pause
|
||||
exit /b 1
|
||||
)
|
||||
|
||||
echo.
|
||||
echo ==================================================
|
||||
echo Installation terminee !
|
||||
echo.
|
||||
echo Pour lancer l'agent :
|
||||
echo .venv_v1_win\Scripts\activate.bat
|
||||
echo python run_agent_v1.py
|
||||
echo.
|
||||
echo Configuration serveur :
|
||||
echo Editer agent_config.json
|
||||
echo ou definir RPA_SERVER_HOST=192.168.1.x
|
||||
echo ==================================================
|
||||
pause
|
||||
Reference in New Issue
Block a user