Dom/rpa_vision_v3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: smart systray Léa (plyer), preflight GPU, fix tests, support qwen3-vl

Browse Source 
- Smart systray (pystray+plyer) remplace PyQt5 : notifications toast,
  menu dynamique avec workflows, chat "Que dois-je faire ?", icône colorée
- Preflight GPU : check_machine_ready() + @pytest.mark.gpu dans conftest
- Correction 63 tests cassés → 0 failed (1200 passed)
- Tests VWB obsolètes déplacés vers _a_trier/
- Support qwen3-vl:8b sur GPU (remplace qwen2.5vl:3b)
  - fix images < 32x32 (Ollama panic)
  - fix force_json=False (qwen3-vl incompatible)
  - fix temperature 0.1 (0.0 bloque avec images)
- Fix captor Windows : Key.esc, _get_key_name()
- Fix LeaServerClient : check_connection, list_workflows format
- deploy_windows.py : packaging propre client Windows
- VWB : edges visibles (#607d8b) + fitView automatique

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Dom
2026-03-16 22:25:12 +01:00
parent cf495dd82f
commit ad15237fe0
36 changed files with 432 additions and 8103 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
core/detection/ollama_client.py
View File

@@ -288,27 +288,31 @@ Respond with just the role name, nothing else."""
Returns:
Dict avec 'type', 'role', 'text', 'confidence', 'success'
"""
# System prompt "zéro tolérance" - Force le VLM à NE produire QUE du JSON
system_prompt = """You are a UI element classifier.
Your ONLY task is to output valid JSON. Never explain. Never comment. Never discuss.
Expected format:
{"type": "...", "role": "...", "text": "..."}"""
# System prompt direct — pas de thinking, JSON uniquement
system_prompt = "You are a JSON-only UI classifier. No thinking. No explanation. Output raw JSON only."
# User prompt simplifié et direct
prompt = """Classify this UI element:
- Type: Choose ONE from [button, text_input, checkbox, radio, dropdown, tab, link, icon, table_row, menu_item]
- Role: Choose ONE from [primary_action, cancel, submit, form_input, search_field, navigation, settings, close, delete, edit, save]
- Text: Any visible text (empty string if none)
# User prompt avec exemples explicites pour guider le modèle
prompt = """/no_think
Look at this UI element image and classify it. Reply with ONLY a JSON object, nothing else.
Output JSON only."""
Types: button, text_input, checkbox, radio, dropdown, tab, link, icon, table_row, menu_item
Roles: primary_action, cancel, submit, form_input, search_field, navigation, settings, close, delete, edit, save
Example 1: {"type": "button", "role": "submit", "text": "OK"}
Example 2: {"type": "text_input", "role": "form_input", "text": ""}
Example 3: {"type": "icon", "role": "close", "text": "X"}
Your answer:"""
# Note: force_json=False car qwen3-vl ne supporte pas format:json
# temperature=0.1 car qwen3-vl bloque à 0.0 avec des images
result = self.generate(
prompt,
image=element_image,
system_prompt=system_prompt,
temperature=0.0,
max_tokens=150,
force_json=True
temperature=0.1,
max_tokens=200,
force_json=False
)
if result["success"]:
@@ -381,6 +385,13 @@ Output JSON only."""
if image.mode != 'RGB':
image = image.convert('RGB')
# 1b. Minimum 32x32 (requis par qwen3-vl, sinon Ollama panic)
min_size = 32
if image.width < min_size or image.height < min_size:
new_w = max(image.width, min_size)
new_h = max(image.height, min_size)
image = image.resize((new_w, new_h), Image.NEAREST)
# 2. Redimensionnement intelligent : max 1280px sur le côté long
max_size = 1280
if max(image.size) > max_size:

17
core/detection/ui_detector.py
View File

@@ -72,7 +72,7 @@ class DetectionConfig:
# - "qwen2.5vl:3b" (léger, tient en GPU 12GB avec split partiel)
# - "qwen2.5vl:7b" (meilleur mais 13GB mémoire, CPU-only sur RTX 5070)
# - "qwen3-vl:8b" (plus gros, supporté mais plus d'erreurs JSON)
vlm_model: str = "qwen2.5vl:3b"
vlm_model: str = "qwen3-vl:8b"
vlm_endpoint: str = "http://localhost:11434"
use_vlm_classification: bool = True # Utiliser VLM pour classifier
@@ -218,7 +218,14 @@ class UIDetector:
logger.debug("Step 2: Classifying regions with VLM...")
ui_elements = []
# Taille minimale pour le VLM Ollama (qwen3-vl exige >= 32x32)
MIN_VLM_SIZE = 32
for i, region in enumerate(regions):
# Ignorer les régions trop petites
if region.w < 5 or region.h < 5:
continue
# Extraire le crop de la région
crop = pil_image.crop((
region.x,
@@ -226,7 +233,13 @@ class UIDetector:
region.x + region.w,
region.y + region.h
))
# Agrandir les crops trop petits pour le VLM (pad ou resize)
if crop.width < MIN_VLM_SIZE or crop.height < MIN_VLM_SIZE:
new_w = max(crop.width, MIN_VLM_SIZE)
new_h = max(crop.height, MIN_VLM_SIZE)
crop = crop.resize((new_w, new_h), Image.NEAREST)
# Classifier avec VLM
element = self._classify_region(
crop,

4
core/gpu/__init__.py
View File

@@ -24,6 +24,7 @@ from .gpu_resource_manager import (
from .ollama_manager import OllamaManager
from .vram_monitor import VRAMMonitor
from .clip_manager import CLIPManager
from .preflight import PreflightResult, check_machine_ready, require_gpu_ready
__all__ = [
"GPUResourceManager",
@@ -37,4 +38,7 @@ __all__ = [
"OllamaManager",
"VRAMMonitor",
"CLIPManager",
"PreflightResult",
"check_machine_ready",
"require_gpu_ready",
]

272
core/gpu/preflight.py Normal file
View File

@@ -0,0 +1,272 @@
"""
Preflight GPU Check — Vérification machine avant tout lancement.
Vérifie que le GPU et la VRAM sont suffisamment libres avant de lancer
des tests, replays, ou tout processus gourmand en ressources.
Usage:
from core.gpu.preflight import check_machine_ready, require_gpu_ready
# Vérification simple
result = check_machine_ready()
if not result.ready:
print(f"Machine pas prête : {result.reason}")
# Avec seuils personnalisés
result = check_machine_ready(min_free_vram_mb=2000, max_gpu_util_percent=50)
# Comme décorateur (skip le test si GPU pas dispo)
@require_gpu_ready(min_free_vram_mb=1000)
def test_something():
...
"""
import functools
import logging
import subprocess
from dataclasses import dataclass, field
from typing import List, Optional
import pytest
logger = logging.getLogger(__name__)
# Seuils par défaut
DEFAULT_MIN_FREE_VRAM_MB = 1000 # 1 GB minimum libre
DEFAULT_MAX_GPU_UTIL_PERCENT = 80 # GPU pas saturé à plus de 80%
DEFAULT_MAX_FOREIGN_PROCESSES = 5 # Alerte si trop de processus GPU
@dataclass
class GPUProcess:
"""Processus utilisant le GPU."""
pid: int
name: str
vram_mb: int
is_own: bool # True si c'est un processus rpa_vision_v3
@dataclass
class PreflightResult:
"""Résultat de la vérification machine."""
ready: bool
reason: Optional[str] = None
# État GPU
gpu_name: str = ""
total_vram_mb: int = 0
used_vram_mb: int = 0
free_vram_mb: int = 0
gpu_utilization_percent: int = 0
# Processus
gpu_processes: List[GPUProcess] = field(default_factory=list)
foreign_processes: List[GPUProcess] = field(default_factory=list)
# Avertissements (non-bloquants)
warnings: List[str] = field(default_factory=list)
def __str__(self) -> str:
status = "PRÊT" if self.ready else "PAS PRÊT"
lines = [
f"[GPU Preflight: {status}]",
f" GPU: {self.gpu_name}",
f" VRAM: {self.used_vram_mb}/{self.total_vram_mb} MB "
f"(libre: {self.free_vram_mb} MB)",
f" Utilisation GPU: {self.gpu_utilization_percent}%",
f" Processus GPU: {len(self.gpu_processes)} "
f"(dont {len(self.foreign_processes)} externes)",
]
if not self.ready:
lines.append(f" Raison: {self.reason}")
for w in self.warnings:
lines.append(f" ⚠ {w}")
if self.foreign_processes:
lines.append(" Processus externes:")
for p in self.foreign_processes:
lines.append(f" - PID {p.pid}: {p.name} ({p.vram_mb} MB)")
return "\n".join(lines)
def _get_gpu_info() -> Optional[dict]:
"""Récupère les infos GPU via nvidia-smi."""
try:
result = subprocess.run(
[
"nvidia-smi",
"--query-gpu=name,memory.total,memory.used,memory.free,utilization.gpu",
"--format=csv,noheader,nounits",
],
capture_output=True,
text=True,
timeout=5,
)
if result.returncode != 0:
return None
parts = [p.strip() for p in result.stdout.strip().split(",")]
if len(parts) < 5:
return None
return {
"name": parts[0],
"total_mb": int(parts[1]),
"used_mb": int(parts[2]),
"free_mb": int(parts[3]),
"utilization": int(parts[4]) if parts[4].isdigit() else 0,
}
except Exception as e:
logger.error(f"nvidia-smi échoué : {e}")
return None
def _get_gpu_processes() -> List[GPUProcess]:
"""Liste les processus utilisant le GPU."""
try:
result = subprocess.run(
[
"nvidia-smi",
"--query-compute-apps=pid,process_name,used_gpu_memory",
"--format=csv,noheader,nounits",
],
capture_output=True,
text=True,
timeout=5,
)
if result.returncode != 0:
return []
processes = []
for line in result.stdout.strip().split("\n"):
if not line.strip():
continue
parts = [p.strip() for p in line.split(",")]
if len(parts) < 3:
continue
pid = int(parts[0])
name = parts[1]
vram = int(parts[2]) if parts[2].strip().isdigit() else 0
is_own = "rpa_vision_v3" in name
processes.append(GPUProcess(
pid=pid,
name=name,
vram_mb=vram,
is_own=is_own,
))
return processes
except Exception as e:
logger.error(f"Impossible de lister les processus GPU : {e}")
return []
def check_machine_ready(
min_free_vram_mb: int = DEFAULT_MIN_FREE_VRAM_MB,
max_gpu_util_percent: int = DEFAULT_MAX_GPU_UTIL_PERCENT,
max_foreign_processes: int = DEFAULT_MAX_FOREIGN_PROCESSES,
) -> PreflightResult:
"""
Vérifie que la machine est prête pour un lancement GPU.
Args:
min_free_vram_mb: VRAM libre minimum requise (défaut: 1000 MB)
max_gpu_util_percent: Utilisation GPU max tolérée (défaut: 80%)
max_foreign_processes: Nombre max de processus externes avant alerte
Returns:
PreflightResult avec l'état détaillé
"""
result = PreflightResult(ready=True)
# 1. Vérifier que le GPU est accessible
gpu_info = _get_gpu_info()
if gpu_info is None:
result.ready = False
result.reason = "GPU inaccessible (nvidia-smi échoué)"
logger.warning(result.reason)
return result
result.gpu_name = gpu_info["name"]
result.total_vram_mb = gpu_info["total_mb"]
result.used_vram_mb = gpu_info["used_mb"]
result.free_vram_mb = gpu_info["free_mb"]
result.gpu_utilization_percent = gpu_info["utilization"]
# 2. Lister les processus GPU
result.gpu_processes = _get_gpu_processes()
result.foreign_processes = [p for p in result.gpu_processes if not p.is_own]
# 3. Vérifier VRAM libre
if result.free_vram_mb < min_free_vram_mb:
result.ready = False
result.reason = (
f"VRAM insuffisante : {result.free_vram_mb} MB libre "
f"(minimum requis : {min_free_vram_mb} MB)"
)
logger.warning(result.reason)
return result
# 4. Vérifier utilisation GPU
if result.gpu_utilization_percent > max_gpu_util_percent:
result.ready = False
result.reason = (
f"GPU surchargé : {result.gpu_utilization_percent}% "
f"(maximum toléré : {max_gpu_util_percent}%)"
)
logger.warning(result.reason)
return result
# 5. Avertissements (non-bloquants)
if len(result.foreign_processes) > max_foreign_processes:
result.warnings.append(
f"{len(result.foreign_processes)} processus externes sur le GPU"
)
foreign_vram = sum(p.vram_mb for p in result.foreign_processes)
if foreign_vram > result.total_vram_mb * 0.5:
result.warnings.append(
f"Processus externes utilisent {foreign_vram} MB "
f"({foreign_vram * 100 // result.total_vram_mb}% de la VRAM)"
)
if result.free_vram_mb < min_free_vram_mb * 2:
result.warnings.append(
f"VRAM libre ({result.free_vram_mb} MB) proche du seuil minimum"
)
if result.warnings:
for w in result.warnings:
logger.info(f"Preflight warning: {w}")
logger.info(
f"GPU preflight OK: {result.free_vram_mb} MB libre, "
f"{result.gpu_utilization_percent}% utilisation"
)
return result
def require_gpu_ready(
min_free_vram_mb: int = DEFAULT_MIN_FREE_VRAM_MB,
max_gpu_util_percent: int = DEFAULT_MAX_GPU_UTIL_PERCENT,
):
"""
Décorateur pytest — skip le test si le GPU n'est pas prêt.
Usage:
@require_gpu_ready(min_free_vram_mb=2000)
def test_heavy_gpu_operation():
...
"""
def decorator(func):
@functools.wraps(func)
def wrapper(*args, **kwargs):
result = check_machine_ready(
min_free_vram_mb=min_free_vram_mb,
max_gpu_util_percent=max_gpu_util_percent,
)
if not result.ready:
pytest.skip(f"GPU pas prêt : {result.reason}")
return func(*args, **kwargs)
return wrapper
return decorator