cf495dd82f
Refonte majeure du système Agent Chat et ajout de nombreux modules : - Chat unifié : suppression du dual Workflows/Agent Libre, tout passe par /api/chat avec résolution en 3 niveaux (workflow → geste → "montre-moi") - GestureCatalog : 38 raccourcis clavier universels Windows avec matching sémantique, substitution automatique dans les replays, et endpoint /api/gestures - Mode Copilot : exécution pas-à-pas des workflows avec validation humaine via WebSocket (approve/skip/abort) avant chaque action - Léa UI (agent_v0/lea_ui/) : interface PyQt5 pour Windows avec overlay transparent pour feedback visuel pendant le replay - Data Extraction (core/extraction/) : moteur d'extraction visuelle de données (OCR + VLM → SQLite), avec schémas YAML et export CSV/Excel - ReplayVerifier (agent_v0/server_v1/) : vérification post-action par comparaison de screenshots, avec logique de retry (max 3) - IntentParser durci : meilleur fallback regex, type GREETING, patterns améliorés - Dashboard : nouvelles pages gestures, streaming, extractions - Tests : 63 tests GestureCatalog, 47 tests extraction, corrections tests existants - Dépréciation : /api/agent/plan et /api/agent/execute retournent HTTP 410, suppression du code hardcodé _plan_to_replay_actions Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
299 lines
10 KiB
Python
299 lines
10 KiB
Python
"""
|
|
Tests unitaires pour UIElement
|
|
|
|
Property 3: UIElement Detection Confidence Bounds
|
|
Validates: Requirements 3.6
|
|
"""
|
|
|
|
import pytest
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
# Ajouter le répertoire racine au path pour les imports
|
|
sys.path.insert(0, str(Path(__file__).parent.parent.parent))
|
|
|
|
from core.models.ui_element import (
|
|
UIElement,
|
|
UIElementEmbeddings,
|
|
VisualFeatures,
|
|
UI_ELEMENT_TYPES,
|
|
UI_ELEMENT_ROLES
|
|
)
|
|
|
|
|
|
class TestUIElementEmbeddings:
|
|
"""Tests pour UIElementEmbeddings"""
|
|
|
|
def test_create_embeddings(self):
|
|
"""Test création UIElementEmbeddings"""
|
|
embeddings = UIElementEmbeddings(
|
|
image={"provider": "openclip", "vector_id": "test_img.npy", "dimensions": 512},
|
|
text={"provider": "openclip", "vector_id": "test_txt.npy", "dimensions": 512}
|
|
)
|
|
|
|
assert embeddings.image is not None
|
|
assert embeddings.text is not None
|
|
|
|
def test_embeddings_serialization(self):
|
|
"""Test sérialisation/désérialisation UIElementEmbeddings"""
|
|
embeddings = UIElementEmbeddings(
|
|
image={"provider": "openclip", "vector_id": "test.npy", "dimensions": 512}
|
|
)
|
|
|
|
data = embeddings.to_dict()
|
|
embeddings2 = UIElementEmbeddings.from_dict(data)
|
|
|
|
assert embeddings2.image == embeddings.image
|
|
|
|
|
|
class TestVisualFeatures:
|
|
"""Tests pour VisualFeatures"""
|
|
|
|
def test_create_visual_features(self):
|
|
"""Test création VisualFeatures"""
|
|
features = VisualFeatures(
|
|
dominant_color="#4CAF50",
|
|
has_icon=False,
|
|
shape="rectangle",
|
|
size_category="medium"
|
|
)
|
|
|
|
assert features.dominant_color == "#4CAF50"
|
|
assert features.has_icon is False
|
|
assert features.shape == "rectangle"
|
|
|
|
def test_visual_features_serialization(self):
|
|
"""Test sérialisation/désérialisation VisualFeatures"""
|
|
features = VisualFeatures(
|
|
dominant_color="#4CAF50",
|
|
has_icon=True,
|
|
shape="rounded_rectangle",
|
|
size_category="large"
|
|
)
|
|
|
|
data = features.to_dict()
|
|
features2 = VisualFeatures.from_dict(data)
|
|
|
|
assert features2.dominant_color == features.dominant_color
|
|
assert features2.has_icon == features.has_icon
|
|
assert features2.shape == features.shape
|
|
assert features2.size_category == features.size_category
|
|
|
|
|
|
class TestUIElement:
|
|
"""Tests pour UIElement"""
|
|
|
|
def create_test_ui_element(self, confidence: float = 0.94) -> UIElement:
|
|
"""Helper pour créer un UIElement de test"""
|
|
embeddings = UIElementEmbeddings(
|
|
image={"provider": "openclip", "vector_id": "test_img.npy", "dimensions": 512},
|
|
text={"provider": "openclip", "vector_id": "test_txt.npy", "dimensions": 512}
|
|
)
|
|
|
|
visual_features = VisualFeatures(
|
|
dominant_color="#4CAF50",
|
|
has_icon=False,
|
|
shape="rectangle",
|
|
size_category="medium"
|
|
)
|
|
|
|
return UIElement(
|
|
element_id="el_btn_001",
|
|
type="button",
|
|
role="primary_action",
|
|
bbox=(100, 200, 150, 40),
|
|
center=(175, 220),
|
|
label="Submit",
|
|
label_confidence=0.96,
|
|
embeddings=embeddings,
|
|
visual_features=visual_features,
|
|
tags=["action", "primary"],
|
|
confidence=confidence
|
|
)
|
|
|
|
def test_create_ui_element(self):
|
|
"""Test création UIElement"""
|
|
element = self.create_test_ui_element()
|
|
|
|
assert element.element_id == "el_btn_001"
|
|
assert element.type == "button"
|
|
assert element.role == "primary_action"
|
|
assert element.label == "Submit"
|
|
assert element.confidence == 0.94
|
|
|
|
def test_ui_element_bbox(self):
|
|
"""Test bbox et center"""
|
|
element = self.create_test_ui_element()
|
|
|
|
assert element.bbox.to_tuple() == (100, 200, 150, 40)
|
|
assert element.center == (175, 220)
|
|
|
|
def test_ui_element_confidence_validation(self):
|
|
"""
|
|
Property 3: UIElement Detection Confidence Bounds
|
|
|
|
Pour tout UIElement, le score de confiance doit être entre 0.0 et 1.0
|
|
|
|
Validates: Requirements 3.6
|
|
"""
|
|
# Confidence valide
|
|
element = self.create_test_ui_element(confidence=0.94)
|
|
assert 0.0 <= element.confidence <= 1.0
|
|
|
|
# Confidence à 0
|
|
element = self.create_test_ui_element(confidence=0.0)
|
|
assert element.confidence == 0.0
|
|
|
|
# Confidence à 1
|
|
element = self.create_test_ui_element(confidence=1.0)
|
|
assert element.confidence == 1.0
|
|
|
|
# Confidence invalide (> 1)
|
|
with pytest.raises(ValueError, match="Confidence must be between 0 and 1"):
|
|
self.create_test_ui_element(confidence=1.5)
|
|
|
|
# Confidence invalide (< 0)
|
|
with pytest.raises(ValueError, match="Confidence must be between 0 and 1"):
|
|
self.create_test_ui_element(confidence=-0.1)
|
|
|
|
def test_ui_element_to_dict(self):
|
|
"""Test sérialisation to_dict"""
|
|
element = self.create_test_ui_element()
|
|
|
|
data = element.to_dict()
|
|
|
|
assert data["element_id"] == "el_btn_001"
|
|
assert data["type"] == "button"
|
|
assert data["role"] == "primary_action"
|
|
assert data["label"] == "Submit"
|
|
assert data["confidence"] == 0.94
|
|
assert "embeddings" in data
|
|
assert "visual_features" in data
|
|
|
|
def test_ui_element_from_dict(self):
|
|
"""Test désérialisation from_dict"""
|
|
element = self.create_test_ui_element()
|
|
|
|
data = element.to_dict()
|
|
element2 = UIElement.from_dict(data)
|
|
|
|
assert element2.element_id == element.element_id
|
|
assert element2.type == element.type
|
|
assert element2.role == element.role
|
|
assert element2.bbox == element.bbox
|
|
assert element2.confidence == element.confidence
|
|
|
|
def test_ui_element_round_trip(self):
|
|
"""Test round trip sérialisation/désérialisation"""
|
|
element = self.create_test_ui_element()
|
|
|
|
data = element.to_dict()
|
|
element2 = UIElement.from_dict(data)
|
|
|
|
# Vérifier tous les champs
|
|
assert element2.element_id == element.element_id
|
|
assert element2.type == element.type
|
|
assert element2.role == element.role
|
|
assert element2.bbox == element.bbox
|
|
assert element2.center == element.center
|
|
assert element2.label == element.label
|
|
assert element2.label_confidence == element.label_confidence
|
|
assert element2.confidence == element.confidence
|
|
assert element2.tags == element.tags
|
|
|
|
# Vérifier embeddings
|
|
assert element2.embeddings.image == element.embeddings.image
|
|
assert element2.embeddings.text == element.embeddings.text
|
|
|
|
# Vérifier visual features
|
|
assert element2.visual_features.dominant_color == element.visual_features.dominant_color
|
|
assert element2.visual_features.has_icon == element.visual_features.has_icon
|
|
|
|
def test_ui_element_to_json_string(self):
|
|
"""Test sérialisation to_json (string)"""
|
|
element = self.create_test_ui_element()
|
|
|
|
json_str = element.to_json()
|
|
assert isinstance(json_str, str)
|
|
|
|
# Vérifier que c'est du JSON valide
|
|
data = json.loads(json_str)
|
|
assert data["element_id"] == "el_btn_001"
|
|
|
|
def test_ui_element_from_json_string(self):
|
|
"""Test désérialisation from_json (string)"""
|
|
element = self.create_test_ui_element()
|
|
|
|
json_str = element.to_json()
|
|
element2 = UIElement.from_json(json_str)
|
|
|
|
assert element2.element_id == element.element_id
|
|
assert element2.confidence == element.confidence
|
|
|
|
def test_ui_element_with_metadata(self):
|
|
"""Test UIElement avec métadonnées"""
|
|
element = self.create_test_ui_element()
|
|
element.metadata = {"detection_method": "qwen_vl", "detection_time_ms": 45}
|
|
|
|
data = element.to_dict()
|
|
element2 = UIElement.from_dict(data)
|
|
|
|
assert element2.metadata["detection_method"] == "qwen_vl"
|
|
assert element2.metadata["detection_time_ms"] == 45
|
|
|
|
def test_ui_element_types_defined(self):
|
|
"""Test que les types UI sont définis"""
|
|
assert "button" in UI_ELEMENT_TYPES
|
|
assert "text_input" in UI_ELEMENT_TYPES
|
|
assert "checkbox" in UI_ELEMENT_TYPES
|
|
assert len(UI_ELEMENT_TYPES) > 0
|
|
|
|
def test_ui_element_roles_defined(self):
|
|
"""Test que les rôles UI sont définis"""
|
|
assert "primary_action" in UI_ELEMENT_ROLES
|
|
assert "cancel" in UI_ELEMENT_ROLES
|
|
assert "form_input" in UI_ELEMENT_ROLES
|
|
assert len(UI_ELEMENT_ROLES) > 0
|
|
|
|
def test_ui_element_different_types(self):
|
|
"""Test création d'éléments de différents types"""
|
|
embeddings = UIElementEmbeddings()
|
|
visual_features = VisualFeatures(
|
|
dominant_color="#FFFFFF",
|
|
has_icon=False,
|
|
shape="rectangle",
|
|
size_category="small"
|
|
)
|
|
|
|
# Button
|
|
button = UIElement(
|
|
element_id="el_001",
|
|
type="button",
|
|
role="primary_action",
|
|
bbox=(0, 0, 100, 30),
|
|
center=(50, 15),
|
|
label="Click",
|
|
label_confidence=0.9,
|
|
embeddings=embeddings,
|
|
visual_features=visual_features,
|
|
confidence=0.9
|
|
)
|
|
assert button.type == "button"
|
|
|
|
# Text input
|
|
text_input = UIElement(
|
|
element_id="el_002",
|
|
type="text_input",
|
|
role="form_input",
|
|
bbox=(0, 0, 200, 30),
|
|
center=(100, 15),
|
|
label="Name",
|
|
label_confidence=0.85,
|
|
embeddings=embeddings,
|
|
visual_features=visual_features,
|
|
confidence=0.85
|
|
)
|
|
assert text_input.type == "text_input"
|
|
assert text_input.role == "form_input"
|