""" UIElement - Couche 2 : Détection Sémantique Représente un élément d'interface détecté avec : - Type sémantique (button, text_input, etc.) - Rôle sémantique (primary_action, cancel, etc.) - Embeddings duaux (image + texte) - Features visuelles Tâche 4 : Contrats de données standardisés avec Pydantic - BBox : Format exclusif (x, y, width, height) - IDs : Strings uniquement - Validation automatique des données """ from dataclasses import dataclass, field from typing import Dict, List, Optional, Tuple, Any from pathlib import Path import json from .base_models import BBox, StandardID, DataConverter @dataclass class UIElementEmbeddings: """Embeddings duaux pour un élément UI""" image: Optional[Dict[str, Any]] = None # Embedding de l'image croppée text: Optional[Dict[str, Any]] = None # Embedding du texte détecté def to_dict(self) -> Dict[str, Any]: return { "image": self.image, "text": self.text } @classmethod def from_dict(cls, data: Dict[str, Any]) -> 'UIElementEmbeddings': return cls( image=data.get("image"), text=data.get("text") ) @dataclass class VisualFeatures: """Features visuelles d'un élément UI""" dominant_color: str has_icon: bool shape: str # "rectangle", "circle", "rounded_rectangle" size_category: str # "small", "medium", "large" def to_dict(self) -> Dict[str, Any]: return { "dominant_color": self.dominant_color, "has_icon": self.has_icon, "shape": self.shape, "size_category": self.size_category } @classmethod def from_dict(cls, data: Dict[str, Any]) -> 'VisualFeatures': return cls( dominant_color=data["dominant_color"], has_icon=data["has_icon"], shape=data["shape"], size_category=data["size_category"] ) @dataclass class UIElement: """ Élément d'interface détecté avec type et rôle sémantiques Types supportés: - button, text_input, checkbox, radio, dropdown - tab, link, icon, table_row, menu_item Rôles sémantiques: - primary_action, cancel, submit, form_input - search_field, navigation, etc. Tâche 4 : Contrats standardisés - element_id : StandardID (string uniquement) - bbox : BBox standardisée (x, y, width, height) """ element_id: str # Migré vers StandardID via DataConverter type: str # Type sémantique role: str # Rôle sémantique bbox: BBox # BBox standardisée (x, y, width, height) center: Tuple[int, int] # (x, y) - calculé depuis bbox label: str label_confidence: float embeddings: UIElementEmbeddings visual_features: VisualFeatures tags: List[str] = field(default_factory=list) confidence: float = 0.0 metadata: Dict[str, Any] = field(default_factory=dict) def __post_init__(self): """Valider les données après initialisation""" # Migrer element_id vers StandardID si nécessaire if not isinstance(self.element_id, str): self.element_id = str(DataConverter.ensure_id(self.element_id)) # Migrer bbox vers BBox si nécessaire if not isinstance(self.bbox, BBox): self.bbox = DataConverter.ensure_bbox(self.bbox) # Recalculer center depuis bbox si nécessaire bbox_center = self.bbox.center() if self.center != bbox_center: self.center = bbox_center # Valider confidence entre 0 et 1 if not 0.0 <= self.confidence <= 1.0: raise ValueError(f"Confidence must be between 0 and 1, got {self.confidence}") if not 0.0 <= self.label_confidence <= 1.0: raise ValueError(f"Label confidence must be between 0 and 1, got {self.label_confidence}") @classmethod def create_with_bbox_tuple(cls, element_id: str, type: str, role: str, bbox_tuple: Tuple[int, int, int, int], **kwargs) -> 'UIElement': """ Méthode de compatibilité pour créer UIElement avec bbox tuple Args: bbox_tuple: (x, y, width, height) """ bbox = BBox.from_tuple(bbox_tuple) center = bbox.center() return cls( element_id=element_id, type=type, role=role, bbox=bbox, center=center, **kwargs ) def to_dict(self) -> Dict[str, Any]: """Sérialiser en JSON""" return { "element_id": self.element_id, "type": self.type, "role": self.role, "bbox": self.bbox.dict(), # BBox Pydantic serialization "center": list(self.center), "label": self.label, "label_confidence": self.label_confidence, "embeddings": self.embeddings.to_dict(), "visual_features": self.visual_features.to_dict(), "tags": self.tags, "confidence": self.confidence, "metadata": self.metadata } @classmethod def from_dict(cls, data: Dict[str, Any]) -> 'UIElement': """Désérialiser depuis JSON avec migration automatique""" # Migrer les données vers les nouveaux contrats migrated_data = DataConverter.migrate_bbox_dict(data, ['bbox']) migrated_data = DataConverter.migrate_id_dict(migrated_data, ['element_id']) embeddings = UIElementEmbeddings.from_dict(migrated_data["embeddings"]) visual_features = VisualFeatures.from_dict(migrated_data["visual_features"]) # Gérer bbox - peut être dict Pydantic ou tuple legacy bbox_data = migrated_data["bbox"] if isinstance(bbox_data, dict): bbox = BBox(**bbox_data) else: bbox = DataConverter.ensure_bbox(bbox_data) # Gérer center - calculer depuis bbox si nécessaire center_data = migrated_data.get("center") if center_data: center = tuple(center_data) else: center = bbox.center() return cls( element_id=migrated_data["element_id"], type=migrated_data["type"], role=migrated_data["role"], bbox=bbox, center=center, label=migrated_data["label"], label_confidence=migrated_data["label_confidence"], embeddings=embeddings, visual_features=visual_features, tags=migrated_data.get("tags", []), confidence=migrated_data.get("confidence", 0.0), metadata=migrated_data.get("metadata", {}) ) def to_json(self) -> str: """Sérialiser en JSON string""" return json.dumps(self.to_dict(), indent=2) @classmethod def from_json(cls, json_str: str) -> 'UIElement': """Désérialiser depuis JSON string""" data = json.loads(json_str) return cls.from_dict(data) # Types d'éléments supportés UI_ELEMENT_TYPES = [ "button", "text_input", "checkbox", "radio", "dropdown", "tab", "link", "icon", "table_row", "menu_item", "label", "image", "container" ] # Rôles sémantiques supportés UI_ELEMENT_ROLES = [ "primary_action", "secondary_action", "cancel", "submit", "form_input", "search_field", "navigation", "data_display", "selectable_item", "action_trigger", "status_indicator", "delete_action", "dangerous_action" ]