Initial commit
This commit is contained in:
Dom
379
geniusia2/core/models.py
Normal file
379
geniusia2/core/models.py
Normal file
@@ -0,0 +1,379 @@
|
||||
"""
|
||||
Modèles de données pour RPA Vision V2
|
||||
Contient les dataclasses pour TaskProfile, Action et Detection
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass, field, asdict
|
||||
from datetime import datetime
|
||||
from typing import List, Dict, Any, Tuple, Optional
|
||||
import json
|
||||
import numpy as np
|
||||
|
||||
|
||||
@dataclass
|
||||
class Detection:
|
||||
"""
|
||||
Représente une détection d'élément UI par un modèle de vision
|
||||
|
||||
Attributes:
|
||||
label: Nom/label de l'élément détecté
|
||||
confidence: Score de confiance de la détection (0-1)
|
||||
bbox: Bounding box (x, y, width, height) en pixels
|
||||
embedding: Embedding visuel 512-d de l'élément
|
||||
model_source: Modèle ayant effectué la détection ("owl-v2", "dino", "yolo")
|
||||
roi_image: Image de la région d'intérêt (optionnel)
|
||||
metadata: Métadonnées additionnelles du modèle
|
||||
"""
|
||||
label: str
|
||||
confidence: float
|
||||
bbox: Tuple[int, int, int, int]
|
||||
embedding: np.ndarray
|
||||
model_source: str
|
||||
roi_image: Optional[np.ndarray] = None
|
||||
metadata: Dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""
|
||||
Convertit la détection en dictionnaire pour sérialisation
|
||||
Note: Les arrays numpy ne sont pas sérialisés directement
|
||||
"""
|
||||
return {
|
||||
"label": self.label,
|
||||
"confidence": float(self.confidence),
|
||||
"bbox": list(self.bbox),
|
||||
"model_source": self.model_source,
|
||||
"metadata": self.metadata,
|
||||
# embedding et roi_image sont exclus car non JSON-sérialisables
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any], embedding: Optional[np.ndarray] = None) -> 'Detection':
|
||||
"""
|
||||
Crée une instance Detection depuis un dictionnaire
|
||||
|
||||
Args:
|
||||
data: Dictionnaire contenant les données de détection
|
||||
embedding: Embedding numpy (doit être fourni séparément)
|
||||
"""
|
||||
return cls(
|
||||
label=data["label"],
|
||||
confidence=data["confidence"],
|
||||
bbox=tuple(data["bbox"]),
|
||||
embedding=embedding if embedding is not None else np.array([]),
|
||||
model_source=data["model_source"],
|
||||
metadata=data.get("metadata", {})
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Action:
|
||||
"""
|
||||
Représente une action UI effectuée ou suggérée
|
||||
|
||||
Attributes:
|
||||
action_type: Type d'action ("click", "type", "scroll", "wait")
|
||||
target_element: Nom de l'élément cible
|
||||
bbox: Bounding box de l'élément cible
|
||||
confidence: Score de confiance pour cette action
|
||||
embedding: Embedding visuel de l'élément cible
|
||||
timestamp: Horodatage de l'action
|
||||
window_title: Titre de la fenêtre où l'action est effectuée
|
||||
parameters: Paramètres additionnels (ex: texte à taper, direction de scroll)
|
||||
result: Résultat de l'exécution ("success", "failed", "pending")
|
||||
"""
|
||||
action_type: str
|
||||
target_element: str
|
||||
bbox: Tuple[int, int, int, int]
|
||||
confidence: float
|
||||
embedding: np.ndarray
|
||||
timestamp: datetime
|
||||
window_title: str
|
||||
parameters: Dict[str, Any] = field(default_factory=dict)
|
||||
result: str = "pending"
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""
|
||||
Convertit l'action en dictionnaire pour sérialisation
|
||||
"""
|
||||
return {
|
||||
"action_type": self.action_type,
|
||||
"target_element": self.target_element,
|
||||
"bbox": list(self.bbox),
|
||||
"confidence": float(self.confidence),
|
||||
"timestamp": self.timestamp.isoformat(),
|
||||
"window_title": self.window_title,
|
||||
"parameters": self.parameters,
|
||||
"result": self.result,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any], embedding: Optional[np.ndarray] = None) -> 'Action':
|
||||
"""
|
||||
Crée une instance Action depuis un dictionnaire
|
||||
|
||||
Args:
|
||||
data: Dictionnaire contenant les données d'action
|
||||
embedding: Embedding numpy (doit être fourni séparément)
|
||||
"""
|
||||
return cls(
|
||||
action_type=data["action_type"],
|
||||
target_element=data["target_element"],
|
||||
bbox=tuple(data["bbox"]),
|
||||
confidence=data["confidence"],
|
||||
embedding=embedding if embedding is not None else np.array([]),
|
||||
timestamp=datetime.fromisoformat(data["timestamp"]),
|
||||
window_title=data["window_title"],
|
||||
parameters=data.get("parameters", {}),
|
||||
result=data.get("result", "pending"),
|
||||
)
|
||||
|
||||
def get_inverse_action(self) -> Optional['Action']:
|
||||
"""
|
||||
Retourne l'action inverse pour rollback (si applicable)
|
||||
"""
|
||||
# Cette méthode sera implémentée plus tard dans input_utils
|
||||
# Pour l'instant, retourne None
|
||||
return None
|
||||
|
||||
|
||||
@dataclass
|
||||
class TaskProfile:
|
||||
"""
|
||||
Profil d'une tâche apprise par le système
|
||||
|
||||
Attributes:
|
||||
task_id: Identifiant unique de la tâche
|
||||
task_name: Nom descriptif de la tâche
|
||||
mode: Mode opérationnel actuel ("shadow", "assist", "auto")
|
||||
observation_count: Nombre d'observations de cette tâche
|
||||
concordance_rate: Taux de concordance (0-1)
|
||||
confidence_score: Score de confiance global (0-1)
|
||||
correction_count: Nombre de corrections reçues
|
||||
last_execution: Horodatage de la dernière exécution
|
||||
window_whitelist: Liste des fenêtres autorisées pour cette tâche
|
||||
action_sequence: Séquence d'actions composant la tâche
|
||||
embeddings: Liste des embeddings visuels associés
|
||||
metadata: Métadonnées additionnelles
|
||||
execution_history: Historique des exécutions récentes
|
||||
"""
|
||||
task_id: str
|
||||
task_name: str
|
||||
mode: str = "shadow"
|
||||
observation_count: int = 0
|
||||
concordance_rate: float = 0.0
|
||||
confidence_score: float = 0.0
|
||||
correction_count: int = 0
|
||||
last_execution: Optional[datetime] = None
|
||||
window_whitelist: List[str] = field(default_factory=list)
|
||||
action_sequence: List[Action] = field(default_factory=list)
|
||||
embeddings: List[np.ndarray] = field(default_factory=list)
|
||||
metadata: Dict[str, Any] = field(default_factory=dict)
|
||||
execution_history: List[Dict[str, Any]] = field(default_factory=list)
|
||||
|
||||
def to_json(self) -> str:
|
||||
"""
|
||||
Sérialise le profil de tâche en JSON
|
||||
Note: Les embeddings numpy ne sont pas inclus dans le JSON
|
||||
"""
|
||||
data = {
|
||||
"task_id": self.task_id,
|
||||
"task_name": self.task_name,
|
||||
"mode": self.mode,
|
||||
"observation_count": self.observation_count,
|
||||
"concordance_rate": float(self.concordance_rate),
|
||||
"confidence_score": float(self.confidence_score),
|
||||
"correction_count": self.correction_count,
|
||||
"last_execution": self.last_execution.isoformat() if self.last_execution else None,
|
||||
"window_whitelist": self.window_whitelist,
|
||||
"action_sequence": [action.to_dict() for action in self.action_sequence],
|
||||
"metadata": self.metadata,
|
||||
"execution_history": self.execution_history,
|
||||
}
|
||||
return json.dumps(data, indent=2, ensure_ascii=False)
|
||||
|
||||
@classmethod
|
||||
def from_json(cls, json_str: str, embeddings: Optional[List[np.ndarray]] = None) -> 'TaskProfile':
|
||||
"""
|
||||
Crée une instance TaskProfile depuis une chaîne JSON
|
||||
|
||||
Args:
|
||||
json_str: Chaîne JSON contenant les données du profil
|
||||
embeddings: Liste d'embeddings numpy (doivent être fournis séparément)
|
||||
"""
|
||||
data = json.loads(json_str)
|
||||
|
||||
# Reconstruire les actions
|
||||
actions = [Action.from_dict(action_data) for action_data in data.get("action_sequence", [])]
|
||||
|
||||
return cls(
|
||||
task_id=data["task_id"],
|
||||
task_name=data["task_name"],
|
||||
mode=data.get("mode", "shadow"),
|
||||
observation_count=data.get("observation_count", 0),
|
||||
concordance_rate=data.get("concordance_rate", 0.0),
|
||||
confidence_score=data.get("confidence_score", 0.0),
|
||||
correction_count=data.get("correction_count", 0),
|
||||
last_execution=datetime.fromisoformat(data["last_execution"]) if data.get("last_execution") else None,
|
||||
window_whitelist=data.get("window_whitelist", []),
|
||||
action_sequence=actions,
|
||||
embeddings=embeddings if embeddings is not None else [],
|
||||
metadata=data.get("metadata", {}),
|
||||
execution_history=data.get("execution_history", []),
|
||||
)
|
||||
|
||||
def get_historical_performance(self) -> float:
|
||||
"""
|
||||
Calcule la performance historique basée sur les exécutions récentes
|
||||
|
||||
Returns:
|
||||
Score de performance (0-1)
|
||||
"""
|
||||
if not self.execution_history:
|
||||
return 0.0
|
||||
|
||||
# Calculer le taux de succès sur les exécutions récentes
|
||||
recent_executions = self.execution_history[-10:] # 10 dernières exécutions
|
||||
success_count = sum(1 for exec in recent_executions if exec.get("result") == "success")
|
||||
|
||||
return success_count / len(recent_executions) if recent_executions else 0.0
|
||||
|
||||
def add_execution(self, result: str, confidence: float, latency_ms: float):
|
||||
"""
|
||||
Ajoute une exécution à l'historique
|
||||
|
||||
Args:
|
||||
result: Résultat de l'exécution ("success", "failed")
|
||||
confidence: Score de confiance de l'exécution
|
||||
latency_ms: Latence en millisecondes
|
||||
"""
|
||||
execution = {
|
||||
"timestamp": datetime.now().isoformat(),
|
||||
"result": result,
|
||||
"confidence": confidence,
|
||||
"latency_ms": latency_ms,
|
||||
}
|
||||
self.execution_history.append(execution)
|
||||
self.last_execution = datetime.now()
|
||||
|
||||
# Limiter l'historique aux 50 dernières exécutions
|
||||
if len(self.execution_history) > 50:
|
||||
self.execution_history = self.execution_history[-50:]
|
||||
|
||||
def update_concordance_rate(self, success: bool):
|
||||
"""
|
||||
Met à jour le taux de concordance basé sur le résultat d'une exécution
|
||||
|
||||
Args:
|
||||
success: True si l'exécution a réussi, False sinon
|
||||
"""
|
||||
# Utiliser une moyenne mobile pour le taux de concordance
|
||||
window_size = 10 # Fenêtre de 10 exécutions
|
||||
recent_executions = self.execution_history[-window_size:]
|
||||
|
||||
if recent_executions:
|
||||
success_count = sum(1 for exec in recent_executions if exec.get("result") == "success")
|
||||
self.concordance_rate = success_count / len(recent_executions)
|
||||
|
||||
def should_transition_to_auto(self, min_observations: int = 20, min_concordance: float = 0.95) -> bool:
|
||||
"""
|
||||
Vérifie si la tâche remplit les critères pour passer en mode Autopilot
|
||||
|
||||
Args:
|
||||
min_observations: Nombre minimum d'observations requises
|
||||
min_concordance: Taux de concordance minimum requis
|
||||
|
||||
Returns:
|
||||
True si les critères sont remplis
|
||||
"""
|
||||
return (self.observation_count >= min_observations and
|
||||
self.concordance_rate >= min_concordance)
|
||||
|
||||
def should_rollback_to_assist(self, min_confidence: float = 0.90) -> bool:
|
||||
"""
|
||||
Vérifie si la tâche doit être rétrogradée au mode Assisté
|
||||
|
||||
Args:
|
||||
min_confidence: Score de confiance minimum requis
|
||||
|
||||
Returns:
|
||||
True si la confiance est trop faible
|
||||
"""
|
||||
return self.mode == "auto" and self.confidence_score < min_confidence
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Tests basiques des modèles
|
||||
print("Test des modèles de données RPA Vision V2")
|
||||
print("=" * 50)
|
||||
|
||||
# Test Detection
|
||||
print("\n1. Test Detection:")
|
||||
detection = Detection(
|
||||
label="valider_button",
|
||||
confidence=0.93,
|
||||
bbox=(450, 320, 120, 40),
|
||||
embedding=np.random.rand(512),
|
||||
model_source="owl-v2",
|
||||
metadata={"class": "button"}
|
||||
)
|
||||
print(f" Label: {detection.label}")
|
||||
print(f" Confidence: {detection.confidence}")
|
||||
print(f" BBox: {detection.bbox}")
|
||||
print(f" Model: {detection.model_source}")
|
||||
det_dict = detection.to_dict()
|
||||
print(f" Dict keys: {list(det_dict.keys())}")
|
||||
|
||||
# Test Action
|
||||
print("\n2. Test Action:")
|
||||
action = Action(
|
||||
action_type="click",
|
||||
target_element="valider_button",
|
||||
bbox=(450, 320, 120, 40),
|
||||
confidence=0.95,
|
||||
embedding=np.random.rand(512),
|
||||
timestamp=datetime.now(),
|
||||
window_title="Dolibarr - Facturation",
|
||||
parameters={"button": "left"},
|
||||
result="success"
|
||||
)
|
||||
print(f" Type: {action.action_type}")
|
||||
print(f" Target: {action.target_element}")
|
||||
print(f" Window: {action.window_title}")
|
||||
print(f" Result: {action.result}")
|
||||
action_dict = action.to_dict()
|
||||
print(f" Dict keys: {list(action_dict.keys())}")
|
||||
|
||||
# Test TaskProfile
|
||||
print("\n3. Test TaskProfile:")
|
||||
task = TaskProfile(
|
||||
task_id="ouvrir_facture_001",
|
||||
task_name="Ouvrir Facture",
|
||||
mode="assist",
|
||||
observation_count=15,
|
||||
concordance_rate=0.87,
|
||||
confidence_score=0.92,
|
||||
window_whitelist=["Dolibarr - Facturation"],
|
||||
action_sequence=[action]
|
||||
)
|
||||
print(f" Task ID: {task.task_id}")
|
||||
print(f" Mode: {task.mode}")
|
||||
print(f" Observations: {task.observation_count}")
|
||||
print(f" Concordance: {task.concordance_rate:.2%}")
|
||||
print(f" Confidence: {task.confidence_score:.2%}")
|
||||
|
||||
# Test sérialisation JSON
|
||||
print("\n4. Test sérialisation JSON:")
|
||||
json_str = task.to_json()
|
||||
print(f" JSON length: {len(json_str)} chars")
|
||||
|
||||
# Test désérialisation
|
||||
task_restored = TaskProfile.from_json(json_str)
|
||||
print(f" Restored task ID: {task_restored.task_id}")
|
||||
print(f" Restored mode: {task_restored.mode}")
|
||||
|
||||
# Test méthodes de transition
|
||||
print("\n5. Test méthodes de transition:")
|
||||
print(f" Should transition to auto: {task.should_transition_to_auto()}")
|
||||
print(f" Should rollback to assist: {task.should_rollback_to_assist()}")
|
||||
|
||||
print("\n✓ Tous les tests basiques réussis!")
|
||||
Reference in New Issue
Block a user