rpa_vision_v3/core/config.py

"""
Configuration centralisée pour RPA Vision V3

Gestionnaire de configuration unifié qui élimine les incohérences entre composants.
Utilise les variables d'environnement avec des valeurs par défaut sensées.
En production, définir ENVIRONMENT=production pour forcer la configuration.
"""

import os
import logging
from pathlib import Path
from dataclasses import dataclass, field
from typing import Optional, List, Dict, Any, Callable
import json
from datetime import datetime

logger = logging.getLogger(__name__)


@dataclass
class ValidationError:
    """Erreur de validation de configuration"""
    field: str
    value: Any
    message: str
    severity: str = "error"  # error, warning


@dataclass
class SystemConfig:
    """Configuration système unifiée - Point central de toute configuration"""
    # Chemins unifiés
    base_path: Path = field(default_factory=lambda: Path.cwd())
    data_path: Path = field(default_factory=lambda: Path("data"))
    logs_path: Path = field(default_factory=lambda: Path("logs"))
    
    # Services
    api_host: str = "0.0.0.0"
    api_port: int = 8000
    dashboard_host: str = "0.0.0.0"
    dashboard_port: int = 5001
    worker_threads: int = 4
    
    # Base de données
    sessions_path: Path = field(default_factory=lambda: Path("data/sessions"))
    workflows_path: Path = field(default_factory=lambda: Path("data/workflows"))
    embeddings_path: Path = field(default_factory=lambda: Path("data/embeddings"))
    faiss_index_path: Path = field(default_factory=lambda: Path("data/faiss_index"))
    screenshots_path: Path = field(default_factory=lambda: Path("data/screenshots"))
    training_path: Path = field(default_factory=lambda: Path("data/training"))
    uploads_path: Path = field(default_factory=lambda: Path("data/training/uploads"))
    
    # Sécurité
    secret_key: str = "dev_secret_key_not_for_production"
    encryption_password: str = "dev_default_key_not_for_production"
    auth_enabled: bool = True
    allowed_origins: List[str] = field(default_factory=lambda: ["*"])
    
    # Monitoring
    health_check_interval: int = 30
    metrics_enabled: bool = True
    
    # Environment
    environment: str = "development"
    debug: bool = False
    
    # Models
    clip_model: str = "ViT-B-32"
    clip_pretrained: str = "openai"
    clip_device: str = "cpu"
    vlm_model: str = "qwen3-vl:8b"
    vlm_endpoint: str = "http://localhost:11434"
    owl_model: str = "google/owlv2-base-patch16-ensemble"
    owl_confidence_threshold: float = 0.1
    
    # FAISS
    faiss_dimensions: int = 512
    faiss_index_type: str = "Flat"
    faiss_metric: str = "cosine"
    faiss_nprobe: int = 8
    faiss_auto_optimize: bool = True
    faiss_migration_threshold: int = 10000
    
    # GPU
    gpu_idle_timeout_seconds: int = 300
    gpu_vram_threshold_mb: int = 1024
    gpu_max_load_retries: int = 3
    gpu_load_timeout_seconds: int = 30
    gpu_unload_timeout_seconds: int = 5
    
    def __post_init__(self):
        """Normalise les chemins après initialisation"""
        # Convertir tous les chemins relatifs en absolus basés sur base_path
        for field_name in self.__dataclass_fields__:
            value = getattr(self, field_name)
            if isinstance(value, Path) and not value.is_absolute():
                if field_name != 'base_path':
                    setattr(self, field_name, self.base_path / value)
    
    def ensure_directories(self):
        """Crée tous les répertoires nécessaires avec les bonnes permissions"""
        directories = [
            self.data_path, self.logs_path, self.sessions_path,
            self.workflows_path, self.embeddings_path, self.faiss_index_path,
            self.screenshots_path, self.training_path, self.uploads_path
        ]
        
        for directory in directories:
            try:
                directory.mkdir(parents=True, exist_ok=True)
                # Vérifier les permissions d'écriture
                test_file = directory / ".write_test"
                test_file.touch()
                test_file.unlink()
                logger.debug(f"Directory created/verified: {directory}")
            except Exception as e:
                logger.error(f"Failed to create/verify directory {directory}: {e}")
                raise


class ConfigurationManager:
    """Gestionnaire centralisé de configuration - Point unique de vérité"""
    
    def __init__(self):
        self._config: Optional[SystemConfig] = None
        self._config_watchers: List[Callable[[SystemConfig], None]] = []
        self._last_loaded: Optional[datetime] = None
    
    def load_config(self) -> SystemConfig:
        """Charge la configuration depuis les variables d'environnement et fichiers"""
        try:
            # Charger depuis les variables d'environnement
            config = self._load_from_env()
            
            # Charger depuis le fichier de config si présent
            config_file = Path(".env")
            if config_file.exists():
                config = self._merge_from_file(config, config_file)
            
            # Valider la configuration
            validation_errors = self.validate_config(config)
            if any(error.severity == "error" for error in validation_errors):
                error_messages = [f"{error.field}: {error.message}" 
                                for error in validation_errors if error.severity == "error"]
                raise ValueError(f"Configuration validation failed: {'; '.join(error_messages)}")
            
            # Afficher les warnings
            warnings = [error for error in validation_errors if error.severity == "warning"]
            for warning in warnings:
                logger.warning(f"Configuration warning - {warning.field}: {warning.message}")
            
            # Créer les répertoires
            config.ensure_directories()
            
            self._config = config
            self._last_loaded = datetime.now()
            
            # Notifier les watchers
            for watcher in self._config_watchers:
                try:
                    watcher(config)
                except Exception as e:
                    logger.error(f"Configuration watcher failed: {e}")
            
            logger.info(f"Configuration loaded successfully (environment: {config.environment})")
            return config
            
        except Exception as e:
            logger.error(f"Failed to load configuration: {e}")
            raise
    
    def _load_from_env(self) -> SystemConfig:
        """Charge la configuration depuis les variables d'environnement"""
        base_path = Path(os.getenv("BASE_PATH", Path.cwd()))
        
        return SystemConfig(
            # Chemins
            base_path=base_path,
            data_path=Path(os.getenv("DATA_PATH", "data")),
            logs_path=Path(os.getenv("LOGS_PATH", "logs")),
            sessions_path=Path(os.getenv("SESSIONS_PATH", "data/sessions")),
            workflows_path=Path(os.getenv("WORKFLOWS_PATH", "data/workflows")),
            embeddings_path=Path(os.getenv("EMBEDDINGS_PATH", "data/embeddings")),
            faiss_index_path=Path(os.getenv("FAISS_INDEX_PATH", "data/faiss_index")),
            screenshots_path=Path(os.getenv("SCREENSHOTS_PATH", "data/screenshots")),
            training_path=Path(os.getenv("TRAINING_PATH", "data/training")),
            uploads_path=Path(os.getenv("UPLOADS_PATH", "data/training/uploads")),
            
            # Services
            api_host=os.getenv("API_HOST", "0.0.0.0"),
            api_port=int(os.getenv("API_PORT", "8000")),
            dashboard_host=os.getenv("DASHBOARD_HOST", "0.0.0.0"),
            dashboard_port=int(os.getenv("DASHBOARD_PORT", "5001")),
            worker_threads=int(os.getenv("WORKER_THREADS", "4")),
            
            # Sécurité
            secret_key=os.getenv("SECRET_KEY", "dev_secret_key_not_for_production"),
            encryption_password=os.getenv("ENCRYPTION_PASSWORD", "dev_default_key_not_for_production"),
            auth_enabled=os.getenv("AUTH_ENABLED", "true").lower() == "true",
            allowed_origins=os.getenv("ALLOWED_ORIGINS", "*").split(","),
            
            # Monitoring
            health_check_interval=int(os.getenv("HEALTH_CHECK_INTERVAL", "30")),
            metrics_enabled=os.getenv("METRICS_ENABLED", "true").lower() == "true",
            
            # Environment
            environment=os.getenv("ENVIRONMENT", "development"),
            debug=os.getenv("DEBUG", "false").lower() == "true",
            
            # Models
            clip_model=os.getenv("CLIP_MODEL", "ViT-B-32"),
            clip_pretrained=os.getenv("CLIP_PRETRAINED", "openai"),
            clip_device=os.getenv("CLIP_DEVICE", "cpu"),
            vlm_model=os.getenv("VLM_MODEL", "qwen3-vl:8b"),
            vlm_endpoint=os.getenv("VLM_ENDPOINT", "http://localhost:11434"),
            owl_model=os.getenv("OWL_MODEL", "google/owlv2-base-patch16-ensemble"),
            owl_confidence_threshold=float(os.getenv("OWL_CONFIDENCE_THRESHOLD", "0.1")),
            
            # FAISS
            faiss_dimensions=int(os.getenv("FAISS_DIMENSIONS", "512")),
            faiss_index_type=os.getenv("FAISS_INDEX_TYPE", "Flat"),
            faiss_metric=os.getenv("FAISS_METRIC", "cosine"),
            faiss_nprobe=int(os.getenv("FAISS_NPROBE", "8")),
            faiss_auto_optimize=os.getenv("FAISS_AUTO_OPTIMIZE", "true").lower() == "true",
            faiss_migration_threshold=int(os.getenv("FAISS_MIGRATION_THRESHOLD", "10000")),
            
            # GPU
            gpu_idle_timeout_seconds=int(os.getenv("GPU_IDLE_TIMEOUT", "300")),
            gpu_vram_threshold_mb=int(os.getenv("GPU_VRAM_THRESHOLD_MB", "1024")),
            gpu_max_load_retries=int(os.getenv("GPU_MAX_LOAD_RETRIES", "3")),
            gpu_load_timeout_seconds=int(os.getenv("GPU_LOAD_TIMEOUT", "30")),
            gpu_unload_timeout_seconds=int(os.getenv("GPU_UNLOAD_TIMEOUT", "5"))
        )
    
    def _merge_from_file(self, config: SystemConfig, config_file: Path) -> SystemConfig:
        """Merge configuration from file (future enhancement)"""
        # Pour l'instant, on utilise seulement les variables d'environnement
        # Cette méthode peut être étendue pour supporter les fichiers JSON/YAML
        return config
    
    def validate_config(self, config: SystemConfig) -> List[ValidationError]:
        """Valide la cohérence de la configuration"""
        errors = []
        
        # Validation de l'environnement de production
        if config.environment == "production":
            if config.secret_key == "dev_secret_key_not_for_production":
                errors.append(ValidationError(
                    "secret_key", config.secret_key,
                    "SECRET_KEY must be set in production environment"
                ))
            
            if config.encryption_password == "dev_default_key_not_for_production":
                errors.append(ValidationError(
                    "encryption_password", config.encryption_password,
                    "ENCRYPTION_PASSWORD must be set in production environment"
                ))
            
            if config.debug:
                errors.append(ValidationError(
                    "debug", config.debug,
                    "DEBUG should be false in production environment",
                    "warning"
                ))
        
        # Validation des ports
        if config.api_port == config.dashboard_port:
            errors.append(ValidationError(
                "ports", f"api:{config.api_port}, dashboard:{config.dashboard_port}",
                "API and Dashboard ports must be different"
            ))
        
        if not (1024 <= config.api_port <= 65535):
            errors.append(ValidationError(
                "api_port", config.api_port,
                "API port must be between 1024 and 65535"
            ))
        
        if not (1024 <= config.dashboard_port <= 65535):
            errors.append(ValidationError(
                "dashboard_port", config.dashboard_port,
                "Dashboard port must be between 1024 and 65535"
            ))
        
        # Validation des chemins
        try:
            config.base_path.resolve()
        except Exception as e:
            errors.append(ValidationError(
                "base_path", config.base_path,
                f"Invalid base path: {e}"
            ))
        
        # Validation des modèles
        if config.faiss_dimensions <= 0:
            errors.append(ValidationError(
                "faiss_dimensions", config.faiss_dimensions,
                "FAISS dimensions must be positive"
            ))
        
        if config.worker_threads <= 0:
            errors.append(ValidationError(
                "worker_threads", config.worker_threads,
                "Worker threads must be positive"
            ))
        
        # Validation des timeouts
        if config.health_check_interval <= 0:
            errors.append(ValidationError(
                "health_check_interval", config.health_check_interval,
                "Health check interval must be positive"
            ))
        
        return errors
    
    def apply_config(self, config: SystemConfig):
        """Applique une nouvelle configuration"""
        # Valider d'abord
        validation_errors = self.validate_config(config)
        if any(error.severity == "error" for error in validation_errors):
            error_messages = [f"{error.field}: {error.message}" 
                            for error in validation_errors if error.severity == "error"]
            raise ValueError(f"Configuration validation failed: {'; '.join(error_messages)}")
        
        # Créer les répertoires
        config.ensure_directories()
        
        # Appliquer la configuration
        old_config = self._config
        self._config = config
        self._last_loaded = datetime.now()
        
        # Notifier les watchers du changement
        for watcher in self._config_watchers:
            try:
                watcher(config)
            except Exception as e:
                logger.error(f"Configuration watcher failed during apply: {e}")
                # En cas d'erreur, restaurer l'ancienne configuration
                self._config = old_config
                raise
        
        logger.info("Configuration applied successfully")
    
    def watch_config_changes(self, callback: Callable[[SystemConfig], None]):
        """Enregistre un callback pour les changements de configuration"""
        self._config_watchers.append(callback)
        
        # Si on a déjà une configuration, appeler immédiatement le callback
        if self._config:
            try:
                callback(self._config)
            except Exception as e:
                logger.error(f"Configuration watcher failed during registration: {e}")
    
    def get_config(self) -> SystemConfig:
        """Récupère la configuration actuelle"""
        if self._config is None:
            return self.load_config()
        return self._config
    
    def reload_config(self) -> SystemConfig:
        """Recharge la configuration depuis les sources"""
        return self.load_config()


# Instance globale du gestionnaire de configuration
_config_manager: Optional[ConfigurationManager] = None


def get_configuration_manager() -> ConfigurationManager:
    """Récupère le gestionnaire de configuration global (singleton)"""
    global _config_manager
    if _config_manager is None:
        _config_manager = ConfigurationManager()
    return _config_manager


def get_config() -> SystemConfig:
    """Récupère la configuration système unifiée"""
    return get_configuration_manager().get_config()


def reload_config() -> SystemConfig:
    """Recharge la configuration système"""
    return get_configuration_manager().reload_config()


# Backward compatibility - Keep old classes for gradual migration
@dataclass
class ServerConfig:
    """Configuration du serveur - DEPRECATED: Use SystemConfig instead"""
    api_host: str = "0.0.0.0"
    api_port: int = 8000
    dashboard_host: str = "0.0.0.0"
    dashboard_port: int = 5001
    environment: str = "development"
    debug: bool = False
    
    @classmethod
    def from_env(cls) -> 'ServerConfig':
        logger.warning("ServerConfig is deprecated. Use SystemConfig via get_config() instead.")
        config = get_config()
        return cls(
            api_host=config.api_host,
            api_port=config.api_port,
            dashboard_host=config.dashboard_host,
            dashboard_port=config.dashboard_port,
            environment=config.environment,
            debug=config.debug
        )


@dataclass
class SecurityConfig:
    """Configuration de sécurité - DEPRECATED: Use SystemConfig instead"""
    encryption_password: Optional[str] = None
    secret_key: Optional[str] = None
    allowed_origins: List[str] = field(default_factory=lambda: ["*"])
    
    @classmethod
    def from_env(cls, environment: str = "development") -> 'SecurityConfig':
        logger.warning("SecurityConfig is deprecated. Use SystemConfig via get_config() instead.")
        config = get_config()
        return cls(
            encryption_password=config.encryption_password,
            secret_key=config.secret_key,
            allowed_origins=config.allowed_origins
        )


@dataclass
class ModelConfig:
    """Configuration des modèles ML - DEPRECATED: Use SystemConfig instead"""
    clip_model: str = "ViT-B-32"
    clip_pretrained: str = "openai"
    clip_device: str = "cpu"
    vlm_model: str = "qwen3-vl:8b"
    vlm_endpoint: str = "http://localhost:11434"
    owl_model: str = "google/owlv2-base-patch16-ensemble"
    owl_confidence_threshold: float = 0.1
    
    @classmethod
    def from_env(cls) -> 'ModelConfig':
        logger.warning("ModelConfig is deprecated. Use SystemConfig via get_config() instead.")
        config = get_config()
        return cls(
            clip_model=config.clip_model,
            clip_pretrained=config.clip_pretrained,
            clip_device=config.clip_device,
            vlm_model=config.vlm_model,
            vlm_endpoint=config.vlm_endpoint,
            owl_model=config.owl_model,
            owl_confidence_threshold=config.owl_confidence_threshold
        )


@dataclass
class PathConfig:
    """Configuration des chemins - DEPRECATED: Use SystemConfig instead"""
    data_path: Path = field(default_factory=lambda: Path("data"))
    models_path: Path = field(default_factory=lambda: Path("models"))
    logs_path: Path = field(default_factory=lambda: Path("logs"))
    uploads_path: Path = field(default_factory=lambda: Path("data/training/uploads"))
    sessions_path: Path = field(default_factory=lambda: Path("data/training/sessions"))
    
    @classmethod
    def from_env(cls) -> 'PathConfig':
        logger.warning("PathConfig is deprecated. Use SystemConfig via get_config() instead.")
        config = get_config()
        return cls(
            data_path=config.data_path,
            models_path=Path("models"),  # Not in SystemConfig yet
            logs_path=config.logs_path,
            uploads_path=config.uploads_path,
            sessions_path=config.sessions_path
        )
    
    def ensure_directories(self):
        """Crée tous les répertoires nécessaires"""
        config = get_config()
        config.ensure_directories()


@dataclass
class FAISSConfig:
    """Configuration FAISS - DEPRECATED: Use SystemConfig instead"""
    dimensions: int = 512
    index_type: str = "Flat"
    metric: str = "cosine"
    nprobe: int = 8
    auto_optimize: bool = True
    migration_threshold: int = 10000
    
    @classmethod
    def from_env(cls) -> 'FAISSConfig':
        logger.warning("FAISSConfig is deprecated. Use SystemConfig via get_config() instead.")
        config = get_config()
        return cls(
            dimensions=config.faiss_dimensions,
            index_type=config.faiss_index_type,
            metric=config.faiss_metric,
            nprobe=config.faiss_nprobe,
            auto_optimize=config.faiss_auto_optimize,
            migration_threshold=config.faiss_migration_threshold
        )


@dataclass
class GPUResourceConfig:
    """Configuration for GPU resource management - DEPRECATED: Use SystemConfig instead"""
    ollama_endpoint: str = "http://localhost:11434"
    vlm_model: str = "qwen3-vl:8b"
    clip_model: str = "ViT-B-32"
    idle_timeout_seconds: int = 300
    vram_threshold_for_clip_gpu_mb: int = 1024
    max_load_retries: int = 3
    load_timeout_seconds: int = 30
    unload_timeout_seconds: int = 5
    
    @classmethod
    def from_env(cls) -> 'GPUResourceConfig':
        logger.warning("GPUResourceConfig is deprecated. Use SystemConfig via get_config() instead.")
        config = get_config()
        return cls(
            ollama_endpoint=config.vlm_endpoint,
            vlm_model=config.vlm_model,
            clip_model=config.clip_model,
            idle_timeout_seconds=config.gpu_idle_timeout_seconds,
            vram_threshold_for_clip_gpu_mb=config.gpu_vram_threshold_mb,
            max_load_retries=config.gpu_max_load_retries,
            load_timeout_seconds=config.gpu_load_timeout_seconds,
            unload_timeout_seconds=config.gpu_unload_timeout_seconds
        )


@dataclass
class AppConfig:
    """Configuration globale de l'application - DEPRECATED: Use SystemConfig instead"""
    server: ServerConfig
    security: SecurityConfig
    models: ModelConfig
    paths: PathConfig
    faiss: FAISSConfig
    gpu: GPUResourceConfig = field(default_factory=GPUResourceConfig)
    
    @classmethod
    def from_env(cls) -> 'AppConfig':
        """Charge la configuration depuis les variables d'environnement"""
        logger.warning("AppConfig is deprecated. Use SystemConfig via get_config() instead.")
        return cls(
            server=ServerConfig.from_env(),
            security=SecurityConfig.from_env(),
            models=ModelConfig.from_env(),
            paths=PathConfig.from_env(),
            faiss=FAISSConfig.from_env(),
            gpu=GPUResourceConfig.from_env()
        )


# Exemple de fichier .env
# Exemple de fichier .env
ENV_TEMPLATE = """
# RPA Vision V3 Configuration Unifiée
# Copier ce fichier en .env et modifier les valeurs

# Environment
ENVIRONMENT=development  # development, staging, production
DEBUG=false

# Base Path
BASE_PATH=/opt/rpa_vision_v3  # Production path, use . for development

# Server
API_HOST=0.0.0.0
API_PORT=8000
DASHBOARD_HOST=0.0.0.0
DASHBOARD_PORT=5001
WORKER_THREADS=4

# Security (REQUIRED in production)
# SECRET_KEY=your_secure_secret_key_here
# ENCRYPTION_PASSWORD=your_secure_password_here
AUTH_ENABLED=true
# ALLOWED_ORIGINS=https://yourdomain.com,https://api.yourdomain.com

# Data Paths (relative to BASE_PATH)
DATA_PATH=data
LOGS_PATH=logs
SESSIONS_PATH=data/sessions
WORKFLOWS_PATH=data/workflows
EMBEDDINGS_PATH=data/embeddings
FAISS_INDEX_PATH=data/faiss_index
SCREENSHOTS_PATH=data/screenshots
TRAINING_PATH=data/training
UPLOADS_PATH=data/training/uploads

# Models
CLIP_MODEL=ViT-B-32
CLIP_PRETRAINED=openai
CLIP_DEVICE=cpu
VLM_MODEL=qwen3-vl:8b
VLM_ENDPOINT=http://localhost:11434
OWL_MODEL=google/owlv2-base-patch16-ensemble
OWL_CONFIDENCE_THRESHOLD=0.1

# FAISS
FAISS_DIMENSIONS=512
FAISS_INDEX_TYPE=Flat
FAISS_METRIC=cosine
FAISS_NPROBE=8
FAISS_AUTO_OPTIMIZE=true
FAISS_MIGRATION_THRESHOLD=10000

# GPU
GPU_IDLE_TIMEOUT=300
GPU_VRAM_THRESHOLD_MB=1024
GPU_MAX_LOAD_RETRIES=3
GPU_LOAD_TIMEOUT=30
GPU_UNLOAD_TIMEOUT=5

# Monitoring
HEALTH_CHECK_INTERVAL=30
METRICS_ENABLED=true
"""


if __name__ == "__main__":
    # Test de la configuration unifiée
    config_manager = get_configuration_manager()
    config = config_manager.load_config()
    
    print("=== Configuration Système Unifiée ===")
    print(f"Environment: {config.environment}")
    print(f"Base Path: {config.base_path}")
    print(f"Data Path: {config.data_path}")
    print(f"API Port: {config.api_port}")
    print(f"Dashboard Port: {config.dashboard_port}")
    print(f"Sessions Path: {config.sessions_path}")
    print(f"CLIP Model: {config.clip_model}")
    print(f"Auth Enabled: {config.auth_enabled}")
    
    # Test de validation
    validation_errors = config_manager.validate_config(config)
    if validation_errors:
        print("\n=== Validation Errors/Warnings ===")
        for error in validation_errors:
            print(f"[{error.severity.upper()}] {error.field}: {error.message}")
    else:
        print("\n✅ Configuration validation passed")
    
    print(f"\n✅ Configuration Manager initialized successfully")