feat(system): Ajouter gestionnaires backup et version pour Dashboard
BackupExporter (backup_exporter.py): - Export complet (workflows, correction packs, coaching sessions, configs) - Export sélectif (workflows only, configs only, etc.) - Export modèles entraînés opt-in (embeddings, FAISS anonymisés) - Sanitisation des configs (masquage des secrets) - Statistiques de backup disponibles VersionManager (version_manager.py): - Suivi de version avec composants - Vérification des mises à jour (manifest local) - Vérification intégrité packages (SHA-256) - Création/restauration de backups pour rollback - Information système complète Ces modules supportent les fonctionnalités Dashboard: - Téléchargement sauvegardes par le client - Mise à jour du système - Rollback en cas de problème Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Dom
278
core/system/cleanup_manager.py
Normal file
278
core/system/cleanup_manager.py
Normal file
@@ -0,0 +1,278 @@
|
||||
"""
|
||||
System Cleanup Manager
|
||||
|
||||
Gestionnaire centralisé pour le nettoyage propre du système à l'arrêt.
|
||||
Exigence 6.5: Libération propre de toutes les ressources
|
||||
"""
|
||||
|
||||
import atexit
|
||||
import gc
|
||||
import logging
|
||||
import signal
|
||||
import sys
|
||||
import threading
|
||||
import weakref
|
||||
from typing import Any, Callable, Dict, List, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CleanupManager:
|
||||
"""
|
||||
Gestionnaire centralisé pour le nettoyage du système.
|
||||
|
||||
Gère l'arrêt propre de tous les composants du système :
|
||||
- Memory managers et caches
|
||||
- GPU resource managers
|
||||
- Threads et timers
|
||||
- Connexions réseau
|
||||
- Fichiers ouverts
|
||||
- Processus enfants
|
||||
"""
|
||||
|
||||
_instance: Optional["CleanupManager"] = None
|
||||
_lock = threading.Lock()
|
||||
|
||||
def __new__(cls):
|
||||
with cls._lock:
|
||||
if cls._instance is None:
|
||||
cls._instance = super().__new__(cls)
|
||||
cls._instance._initialized = False
|
||||
return cls._instance
|
||||
|
||||
def __init__(self):
|
||||
if self._initialized:
|
||||
return
|
||||
|
||||
self._cleanup_functions: List[Callable[[], None]] = []
|
||||
self._cleanup_objects: List[weakref.ReferenceType] = []
|
||||
self._shutdown_in_progress = False
|
||||
self._lock = threading.Lock()
|
||||
|
||||
# Enregistrer les handlers de signaux
|
||||
self._register_signal_handlers()
|
||||
|
||||
# Enregistrer atexit
|
||||
atexit.register(self.shutdown)
|
||||
|
||||
self._initialized = True
|
||||
logger.info("CleanupManager initialized")
|
||||
|
||||
def _register_signal_handlers(self) -> None:
|
||||
"""Enregistre les handlers pour les signaux système."""
|
||||
def signal_handler(signum, frame):
|
||||
logger.info(f"Received signal {signum}, initiating shutdown...")
|
||||
self.shutdown()
|
||||
sys.exit(0)
|
||||
|
||||
# Enregistrer les signaux principaux
|
||||
try:
|
||||
signal.signal(signal.SIGINT, signal_handler) # Ctrl+C
|
||||
signal.signal(signal.SIGTERM, signal_handler) # Termination
|
||||
if hasattr(signal, 'SIGHUP'):
|
||||
signal.signal(signal.SIGHUP, signal_handler) # Hangup (Unix)
|
||||
except (OSError, ValueError) as e:
|
||||
logger.warning(f"Could not register signal handler: {e}")
|
||||
|
||||
def register_cleanup_function(self, cleanup_func: Callable[[], None], name: str = None) -> None:
|
||||
"""
|
||||
Enregistre une fonction de nettoyage.
|
||||
|
||||
Args:
|
||||
cleanup_func: Fonction à appeler lors du shutdown
|
||||
name: Nom optionnel pour le logging
|
||||
"""
|
||||
with self._lock:
|
||||
if self._shutdown_in_progress:
|
||||
logger.warning("Cannot register cleanup function during shutdown")
|
||||
return
|
||||
|
||||
self._cleanup_functions.append(cleanup_func)
|
||||
logger.debug(f"Registered cleanup function: {name or 'unnamed'}")
|
||||
|
||||
def register_cleanup_object(self, obj: Any, method_name: str = "shutdown") -> None:
|
||||
"""
|
||||
Enregistre un objet avec une méthode de nettoyage.
|
||||
|
||||
Args:
|
||||
obj: Objet à nettoyer
|
||||
method_name: Nom de la méthode à appeler (défaut: "shutdown")
|
||||
"""
|
||||
with self._lock:
|
||||
if self._shutdown_in_progress:
|
||||
logger.warning("Cannot register cleanup object during shutdown")
|
||||
return
|
||||
|
||||
if not hasattr(obj, method_name):
|
||||
logger.warning(f"Object {obj} does not have method {method_name}")
|
||||
return
|
||||
|
||||
# Utiliser une weak reference pour éviter les cycles
|
||||
def cleanup_callback(ref):
|
||||
# Callback appelé quand l'objet est garbage collected
|
||||
with self._lock:
|
||||
if ref in self._cleanup_objects:
|
||||
self._cleanup_objects.remove(ref)
|
||||
|
||||
weak_ref = weakref.ref(obj, cleanup_callback)
|
||||
self._cleanup_objects.append(weak_ref)
|
||||
logger.debug(f"Registered cleanup object: {obj.__class__.__name__}")
|
||||
|
||||
def register_memory_manager(self) -> None:
|
||||
"""Enregistre le memory manager global pour nettoyage."""
|
||||
try:
|
||||
from core.execution.memory_cache import shutdown_memory_manager
|
||||
self.register_cleanup_function(shutdown_memory_manager, "MemoryManager")
|
||||
except ImportError:
|
||||
logger.warning("Could not import memory manager for cleanup")
|
||||
|
||||
def register_gpu_manager(self) -> None:
|
||||
"""Enregistre le GPU resource manager pour nettoyage."""
|
||||
try:
|
||||
from core.gpu.gpu_resource_manager import get_gpu_resource_manager
|
||||
gpu_manager = get_gpu_resource_manager()
|
||||
self.register_cleanup_object(gpu_manager, "shutdown")
|
||||
except ImportError:
|
||||
logger.warning("Could not import GPU manager for cleanup")
|
||||
|
||||
def register_analytics_system(self) -> None:
|
||||
"""Enregistre le système d'analytics pour nettoyage."""
|
||||
try:
|
||||
from core.analytics.analytics_system import get_analytics_system
|
||||
analytics = get_analytics_system()
|
||||
self.register_cleanup_object(analytics, "shutdown")
|
||||
except ImportError:
|
||||
logger.debug("Analytics system not available for cleanup")
|
||||
|
||||
def register_all_core_systems(self) -> None:
|
||||
"""Enregistre tous les systèmes core pour nettoyage automatique."""
|
||||
logger.info("Registering all core systems for cleanup...")
|
||||
|
||||
self.register_memory_manager()
|
||||
self.register_gpu_manager()
|
||||
self.register_analytics_system()
|
||||
|
||||
logger.info("Core systems registered for cleanup")
|
||||
|
||||
def shutdown(self) -> None:
|
||||
"""
|
||||
Effectue le nettoyage complet du système.
|
||||
|
||||
Ordre de nettoyage :
|
||||
1. Fonctions de nettoyage personnalisées
|
||||
2. Objets enregistrés
|
||||
3. Garbage collection forcé
|
||||
"""
|
||||
with self._lock:
|
||||
if self._shutdown_in_progress:
|
||||
return
|
||||
|
||||
self._shutdown_in_progress = True
|
||||
|
||||
logger.info("Starting system cleanup...")
|
||||
|
||||
# 1. Nettoyer les fonctions personnalisées
|
||||
cleanup_count = 0
|
||||
for cleanup_func in self._cleanup_functions:
|
||||
try:
|
||||
cleanup_func()
|
||||
cleanup_count += 1
|
||||
except Exception as e:
|
||||
logger.error(f"Error in cleanup function: {e}")
|
||||
|
||||
logger.info(f"Executed {cleanup_count} cleanup functions")
|
||||
|
||||
# 2. Nettoyer les objets enregistrés
|
||||
object_count = 0
|
||||
for weak_ref in self._cleanup_objects[:]: # Copie pour éviter modification pendant itération
|
||||
obj = weak_ref()
|
||||
if obj is not None:
|
||||
try:
|
||||
if hasattr(obj, 'shutdown'):
|
||||
obj.shutdown()
|
||||
elif hasattr(obj, 'close'):
|
||||
obj.close()
|
||||
elif hasattr(obj, 'cleanup'):
|
||||
obj.cleanup()
|
||||
object_count += 1
|
||||
except Exception as e:
|
||||
logger.error(f"Error cleaning up object {obj}: {e}")
|
||||
|
||||
logger.info(f"Cleaned up {object_count} objects")
|
||||
|
||||
# 3. Garbage collection forcé
|
||||
try:
|
||||
collected = gc.collect()
|
||||
logger.info(f"Garbage collection freed {collected} objects")
|
||||
except Exception as e:
|
||||
logger.error(f"Error during garbage collection: {e}")
|
||||
|
||||
# 4. Nettoyer les listes
|
||||
with self._lock:
|
||||
self._cleanup_functions.clear()
|
||||
self._cleanup_objects.clear()
|
||||
|
||||
logger.info("System cleanup completed")
|
||||
|
||||
def is_shutdown_in_progress(self) -> bool:
|
||||
"""Vérifie si le shutdown est en cours."""
|
||||
return self._shutdown_in_progress
|
||||
|
||||
|
||||
# Instance globale
|
||||
_cleanup_manager: Optional[CleanupManager] = None
|
||||
|
||||
|
||||
def get_cleanup_manager() -> CleanupManager:
|
||||
"""
|
||||
Retourne l'instance globale du cleanup manager.
|
||||
|
||||
Returns:
|
||||
Instance du CleanupManager
|
||||
"""
|
||||
global _cleanup_manager
|
||||
if _cleanup_manager is None:
|
||||
_cleanup_manager = CleanupManager()
|
||||
return _cleanup_manager
|
||||
|
||||
|
||||
def register_cleanup_function(cleanup_func: Callable[[], None], name: str = None) -> None:
|
||||
"""
|
||||
Fonction utilitaire pour enregistrer une fonction de nettoyage.
|
||||
|
||||
Args:
|
||||
cleanup_func: Fonction à appeler lors du shutdown
|
||||
name: Nom optionnel pour le logging
|
||||
"""
|
||||
get_cleanup_manager().register_cleanup_function(cleanup_func, name)
|
||||
|
||||
|
||||
def register_cleanup_object(obj: Any, method_name: str = "shutdown") -> None:
|
||||
"""
|
||||
Fonction utilitaire pour enregistrer un objet avec nettoyage.
|
||||
|
||||
Args:
|
||||
obj: Objet à nettoyer
|
||||
method_name: Nom de la méthode à appeler
|
||||
"""
|
||||
get_cleanup_manager().register_cleanup_object(obj, method_name)
|
||||
|
||||
|
||||
def initialize_system_cleanup() -> None:
|
||||
"""
|
||||
Initialise le système de cleanup avec tous les composants core.
|
||||
|
||||
À appeler au démarrage de l'application.
|
||||
"""
|
||||
cleanup_manager = get_cleanup_manager()
|
||||
cleanup_manager.register_all_core_systems()
|
||||
logger.info("System cleanup initialized")
|
||||
|
||||
|
||||
def shutdown_system() -> None:
|
||||
"""
|
||||
Force le shutdown du système.
|
||||
|
||||
À appeler pour un arrêt propre programmé.
|
||||
"""
|
||||
get_cleanup_manager().shutdown()
|
||||
Reference in New Issue
Block a user