Files
Geniusia_v2/test_enhanced_matcher_integration.py
2026-03-05 00:20:25 +01:00

683 lines
22 KiB
Python

#!/usr/bin/env python3
"""
Test d'intégration pour EnhancedWorkflowMatcher - Task 7.10
Tests de matching avec workflows legacy et enrichis, et routage automatique.
Valide les exigences 9.1, 9.2, 9.3:
- 9.1: Compatibilité arrière avec workflows legacy
- 9.2: Routage automatique (legacy vs enriched)
- 9.3: Support des workflows enrichis avec éléments
"""
import sys
import numpy as np
from pathlib import Path
import shutil
from datetime import datetime
from dataclasses import dataclass
from typing import List, Optional
# Ajouter le répertoire parent au path
sys.path.insert(0, str(Path(__file__).parent))
from geniusia2.core.enhanced_workflow_matcher import EnhancedWorkflowMatcher
from geniusia2.core.multimodal_embedding_manager import MultiModalEmbeddingManager
from geniusia2.core.ui_element_models import (
EnrichedScreenState,
WindowInfo,
RawData,
PerceptionData,
UIElement,
UIElementType,
VisualData,
TextData,
ElementProperties,
ElementContext
)
from geniusia2.core.logger import Logger
# Structures de workflow (compatibles avec workflow_detector.py)
@dataclass
class WorkflowStep:
"""Représente une étape dans un workflow."""
step_id: int
action_type: str
target_description: str
position: tuple
window: str
embedding: Optional[np.ndarray] = None
screenshot: Optional[np.ndarray] = None
@dataclass
class Workflow:
"""Représente un workflow détecté."""
workflow_id: str
name: str
steps: List[WorkflowStep]
repetitions: int
confidence: float
def create_test_screen_state(
screen_id: str,
ui_elements: List[UIElement],
window_title: str = "Test Window",
app_name: str = "TestApp"
) -> EnrichedScreenState:
"""Crée un EnrichedScreenState pour les tests."""
return EnrichedScreenState(
screen_state_id=screen_id,
timestamp=datetime.now(),
session_id="test_session",
window=WindowInfo(app_name, window_title, True),
raw=RawData("test.png"),
perception=PerceptionData(["Test", "Button", "Submit"]),
ui_elements=ui_elements,
state_embedding=None,
context=None
)
def create_test_ui_element(
element_id: str,
label: str,
element_type: UIElementType = UIElementType.BUTTON,
bbox: tuple = (100, 200, 80, 30),
confidence: float = 0.9
) -> UIElement:
"""Crée un UIElement pour les tests."""
return UIElement(
element_id=element_id,
type=element_type,
role="primary_action",
bbox=bbox,
label=label,
visual=VisualData("test.png", "test", "test.npy"),
text=TextData(label, label.lower(), "test", "test.npy"),
properties=ElementProperties(is_clickable=True),
context=ElementContext("TestApp", "Test Window"),
confidence=confidence
)
def test_legacy_workflow_compatibility():
"""
Test 1: Compatibilité avec workflows legacy (sans éléments UI).
Valide l'exigence 9.1.
"""
print("\n" + "="*70)
print("Test 1: Compatibilité avec Workflows Legacy")
print("="*70)
logger = Logger(log_dir="test_logs")
multimodal_manager = MultiModalEmbeddingManager(
logger=logger,
data_dir="test_data"
)
matcher = EnhancedWorkflowMatcher(
multimodal_manager=multimodal_manager,
logger=logger
)
# Créer un workflow legacy (ancien style, sans descripteurs d'éléments)
# Ces workflows ont seulement des embeddings d'écran complet
legacy_workflow = Workflow(
workflow_id="legacy_wf_001",
name="Legacy Invoice Workflow",
steps=[
WorkflowStep(
step_id=1,
action_type="click",
target_description="validate button",
position=(1700, 920),
window="Invoice App",
embedding=np.random.rand(512) # Embedding d'écran complet
),
WorkflowStep(
step_id=2,
action_type="type",
target_description="amount field",
position=(500, 400),
window="Invoice App",
embedding=np.random.rand(512)
)
],
repetitions=5,
confidence=0.85
)
# Créer un écran actuel (sans éléments UI détectés - mode light)
screen_state = create_test_screen_state(
screen_id="screen_legacy_001",
ui_elements=[], # Pas d'éléments UI détectés
window_title="Invoice App",
app_name="InvoiceApp"
)
print(f"✓ Workflow legacy créé: {legacy_workflow.name}")
print(f" - Steps: {len(legacy_workflow.steps)}")
print(f" - Répétitions: {legacy_workflow.repetitions}")
print(f" - Confiance: {legacy_workflow.confidence}")
print(f"\n✓ Screen state créé (mode light)")
print(f" - UI Elements: {len(screen_state.ui_elements)}")
print(f" - Window: {screen_state.window.window_title}")
# Tester le matching
try:
matches = matcher.find_matching_workflows(
screen_state=screen_state,
screenshot=None,
workflows=[legacy_workflow],
top_k=5
)
print(f"\n✓ Matching exécuté avec succès")
print(f" - Matches trouvés: {len(matches)}")
if matches:
best_match = matches[0]
print(f" - Meilleur match: {best_match.workflow_name}")
print(f" - Score composite: {best_match.composite_score:.3f}")
print(f" - Screen similarity: {best_match.screen_similarity:.3f}")
print(f" - Confiance: {best_match.confidence:.3f}")
# Vérifications
assert isinstance(matches, list), "Le résultat doit être une liste"
print("\n✅ Test 1 réussi: Workflows legacy supportés!")
except Exception as e:
print(f"\n❌ Test 1 échoué: {e}")
import traceback
traceback.print_exc()
return False
return True
def test_enriched_workflow_with_elements():
"""
Test 2: Matching avec workflows enrichis (avec éléments UI).
Valide l'exigence 9.3.
"""
print("\n" + "="*70)
print("Test 2: Matching avec Workflows Enrichis (Éléments UI)")
print("="*70)
logger = Logger(log_dir="test_logs")
multimodal_manager = MultiModalEmbeddingManager(
logger=logger,
data_dir="test_data"
)
matcher = EnhancedWorkflowMatcher(
multimodal_manager=multimodal_manager,
logger=logger,
config={
"screen_weight": 0.5,
"elements_weight": 0.5 # Poids égal pour éléments
}
)
# Créer un workflow enrichi (nouveau style, avec descripteurs d'éléments)
enriched_workflow = Workflow(
workflow_id="enriched_wf_001",
name="Enriched Form Workflow",
steps=[
WorkflowStep(
step_id=1,
action_type="type",
target_description="username field",
position=(300, 200),
window="Login Form",
embedding=np.random.rand(512)
),
WorkflowStep(
step_id=2,
action_type="type",
target_description="password field",
position=(300, 250),
window="Login Form",
embedding=np.random.rand(512)
),
WorkflowStep(
step_id=3,
action_type="click",
target_description="submit button",
position=(300, 320),
window="Login Form",
embedding=np.random.rand(512)
)
],
repetitions=10,
confidence=0.92
)
# Créer des éléments UI correspondants
ui_elements = [
create_test_ui_element(
"elem_username",
"Username",
UIElementType.TEXT_INPUT,
(250, 180, 100, 40),
0.95
),
create_test_ui_element(
"elem_password",
"Password",
UIElementType.TEXT_INPUT,
(250, 230, 100, 40),
0.93
),
create_test_ui_element(
"elem_submit",
"Submit",
UIElementType.BUTTON,
(280, 300, 60, 40),
0.97
)
]
# Créer un écran actuel (avec éléments UI détectés - mode enriched)
screen_state = create_test_screen_state(
screen_id="screen_enriched_001",
ui_elements=ui_elements,
window_title="Login Form",
app_name="LoginApp"
)
print(f"✓ Workflow enrichi créé: {enriched_workflow.name}")
print(f" - Steps: {len(enriched_workflow.steps)}")
print(f" - Répétitions: {enriched_workflow.repetitions}")
print(f"\n✓ Screen state créé (mode enriched)")
print(f" - UI Elements: {len(screen_state.ui_elements)}")
for elem in screen_state.ui_elements:
print(f"{elem.type.value}: {elem.label} (conf: {elem.confidence:.2f})")
# Tester le matching
try:
matches = matcher.find_matching_workflows(
screen_state=screen_state,
screenshot=None,
workflows=[enriched_workflow],
top_k=5
)
print(f"\n✓ Matching exécuté avec succès")
print(f" - Matches trouvés: {len(matches)}")
if matches:
best_match = matches[0]
print(f"\n 📊 Meilleur match: {best_match.workflow_name}")
print(f" - Score composite: {best_match.composite_score:.3f}")
print(f" - Screen similarity: {best_match.screen_similarity:.3f}")
print(f" - Confiance: {best_match.confidence:.3f}")
print(f" - Element matches: {len(best_match.element_matches)}")
# Afficher les matches d'éléments
if best_match.element_matches:
print(f"\n 🎯 Matches d'éléments:")
for elem_match in best_match.element_matches:
print(f"{elem_match.ui_element.label}: "
f"{elem_match.match_type} "
f"(score: {elem_match.similarity_score:.2f})")
# Vérifications
assert isinstance(matches, list), "Le résultat doit être une liste"
if matches:
assert len(matches[0].element_matches) > 0, "Des matches d'éléments doivent être trouvés"
print("\n✅ Test 2 réussi: Workflows enrichis supportés!")
except Exception as e:
print(f"\n❌ Test 2 échoué: {e}")
import traceback
traceback.print_exc()
return False
return True
def test_automatic_routing():
"""
Test 3: Routage automatique entre legacy et enriched.
Valide l'exigence 9.2.
"""
print("\n" + "="*70)
print("Test 3: Routage Automatique (Legacy vs Enriched)")
print("="*70)
logger = Logger(log_dir="test_logs")
multimodal_manager = MultiModalEmbeddingManager(
logger=logger,
data_dir="test_data"
)
matcher = EnhancedWorkflowMatcher(
multimodal_manager=multimodal_manager,
logger=logger
)
# Créer un workflow legacy
legacy_workflow = Workflow(
workflow_id="legacy_wf_002",
name="Legacy Workflow",
steps=[
WorkflowStep(
step_id=1,
action_type="click",
target_description="button",
position=(100, 200),
window="App",
embedding=np.random.rand(512)
)
],
repetitions=3,
confidence=0.8
)
# Créer un workflow enrichi
enriched_workflow = Workflow(
workflow_id="enriched_wf_002",
name="Enriched Workflow",
steps=[
WorkflowStep(
step_id=1,
action_type="click",
target_description="save button",
position=(500, 600),
window="App",
embedding=np.random.rand(512)
),
WorkflowStep(
step_id=2,
action_type="click",
target_description="close button",
position=(600, 600),
window="App",
embedding=np.random.rand(512)
)
],
repetitions=5,
confidence=0.9
)
# Créer un écran avec éléments UI
ui_elements = [
create_test_ui_element("elem_save", "Save", UIElementType.BUTTON),
create_test_ui_element("elem_close", "Close", UIElementType.BUTTON)
]
screen_state = create_test_screen_state(
screen_id="screen_routing_001",
ui_elements=ui_elements,
window_title="App",
app_name="TestApp"
)
print(f"✓ Workflows créés:")
print(f" - Legacy: {legacy_workflow.name} ({len(legacy_workflow.steps)} steps)")
print(f" - Enriched: {enriched_workflow.name} ({len(enriched_workflow.steps)} steps)")
print(f"\n✓ Screen state créé:")
print(f" - UI Elements: {len(screen_state.ui_elements)}")
# Tester le matching avec les deux types de workflows
try:
matches = matcher.find_matching_workflows(
screen_state=screen_state,
screenshot=None,
workflows=[legacy_workflow, enriched_workflow],
top_k=5
)
print(f"\n✓ Matching exécuté avec succès")
print(f" - Matches trouvés: {len(matches)}")
# Afficher tous les matches
for i, match in enumerate(matches, 1):
print(f"\n {i}. {match.workflow_name}")
print(f" - Type: {'Enriched' if match.element_matches else 'Legacy'}")
print(f" - Score: {match.composite_score:.3f}")
print(f" - Element matches: {len(match.element_matches)}")
# Vérifications
assert isinstance(matches, list), "Le résultat doit être une liste"
# Vérifier que les deux types de workflows sont traités
has_legacy = any(len(m.element_matches) == 0 for m in matches)
has_enriched = any(len(m.element_matches) > 0 for m in matches)
print(f"\n ✓ Legacy workflows traités: {has_legacy}")
print(f" ✓ Enriched workflows traités: {has_enriched}")
print("\n✅ Test 3 réussi: Routage automatique fonctionne!")
except Exception as e:
print(f"\n❌ Test 3 échoué: {e}")
import traceback
traceback.print_exc()
return False
return True
def test_mixed_workflow_scenarios():
"""
Test 4: Scénarios mixtes avec différents niveaux de matching.
"""
print("\n" + "="*70)
print("Test 4: Scénarios Mixtes")
print("="*70)
logger = Logger(log_dir="test_logs")
multimodal_manager = MultiModalEmbeddingManager(
logger=logger,
data_dir="test_data"
)
matcher = EnhancedWorkflowMatcher(
multimodal_manager=multimodal_manager,
logger=logger
)
# Scénario 1: Écran sans éléments, workflow legacy
print("\n📋 Scénario 1: Écran vide + Workflow legacy")
screen_empty = create_test_screen_state("screen_empty", [])
workflow_legacy = Workflow(
"wf_legacy", "Legacy",
[WorkflowStep(1, "click", "button", (100, 100), "App", np.random.rand(512))],
3, 0.8
)
matches1 = matcher.find_matching_workflows(
screen_empty, None, [workflow_legacy], 5
)
print(f" ✓ Matches: {len(matches1)}")
# Scénario 2: Écran avec éléments, workflow enrichi
print("\n📋 Scénario 2: Écran enrichi + Workflow enrichi")
screen_enriched = create_test_screen_state(
"screen_enriched",
[create_test_ui_element("elem1", "Button")]
)
workflow_enriched = Workflow(
"wf_enriched", "Enriched",
[WorkflowStep(1, "click", "button", (100, 100), "App", np.random.rand(512))],
5, 0.9
)
matches2 = matcher.find_matching_workflows(
screen_enriched, None, [workflow_enriched], 5
)
print(f" ✓ Matches: {len(matches2)}")
if matches2:
print(f" ✓ Element matches: {len(matches2[0].element_matches)}")
# Scénario 3: Écran enrichi, workflow legacy (compatibilité)
print("\n📋 Scénario 3: Écran enrichi + Workflow legacy (compatibilité)")
matches3 = matcher.find_matching_workflows(
screen_enriched, None, [workflow_legacy], 5
)
print(f" ✓ Matches: {len(matches3)}")
# Scénario 4: Aucun workflow
print("\n📋 Scénario 4: Aucun workflow disponible")
matches4 = matcher.find_matching_workflows(
screen_enriched, None, [], 5
)
print(f" ✓ Matches: {len(matches4)}")
assert len(matches4) == 0, "Aucun match ne devrait être trouvé"
print("\n✅ Test 4 réussi: Tous les scénarios mixtes fonctionnent!")
return True
def test_feedback_on_partial_match():
"""
Test 5: Feedback détaillé sur match partiel.
"""
print("\n" + "="*70)
print("Test 5: Feedback Détaillé sur Match Partiel")
print("="*70)
logger = Logger(log_dir="test_logs")
multimodal_manager = MultiModalEmbeddingManager(
logger=logger,
data_dir="test_data"
)
matcher = EnhancedWorkflowMatcher(
multimodal_manager=multimodal_manager,
logger=logger
)
# Créer un workflow avec 3 steps
workflow = Workflow(
workflow_id="wf_partial",
name="Partial Match Workflow",
steps=[
WorkflowStep(1, "type", "username", (100, 100), "App", np.random.rand(512)),
WorkflowStep(2, "type", "password", (100, 150), "App", np.random.rand(512)),
WorkflowStep(3, "click", "submit", (100, 200), "App", np.random.rand(512))
],
repetitions=5,
confidence=0.9
)
# Créer un écran avec seulement 1 élément (match partiel)
screen_state = create_test_screen_state(
"screen_partial",
[create_test_ui_element("elem_submit", "Submit")]
)
print(f"✓ Workflow créé: {len(workflow.steps)} steps attendus")
print(f"✓ Screen state créé: {len(screen_state.ui_elements)} éléments détectés")
# Tester le matching
matches = matcher.find_matching_workflows(
screen_state, None, [workflow], 5
)
if matches:
match = matches[0]
print(f"\n✓ Match trouvé:")
print(f" - Score: {match.composite_score:.3f}")
print(f" - Confiance: {match.confidence:.3f}")
# Vérifier le feedback
if match.differences:
print(f"\n📋 Feedback détaillé ({len(match.differences)} différences):")
for diff in match.differences:
print(f" • [{diff.severity.upper()}] {diff.difference_type}")
print(f" {diff.description}")
if diff.suggestion:
print(f" 💡 {diff.suggestion}")
# Tester le résumé
print(f"\n📝 Résumé du feedback:")
summary = match.get_feedback_summary()
print(summary)
assert len(match.differences) > 0, "Des différences devraient être détectées"
print("\n✅ Test 5 réussi: Feedback détaillé généré!")
else:
print("\n⚠️ Aucun feedback généré (score trop élevé)")
else:
print("\n⚠️ Aucun match trouvé")
return True
def main():
"""Exécute tous les tests d'intégration."""
print("\n" + "="*70)
print("TESTS D'INTÉGRATION - EnhancedWorkflowMatcher")
print("Task 7.10: Matching avec workflows legacy et enrichis")
print("="*70)
tests = [
("Compatibilité Workflows Legacy", test_legacy_workflow_compatibility),
("Workflows Enrichis avec Éléments", test_enriched_workflow_with_elements),
("Routage Automatique", test_automatic_routing),
("Scénarios Mixtes", test_mixed_workflow_scenarios),
("Feedback sur Match Partiel", test_feedback_on_partial_match)
]
results = []
for test_name, test_func in tests:
try:
success = test_func()
results.append((test_name, success, None))
except Exception as e:
print(f"\n❌ Test '{test_name}' échoué: {e}")
import traceback
traceback.print_exc()
results.append((test_name, False, str(e)))
# Nettoyage
print("\n" + "="*70)
print("Nettoyage...")
if Path("test_data").exists():
shutil.rmtree("test_data")
if Path("test_logs").exists():
shutil.rmtree("test_logs")
print("✓ Nettoyage terminé")
# Résumé
print("\n" + "="*70)
print("RÉSUMÉ DES TESTS D'INTÉGRATION")
print("="*70)
passed = sum(1 for _, success, _ in results if success)
total = len(results)
for test_name, success, error in results:
status = "✅ RÉUSSI" if success else "❌ ÉCHOUÉ"
print(f"{status}: {test_name}")
if error:
print(f" Erreur: {error}")
print(f"\nRésultat: {passed}/{total} tests réussis")
if passed == total:
print("\n🎉 TOUS LES TESTS D'INTÉGRATION SONT RÉUSSIS! 🎉")
print("\nValidation des exigences:")
print(" ✓ 9.1: Compatibilité arrière avec workflows legacy")
print(" ✓ 9.2: Routage automatique (legacy vs enriched)")
print(" ✓ 9.3: Support des workflows enrichis avec éléments")
print("\nLe système de matching amélioré est opérationnel!")
return 0
else:
print(f"\n⚠️ {total - passed} test(s) ont échoué")
return 1
if __name__ == "__main__":
sys.exit(main())