rpa_vision_v3/test_circuit_breaker_simple.py

#!/usr/bin/env python3
"""
Direct Real Functionality Test for CircuitBreaker

Tests actual CircuitBreaker behavior by importing directly from the module,
bypassing any __init__.py import issues.
"""

import sys
import os
import time
from datetime import datetime, timedelta

# Add the specific module path
sys.path.insert(0, '.')
sys.path.insert(0, 'core/system')

def test_circuit_breaker_real_functionality():
    """Test CircuitBreaker with real failure scenarios and state transitions."""
    
    print("🔧 Testing CircuitBreaker Real Functionality")
    print("=" * 50)
    
    # Test 1: Direct imports
    try:
        # Import models directly
        from core.system.models import SimpleFailureEvent
        
        # Import CircuitBreaker directly from the module file
        import importlib.util
        spec = importlib.util.spec_from_file_location("circuit_breaker", "core/system/circuit_breaker.py")
        circuit_breaker_module = importlib.util.module_from_spec(spec)
        spec.loader.exec_module(circuit_breaker_module)
        CircuitBreaker = circuit_breaker_module.CircuitBreaker
        
        print("✓ Direct imports successful")
    except Exception as e:
        print(f"✗ Import failed: {e}")
        import traceback
        traceback.print_exc()
        return False
    
    # Test 2: Create CircuitBreaker with real policy
    try:
        policy = {
            'step_fail_streak_to_degraded': 3,
            'workflow_fail_window_s': 60,
            'workflow_fail_max_in_window': 5,
            'global_fail_max_in_window': 15,
            'success_reset_threshold': 2
        }
        cb = CircuitBreaker(policy)
        print("✓ CircuitBreaker created with real policy")
    except Exception as e:
        print(f"✗ CircuitBreaker creation failed: {e}")
        import traceback
        traceback.print_exc()
        return False
    
    # Test 3: Record real failures for step degradation
    try:
        workflow_id = "test_workflow"
        step_id = "login_step"
        
        # Record consecutive failures for the same step
        for i in range(3):
            cb.record_failure(workflow_id, step_id, "TARGET_NOT_FOUND")
        
        # Should trigger degraded mode
        should_degrade = cb.should_trigger_degraded(workflow_id, step_id)
        assert should_degrade, "Step should be degraded after 3 consecutive failures"
        print("✓ Step degradation triggered after consecutive failures")
        
    except Exception as e:
        print(f"✗ Step degradation test failed: {e}")
        import traceback
        traceback.print_exc()
        return False
    
    # Test 4: Test success recovery
    try:
        # Record successes to reset failures
        cb.record_success(workflow_id, step_id)
        cb.record_success(workflow_id, step_id)
        
        # Should no longer trigger degraded mode
        should_degrade = cb.should_trigger_degraded(workflow_id, step_id)
        assert not should_degrade, "Step should recover after successful operations"
        print("✓ Step recovered after successful operations")
        
    except Exception as e:
        print(f"✗ Success recovery test failed: {e}")
        import traceback
        traceback.print_exc()
        return False
    
    # Test 5: Test workflow quarantine
    try:
        # Record many failures for workflow quarantine
        for i in range(6):
            cb.record_failure(workflow_id, f"step_{i}", "TIMEOUT")
        
        # Should trigger quarantine
        should_quarantine = cb.should_trigger_quarantine(workflow_id)
        assert should_quarantine, "Workflow should be quarantined after multiple failures"
        print("✓ Workflow quarantine triggered after multiple failures")
        
    except Exception as e:
        print(f"✗ Workflow quarantine test failed: {e}")
        import traceback
        traceback.print_exc()
        return False
    
    # Test 6: Test global pause
    try:
        # Record many global failures
        for i in range(16):
            cb.record_failure(f"workflow_{i}", "global_step", "SYSTEM_ERROR")
        
        # Should trigger global pause
        should_pause = cb.should_trigger_global_pause()
        assert should_pause, "Global pause should be triggered after system-wide failures"
        print("✓ Global pause triggered after system-wide failures")
        
    except Exception as e:
        print(f"✗ Global pause test failed: {e}")
        import traceback
        traceback.print_exc()
        return False
    
    # Test 7: Test failure counts and statistics
    try:
        failure_counts = cb.get_failure_counts(workflow_id)
        
        # Verify structure
        required_keys = ['step_consecutive', 'workflow_window', 'global_window', 'window_duration_s']
        for key in required_keys:
            assert key in failure_counts, f"Missing key: {key}"
        
        # Verify data
        assert failure_counts['workflow_window'] >= 5, f"Expected >= 5 workflow failures, got {failure_counts['workflow_window']}"
        assert failure_counts['global_window'] >= 15, f"Expected >= 15 global failures, got {failure_counts['global_window']}"
        
        print("✓ Failure counts tracking works correctly")
        print(f"   - Workflow window failures: {failure_counts['workflow_window']}")
        print(f"   - Global window failures: {failure_counts['global_window']}")
        
    except Exception as e:
        print(f"✗ Failure counts test failed: {e}")
        import traceback
        traceback.print_exc()
        return False
    
    # Test 8: Test failure history
    try:
        # Get failure history for a specific step
        history = cb.get_step_failure_history(workflow_id, step_id, limit=5)
        
        # Should have some failures recorded (may be empty due to success reset)
        assert isinstance(history, list), "History should be a list"
        
        # Get workflow failure types
        failure_types = cb.get_workflow_failure_types(workflow_id)
        assert isinstance(failure_types, dict), "Failure types should be a dict"
        
        print("✓ Failure history and types tracking works")
        print(f"   - Step failure history length: {len(history)}")
        print(f"   - Workflow failure types: {failure_types}")
        
    except Exception as e:
        print(f"✗ Failure history test failed: {e}")
        import traceback
        traceback.print_exc()
        return False
    
    # Test 9: Test status summary
    try:
        status = cb.get_status_summary()
        
        # Verify structure
        required_keys = ['timestamp', 'policy', 'global_stats', 'thresholds']
        for key in required_keys:
            assert key in status, f"Missing key in status: {key}"
        
        # Verify global stats
        global_stats = status['global_stats']
        required_global_keys = ['global_failures_in_window', 'workflows_with_failures', 'steps_with_consecutive_failures']
        for key in required_global_keys:
            assert key in global_stats, f"Missing key in global_stats: {key}"
        
        print("✓ Status summary provides comprehensive information")
        print(f"   - Global failures: {global_stats['global_failures_in_window']}")
        print(f"   - Workflows with failures: {global_stats['workflows_with_failures']}")
        
    except Exception as e:
        print(f"✗ Status summary test failed: {e}")
        import traceback
        traceback.print_exc()
        return False
    
    print("\n🎉 All CircuitBreaker real functionality tests passed!")
    return True


def test_circuit_breaker_rpa_integration():
    """Test CircuitBreaker integration with RPA Vision V3 scenarios."""
    
    print("\n🔗 Testing RPA Integration Scenarios")
    print("=" * 40)
    
    try:
        # Import models directly
        from core.system.models import SimpleFailureEvent
        
        # Import CircuitBreaker directly from the module file
        import importlib.util
        spec = importlib.util.spec_from_file_location("circuit_breaker", "core/system/circuit_breaker.py")
        circuit_breaker_module = importlib.util.module_from_spec(spec)
        spec.loader.exec_module(circuit_breaker_module)
        CircuitBreaker = circuit_breaker_module.CircuitBreaker
        
        # Create circuit breaker with RPA-specific policy
        rpa_policy = {
            'step_fail_streak_to_degraded': 2,
            'workflow_fail_window_s': 300,  # 5 minutes
            'workflow_fail_max_in_window': 8,
            'global_fail_max_in_window': 25,
            'success_reset_threshold': 3
        }
        rpa_cb = CircuitBreaker(rpa_policy)
        
        # Test with real RPA workflow scenarios
        login_workflow = "user_login_workflow"
        form_workflow = "data_entry_workflow"
        
        # Simulate login failures (UI detection issues)
        for i in range(3):
            rpa_cb.record_failure(login_workflow, "find_username_field", "TARGET_NOT_FOUND")
            rpa_cb.record_failure(login_workflow, "find_password_field", "TARGET_NOT_FOUND")
        
        # Should trigger degraded mode for these steps
        assert rpa_cb.should_trigger_degraded(login_workflow, "find_username_field")
        assert rpa_cb.should_trigger_degraded(login_workflow, "find_password_field")
        print("✓ RPA step degradation works for UI detection failures")
        
        # Simulate form entry timeout failures
        for i in range(9):
            rpa_cb.record_failure(form_workflow, f"fill_field_{i}", "TIMEOUT")
        
        # Should trigger workflow quarantine
        assert rpa_cb.should_trigger_quarantine(form_workflow)
        print("✓ RPA workflow quarantine works for timeout failures")
        
        # Test failure type analysis
        form_failure_types = rpa_cb.get_workflow_failure_types(form_workflow)
        assert "TIMEOUT" in form_failure_types
        assert form_failure_types["TIMEOUT"] >= 9
        print("✓ RPA failure type analysis works correctly")
        
        # Test recovery with successful operations
        for i in range(3):
            rpa_cb.record_success(login_workflow, "find_username_field")
        
        # Should no longer be degraded
        assert not rpa_cb.should_trigger_degraded(login_workflow, "find_username_field")
        print("✓ RPA step recovery works after successful operations")
        
        # Test comprehensive status for RPA monitoring
        status = rpa_cb.get_status_summary()
        global_stats = status['global_stats']
        
        # Verify monitoring data is useful for RPA operations
        assert global_stats['workflows_with_failures'] >= 2
        assert global_stats['global_failures_in_window'] >= 15
        
        print("✓ RPA monitoring integration provides actionable data")
        print(f"   - Workflows affected: {global_stats['workflows_with_failures']}")
        print(f"   - Total system failures: {global_stats['global_failures_in_window']}")
        print(f"   - Failure types: {global_stats.get('global_failure_types', {})}")
        
        return True
        
    except Exception as e:
        print(f"✗ RPA integration test failed: {e}")
        import traceback
        traceback.print_exc()
        return False


if __name__ == "__main__":
    success = True
    
    # Run main functionality tests
    if not test_circuit_breaker_real_functionality():
        success = False
    
    # Run RPA integration tests
    if not test_circuit_breaker_rpa_integration():
        success = False
    
    if success:
        print("\n✅ All tests passed - CircuitBreaker is working correctly!")
        sys.exit(0)
    else:
        print("\n❌ Some tests failed - CircuitBreaker needs attention")
        sys.exit(1)