#!/usr/bin/env python3 """ Direct Real Functionality Test for CircuitBreaker Tests actual CircuitBreaker behavior by importing directly from the module, bypassing any __init__.py import issues. """ import sys import os import time from datetime import datetime, timedelta # Add the specific module path sys.path.insert(0, '.') sys.path.insert(0, 'core/system') def test_circuit_breaker_real_functionality(): """Test CircuitBreaker with real failure scenarios and state transitions.""" print("šŸ”§ Testing CircuitBreaker Real Functionality") print("=" * 50) # Test 1: Direct imports try: # Import models directly from core.system.models import SimpleFailureEvent # Import CircuitBreaker directly from the module file import importlib.util spec = importlib.util.spec_from_file_location("circuit_breaker", "core/system/circuit_breaker.py") circuit_breaker_module = importlib.util.module_from_spec(spec) spec.loader.exec_module(circuit_breaker_module) CircuitBreaker = circuit_breaker_module.CircuitBreaker print("āœ“ Direct imports successful") except Exception as e: print(f"āœ— Import failed: {e}") import traceback traceback.print_exc() return False # Test 2: Create CircuitBreaker with real policy try: policy = { 'step_fail_streak_to_degraded': 3, 'workflow_fail_window_s': 60, 'workflow_fail_max_in_window': 5, 'global_fail_max_in_window': 15, 'success_reset_threshold': 2 } cb = CircuitBreaker(policy) print("āœ“ CircuitBreaker created with real policy") except Exception as e: print(f"āœ— CircuitBreaker creation failed: {e}") import traceback traceback.print_exc() return False # Test 3: Record real failures for step degradation try: workflow_id = "test_workflow" step_id = "login_step" # Record consecutive failures for the same step for i in range(3): cb.record_failure(workflow_id, step_id, "TARGET_NOT_FOUND") # Should trigger degraded mode should_degrade = cb.should_trigger_degraded(workflow_id, step_id) assert should_degrade, "Step should be degraded after 3 consecutive failures" print("āœ“ Step degradation triggered after consecutive failures") except Exception as e: print(f"āœ— Step degradation test failed: {e}") import traceback traceback.print_exc() return False # Test 4: Test success recovery try: # Record successes to reset failures cb.record_success(workflow_id, step_id) cb.record_success(workflow_id, step_id) # Should no longer trigger degraded mode should_degrade = cb.should_trigger_degraded(workflow_id, step_id) assert not should_degrade, "Step should recover after successful operations" print("āœ“ Step recovered after successful operations") except Exception as e: print(f"āœ— Success recovery test failed: {e}") import traceback traceback.print_exc() return False # Test 5: Test workflow quarantine try: # Record many failures for workflow quarantine for i in range(6): cb.record_failure(workflow_id, f"step_{i}", "TIMEOUT") # Should trigger quarantine should_quarantine = cb.should_trigger_quarantine(workflow_id) assert should_quarantine, "Workflow should be quarantined after multiple failures" print("āœ“ Workflow quarantine triggered after multiple failures") except Exception as e: print(f"āœ— Workflow quarantine test failed: {e}") import traceback traceback.print_exc() return False # Test 6: Test global pause try: # Record many global failures for i in range(16): cb.record_failure(f"workflow_{i}", "global_step", "SYSTEM_ERROR") # Should trigger global pause should_pause = cb.should_trigger_global_pause() assert should_pause, "Global pause should be triggered after system-wide failures" print("āœ“ Global pause triggered after system-wide failures") except Exception as e: print(f"āœ— Global pause test failed: {e}") import traceback traceback.print_exc() return False # Test 7: Test failure counts and statistics try: failure_counts = cb.get_failure_counts(workflow_id) # Verify structure required_keys = ['step_consecutive', 'workflow_window', 'global_window', 'window_duration_s'] for key in required_keys: assert key in failure_counts, f"Missing key: {key}" # Verify data assert failure_counts['workflow_window'] >= 5, f"Expected >= 5 workflow failures, got {failure_counts['workflow_window']}" assert failure_counts['global_window'] >= 15, f"Expected >= 15 global failures, got {failure_counts['global_window']}" print("āœ“ Failure counts tracking works correctly") print(f" - Workflow window failures: {failure_counts['workflow_window']}") print(f" - Global window failures: {failure_counts['global_window']}") except Exception as e: print(f"āœ— Failure counts test failed: {e}") import traceback traceback.print_exc() return False # Test 8: Test failure history try: # Get failure history for a specific step history = cb.get_step_failure_history(workflow_id, step_id, limit=5) # Should have some failures recorded (may be empty due to success reset) assert isinstance(history, list), "History should be a list" # Get workflow failure types failure_types = cb.get_workflow_failure_types(workflow_id) assert isinstance(failure_types, dict), "Failure types should be a dict" print("āœ“ Failure history and types tracking works") print(f" - Step failure history length: {len(history)}") print(f" - Workflow failure types: {failure_types}") except Exception as e: print(f"āœ— Failure history test failed: {e}") import traceback traceback.print_exc() return False # Test 9: Test status summary try: status = cb.get_status_summary() # Verify structure required_keys = ['timestamp', 'policy', 'global_stats', 'thresholds'] for key in required_keys: assert key in status, f"Missing key in status: {key}" # Verify global stats global_stats = status['global_stats'] required_global_keys = ['global_failures_in_window', 'workflows_with_failures', 'steps_with_consecutive_failures'] for key in required_global_keys: assert key in global_stats, f"Missing key in global_stats: {key}" print("āœ“ Status summary provides comprehensive information") print(f" - Global failures: {global_stats['global_failures_in_window']}") print(f" - Workflows with failures: {global_stats['workflows_with_failures']}") except Exception as e: print(f"āœ— Status summary test failed: {e}") import traceback traceback.print_exc() return False print("\nšŸŽ‰ All CircuitBreaker real functionality tests passed!") return True def test_circuit_breaker_rpa_integration(): """Test CircuitBreaker integration with RPA Vision V3 scenarios.""" print("\nšŸ”— Testing RPA Integration Scenarios") print("=" * 40) try: # Import models directly from core.system.models import SimpleFailureEvent # Import CircuitBreaker directly from the module file import importlib.util spec = importlib.util.spec_from_file_location("circuit_breaker", "core/system/circuit_breaker.py") circuit_breaker_module = importlib.util.module_from_spec(spec) spec.loader.exec_module(circuit_breaker_module) CircuitBreaker = circuit_breaker_module.CircuitBreaker # Create circuit breaker with RPA-specific policy rpa_policy = { 'step_fail_streak_to_degraded': 2, 'workflow_fail_window_s': 300, # 5 minutes 'workflow_fail_max_in_window': 8, 'global_fail_max_in_window': 25, 'success_reset_threshold': 3 } rpa_cb = CircuitBreaker(rpa_policy) # Test with real RPA workflow scenarios login_workflow = "user_login_workflow" form_workflow = "data_entry_workflow" # Simulate login failures (UI detection issues) for i in range(3): rpa_cb.record_failure(login_workflow, "find_username_field", "TARGET_NOT_FOUND") rpa_cb.record_failure(login_workflow, "find_password_field", "TARGET_NOT_FOUND") # Should trigger degraded mode for these steps assert rpa_cb.should_trigger_degraded(login_workflow, "find_username_field") assert rpa_cb.should_trigger_degraded(login_workflow, "find_password_field") print("āœ“ RPA step degradation works for UI detection failures") # Simulate form entry timeout failures for i in range(9): rpa_cb.record_failure(form_workflow, f"fill_field_{i}", "TIMEOUT") # Should trigger workflow quarantine assert rpa_cb.should_trigger_quarantine(form_workflow) print("āœ“ RPA workflow quarantine works for timeout failures") # Test failure type analysis form_failure_types = rpa_cb.get_workflow_failure_types(form_workflow) assert "TIMEOUT" in form_failure_types assert form_failure_types["TIMEOUT"] >= 9 print("āœ“ RPA failure type analysis works correctly") # Test recovery with successful operations for i in range(3): rpa_cb.record_success(login_workflow, "find_username_field") # Should no longer be degraded assert not rpa_cb.should_trigger_degraded(login_workflow, "find_username_field") print("āœ“ RPA step recovery works after successful operations") # Test comprehensive status for RPA monitoring status = rpa_cb.get_status_summary() global_stats = status['global_stats'] # Verify monitoring data is useful for RPA operations assert global_stats['workflows_with_failures'] >= 2 assert global_stats['global_failures_in_window'] >= 15 print("āœ“ RPA monitoring integration provides actionable data") print(f" - Workflows affected: {global_stats['workflows_with_failures']}") print(f" - Total system failures: {global_stats['global_failures_in_window']}") print(f" - Failure types: {global_stats.get('global_failure_types', {})}") return True except Exception as e: print(f"āœ— RPA integration test failed: {e}") import traceback traceback.print_exc() return False if __name__ == "__main__": success = True # Run main functionality tests if not test_circuit_breaker_real_functionality(): success = False # Run RPA integration tests if not test_circuit_breaker_rpa_integration(): success = False if success: print("\nāœ… All tests passed - CircuitBreaker is working correctly!") sys.exit(0) else: print("\nāŒ Some tests failed - CircuitBreaker needs attention") sys.exit(1)