#!/usr/bin/env python3 """ Test Action Execution - Phase 6 Tests the ActionExecutor and TargetResolver with synthetic data. """ import sys from pathlib import Path sys.path.insert(0, str(Path(__file__).parent.parent)) import logging from dataclasses import dataclass from core.models.screen_state import ScreenState, RawLevel, PerceptionLevel from core.models.ui_element import UIElement, UIElementEmbeddings, VisualFeatures from core.models.workflow_graph import ( WorkflowEdge, Action, ActionType, TargetSpec, WindowConstraint ) from core.execution.action_executor import ActionExecutor from core.execution.target_resolver import TargetResolver logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) def create_test_screen_state() -> ScreenState: """Create a synthetic ScreenState for testing.""" # Create UI elements ui_elements = [ UIElement( element_id="btn_submit", type="button", bbox=(100, 100, 200, 150), center=(150, 125), label="Submit", label_confidence=0.95, role="submit_button", confidence=0.95, embeddings=UIElementEmbeddings(), visual_features=VisualFeatures( dominant_color="blue", has_icon=False, shape="rectangle", size_category="medium" ) ), UIElement( element_id="input_email", type="text_input", bbox=(100, 50, 300, 80), center=(200, 65), label="", label_confidence=0.92, role="email_input", confidence=0.92, embeddings=UIElementEmbeddings(), visual_features=VisualFeatures( dominant_color="white", has_icon=False, shape="rectangle", size_category="medium" ) ), UIElement( element_id="btn_cancel", type="button", bbox=(220, 100, 320, 150), center=(270, 125), label="Cancel", label_confidence=0.90, role="cancel_button", confidence=0.90, embeddings=UIElementEmbeddings(), visual_features=VisualFeatures( dominant_color="gray", has_icon=False, shape="rectangle", size_category="medium" ) ) ] # Create ScreenState raw = RawLevel( screenshot_path="test_screenshot.png", capture_method="test", file_size_bytes=1024 ) perception = PerceptionLevel( ui_elements=ui_elements, ocr_text="Submit Cancel", detected_at=1234567890.0 ) return ScreenState( id="test_state_1", raw=raw, perception=perception ) def test_target_resolver(): """Test TargetResolver with different strategies.""" logger.info("\n=== Testing TargetResolver ===") screen_state = create_test_screen_state() resolver = TargetResolver() # Test 1: Resolve by role logger.info("\nTest 1: Resolve by role (submit_button)") target_spec = TargetSpec( by_role="submit_button", selection_policy="first" ) result = resolver.resolve_target(target_spec, screen_state) if result: logger.info(f"✓ Resolved: {result.element.text} (confidence={result.confidence:.2f})") logger.info(f" Strategy: {result.strategy_used}, Fallback: {result.fallback_applied}") else: logger.error("✗ Failed to resolve target") # Test 2: Resolve by text logger.info("\nTest 2: Resolve by text (Cancel)") target_spec = TargetSpec( by_text="Cancel", selection_policy="first" ) result = resolver.resolve_target(target_spec, screen_state) if result: logger.info(f"✓ Resolved: {result.element.text} (confidence={result.confidence:.2f})") logger.info(f" Strategy: {result.strategy_used}, Fallback: {result.fallback_applied}") else: logger.error("✗ Failed to resolve target") # Test 3: Resolve by position logger.info("\nTest 3: Resolve by position (150, 125)") target_spec = TargetSpec( by_position=(150, 125), selection_policy="first" ) result = resolver.resolve_target(target_spec, screen_state) if result: logger.info(f"✓ Resolved: {result.element.text} (confidence={result.confidence:.2f})") logger.info(f" Strategy: {result.strategy_used}, Fallback: {result.fallback_applied}") else: logger.error("✗ Failed to resolve target") # Test 4: Selection policy - last logger.info("\nTest 4: Selection policy (last button)") target_spec = TargetSpec( by_role="cancel_button", selection_policy="last" ) result = resolver.resolve_target(target_spec, screen_state) if result: logger.info(f"✓ Resolved: {result.element.text} (confidence={result.confidence:.2f})") else: logger.error("✗ Failed to resolve target") def test_action_executor_dry_run(): """Test ActionExecutor without actually executing (dry run).""" logger.info("\n=== Testing ActionExecutor (Dry Run) ===") screen_state = create_test_screen_state() executor = ActionExecutor(verify_postconditions=False) # Test 1: Mouse click action logger.info("\nTest 1: Mouse click action") action = Action( type=ActionType.MOUSE_CLICK, target=TargetSpec( by_role="submit_button", selection_policy="first" ), params={'wait_after_ms': 100} ) edge = WorkflowEdge( from_node="node_1", to_node="node_2", action=action ) # Note: This will actually try to click if pyautogui is available # In production, we'd mock pyautogui for testing logger.info(" Action configured: mouse_click on submit_button") logger.info(" (Skipping actual execution in test)") # Test 2: Text input action logger.info("\nTest 2: Text input action") action = Action( type=ActionType.TEXT_INPUT, target=TargetSpec( by_role="email_input", selection_policy="first" ), params={ 'text': 'test@example.com', 'wait_after_ms': 100 } ) edge = WorkflowEdge( from_node="node_2", to_node="node_3", action=action ) logger.info(" Action configured: text_input on email_input") logger.info(" Text: test@example.com") logger.info(" (Skipping actual execution in test)") # Test 3: Window constraints logger.info("\nTest 3: Window constraint validation") window_constraint = WindowConstraint(title_contains='Test Form') matches = window_constraint.matches('Test Form', 'test_process') logger.info(f" Window constraint check: {'✓ PASS' if matches else '✗ FAIL'}") # Test 4: Window constraint failure logger.info("\nTest 4: Window constraint failure") window_constraint_bad = WindowConstraint(title_contains='Wrong Title') matches = window_constraint_bad.matches('Test Form', 'test_process') logger.info(f" Window constraint check: {'✓ PASS' if matches else '✗ FAIL (expected)'}") def test_compound_action(): """Test compound action structure.""" logger.info("\n=== Testing Compound Action ===") # Create compound action sub_action1 = Action( type=ActionType.MOUSE_CLICK, target=TargetSpec(by_role="email_input", selection_policy="first") ) sub_action2 = Action( type=ActionType.TEXT_INPUT, target=TargetSpec(by_role="email_input", selection_policy="first"), params={'text': 'test@example.com'} ) sub_action3 = Action( type=ActionType.MOUSE_CLICK, target=TargetSpec(by_role="submit_button", selection_policy="first") ) compound = Action( type=ActionType.COMPOUND, target=TargetSpec(by_role="form", selection_policy="first"), params={ 'actions': [sub_action1, sub_action2, sub_action3], 'repeat_policy': 'all' } ) logger.info("Compound action created:") logger.info(f" - {len(compound.params['actions'])} sub-actions") logger.info(f" - Repeat policy: {compound.params['repeat_policy']}") logger.info(" Steps:") logger.info(" 1. Click email input") logger.info(" 2. Type email") logger.info(" 3. Click submit") def main(): """Run all tests.""" logger.info("=" * 60) logger.info("Phase 6 - Action Execution Tests") logger.info("=" * 60) try: test_target_resolver() test_action_executor_dry_run() test_compound_action() logger.info("\n" + "=" * 60) logger.info("✓ All tests completed successfully") logger.info("=" * 60) except Exception as e: logger.error(f"\n✗ Test failed: {e}", exc_info=True) return 1 return 0 if __name__ == '__main__': sys.exit(main())