rpa_vision_v3/examples/test_action_execution.py

#!/usr/bin/env python3
"""
Test Action Execution - Phase 6

Tests the ActionExecutor and TargetResolver with synthetic data.
"""

import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).parent.parent))

import logging
from dataclasses import dataclass

from core.models.screen_state import ScreenState, RawLevel, PerceptionLevel
from core.models.ui_element import UIElement, UIElementEmbeddings, VisualFeatures
from core.models.workflow_graph import (
    WorkflowEdge, Action, ActionType, TargetSpec, WindowConstraint
)
from core.execution.action_executor import ActionExecutor
from core.execution.target_resolver import TargetResolver

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)


def create_test_screen_state() -> ScreenState:
    """Create a synthetic ScreenState for testing."""
    
    # Create UI elements
    ui_elements = [
        UIElement(
            element_id="btn_submit",
            type="button",
            bbox=(100, 100, 200, 150),
            center=(150, 125),
            label="Submit",
            label_confidence=0.95,
            role="submit_button",
            confidence=0.95,
            embeddings=UIElementEmbeddings(),
            visual_features=VisualFeatures(
                dominant_color="blue",
                has_icon=False,
                shape="rectangle",
                size_category="medium"
            )
        ),
        UIElement(
            element_id="input_email",
            type="text_input",
            bbox=(100, 50, 300, 80),
            center=(200, 65),
            label="",
            label_confidence=0.92,
            role="email_input",
            confidence=0.92,
            embeddings=UIElementEmbeddings(),
            visual_features=VisualFeatures(
                dominant_color="white",
                has_icon=False,
                shape="rectangle",
                size_category="medium"
            )
        ),
        UIElement(
            element_id="btn_cancel",
            type="button",
            bbox=(220, 100, 320, 150),
            center=(270, 125),
            label="Cancel",
            label_confidence=0.90,
            role="cancel_button",
            confidence=0.90,
            embeddings=UIElementEmbeddings(),
            visual_features=VisualFeatures(
                dominant_color="gray",
                has_icon=False,
                shape="rectangle",
                size_category="medium"
            )
        )
    ]
    
    # Create ScreenState
    raw = RawLevel(
        screenshot_path="test_screenshot.png",
        capture_method="test",
        file_size_bytes=1024
    )
    
    perception = PerceptionLevel(
        ui_elements=ui_elements,
        ocr_text="Submit Cancel",
        detected_at=1234567890.0
    )
    
    return ScreenState(
        id="test_state_1",
        raw=raw,
        perception=perception
    )


def test_target_resolver():
    """Test TargetResolver with different strategies."""
    logger.info("\n=== Testing TargetResolver ===")
    
    screen_state = create_test_screen_state()
    resolver = TargetResolver()
    
    # Test 1: Resolve by role
    logger.info("\nTest 1: Resolve by role (submit_button)")
    target_spec = TargetSpec(
        by_role="submit_button",
        selection_policy="first"
    )
    result = resolver.resolve_target(target_spec, screen_state)
    
    if result:
        logger.info(f"✓ Resolved: {result.element.text} (confidence={result.confidence:.2f})")
        logger.info(f"  Strategy: {result.strategy_used}, Fallback: {result.fallback_applied}")
    else:
        logger.error("✗ Failed to resolve target")
    
    # Test 2: Resolve by text
    logger.info("\nTest 2: Resolve by text (Cancel)")
    target_spec = TargetSpec(
        by_text="Cancel",
        selection_policy="first"
    )
    result = resolver.resolve_target(target_spec, screen_state)
    
    if result:
        logger.info(f"✓ Resolved: {result.element.text} (confidence={result.confidence:.2f})")
        logger.info(f"  Strategy: {result.strategy_used}, Fallback: {result.fallback_applied}")
    else:
        logger.error("✗ Failed to resolve target")
    
    # Test 3: Resolve by position
    logger.info("\nTest 3: Resolve by position (150, 125)")
    target_spec = TargetSpec(
        by_position=(150, 125),
        selection_policy="first"
    )
    result = resolver.resolve_target(target_spec, screen_state)
    
    if result:
        logger.info(f"✓ Resolved: {result.element.text} (confidence={result.confidence:.2f})")
        logger.info(f"  Strategy: {result.strategy_used}, Fallback: {result.fallback_applied}")
    else:
        logger.error("✗ Failed to resolve target")
    
    # Test 4: Selection policy - last
    logger.info("\nTest 4: Selection policy (last button)")
    target_spec = TargetSpec(
        by_role="cancel_button",
        selection_policy="last"
    )
    result = resolver.resolve_target(target_spec, screen_state)
    
    if result:
        logger.info(f"✓ Resolved: {result.element.text} (confidence={result.confidence:.2f})")
    else:
        logger.error("✗ Failed to resolve target")


def test_action_executor_dry_run():
    """Test ActionExecutor without actually executing (dry run)."""
    logger.info("\n=== Testing ActionExecutor (Dry Run) ===")
    
    screen_state = create_test_screen_state()
    executor = ActionExecutor(verify_postconditions=False)
    
    # Test 1: Mouse click action
    logger.info("\nTest 1: Mouse click action")
    action = Action(
        type=ActionType.MOUSE_CLICK,
        target=TargetSpec(
            by_role="submit_button",
            selection_policy="first"
        ),
        params={'wait_after_ms': 100}
    )
    
    edge = WorkflowEdge(
        from_node="node_1",
        to_node="node_2",
        action=action
    )
    
    # Note: This will actually try to click if pyautogui is available
    # In production, we'd mock pyautogui for testing
    logger.info("  Action configured: mouse_click on submit_button")
    logger.info("  (Skipping actual execution in test)")
    
    # Test 2: Text input action
    logger.info("\nTest 2: Text input action")
    action = Action(
        type=ActionType.TEXT_INPUT,
        target=TargetSpec(
            by_role="email_input",
            selection_policy="first"
        ),
        params={
            'text': 'test@example.com',
            'wait_after_ms': 100
        }
    )
    
    edge = WorkflowEdge(
        from_node="node_2",
        to_node="node_3",
        action=action
    )
    
    logger.info("  Action configured: text_input on email_input")
    logger.info("  Text: test@example.com")
    logger.info("  (Skipping actual execution in test)")
    
    # Test 3: Window constraints
    logger.info("\nTest 3: Window constraint validation")
    window_constraint = WindowConstraint(title_contains='Test Form')
    matches = window_constraint.matches('Test Form', 'test_process')
    logger.info(f"  Window constraint check: {'✓ PASS' if matches else '✗ FAIL'}")
    
    # Test 4: Window constraint failure
    logger.info("\nTest 4: Window constraint failure")
    window_constraint_bad = WindowConstraint(title_contains='Wrong Title')
    matches = window_constraint_bad.matches('Test Form', 'test_process')
    logger.info(f"  Window constraint check: {'✓ PASS' if matches else '✗ FAIL (expected)'}")


def test_compound_action():
    """Test compound action structure."""
    logger.info("\n=== Testing Compound Action ===")
    
    # Create compound action
    sub_action1 = Action(
        type=ActionType.MOUSE_CLICK,
        target=TargetSpec(by_role="email_input", selection_policy="first")
    )
    
    sub_action2 = Action(
        type=ActionType.TEXT_INPUT,
        target=TargetSpec(by_role="email_input", selection_policy="first"),
        params={'text': 'test@example.com'}
    )
    
    sub_action3 = Action(
        type=ActionType.MOUSE_CLICK,
        target=TargetSpec(by_role="submit_button", selection_policy="first")
    )
    
    compound = Action(
        type=ActionType.COMPOUND,
        target=TargetSpec(by_role="form", selection_policy="first"),
        params={
            'actions': [sub_action1, sub_action2, sub_action3],
            'repeat_policy': 'all'
        }
    )
    
    logger.info("Compound action created:")
    logger.info(f"  - {len(compound.params['actions'])} sub-actions")
    logger.info(f"  - Repeat policy: {compound.params['repeat_policy']}")
    logger.info("  Steps:")
    logger.info("    1. Click email input")
    logger.info("    2. Type email")
    logger.info("    3. Click submit")


def main():
    """Run all tests."""
    logger.info("=" * 60)
    logger.info("Phase 6 - Action Execution Tests")
    logger.info("=" * 60)
    
    try:
        test_target_resolver()
        test_action_executor_dry_run()
        test_compound_action()
        
        logger.info("\n" + "=" * 60)
        logger.info("✓ All tests completed successfully")
        logger.info("=" * 60)
        
    except Exception as e:
        logger.error(f"\n✗ Test failed: {e}", exc_info=True)
        return 1
    
    return 0


if __name__ == '__main__':
    sys.exit(main())