v1.0 - Version stable: multi-PC, détection UI-DETR-1, 3 modes exécution
- Frontend v4 accessible sur réseau local (192.168.1.40) - Ports ouverts: 3002 (frontend), 5001 (backend), 5004 (dashboard) - Ollama GPU fonctionnel - Self-healing interactif - Dashboard confiance Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Dom
296
examples/test_action_execution.py
Normal file
296
examples/test_action_execution.py
Normal file
@@ -0,0 +1,296 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Test Action Execution - Phase 6
|
||||
|
||||
Tests the ActionExecutor and TargetResolver with synthetic data.
|
||||
"""
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
|
||||
from core.models.screen_state import ScreenState, RawLevel, PerceptionLevel
|
||||
from core.models.ui_element import UIElement, UIElementEmbeddings, VisualFeatures
|
||||
from core.models.workflow_graph import (
|
||||
WorkflowEdge, Action, ActionType, TargetSpec, WindowConstraint
|
||||
)
|
||||
from core.execution.action_executor import ActionExecutor
|
||||
from core.execution.target_resolver import TargetResolver
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def create_test_screen_state() -> ScreenState:
|
||||
"""Create a synthetic ScreenState for testing."""
|
||||
|
||||
# Create UI elements
|
||||
ui_elements = [
|
||||
UIElement(
|
||||
element_id="btn_submit",
|
||||
type="button",
|
||||
bbox=(100, 100, 200, 150),
|
||||
center=(150, 125),
|
||||
label="Submit",
|
||||
label_confidence=0.95,
|
||||
role="submit_button",
|
||||
confidence=0.95,
|
||||
embeddings=UIElementEmbeddings(),
|
||||
visual_features=VisualFeatures(
|
||||
dominant_color="blue",
|
||||
has_icon=False,
|
||||
shape="rectangle",
|
||||
size_category="medium"
|
||||
)
|
||||
),
|
||||
UIElement(
|
||||
element_id="input_email",
|
||||
type="text_input",
|
||||
bbox=(100, 50, 300, 80),
|
||||
center=(200, 65),
|
||||
label="",
|
||||
label_confidence=0.92,
|
||||
role="email_input",
|
||||
confidence=0.92,
|
||||
embeddings=UIElementEmbeddings(),
|
||||
visual_features=VisualFeatures(
|
||||
dominant_color="white",
|
||||
has_icon=False,
|
||||
shape="rectangle",
|
||||
size_category="medium"
|
||||
)
|
||||
),
|
||||
UIElement(
|
||||
element_id="btn_cancel",
|
||||
type="button",
|
||||
bbox=(220, 100, 320, 150),
|
||||
center=(270, 125),
|
||||
label="Cancel",
|
||||
label_confidence=0.90,
|
||||
role="cancel_button",
|
||||
confidence=0.90,
|
||||
embeddings=UIElementEmbeddings(),
|
||||
visual_features=VisualFeatures(
|
||||
dominant_color="gray",
|
||||
has_icon=False,
|
||||
shape="rectangle",
|
||||
size_category="medium"
|
||||
)
|
||||
)
|
||||
]
|
||||
|
||||
# Create ScreenState
|
||||
raw = RawLevel(
|
||||
screenshot_path="test_screenshot.png",
|
||||
capture_method="test",
|
||||
file_size_bytes=1024
|
||||
)
|
||||
|
||||
perception = PerceptionLevel(
|
||||
ui_elements=ui_elements,
|
||||
ocr_text="Submit Cancel",
|
||||
detected_at=1234567890.0
|
||||
)
|
||||
|
||||
return ScreenState(
|
||||
id="test_state_1",
|
||||
raw=raw,
|
||||
perception=perception
|
||||
)
|
||||
|
||||
|
||||
def test_target_resolver():
|
||||
"""Test TargetResolver with different strategies."""
|
||||
logger.info("\n=== Testing TargetResolver ===")
|
||||
|
||||
screen_state = create_test_screen_state()
|
||||
resolver = TargetResolver()
|
||||
|
||||
# Test 1: Resolve by role
|
||||
logger.info("\nTest 1: Resolve by role (submit_button)")
|
||||
target_spec = TargetSpec(
|
||||
by_role="submit_button",
|
||||
selection_policy="first"
|
||||
)
|
||||
result = resolver.resolve_target(target_spec, screen_state)
|
||||
|
||||
if result:
|
||||
logger.info(f"✓ Resolved: {result.element.text} (confidence={result.confidence:.2f})")
|
||||
logger.info(f" Strategy: {result.strategy_used}, Fallback: {result.fallback_applied}")
|
||||
else:
|
||||
logger.error("✗ Failed to resolve target")
|
||||
|
||||
# Test 2: Resolve by text
|
||||
logger.info("\nTest 2: Resolve by text (Cancel)")
|
||||
target_spec = TargetSpec(
|
||||
by_text="Cancel",
|
||||
selection_policy="first"
|
||||
)
|
||||
result = resolver.resolve_target(target_spec, screen_state)
|
||||
|
||||
if result:
|
||||
logger.info(f"✓ Resolved: {result.element.text} (confidence={result.confidence:.2f})")
|
||||
logger.info(f" Strategy: {result.strategy_used}, Fallback: {result.fallback_applied}")
|
||||
else:
|
||||
logger.error("✗ Failed to resolve target")
|
||||
|
||||
# Test 3: Resolve by position
|
||||
logger.info("\nTest 3: Resolve by position (150, 125)")
|
||||
target_spec = TargetSpec(
|
||||
by_position=(150, 125),
|
||||
selection_policy="first"
|
||||
)
|
||||
result = resolver.resolve_target(target_spec, screen_state)
|
||||
|
||||
if result:
|
||||
logger.info(f"✓ Resolved: {result.element.text} (confidence={result.confidence:.2f})")
|
||||
logger.info(f" Strategy: {result.strategy_used}, Fallback: {result.fallback_applied}")
|
||||
else:
|
||||
logger.error("✗ Failed to resolve target")
|
||||
|
||||
# Test 4: Selection policy - last
|
||||
logger.info("\nTest 4: Selection policy (last button)")
|
||||
target_spec = TargetSpec(
|
||||
by_role="cancel_button",
|
||||
selection_policy="last"
|
||||
)
|
||||
result = resolver.resolve_target(target_spec, screen_state)
|
||||
|
||||
if result:
|
||||
logger.info(f"✓ Resolved: {result.element.text} (confidence={result.confidence:.2f})")
|
||||
else:
|
||||
logger.error("✗ Failed to resolve target")
|
||||
|
||||
|
||||
def test_action_executor_dry_run():
|
||||
"""Test ActionExecutor without actually executing (dry run)."""
|
||||
logger.info("\n=== Testing ActionExecutor (Dry Run) ===")
|
||||
|
||||
screen_state = create_test_screen_state()
|
||||
executor = ActionExecutor(verify_postconditions=False)
|
||||
|
||||
# Test 1: Mouse click action
|
||||
logger.info("\nTest 1: Mouse click action")
|
||||
action = Action(
|
||||
type=ActionType.MOUSE_CLICK,
|
||||
target=TargetSpec(
|
||||
by_role="submit_button",
|
||||
selection_policy="first"
|
||||
),
|
||||
params={'wait_after_ms': 100}
|
||||
)
|
||||
|
||||
edge = WorkflowEdge(
|
||||
from_node="node_1",
|
||||
to_node="node_2",
|
||||
action=action
|
||||
)
|
||||
|
||||
# Note: This will actually try to click if pyautogui is available
|
||||
# In production, we'd mock pyautogui for testing
|
||||
logger.info(" Action configured: mouse_click on submit_button")
|
||||
logger.info(" (Skipping actual execution in test)")
|
||||
|
||||
# Test 2: Text input action
|
||||
logger.info("\nTest 2: Text input action")
|
||||
action = Action(
|
||||
type=ActionType.TEXT_INPUT,
|
||||
target=TargetSpec(
|
||||
by_role="email_input",
|
||||
selection_policy="first"
|
||||
),
|
||||
params={
|
||||
'text': 'test@example.com',
|
||||
'wait_after_ms': 100
|
||||
}
|
||||
)
|
||||
|
||||
edge = WorkflowEdge(
|
||||
from_node="node_2",
|
||||
to_node="node_3",
|
||||
action=action
|
||||
)
|
||||
|
||||
logger.info(" Action configured: text_input on email_input")
|
||||
logger.info(" Text: test@example.com")
|
||||
logger.info(" (Skipping actual execution in test)")
|
||||
|
||||
# Test 3: Window constraints
|
||||
logger.info("\nTest 3: Window constraint validation")
|
||||
window_constraint = WindowConstraint(title_contains='Test Form')
|
||||
matches = window_constraint.matches('Test Form', 'test_process')
|
||||
logger.info(f" Window constraint check: {'✓ PASS' if matches else '✗ FAIL'}")
|
||||
|
||||
# Test 4: Window constraint failure
|
||||
logger.info("\nTest 4: Window constraint failure")
|
||||
window_constraint_bad = WindowConstraint(title_contains='Wrong Title')
|
||||
matches = window_constraint_bad.matches('Test Form', 'test_process')
|
||||
logger.info(f" Window constraint check: {'✓ PASS' if matches else '✗ FAIL (expected)'}")
|
||||
|
||||
|
||||
def test_compound_action():
|
||||
"""Test compound action structure."""
|
||||
logger.info("\n=== Testing Compound Action ===")
|
||||
|
||||
# Create compound action
|
||||
sub_action1 = Action(
|
||||
type=ActionType.MOUSE_CLICK,
|
||||
target=TargetSpec(by_role="email_input", selection_policy="first")
|
||||
)
|
||||
|
||||
sub_action2 = Action(
|
||||
type=ActionType.TEXT_INPUT,
|
||||
target=TargetSpec(by_role="email_input", selection_policy="first"),
|
||||
params={'text': 'test@example.com'}
|
||||
)
|
||||
|
||||
sub_action3 = Action(
|
||||
type=ActionType.MOUSE_CLICK,
|
||||
target=TargetSpec(by_role="submit_button", selection_policy="first")
|
||||
)
|
||||
|
||||
compound = Action(
|
||||
type=ActionType.COMPOUND,
|
||||
target=TargetSpec(by_role="form", selection_policy="first"),
|
||||
params={
|
||||
'actions': [sub_action1, sub_action2, sub_action3],
|
||||
'repeat_policy': 'all'
|
||||
}
|
||||
)
|
||||
|
||||
logger.info("Compound action created:")
|
||||
logger.info(f" - {len(compound.params['actions'])} sub-actions")
|
||||
logger.info(f" - Repeat policy: {compound.params['repeat_policy']}")
|
||||
logger.info(" Steps:")
|
||||
logger.info(" 1. Click email input")
|
||||
logger.info(" 2. Type email")
|
||||
logger.info(" 3. Click submit")
|
||||
|
||||
|
||||
def main():
|
||||
"""Run all tests."""
|
||||
logger.info("=" * 60)
|
||||
logger.info("Phase 6 - Action Execution Tests")
|
||||
logger.info("=" * 60)
|
||||
|
||||
try:
|
||||
test_target_resolver()
|
||||
test_action_executor_dry_run()
|
||||
test_compound_action()
|
||||
|
||||
logger.info("\n" + "=" * 60)
|
||||
logger.info("✓ All tests completed successfully")
|
||||
logger.info("=" * 60)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"\n✗ Test failed: {e}", exc_info=True)
|
||||
return 1
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user