// lea_uia — Helper Windows UI Automation pour Léa // // Binaire standalone qui expose 3 commandes UIA : // query → retourne l'élément UIA à une position (x, y) // find → retrouve un élément par son chemin logique // capture → liste les éléments visibles (debug) // // Communication avec l'agent Python via stdin/stdout JSON. // Tous les appels sont non-bloquants et retournent du JSON structuré. // // Sur Linux (développement) : retourne des stubs d'erreur. // Sur Windows : utilise UIAutomationCore via `windows-rs`. use clap::{Parser, Subcommand}; use serde::{Deserialize, Serialize}; #[derive(Parser)] #[command(name = "lea_uia")] #[command(about = "Helper UI Automation pour Léa", long_about = None)] #[command(version)] struct Cli { #[command(subcommand)] command: Commands, } #[derive(Subcommand)] enum Commands { /// Retourner l'élément UIA à une position donnée (x, y en pixels écran) Query { /// Coordonnée X (pixels) #[arg(long)] x: i32, /// Coordonnée Y (pixels) #[arg(long)] y: i32, /// Inclure la hiérarchie des parents (peut être lent) #[arg(long, default_value_t = true)] with_parents: bool, }, /// Rechercher un élément par son chemin logique ou son nom Find { /// Nom de l'élément (Name property) #[arg(long)] name: Option, /// Type de contrôle (Button, Edit, MenuItem, etc.) #[arg(long)] control_type: Option, /// AutomationId #[arg(long)] automation_id: Option, /// Limite la recherche à cette fenêtre (titre exact) #[arg(long)] window: Option, /// Timeout en millisecondes #[arg(long, default_value_t = 2000)] timeout_ms: u32, }, /// Lister tous les éléments visibles de la fenêtre active (debug) Capture { /// Profondeur maximale de l'arbre #[arg(long, default_value_t = 3)] max_depth: u32, }, /// Vérifier que UIA est disponible et fonctionnel Health, } // ========================================================================= // Modèles de sortie JSON // ========================================================================= #[derive(Serialize, Deserialize, Debug, Clone)] struct UiaElement { /// Nom visible de l'élément name: String, /// Type de contrôle (Button, Edit, MenuItem, Window, ...) control_type: String, /// Classe Windows (Edit, Static, #32770, ...) class_name: String, /// AutomationId (ID interne, parfois vide) automation_id: String, /// Rectangle absolu [x1, y1, x2, y2] en pixels écran bounding_rect: [i32; 4], /// Est-ce que l'élément est activable is_enabled: bool, /// Est-ce que l'élément est visible is_offscreen: bool, /// Hiérarchie des parents (chemin logique) #[serde(skip_serializing_if = "Vec::is_empty")] parent_path: Vec, /// Process owning this element #[serde(skip_serializing_if = "String::is_empty")] process_name: String, } #[derive(Serialize, Deserialize, Debug, Clone)] struct ParentHint { name: String, control_type: String, } #[derive(Serialize, Deserialize, Debug)] #[serde(tag = "status")] enum UiaResponse { #[serde(rename = "ok")] Ok { element: Option, #[serde(skip_serializing_if = "Vec::is_empty")] elements: Vec, elapsed_ms: u64, }, #[serde(rename = "not_found")] NotFound { reason: String, elapsed_ms: u64, }, #[serde(rename = "error")] Error { message: String, code: String, }, #[serde(rename = "unavailable")] Unavailable { reason: String, }, } // ========================================================================= // Implémentation Windows // ========================================================================= #[cfg(windows)] mod uia_impl { use super::*; use std::time::Instant; use windows::Win32::Foundation::POINT; use windows::Win32::System::Com::{ CoCreateInstance, CoInitializeEx, CoUninitialize, CLSCTX_INPROC_SERVER, COINIT_APARTMENTTHREADED, }; use windows::Win32::UI::Accessibility::{ CUIAutomation, IUIAutomation, IUIAutomationElement, IUIAutomationTreeWalker, }; struct ComGuard; impl ComGuard { fn new() -> windows::core::Result { unsafe { let hr = CoInitializeEx(None, COINIT_APARTMENTTHREADED); if hr.is_err() { // RPC_E_CHANGED_MODE : le thread est déjà initialisé → OK let code = hr.0 as u32; if code != 0x80010106 { return Err(windows::core::Error::from(hr)); } } } Ok(Self) } } impl Drop for ComGuard { fn drop(&mut self) { unsafe { CoUninitialize() }; } } fn get_automation() -> windows::core::Result { unsafe { CoCreateInstance(&CUIAutomation, None, CLSCTX_INPROC_SERVER) } } fn element_to_struct( element: &IUIAutomationElement, with_parents: bool, ) -> windows::core::Result { let mut result = UiaElement { name: String::new(), control_type: String::new(), class_name: String::new(), automation_id: String::new(), bounding_rect: [0, 0, 0, 0], is_enabled: false, is_offscreen: true, parent_path: Vec::new(), process_name: String::new(), }; unsafe { if let Ok(name) = element.CurrentName() { result.name = name.to_string(); } if let Ok(ct) = element.CurrentLocalizedControlType() { result.control_type = ct.to_string(); } if let Ok(cn) = element.CurrentClassName() { result.class_name = cn.to_string(); } if let Ok(aid) = element.CurrentAutomationId() { result.automation_id = aid.to_string(); } if let Ok(rect) = element.CurrentBoundingRectangle() { result.bounding_rect = [rect.left, rect.top, rect.right, rect.bottom]; } if let Ok(enabled) = element.CurrentIsEnabled() { result.is_enabled = enabled.as_bool(); } if let Ok(offscreen) = element.CurrentIsOffscreen() { result.is_offscreen = offscreen.as_bool(); } if with_parents { // Remonter la hiérarchie jusqu'à la Window root if let Ok(automation) = get_automation() { let walker = automation.ControlViewWalker(); if let Ok(walker) = walker { let mut current = element.clone(); for _ in 0..10 { match walker.GetParentElement(¤t) { Ok(parent) => { let name = parent .CurrentName() .map(|n| n.to_string()) .unwrap_or_default(); let ct = parent .CurrentLocalizedControlType() .map(|c| c.to_string()) .unwrap_or_default(); if name.is_empty() && ct.is_empty() { break; } result.parent_path.insert( 0, ParentHint { name, control_type: ct, }, ); current = parent; } Err(_) => break, } } } } } } Ok(result) } pub fn query_at_point(x: i32, y: i32, with_parents: bool) -> UiaResponse { let start = Instant::now(); let _com = match ComGuard::new() { Ok(g) => g, Err(e) => { return UiaResponse::Error { message: format!("CoInitializeEx: {}", e), code: "com_init_failed".into(), } } }; let automation = match get_automation() { Ok(a) => a, Err(e) => { return UiaResponse::Error { message: format!("CUIAutomation: {}", e), code: "automation_failed".into(), } } }; let point = POINT { x, y }; let element = unsafe { automation.ElementFromPoint(point) }; match element { Ok(el) => match element_to_struct(&el, with_parents) { Ok(e) => UiaResponse::Ok { element: Some(e), elements: Vec::new(), elapsed_ms: start.elapsed().as_millis() as u64, }, Err(e) => UiaResponse::Error { message: format!("element_to_struct: {}", e), code: "extract_failed".into(), }, }, Err(_) => UiaResponse::NotFound { reason: format!("Aucun élément UIA à ({}, {})", x, y), elapsed_ms: start.elapsed().as_millis() as u64, }, } } pub fn find_element( name: Option, _control_type: Option, _automation_id: Option, _window: Option, _timeout_ms: u32, ) -> UiaResponse { let start = Instant::now(); let _com = match ComGuard::new() { Ok(g) => g, Err(e) => { return UiaResponse::Error { message: format!("CoInitializeEx: {}", e), code: "com_init_failed".into(), } } }; let automation = match get_automation() { Ok(a) => a, Err(e) => { return UiaResponse::Error { message: format!("CUIAutomation: {}", e), code: "automation_failed".into(), } } }; let root = match unsafe { automation.GetRootElement() } { Ok(r) => r, Err(e) => { return UiaResponse::Error { message: format!("GetRootElement: {}", e), code: "root_failed".into(), } } }; // Recherche simple par parcours d'arbre (MVP) // L'arbre UIA peut être énorme → on limite la profondeur if let Some(target_name) = name { let walker = unsafe { automation.ControlViewWalker() }; if let Ok(walker) = walker { if let Some(found) = walk_and_find(&walker, &root, &target_name, 0, 6, &_control_type, &_automation_id) { match element_to_struct(&found, true) { Ok(e) => { return UiaResponse::Ok { element: Some(e), elements: Vec::new(), elapsed_ms: start.elapsed().as_millis() as u64, } } Err(e) => { return UiaResponse::Error { message: format!("element_to_struct: {}", e), code: "extract_failed".into(), } } } } } } UiaResponse::NotFound { reason: "Aucun élément trouvé".into(), elapsed_ms: start.elapsed().as_millis() as u64, } } /// Parcours récursif de l'arbre UIA pour trouver un élément par nom fn walk_and_find( walker: &IUIAutomationTreeWalker, element: &IUIAutomationElement, target_name: &str, depth: u32, max_depth: u32, target_control_type: &Option, target_automation_id: &Option, ) -> Option { if depth > max_depth { return None; } // Tester l'élément courant unsafe { if let Ok(name) = element.CurrentName() { if name.to_string() == target_name { // Vérifier les filtres additionnels let mut matches = true; if let Some(ct) = target_control_type { if let Ok(local_ct) = element.CurrentLocalizedControlType() { if !local_ct.to_string().to_lowercase().contains(&ct.to_lowercase()) { matches = false; } } } if matches { if let Some(aid) = target_automation_id { if let Ok(local_aid) = element.CurrentAutomationId() { if local_aid.to_string() != *aid { matches = false; } } } } if matches { return Some(element.clone()); } } } // Parcourir les enfants if let Ok(first_child) = walker.GetFirstChildElement(element) { let mut current = first_child; loop { if let Some(found) = walk_and_find( walker, ¤t, target_name, depth + 1, max_depth, target_control_type, target_automation_id, ) { return Some(found); } match walker.GetNextSiblingElement(¤t) { Ok(next) => current = next, Err(_) => break, } } } } None } pub fn capture_tree(_max_depth: u32) -> UiaResponse { let start = Instant::now(); let _com = match ComGuard::new() { Ok(g) => g, Err(e) => { return UiaResponse::Error { message: format!("CoInitializeEx: {}", e), code: "com_init_failed".into(), } } }; let automation = match get_automation() { Ok(a) => a, Err(e) => { return UiaResponse::Error { message: format!("CUIAutomation: {}", e), code: "automation_failed".into(), } } }; let focused = unsafe { automation.GetFocusedElement() }; match focused { Ok(el) => match element_to_struct(&el, true) { Ok(e) => UiaResponse::Ok { element: Some(e), elements: Vec::new(), elapsed_ms: start.elapsed().as_millis() as u64, }, Err(e) => UiaResponse::Error { message: format!("element_to_struct: {}", e), code: "extract_failed".into(), }, }, Err(e) => UiaResponse::Error { message: format!("GetFocusedElement: {}", e), code: "focused_failed".into(), }, } } pub fn health_check() -> UiaResponse { let _com = match ComGuard::new() { Ok(g) => g, Err(e) => { return UiaResponse::Unavailable { reason: format!("COM init failed: {}", e), } } }; match get_automation() { Ok(_) => UiaResponse::Ok { element: None, elements: Vec::new(), elapsed_ms: 0, }, Err(e) => UiaResponse::Unavailable { reason: format!("UIA not available: {}", e), }, } } } // ========================================================================= // Stub Linux (pour développement et tests) // ========================================================================= #[cfg(not(windows))] mod uia_impl { use super::*; pub fn query_at_point(_x: i32, _y: i32, _with_parents: bool) -> UiaResponse { UiaResponse::Unavailable { reason: "UIA n'est disponible que sur Windows".into(), } } pub fn find_element( _name: Option, _control_type: Option, _automation_id: Option, _window: Option, _timeout_ms: u32, ) -> UiaResponse { UiaResponse::Unavailable { reason: "UIA n'est disponible que sur Windows".into(), } } pub fn capture_tree(_max_depth: u32) -> UiaResponse { UiaResponse::Unavailable { reason: "UIA n'est disponible que sur Windows".into(), } } pub fn health_check() -> UiaResponse { UiaResponse::Unavailable { reason: "UIA n'est disponible que sur Windows".into(), } } } // ========================================================================= // Main // ========================================================================= fn main() { let cli = Cli::parse(); let response = match cli.command { Commands::Query { x, y, with_parents, } => uia_impl::query_at_point(x, y, with_parents), Commands::Find { name, control_type, automation_id, window, timeout_ms, } => uia_impl::find_element(name, control_type, automation_id, window, timeout_ms), Commands::Capture { max_depth } => uia_impl::capture_tree(max_depth), Commands::Health => uia_impl::health_check(), }; // Sortie JSON sur stdout match serde_json::to_string(&response) { Ok(json) => println!("{}", json), Err(e) => { eprintln!("{{\"status\":\"error\",\"message\":\"JSON serialization: {}\"}}", e); std::process::exit(1); } } }