feat: agent Rust Phase 2 — visual mode (template matching serveur)
- visual.rs : resolve via POST /replay/resolve_target - executor.rs : resolve avant chaque clic si visual_mode=true - Fallback blind si matching échoue - Binaire toujours 1.8 MB (pas de nouvelle dépendance) Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
110
agent_rust/src/visual.rs
Normal file
110
agent_rust/src/visual.rs
Normal file
@@ -0,0 +1,110 @@
|
||||
//! Résolution visuelle des cibles via le serveur.
|
||||
//!
|
||||
//! Envoie un screenshot + target_spec au serveur qui effectue le template
|
||||
//! matching OpenCV et retourne les coordonnées résolues (x_pct, y_pct).
|
||||
//! Approche server-side : pas de dépendance OpenCV dans le binaire Rust.
|
||||
|
||||
use crate::capture;
|
||||
use crate::config::Config;
|
||||
use reqwest::blocking::Client;
|
||||
|
||||
/// Résout visuellement une cible en envoyant le screenshot courant au serveur.
|
||||
///
|
||||
/// Capture l'écran, l'encode en JPEG base64, envoie au endpoint
|
||||
/// `/traces/stream/replay/resolve_target` qui fait le template matching.
|
||||
///
|
||||
/// Retourne Some((x_pct, y_pct)) si la cible est trouvée, None sinon.
|
||||
pub fn resolve_target_visual(
|
||||
config: &Config,
|
||||
target_spec: &serde_json::Value,
|
||||
fallback_x: f64,
|
||||
fallback_y: f64,
|
||||
screen_width: u32,
|
||||
screen_height: u32,
|
||||
) -> Option<(f64, f64)> {
|
||||
// 1. Capturer le screenshot actuel
|
||||
let screenshot = match capture::capture_screenshot() {
|
||||
Some(img) => img,
|
||||
None => {
|
||||
eprintln!(" [VISUAL] Echec capture screenshot pour résolution visuelle");
|
||||
return None;
|
||||
}
|
||||
};
|
||||
|
||||
// Encoder en JPEG base64 (qualité 75 — bon compromis taille/précision)
|
||||
let screenshot_b64 = capture::screenshot_to_jpeg_base64(&screenshot, 75);
|
||||
if screenshot_b64.is_empty() {
|
||||
eprintln!(" [VISUAL] Echec encodage JPEG");
|
||||
return None;
|
||||
}
|
||||
|
||||
println!(
|
||||
" [VISUAL] Screenshot capture ({}x{}), envoi au serveur...",
|
||||
screen_width, screen_height
|
||||
);
|
||||
|
||||
// 2. Envoyer au serveur /replay/resolve_target
|
||||
let client = Client::new();
|
||||
let payload = serde_json::json!({
|
||||
"session_id": config.agent_session_id(),
|
||||
"screenshot_b64": screenshot_b64,
|
||||
"target_spec": target_spec,
|
||||
"fallback_x_pct": fallback_x,
|
||||
"fallback_y_pct": fallback_y,
|
||||
"screen_width": screen_width,
|
||||
"screen_height": screen_height,
|
||||
});
|
||||
|
||||
let url = format!("{}/traces/stream/replay/resolve_target", config.server_url);
|
||||
|
||||
let resp = match client
|
||||
.post(&url)
|
||||
.json(&payload)
|
||||
.timeout(std::time::Duration::from_secs(30))
|
||||
.send()
|
||||
{
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
eprintln!(" [VISUAL] Erreur reseau vers {} : {}", url, e);
|
||||
return None;
|
||||
}
|
||||
};
|
||||
|
||||
if !resp.status().is_success() {
|
||||
eprintln!(
|
||||
" [VISUAL] Serveur a repondu HTTP {}",
|
||||
resp.status()
|
||||
);
|
||||
return None;
|
||||
}
|
||||
|
||||
// 3. Parser la réponse
|
||||
let data: serde_json::Value = match resp.json() {
|
||||
Ok(d) => d,
|
||||
Err(e) => {
|
||||
eprintln!(" [VISUAL] Erreur parsing reponse JSON : {}", e);
|
||||
return None;
|
||||
}
|
||||
};
|
||||
|
||||
let resolved = data["resolved"].as_bool().unwrap_or(false);
|
||||
if resolved {
|
||||
let x = data["x_pct"].as_f64()?;
|
||||
let y = data["y_pct"].as_f64()?;
|
||||
let method = data["method"].as_str().unwrap_or("?");
|
||||
let score = data["score"].as_f64().unwrap_or(0.0);
|
||||
println!(
|
||||
" [VISUAL] Resolu par {} (score={:.3}) : ({:.4}, {:.4})",
|
||||
method, score, x, y
|
||||
);
|
||||
Some((x, y))
|
||||
} else {
|
||||
let reason = data["reason"].as_str().unwrap_or("inconnu");
|
||||
let method = data["method"].as_str().unwrap_or("?");
|
||||
println!(
|
||||
" [VISUAL] Non resolu (methode={}, raison={})",
|
||||
method, reason
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user