Compare commits
15 Commits
v3.0
...
e66629ce1a
| Author | SHA1 | Date | |
|---|---|---|---|
|
e66629ce1a | ||
|
|
cecdf417b7 | ||
|
|
56e3cc052a | ||
|
|
332366b58c | ||
|
|
ac9c207474 | ||
|
|
f85d56ac05 | ||
|
|
172167f6c0 | ||
|
|
42d49dd8bd | ||
|
|
f541bb8ce4 | ||
|
|
a6eb4c168f | ||
|
|
f6ad5ff2b2 | ||
|
|
2ac781343a | ||
|
|
bffcfb2db3 | ||
|
|
cc673755f7 | ||
|
|
4509038bf0 |
3
agent_rust/lea_uia/.gitignore
vendored
Normal file
3
agent_rust/lea_uia/.gitignore
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
target/
|
||||
**/target/
|
||||
|
||||
384
agent_rust/lea_uia/Cargo.lock
generated
Normal file
384
agent_rust/lea_uia/Cargo.lock
generated
Normal file
@@ -0,0 +1,384 @@
|
||||
# This file is automatically @generated by Cargo.
|
||||
# It is not intended for manual editing.
|
||||
version = 4
|
||||
|
||||
[[package]]
|
||||
name = "anstream"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d"
|
||||
dependencies = [
|
||||
"anstyle",
|
||||
"anstyle-parse",
|
||||
"anstyle-query",
|
||||
"anstyle-wincon",
|
||||
"colorchoice",
|
||||
"is_terminal_polyfill",
|
||||
"utf8parse",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstyle"
|
||||
version = "1.0.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000"
|
||||
|
||||
[[package]]
|
||||
name = "anstyle-parse"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e"
|
||||
dependencies = [
|
||||
"utf8parse",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstyle-query"
|
||||
version = "1.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
|
||||
dependencies = [
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstyle-wincon"
|
||||
version = "3.0.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
|
||||
dependencies = [
|
||||
"anstyle",
|
||||
"once_cell_polyfill",
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap"
|
||||
version = "4.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b193af5b67834b676abd72466a96c1024e6a6ad978a1f484bd90b85c94041351"
|
||||
dependencies = [
|
||||
"clap_builder",
|
||||
"clap_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap_builder"
|
||||
version = "4.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f"
|
||||
dependencies = [
|
||||
"anstream",
|
||||
"anstyle",
|
||||
"clap_lex",
|
||||
"strsim",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap_derive"
|
||||
version = "4.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1110bd8a634a1ab8cb04345d8d878267d57c3cf1b38d91b71af6686408bbca6a"
|
||||
dependencies = [
|
||||
"heck",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap_lex"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9"
|
||||
|
||||
[[package]]
|
||||
name = "colorchoice"
|
||||
version = "1.0.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570"
|
||||
|
||||
[[package]]
|
||||
name = "heck"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
|
||||
|
||||
[[package]]
|
||||
name = "is_terminal_polyfill"
|
||||
version = "1.70.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695"
|
||||
|
||||
[[package]]
|
||||
name = "itoa"
|
||||
version = "1.0.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
|
||||
|
||||
[[package]]
|
||||
name = "lea_uia"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"clap",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"windows",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"
|
||||
|
||||
[[package]]
|
||||
name = "once_cell_polyfill"
|
||||
version = "1.70.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.106"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934"
|
||||
dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.45"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.228"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"
|
||||
dependencies = [
|
||||
"serde_core",
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_core"
|
||||
version = "1.0.228"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"
|
||||
dependencies = [
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_derive"
|
||||
version = "1.0.228"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_json"
|
||||
version = "1.0.149"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86"
|
||||
dependencies = [
|
||||
"itoa",
|
||||
"memchr",
|
||||
"serde",
|
||||
"serde_core",
|
||||
"zmij",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "strsim"
|
||||
version = "0.11.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "2.0.117"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "unicode-ident"
|
||||
version = "1.0.24"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
|
||||
|
||||
[[package]]
|
||||
name = "utf8parse"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
|
||||
|
||||
[[package]]
|
||||
name = "windows"
|
||||
version = "0.59.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7f919aee0a93304be7f62e8e5027811bbba96bcb1de84d6618be56e43f8a32a1"
|
||||
dependencies = [
|
||||
"windows-core",
|
||||
"windows-targets",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-core"
|
||||
version = "0.59.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "810ce18ed2112484b0d4e15d022e5f598113e220c53e373fb31e67e21670c1ce"
|
||||
dependencies = [
|
||||
"windows-implement",
|
||||
"windows-interface",
|
||||
"windows-result",
|
||||
"windows-strings",
|
||||
"windows-targets",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-implement"
|
||||
version = "0.59.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "83577b051e2f49a058c308f17f273b570a6a758386fc291b5f6a934dd84e48c1"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-interface"
|
||||
version = "0.59.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-link"
|
||||
version = "0.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a"
|
||||
|
||||
[[package]]
|
||||
name = "windows-link"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
|
||||
|
||||
[[package]]
|
||||
name = "windows-result"
|
||||
version = "0.3.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6"
|
||||
dependencies = [
|
||||
"windows-link 0.1.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-strings"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "87fa48cc5d406560701792be122a10132491cff9d0aeb23583cc2dcafc847319"
|
||||
dependencies = [
|
||||
"windows-link 0.1.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-sys"
|
||||
version = "0.61.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc"
|
||||
dependencies = [
|
||||
"windows-link 0.2.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-targets"
|
||||
version = "0.53.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3"
|
||||
dependencies = [
|
||||
"windows-link 0.2.1",
|
||||
"windows_aarch64_gnullvm",
|
||||
"windows_aarch64_msvc",
|
||||
"windows_i686_gnu",
|
||||
"windows_i686_gnullvm",
|
||||
"windows_i686_msvc",
|
||||
"windows_x86_64_gnu",
|
||||
"windows_x86_64_gnullvm",
|
||||
"windows_x86_64_msvc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows_aarch64_gnullvm"
|
||||
version = "0.53.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53"
|
||||
|
||||
[[package]]
|
||||
name = "windows_aarch64_msvc"
|
||||
version = "0.53.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_gnu"
|
||||
version = "0.53.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_gnullvm"
|
||||
version = "0.53.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_msvc"
|
||||
version = "0.53.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_gnu"
|
||||
version = "0.53.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_gnullvm"
|
||||
version = "0.53.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_msvc"
|
||||
version = "0.53.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650"
|
||||
|
||||
[[package]]
|
||||
name = "zmij"
|
||||
version = "1.0.21"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa"
|
||||
34
agent_rust/lea_uia/Cargo.toml
Normal file
34
agent_rust/lea_uia/Cargo.toml
Normal file
@@ -0,0 +1,34 @@
|
||||
[package]
|
||||
name = "lea_uia"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
authors = ["Dom <dom@rpa-vision-v3>"]
|
||||
description = "Helper Windows UI Automation pour Léa (agent RPA V3)"
|
||||
license = "Proprietary"
|
||||
|
||||
[[bin]]
|
||||
name = "lea_uia"
|
||||
path = "src/main.rs"
|
||||
|
||||
[dependencies]
|
||||
clap = { version = "4.5", features = ["derive"] }
|
||||
serde = { version = "1.0", features = ["derive"] }
|
||||
serde_json = "1.0"
|
||||
|
||||
[target.'cfg(windows)'.dependencies]
|
||||
windows = { version = "0.59", features = [
|
||||
"Win32_Foundation",
|
||||
"Win32_System_Com",
|
||||
"Win32_System_Ole",
|
||||
"Win32_System_Variant",
|
||||
"Win32_UI_Accessibility",
|
||||
"Win32_UI_WindowsAndMessaging",
|
||||
"Win32_Graphics_Gdi",
|
||||
] }
|
||||
|
||||
[profile.release]
|
||||
opt-level = "z" # Taille minimale
|
||||
lto = true # Link-time optimization
|
||||
codegen-units = 1 # Meilleure optimisation
|
||||
strip = true # Retirer les symboles
|
||||
panic = "abort" # Pas d'unwinding → binaire plus petit
|
||||
564
agent_rust/lea_uia/src/main.rs
Normal file
564
agent_rust/lea_uia/src/main.rs
Normal file
@@ -0,0 +1,564 @@
|
||||
// lea_uia — Helper Windows UI Automation pour Léa
|
||||
//
|
||||
// Binaire standalone qui expose 3 commandes UIA :
|
||||
// query → retourne l'élément UIA à une position (x, y)
|
||||
// find → retrouve un élément par son chemin logique
|
||||
// capture → liste les éléments visibles (debug)
|
||||
//
|
||||
// Communication avec l'agent Python via stdin/stdout JSON.
|
||||
// Tous les appels sont non-bloquants et retournent du JSON structuré.
|
||||
//
|
||||
// Sur Linux (développement) : retourne des stubs d'erreur.
|
||||
// Sur Windows : utilise UIAutomationCore via `windows-rs`.
|
||||
|
||||
use clap::{Parser, Subcommand};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Parser)]
|
||||
#[command(name = "lea_uia")]
|
||||
#[command(about = "Helper UI Automation pour Léa", long_about = None)]
|
||||
#[command(version)]
|
||||
struct Cli {
|
||||
#[command(subcommand)]
|
||||
command: Commands,
|
||||
}
|
||||
|
||||
#[derive(Subcommand)]
|
||||
enum Commands {
|
||||
/// Retourner l'élément UIA à une position donnée (x, y en pixels écran)
|
||||
Query {
|
||||
/// Coordonnée X (pixels)
|
||||
#[arg(long)]
|
||||
x: i32,
|
||||
/// Coordonnée Y (pixels)
|
||||
#[arg(long)]
|
||||
y: i32,
|
||||
/// Inclure la hiérarchie des parents (peut être lent)
|
||||
#[arg(long, default_value_t = true)]
|
||||
with_parents: bool,
|
||||
},
|
||||
/// Rechercher un élément par son chemin logique ou son nom
|
||||
Find {
|
||||
/// Nom de l'élément (Name property)
|
||||
#[arg(long)]
|
||||
name: Option<String>,
|
||||
/// Type de contrôle (Button, Edit, MenuItem, etc.)
|
||||
#[arg(long)]
|
||||
control_type: Option<String>,
|
||||
/// AutomationId
|
||||
#[arg(long)]
|
||||
automation_id: Option<String>,
|
||||
/// Limite la recherche à cette fenêtre (titre exact)
|
||||
#[arg(long)]
|
||||
window: Option<String>,
|
||||
/// Timeout en millisecondes
|
||||
#[arg(long, default_value_t = 2000)]
|
||||
timeout_ms: u32,
|
||||
},
|
||||
/// Lister tous les éléments visibles de la fenêtre active (debug)
|
||||
Capture {
|
||||
/// Profondeur maximale de l'arbre
|
||||
#[arg(long, default_value_t = 3)]
|
||||
max_depth: u32,
|
||||
},
|
||||
/// Vérifier que UIA est disponible et fonctionnel
|
||||
Health,
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// Modèles de sortie JSON
|
||||
// =========================================================================
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone)]
|
||||
struct UiaElement {
|
||||
/// Nom visible de l'élément
|
||||
name: String,
|
||||
/// Type de contrôle (Button, Edit, MenuItem, Window, ...)
|
||||
control_type: String,
|
||||
/// Classe Windows (Edit, Static, #32770, ...)
|
||||
class_name: String,
|
||||
/// AutomationId (ID interne, parfois vide)
|
||||
automation_id: String,
|
||||
/// Rectangle absolu [x1, y1, x2, y2] en pixels écran
|
||||
bounding_rect: [i32; 4],
|
||||
/// Est-ce que l'élément est activable
|
||||
is_enabled: bool,
|
||||
/// Est-ce que l'élément est visible
|
||||
is_offscreen: bool,
|
||||
/// Hiérarchie des parents (chemin logique)
|
||||
#[serde(skip_serializing_if = "Vec::is_empty")]
|
||||
parent_path: Vec<ParentHint>,
|
||||
/// Process owning this element
|
||||
#[serde(skip_serializing_if = "String::is_empty")]
|
||||
process_name: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone)]
|
||||
struct ParentHint {
|
||||
name: String,
|
||||
control_type: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug)]
|
||||
#[serde(tag = "status")]
|
||||
enum UiaResponse {
|
||||
#[serde(rename = "ok")]
|
||||
Ok {
|
||||
element: Option<UiaElement>,
|
||||
#[serde(skip_serializing_if = "Vec::is_empty")]
|
||||
elements: Vec<UiaElement>,
|
||||
elapsed_ms: u64,
|
||||
},
|
||||
#[serde(rename = "not_found")]
|
||||
NotFound {
|
||||
reason: String,
|
||||
elapsed_ms: u64,
|
||||
},
|
||||
#[serde(rename = "error")]
|
||||
Error {
|
||||
message: String,
|
||||
code: String,
|
||||
},
|
||||
#[serde(rename = "unavailable")]
|
||||
Unavailable {
|
||||
reason: String,
|
||||
},
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// Implémentation Windows
|
||||
// =========================================================================
|
||||
|
||||
#[cfg(windows)]
|
||||
mod uia_impl {
|
||||
use super::*;
|
||||
use std::time::Instant;
|
||||
use windows::Win32::Foundation::POINT;
|
||||
use windows::Win32::System::Com::{
|
||||
CoCreateInstance, CoInitializeEx, CoUninitialize, CLSCTX_INPROC_SERVER,
|
||||
COINIT_APARTMENTTHREADED,
|
||||
};
|
||||
use windows::Win32::UI::Accessibility::{
|
||||
CUIAutomation, IUIAutomation, IUIAutomationElement, IUIAutomationTreeWalker,
|
||||
};
|
||||
|
||||
struct ComGuard;
|
||||
impl ComGuard {
|
||||
fn new() -> windows::core::Result<Self> {
|
||||
unsafe {
|
||||
let hr = CoInitializeEx(None, COINIT_APARTMENTTHREADED);
|
||||
if hr.is_err() {
|
||||
// RPC_E_CHANGED_MODE : le thread est déjà initialisé → OK
|
||||
let code = hr.0 as u32;
|
||||
if code != 0x80010106 {
|
||||
return Err(windows::core::Error::from(hr));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(Self)
|
||||
}
|
||||
}
|
||||
impl Drop for ComGuard {
|
||||
fn drop(&mut self) {
|
||||
unsafe { CoUninitialize() };
|
||||
}
|
||||
}
|
||||
|
||||
fn get_automation() -> windows::core::Result<IUIAutomation> {
|
||||
unsafe { CoCreateInstance(&CUIAutomation, None, CLSCTX_INPROC_SERVER) }
|
||||
}
|
||||
|
||||
fn element_to_struct(
|
||||
element: &IUIAutomationElement,
|
||||
with_parents: bool,
|
||||
) -> windows::core::Result<UiaElement> {
|
||||
let mut result = UiaElement {
|
||||
name: String::new(),
|
||||
control_type: String::new(),
|
||||
class_name: String::new(),
|
||||
automation_id: String::new(),
|
||||
bounding_rect: [0, 0, 0, 0],
|
||||
is_enabled: false,
|
||||
is_offscreen: true,
|
||||
parent_path: Vec::new(),
|
||||
process_name: String::new(),
|
||||
};
|
||||
|
||||
unsafe {
|
||||
if let Ok(name) = element.CurrentName() {
|
||||
result.name = name.to_string();
|
||||
}
|
||||
if let Ok(ct) = element.CurrentLocalizedControlType() {
|
||||
result.control_type = ct.to_string();
|
||||
}
|
||||
if let Ok(cn) = element.CurrentClassName() {
|
||||
result.class_name = cn.to_string();
|
||||
}
|
||||
if let Ok(aid) = element.CurrentAutomationId() {
|
||||
result.automation_id = aid.to_string();
|
||||
}
|
||||
if let Ok(rect) = element.CurrentBoundingRectangle() {
|
||||
result.bounding_rect = [rect.left, rect.top, rect.right, rect.bottom];
|
||||
}
|
||||
if let Ok(enabled) = element.CurrentIsEnabled() {
|
||||
result.is_enabled = enabled.as_bool();
|
||||
}
|
||||
if let Ok(offscreen) = element.CurrentIsOffscreen() {
|
||||
result.is_offscreen = offscreen.as_bool();
|
||||
}
|
||||
if with_parents {
|
||||
// Remonter la hiérarchie jusqu'à la Window root
|
||||
if let Ok(automation) = get_automation() {
|
||||
let walker = automation.ControlViewWalker();
|
||||
if let Ok(walker) = walker {
|
||||
let mut current = element.clone();
|
||||
for _ in 0..10 {
|
||||
match walker.GetParentElement(¤t) {
|
||||
Ok(parent) => {
|
||||
let name = parent
|
||||
.CurrentName()
|
||||
.map(|n| n.to_string())
|
||||
.unwrap_or_default();
|
||||
let ct = parent
|
||||
.CurrentLocalizedControlType()
|
||||
.map(|c| c.to_string())
|
||||
.unwrap_or_default();
|
||||
if name.is_empty() && ct.is_empty() {
|
||||
break;
|
||||
}
|
||||
result.parent_path.insert(
|
||||
0,
|
||||
ParentHint {
|
||||
name,
|
||||
control_type: ct,
|
||||
},
|
||||
);
|
||||
current = parent;
|
||||
}
|
||||
Err(_) => break,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
pub fn query_at_point(x: i32, y: i32, with_parents: bool) -> UiaResponse {
|
||||
let start = Instant::now();
|
||||
let _com = match ComGuard::new() {
|
||||
Ok(g) => g,
|
||||
Err(e) => {
|
||||
return UiaResponse::Error {
|
||||
message: format!("CoInitializeEx: {}", e),
|
||||
code: "com_init_failed".into(),
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let automation = match get_automation() {
|
||||
Ok(a) => a,
|
||||
Err(e) => {
|
||||
return UiaResponse::Error {
|
||||
message: format!("CUIAutomation: {}", e),
|
||||
code: "automation_failed".into(),
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let point = POINT { x, y };
|
||||
let element = unsafe { automation.ElementFromPoint(point) };
|
||||
match element {
|
||||
Ok(el) => match element_to_struct(&el, with_parents) {
|
||||
Ok(e) => UiaResponse::Ok {
|
||||
element: Some(e),
|
||||
elements: Vec::new(),
|
||||
elapsed_ms: start.elapsed().as_millis() as u64,
|
||||
},
|
||||
Err(e) => UiaResponse::Error {
|
||||
message: format!("element_to_struct: {}", e),
|
||||
code: "extract_failed".into(),
|
||||
},
|
||||
},
|
||||
Err(_) => UiaResponse::NotFound {
|
||||
reason: format!("Aucun élément UIA à ({}, {})", x, y),
|
||||
elapsed_ms: start.elapsed().as_millis() as u64,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub fn find_element(
|
||||
name: Option<String>,
|
||||
_control_type: Option<String>,
|
||||
_automation_id: Option<String>,
|
||||
_window: Option<String>,
|
||||
_timeout_ms: u32,
|
||||
) -> UiaResponse {
|
||||
let start = Instant::now();
|
||||
let _com = match ComGuard::new() {
|
||||
Ok(g) => g,
|
||||
Err(e) => {
|
||||
return UiaResponse::Error {
|
||||
message: format!("CoInitializeEx: {}", e),
|
||||
code: "com_init_failed".into(),
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let automation = match get_automation() {
|
||||
Ok(a) => a,
|
||||
Err(e) => {
|
||||
return UiaResponse::Error {
|
||||
message: format!("CUIAutomation: {}", e),
|
||||
code: "automation_failed".into(),
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let root = match unsafe { automation.GetRootElement() } {
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
return UiaResponse::Error {
|
||||
message: format!("GetRootElement: {}", e),
|
||||
code: "root_failed".into(),
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// Recherche simple par parcours d'arbre (MVP)
|
||||
// L'arbre UIA peut être énorme → on limite la profondeur
|
||||
if let Some(target_name) = name {
|
||||
let walker = unsafe { automation.ControlViewWalker() };
|
||||
if let Ok(walker) = walker {
|
||||
if let Some(found) =
|
||||
walk_and_find(&walker, &root, &target_name, 0, 6, &_control_type, &_automation_id)
|
||||
{
|
||||
match element_to_struct(&found, true) {
|
||||
Ok(e) => {
|
||||
return UiaResponse::Ok {
|
||||
element: Some(e),
|
||||
elements: Vec::new(),
|
||||
elapsed_ms: start.elapsed().as_millis() as u64,
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
return UiaResponse::Error {
|
||||
message: format!("element_to_struct: {}", e),
|
||||
code: "extract_failed".into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
UiaResponse::NotFound {
|
||||
reason: "Aucun élément trouvé".into(),
|
||||
elapsed_ms: start.elapsed().as_millis() as u64,
|
||||
}
|
||||
}
|
||||
|
||||
/// Parcours récursif de l'arbre UIA pour trouver un élément par nom
|
||||
fn walk_and_find(
|
||||
walker: &IUIAutomationTreeWalker,
|
||||
element: &IUIAutomationElement,
|
||||
target_name: &str,
|
||||
depth: u32,
|
||||
max_depth: u32,
|
||||
target_control_type: &Option<String>,
|
||||
target_automation_id: &Option<String>,
|
||||
) -> Option<IUIAutomationElement> {
|
||||
if depth > max_depth {
|
||||
return None;
|
||||
}
|
||||
|
||||
// Tester l'élément courant
|
||||
unsafe {
|
||||
if let Ok(name) = element.CurrentName() {
|
||||
if name.to_string() == target_name {
|
||||
// Vérifier les filtres additionnels
|
||||
let mut matches = true;
|
||||
if let Some(ct) = target_control_type {
|
||||
if let Ok(local_ct) = element.CurrentLocalizedControlType() {
|
||||
if !local_ct.to_string().to_lowercase().contains(&ct.to_lowercase()) {
|
||||
matches = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
if matches {
|
||||
if let Some(aid) = target_automation_id {
|
||||
if let Ok(local_aid) = element.CurrentAutomationId() {
|
||||
if local_aid.to_string() != *aid {
|
||||
matches = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if matches {
|
||||
return Some(element.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Parcourir les enfants
|
||||
if let Ok(first_child) = walker.GetFirstChildElement(element) {
|
||||
let mut current = first_child;
|
||||
loop {
|
||||
if let Some(found) = walk_and_find(
|
||||
walker,
|
||||
¤t,
|
||||
target_name,
|
||||
depth + 1,
|
||||
max_depth,
|
||||
target_control_type,
|
||||
target_automation_id,
|
||||
) {
|
||||
return Some(found);
|
||||
}
|
||||
match walker.GetNextSiblingElement(¤t) {
|
||||
Ok(next) => current = next,
|
||||
Err(_) => break,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
pub fn capture_tree(_max_depth: u32) -> UiaResponse {
|
||||
let start = Instant::now();
|
||||
let _com = match ComGuard::new() {
|
||||
Ok(g) => g,
|
||||
Err(e) => {
|
||||
return UiaResponse::Error {
|
||||
message: format!("CoInitializeEx: {}", e),
|
||||
code: "com_init_failed".into(),
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let automation = match get_automation() {
|
||||
Ok(a) => a,
|
||||
Err(e) => {
|
||||
return UiaResponse::Error {
|
||||
message: format!("CUIAutomation: {}", e),
|
||||
code: "automation_failed".into(),
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let focused = unsafe { automation.GetFocusedElement() };
|
||||
match focused {
|
||||
Ok(el) => match element_to_struct(&el, true) {
|
||||
Ok(e) => UiaResponse::Ok {
|
||||
element: Some(e),
|
||||
elements: Vec::new(),
|
||||
elapsed_ms: start.elapsed().as_millis() as u64,
|
||||
},
|
||||
Err(e) => UiaResponse::Error {
|
||||
message: format!("element_to_struct: {}", e),
|
||||
code: "extract_failed".into(),
|
||||
},
|
||||
},
|
||||
Err(e) => UiaResponse::Error {
|
||||
message: format!("GetFocusedElement: {}", e),
|
||||
code: "focused_failed".into(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub fn health_check() -> UiaResponse {
|
||||
let _com = match ComGuard::new() {
|
||||
Ok(g) => g,
|
||||
Err(e) => {
|
||||
return UiaResponse::Unavailable {
|
||||
reason: format!("COM init failed: {}", e),
|
||||
}
|
||||
}
|
||||
};
|
||||
match get_automation() {
|
||||
Ok(_) => UiaResponse::Ok {
|
||||
element: None,
|
||||
elements: Vec::new(),
|
||||
elapsed_ms: 0,
|
||||
},
|
||||
Err(e) => UiaResponse::Unavailable {
|
||||
reason: format!("UIA not available: {}", e),
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// Stub Linux (pour développement et tests)
|
||||
// =========================================================================
|
||||
|
||||
#[cfg(not(windows))]
|
||||
mod uia_impl {
|
||||
use super::*;
|
||||
|
||||
pub fn query_at_point(_x: i32, _y: i32, _with_parents: bool) -> UiaResponse {
|
||||
UiaResponse::Unavailable {
|
||||
reason: "UIA n'est disponible que sur Windows".into(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn find_element(
|
||||
_name: Option<String>,
|
||||
_control_type: Option<String>,
|
||||
_automation_id: Option<String>,
|
||||
_window: Option<String>,
|
||||
_timeout_ms: u32,
|
||||
) -> UiaResponse {
|
||||
UiaResponse::Unavailable {
|
||||
reason: "UIA n'est disponible que sur Windows".into(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn capture_tree(_max_depth: u32) -> UiaResponse {
|
||||
UiaResponse::Unavailable {
|
||||
reason: "UIA n'est disponible que sur Windows".into(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn health_check() -> UiaResponse {
|
||||
UiaResponse::Unavailable {
|
||||
reason: "UIA n'est disponible que sur Windows".into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// Main
|
||||
// =========================================================================
|
||||
|
||||
fn main() {
|
||||
let cli = Cli::parse();
|
||||
|
||||
let response = match cli.command {
|
||||
Commands::Query {
|
||||
x,
|
||||
y,
|
||||
with_parents,
|
||||
} => uia_impl::query_at_point(x, y, with_parents),
|
||||
Commands::Find {
|
||||
name,
|
||||
control_type,
|
||||
automation_id,
|
||||
window,
|
||||
timeout_ms,
|
||||
} => uia_impl::find_element(name, control_type, automation_id, window, timeout_ms),
|
||||
Commands::Capture { max_depth } => uia_impl::capture_tree(max_depth),
|
||||
Commands::Health => uia_impl::health_check(),
|
||||
};
|
||||
|
||||
// Sortie JSON sur stdout
|
||||
match serde_json::to_string(&response) {
|
||||
Ok(json) => println!("{}", json),
|
||||
Err(e) => {
|
||||
eprintln!("{{\"status\":\"error\",\"message\":\"JSON serialization: {}\"}}", e);
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -178,8 +178,41 @@ class EventCaptorV1:
|
||||
"timestamp": now,
|
||||
}
|
||||
self._inject_screen_metadata(event)
|
||||
# Capturer le snapshot UIA à la position du clic (si helper dispo)
|
||||
# Non-bloquant : si UIA échoue, l'event est enrichi uniquement
|
||||
# des données vision comme aujourd'hui.
|
||||
self._inject_uia_snapshot(event, x, y)
|
||||
self.on_event(event)
|
||||
|
||||
def _inject_uia_snapshot(self, event: dict, x: int, y: int) -> None:
|
||||
"""Ajouter un uia_snapshot à l'événement si le helper UIA est dispo.
|
||||
|
||||
Appelle lea_uia.exe query --x N --y N en ~10-20ms.
|
||||
Fallback silencieux si le helper n'est pas dispo ou échoue.
|
||||
"""
|
||||
try:
|
||||
from .uia_helper import get_shared_helper
|
||||
helper = get_shared_helper()
|
||||
if not helper.available:
|
||||
return
|
||||
element = helper.query_at(int(x), int(y), with_parents=True)
|
||||
if element is None:
|
||||
return
|
||||
event["uia_snapshot"] = {
|
||||
"name": element.name,
|
||||
"control_type": element.control_type,
|
||||
"class_name": element.class_name,
|
||||
"automation_id": element.automation_id,
|
||||
"bounding_rect": list(element.bounding_rect),
|
||||
"is_enabled": element.is_enabled,
|
||||
"is_offscreen": element.is_offscreen,
|
||||
"parent_path": element.parent_path,
|
||||
}
|
||||
except Exception as e:
|
||||
# Non bloquant — on continue sans UIA
|
||||
import logging
|
||||
logging.getLogger(__name__).debug(f"UIA snapshot skip: {e}")
|
||||
|
||||
def _on_scroll(self, x, y, dx, dy):
|
||||
event = {
|
||||
"type": "mouse_scroll",
|
||||
|
||||
@@ -98,20 +98,43 @@ class ActionExecutorV1:
|
||||
|
||||
@property
|
||||
def notifier(self):
|
||||
"""Instance NotificationManager paresseuse."""
|
||||
"""Instance NotificationManager paresseuse.
|
||||
|
||||
Retourne un objet avec des méthodes no-op si NotificationManager
|
||||
n'est pas disponible (tkinter / plyer absents), pour que l'executor
|
||||
ne plante jamais à cause de l'UI.
|
||||
"""
|
||||
if self._notification_manager is None:
|
||||
try:
|
||||
from ..ui.notifications import NotificationManager
|
||||
self._notification_manager = NotificationManager()
|
||||
except Exception as e:
|
||||
logger.debug(f"NotificationManager indisponible : {e}")
|
||||
# Retourner un objet factice qui ne fait rien
|
||||
# Retourner un objet factice qui ne fait rien — couvre toutes
|
||||
# les méthodes possibles via __getattr__.
|
||||
class _Noop:
|
||||
def replay_target_not_found(self, *a, **kw):
|
||||
return False
|
||||
def __getattr__(self, name):
|
||||
return lambda *a, **kw: False
|
||||
self._notification_manager = _Noop()
|
||||
return self._notification_manager
|
||||
|
||||
@property
|
||||
def activity_panel(self):
|
||||
"""Instance ActivityPanel paresseuse (singleton).
|
||||
|
||||
Fallback silencieux si le panel ne peut pas être créé.
|
||||
"""
|
||||
try:
|
||||
from ..ui.activity_panel import get_activity_panel
|
||||
return get_activity_panel()
|
||||
except Exception as e:
|
||||
logger.debug(f"ActivityPanel indisponible : {e}")
|
||||
|
||||
class _Noop:
|
||||
def __getattr__(self, name):
|
||||
return lambda *a, **kw: None
|
||||
return _Noop()
|
||||
|
||||
def _auth_headers(self) -> dict:
|
||||
"""Headers d'authentification Bearer pour les requetes au serveur."""
|
||||
if self._api_token:
|
||||
@@ -241,6 +264,97 @@ class ActionExecutorV1:
|
||||
logger.warning(f"Acteur gemma4 indisponible : {e}")
|
||||
return "EXECUTER"
|
||||
|
||||
# =========================================================================
|
||||
# UIA local — résolution via lea_uia.exe (helper Rust)
|
||||
# =========================================================================
|
||||
|
||||
def _resolve_via_uia_local(
|
||||
self, uia_target: dict, screen_width: int, screen_height: int,
|
||||
):
|
||||
"""Résoudre une cible via UIA local (lea_uia.exe).
|
||||
|
||||
Le plan V4 contient un uia_target (nom, control_type, parent_path).
|
||||
On appelle le helper Rust qui interroge UIAutomationCore.dll et
|
||||
retourne les coordonnées pixel-perfect de l'élément.
|
||||
|
||||
STRICT : si l'élément trouvé n'appartient pas à la bonne fenêtre
|
||||
parente (comparaison du parent_path), on REFUSE — sinon on clique
|
||||
au mauvais endroit (ex: 'Rechercher' de la taskbar au lieu de
|
||||
l'explorateur).
|
||||
|
||||
Retourne (x_pct, y_pct) si trouvé ET validé, None sinon.
|
||||
"""
|
||||
try:
|
||||
from .uia_helper import get_shared_helper
|
||||
helper = get_shared_helper()
|
||||
if not helper.available:
|
||||
return None
|
||||
|
||||
name = uia_target.get("name", "")
|
||||
control_type = uia_target.get("control_type", "") or None
|
||||
automation_id = uia_target.get("automation_id", "") or None
|
||||
expected_parent_path = uia_target.get("parent_path", []) or []
|
||||
|
||||
if not name:
|
||||
return None
|
||||
|
||||
element = helper.find_by_name(
|
||||
name=name,
|
||||
control_type=control_type,
|
||||
automation_id=automation_id,
|
||||
timeout_ms=1500,
|
||||
)
|
||||
if element is None or not element.is_clickable():
|
||||
logger.debug(f"UIA: '{name}' non trouvé ou non cliquable")
|
||||
return None
|
||||
|
||||
# ── VÉRIFICATION STRICTE du parent_path ──
|
||||
# Si l'élément a été enregistré dans une fenêtre spécifique,
|
||||
# il doit être trouvé dans la MÊME fenêtre au replay.
|
||||
# Sinon on clique sur un homonyme dans une autre app.
|
||||
if expected_parent_path:
|
||||
expected_root = None
|
||||
for p in expected_parent_path:
|
||||
if p.get("control_type", "").lower() in ("fenêtre", "window"):
|
||||
expected_root = p.get("name", "").strip()
|
||||
break
|
||||
|
||||
if expected_root:
|
||||
found_root = None
|
||||
for p in element.parent_path:
|
||||
if p.get("control_type", "").lower() in ("fenêtre", "window"):
|
||||
found_root = p.get("name", "").strip()
|
||||
break
|
||||
|
||||
if found_root and expected_root != found_root:
|
||||
# Match souple : une sous-partie commune (ex: "Bloc-notes")
|
||||
if (expected_root.lower() not in found_root.lower()
|
||||
and found_root.lower() not in expected_root.lower()):
|
||||
logger.warning(
|
||||
f"UIA REJET : '{name}' trouvé dans '{found_root}' "
|
||||
f"mais attendu dans '{expected_root}'"
|
||||
)
|
||||
print(
|
||||
f" [UIA] REJET — '{name}' trouvé dans mauvaise fenêtre "
|
||||
f"({found_root} ≠ {expected_root})"
|
||||
)
|
||||
return None
|
||||
|
||||
cx, cy = element.center()
|
||||
if screen_width <= 0 or screen_height <= 0:
|
||||
return None
|
||||
|
||||
x_pct = cx / screen_width
|
||||
y_pct = cy / screen_height
|
||||
if not (0.0 <= x_pct <= 1.0 and 0.0 <= y_pct <= 1.0):
|
||||
return None
|
||||
|
||||
return (x_pct, y_pct)
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"UIA local resolve erreur : {e}")
|
||||
return None
|
||||
|
||||
# =========================================================================
|
||||
# Observer — pré-analyse écran avant chaque action
|
||||
# =========================================================================
|
||||
@@ -403,12 +517,38 @@ class ActionExecutorV1:
|
||||
# ── Pré-vérification : titre fenêtre ──
|
||||
# Vérifier que l'écran est dans l'état attendu AVANT de cliquer.
|
||||
if visual_mode and target_spec:
|
||||
expected_title = target_spec.get("window_title", "")
|
||||
# Le champ explicite `expected_window_before` a priorité
|
||||
# (il vient du plan V4 et indique la fenêtre STRICTEMENT
|
||||
# attendue avant l'action). Sinon fallback sur target_spec.
|
||||
expected_title = (
|
||||
action.get("expected_window_before", "")
|
||||
or target_spec.get("window_title", "")
|
||||
)
|
||||
if expected_title and expected_title != "unknown_window":
|
||||
from ..window_info_crossplatform import get_active_window_info
|
||||
from ..ui.messages import est_fenetre_lea
|
||||
|
||||
# Polling court pour laisser le temps à la fenêtre de
|
||||
# se stabiliser (évite les faux négatifs sur transitions
|
||||
# rapides : menu qui se ferme, taskbar qui perd le focus, etc.)
|
||||
current_title = ""
|
||||
title_match = False
|
||||
is_lea_window = False
|
||||
for attempt in range(5):
|
||||
current_info = get_active_window_info()
|
||||
current_title = current_info.get("title", "")
|
||||
|
||||
# Si on tombe sur Léa elle-même → on attend un peu
|
||||
if est_fenetre_lea(current_title):
|
||||
is_lea_window = True
|
||||
time.sleep(0.3)
|
||||
continue
|
||||
|
||||
# Si on tombe sur unknown_window → on attend aussi
|
||||
if not current_title or current_title == "unknown_window":
|
||||
time.sleep(0.3)
|
||||
continue
|
||||
|
||||
current_app = _app_name(current_title)
|
||||
expected_app = _app_name(expected_title)
|
||||
title_match = (
|
||||
@@ -416,23 +556,38 @@ class ActionExecutorV1:
|
||||
or expected_title.lower() in current_title.lower()
|
||||
or current_title.lower() in expected_title.lower()
|
||||
)
|
||||
# Ignorer la fenêtre de Léa elle-même (overlay agent)
|
||||
_lea_windows = ("léa", "lea —", "léa —", "lea -", "léa -", "lea assistante", "léa assistante")
|
||||
is_lea_window = any(p in current_title.lower() for p in _lea_windows)
|
||||
if title_match:
|
||||
break
|
||||
# Sinon on retente un peu au cas où la fenêtre
|
||||
# est en cours de transition
|
||||
time.sleep(0.3)
|
||||
|
||||
if not title_match and not is_lea_window:
|
||||
if not title_match:
|
||||
if is_lea_window:
|
||||
# Si après 5 essais on est encore sur Léa,
|
||||
# on ignore (l'utilisateur a Léa au premier plan)
|
||||
logger.info("[LEA] Fenêtre de Léa persistante — ignorée, on continue")
|
||||
elif not current_title or current_title == "unknown_window":
|
||||
# unknown_window persistant : on continue avec un
|
||||
# warning, UIA décidera peut-être
|
||||
logger.warning(
|
||||
f"PRÉ-VÉRIF ÉCHOUÉE : attendu '{expected_title}', "
|
||||
f"actuel '{current_title}' — STOP"
|
||||
f"[LEA] Fenêtre active inconnue — on tente quand même"
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
f"[LEA] Fenêtre incorrecte : attendu '{expected_title}', "
|
||||
f"actuel '{current_title}'"
|
||||
)
|
||||
print(f" [PRÉ-VÉRIF] STOP — fenêtre '{current_title}' ≠ attendu '{expected_title}'")
|
||||
try:
|
||||
self.notifier.replay_wrong_window(current_title, expected_title)
|
||||
except Exception:
|
||||
pass
|
||||
result["success"] = False
|
||||
result["error"] = f"Fenêtre incorrecte: '{current_title}' (attendu: '{expected_title}')"
|
||||
return result
|
||||
elif is_lea_window:
|
||||
logger.info(f"PRÉ-VÉRIF : fenêtre Léa détectée, ignorée — on continue")
|
||||
else:
|
||||
logger.info(f"PRÉ-VÉRIF OK : '{current_title}'")
|
||||
logger.info(f"[LEA] Pré-vérif OK : '{current_title}'")
|
||||
|
||||
# ── OBSERVER : pré-analyse écran avant résolution ──
|
||||
# Détecte popups, dialogues, états inattendus AVANT de chercher la cible.
|
||||
@@ -476,8 +631,29 @@ class ActionExecutorV1:
|
||||
return result
|
||||
# EXECUTER → continuer normalement
|
||||
|
||||
if visual_mode and target_spec and server_url:
|
||||
# ── GROUNDING : localisation pure via GroundingEngine ──
|
||||
# ── V4 : UIA local (si resolve_order l'indique et helper dispo) ──
|
||||
# Court-circuite le grounding serveur pour les clicks sur Windows natif.
|
||||
# 10-20ms au lieu de 2-5s pour un clic — c'est le cœur du V4.
|
||||
uia_resolved = False
|
||||
if visual_mode and target_spec and action_type == "click":
|
||||
resolve_order = target_spec.get("resolve_order", [])
|
||||
uia_target = target_spec.get("uia_target")
|
||||
if resolve_order and resolve_order[0] == "uia" and uia_target:
|
||||
uia_coords = self._resolve_via_uia_local(uia_target, width, height)
|
||||
if uia_coords:
|
||||
x_pct, y_pct = uia_coords
|
||||
result["visual_resolved"] = True
|
||||
result["resolution_method"] = "v4_uia_local"
|
||||
result["resolution_score"] = 0.95
|
||||
uia_resolved = True
|
||||
print(f" [UIA] résolu en local: ({x_pct:.4f}, {y_pct:.4f})")
|
||||
logger.info(
|
||||
f"V4 UIA local OK : {uia_target.get('name', '?')} "
|
||||
f"→ ({x_pct:.4f}, {y_pct:.4f})"
|
||||
)
|
||||
|
||||
if not uia_resolved and visual_mode and target_spec and server_url:
|
||||
# ── GROUNDING : localisation pure via GroundingEngine (fallback) ──
|
||||
from .grounding import GroundingEngine
|
||||
grounding = GroundingEngine(self)
|
||||
grounding_result = grounding.locate(
|
||||
@@ -549,7 +725,10 @@ class ActionExecutorV1:
|
||||
result["target_spec"] = target_spec
|
||||
result["screenshot"] = self._capture_screenshot_b64()
|
||||
result["warning"] = "visual_resolve_failed"
|
||||
self.notifier.replay_target_not_found(target_desc)
|
||||
self.notifier.replay_target_not_found(
|
||||
target_desc,
|
||||
target_spec.get("window_title", ""),
|
||||
)
|
||||
return result
|
||||
|
||||
elif policy_decision.decision == Decision.SKIP:
|
||||
@@ -560,7 +739,10 @@ class ActionExecutorV1:
|
||||
elif policy_decision.decision == Decision.ABORT:
|
||||
result["success"] = False
|
||||
result["error"] = f"policy_abort:{target_desc}"
|
||||
self.notifier.replay_target_not_found(target_desc)
|
||||
self.notifier.replay_target_not_found(
|
||||
target_desc,
|
||||
target_spec.get("window_title", ""),
|
||||
)
|
||||
return result
|
||||
|
||||
else: # SUPERVISE ou CONTINUE
|
||||
@@ -570,7 +752,10 @@ class ActionExecutorV1:
|
||||
result["target_spec"] = target_spec
|
||||
result["screenshot"] = self._capture_screenshot_b64()
|
||||
result["warning"] = "visual_resolve_failed"
|
||||
self.notifier.replay_target_not_found(target_desc)
|
||||
self.notifier.replay_target_not_found(
|
||||
target_desc,
|
||||
target_spec.get("window_title", ""),
|
||||
)
|
||||
return result
|
||||
|
||||
real_x = int(x_pct * width)
|
||||
@@ -615,6 +800,27 @@ class ActionExecutorV1:
|
||||
else:
|
||||
print(f" [POST-VÉRIF] TIMEOUT {max_wait}s — '{post_title}' ≠ '{expected_after}'")
|
||||
logger.warning(f"POST-VÉRIF TIMEOUT : '{post_title}' ≠ '{expected_after}'")
|
||||
# Contrôle strict : si success_strict, on STOP.
|
||||
# Sinon on continue avec un warning (legacy).
|
||||
is_strict = bool(action.get("success_strict"))
|
||||
if is_strict:
|
||||
result["success"] = False
|
||||
result["error"] = (
|
||||
f"Post-vérif échouée : fenêtre '{post_title}' "
|
||||
f"au lieu de '{expected_after}'"
|
||||
)
|
||||
print(
|
||||
f" [POST-VÉRIF] STOP STRICT — l'étape ne s'est "
|
||||
f"pas déroulée comme prévu, arrêt du replay"
|
||||
)
|
||||
try:
|
||||
self.notifier.replay_wrong_window(
|
||||
post_title, expected_after,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
return result
|
||||
else:
|
||||
result["warning"] = f"post_verif_timeout:{post_title}"
|
||||
else:
|
||||
print(f" [CLICK] Terminé.")
|
||||
@@ -729,9 +935,14 @@ class ActionExecutorV1:
|
||||
f"l'action n'a pas eu d'effet visible"
|
||||
)
|
||||
logger.warning(
|
||||
f"Action {action_id} ({action_type}) : ecran inchange "
|
||||
f"— action sans effet visible"
|
||||
f"[LEA] Écran inchangé après {action_type} "
|
||||
f"(action_id={action_id}) — pas d'effet visible"
|
||||
)
|
||||
# Notifier l'utilisateur en français naturel (niveau ATTENTION)
|
||||
try:
|
||||
self.notifier.replay_no_screen_change(action_type)
|
||||
except Exception:
|
||||
pass
|
||||
else:
|
||||
print(f" [OK] Changement d'ecran detecte apres {action_type}")
|
||||
else:
|
||||
|
||||
278
agent_v0/agent_v1/core/uia_helper.py
Normal file
278
agent_v0/agent_v1/core/uia_helper.py
Normal file
@@ -0,0 +1,278 @@
|
||||
# core/workflow/uia_helper.py
|
||||
"""
|
||||
UIAHelper — Wrapper Python pour lea_uia.exe (helper Rust UI Automation).
|
||||
|
||||
Expose une API Python simple pour interroger UIA via le binaire Rust.
|
||||
Communique via subprocess + stdin/stdout JSON.
|
||||
|
||||
Pourquoi un helper Rust ?
|
||||
- 5-10x plus rapide que pywinauto (10-20ms vs 50-200ms)
|
||||
- Binaire standalone ~500 Ko, aucune dépendance runtime
|
||||
- Pas de problèmes de threading COM en Python
|
||||
- Crash-safe (le crash du helper n'affecte pas l'agent Python)
|
||||
|
||||
Architecture :
|
||||
Python executor
|
||||
↓ subprocess.run
|
||||
lea_uia.exe query --x 812 --y 436
|
||||
↓ UIA API Windows
|
||||
JSON response
|
||||
↓ stdout
|
||||
Python executor parse JSON
|
||||
|
||||
Si lea_uia.exe n'est pas disponible (Linux, binaire absent, crash) :
|
||||
toutes les méthodes retournent None → fallback vision automatique.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import platform
|
||||
import subprocess
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Timeout par défaut pour les appels UIA (en secondes)
|
||||
_DEFAULT_TIMEOUT = 5.0
|
||||
|
||||
|
||||
@dataclass
|
||||
class UiaElement:
|
||||
"""Représentation Python d'un élément UIA."""
|
||||
name: str = ""
|
||||
control_type: str = ""
|
||||
class_name: str = ""
|
||||
automation_id: str = ""
|
||||
bounding_rect: Tuple[int, int, int, int] = (0, 0, 0, 0)
|
||||
is_enabled: bool = False
|
||||
is_offscreen: bool = True
|
||||
parent_path: List[Dict[str, str]] = field(default_factory=list)
|
||||
process_name: str = ""
|
||||
|
||||
def center(self) -> Tuple[int, int]:
|
||||
"""Retourner le centre du rectangle (pixels)."""
|
||||
x1, y1, x2, y2 = self.bounding_rect
|
||||
return ((x1 + x2) // 2, (y1 + y2) // 2)
|
||||
|
||||
def width(self) -> int:
|
||||
return self.bounding_rect[2] - self.bounding_rect[0]
|
||||
|
||||
def height(self) -> int:
|
||||
return self.bounding_rect[3] - self.bounding_rect[1]
|
||||
|
||||
def is_clickable(self) -> bool:
|
||||
"""Peut-on cliquer dessus ?"""
|
||||
return (
|
||||
self.is_enabled
|
||||
and not self.is_offscreen
|
||||
and self.width() > 0
|
||||
and self.height() > 0
|
||||
)
|
||||
|
||||
def path_signature(self) -> str:
|
||||
"""Signature du chemin parent (pour retrouver l'élément)."""
|
||||
parts = [f"{p['control_type']}[{p['name']}]" for p in self.parent_path if p.get("name")]
|
||||
parts.append(f"{self.control_type}[{self.name}]")
|
||||
return " > ".join(parts)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"name": self.name,
|
||||
"control_type": self.control_type,
|
||||
"class_name": self.class_name,
|
||||
"automation_id": self.automation_id,
|
||||
"bounding_rect": list(self.bounding_rect),
|
||||
"is_enabled": self.is_enabled,
|
||||
"is_offscreen": self.is_offscreen,
|
||||
"parent_path": self.parent_path,
|
||||
"process_name": self.process_name,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, d: Dict[str, Any]) -> "UiaElement":
|
||||
rect = d.get("bounding_rect", [0, 0, 0, 0])
|
||||
if isinstance(rect, list) and len(rect) >= 4:
|
||||
rect = tuple(rect[:4])
|
||||
else:
|
||||
rect = (0, 0, 0, 0)
|
||||
return cls(
|
||||
name=d.get("name", ""),
|
||||
control_type=d.get("control_type", ""),
|
||||
class_name=d.get("class_name", ""),
|
||||
automation_id=d.get("automation_id", ""),
|
||||
bounding_rect=rect,
|
||||
is_enabled=d.get("is_enabled", False),
|
||||
is_offscreen=d.get("is_offscreen", True),
|
||||
parent_path=d.get("parent_path", []),
|
||||
process_name=d.get("process_name", ""),
|
||||
)
|
||||
|
||||
|
||||
class UIAHelper:
|
||||
"""Wrapper Python pour lea_uia.exe."""
|
||||
|
||||
def __init__(self, helper_path: str = "", timeout: float = _DEFAULT_TIMEOUT):
|
||||
self._helper_path = helper_path or self._find_helper()
|
||||
self._timeout = timeout
|
||||
self._available = self._check_available()
|
||||
|
||||
def _find_helper(self) -> str:
|
||||
"""Trouver lea_uia.exe dans les emplacements standards."""
|
||||
candidates = [
|
||||
r"C:\Lea\helpers\lea_uia.exe",
|
||||
os.path.join(os.path.dirname(__file__), "..", "..",
|
||||
"agent_rust", "lea_uia", "target",
|
||||
"x86_64-pc-windows-gnu", "release", "lea_uia.exe"),
|
||||
"./helpers/lea_uia.exe",
|
||||
"lea_uia.exe",
|
||||
]
|
||||
for path in candidates:
|
||||
if os.path.isfile(path):
|
||||
return os.path.abspath(path)
|
||||
return ""
|
||||
|
||||
def _check_available(self) -> bool:
|
||||
"""Vérifier que le helper est utilisable (Windows + binaire + health OK)."""
|
||||
if platform.system() != "Windows":
|
||||
logger.debug("UIAHelper: Linux/Mac — helper désactivé")
|
||||
return False
|
||||
if not self._helper_path:
|
||||
logger.debug("UIAHelper: lea_uia.exe introuvable")
|
||||
return False
|
||||
if not os.path.isfile(self._helper_path):
|
||||
logger.debug(f"UIAHelper: chemin invalide {self._helper_path}")
|
||||
return False
|
||||
return True
|
||||
|
||||
@property
|
||||
def available(self) -> bool:
|
||||
return self._available
|
||||
|
||||
@property
|
||||
def helper_path(self) -> str:
|
||||
return self._helper_path
|
||||
|
||||
def _run(self, args: List[str]) -> Optional[Dict[str, Any]]:
|
||||
"""Exécuter lea_uia.exe avec les arguments et parser le JSON."""
|
||||
if not self._available:
|
||||
return None
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[self._helper_path] + args,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=self._timeout,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
)
|
||||
if result.returncode != 0:
|
||||
logger.debug(
|
||||
f"UIAHelper: exit code {result.returncode}, "
|
||||
f"stderr: {result.stderr[:200]}"
|
||||
)
|
||||
return None
|
||||
output = result.stdout.strip()
|
||||
if not output:
|
||||
return None
|
||||
return json.loads(output)
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.debug(f"UIAHelper: timeout ({self._timeout}s) sur {args}")
|
||||
return None
|
||||
except json.JSONDecodeError as e:
|
||||
logger.debug(f"UIAHelper: JSON invalide — {e}")
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.debug(f"UIAHelper: erreur {e}")
|
||||
return None
|
||||
|
||||
def health(self) -> bool:
|
||||
"""Vérifier que UIA répond."""
|
||||
data = self._run(["health"])
|
||||
return data is not None and data.get("status") == "ok"
|
||||
|
||||
def query_at(
|
||||
self,
|
||||
x: int,
|
||||
y: int,
|
||||
with_parents: bool = True,
|
||||
) -> Optional[UiaElement]:
|
||||
"""Récupérer l'élément UIA à une position écran.
|
||||
|
||||
Args:
|
||||
x, y: Coordonnées pixel absolues
|
||||
with_parents: Inclure la hiérarchie des parents
|
||||
|
||||
Returns:
|
||||
UiaElement si trouvé, None sinon (pas d'élément ou UIA indispo)
|
||||
"""
|
||||
args = ["query", "--x", str(x), "--y", str(y)]
|
||||
if not with_parents:
|
||||
args.append("--with-parents=false")
|
||||
|
||||
data = self._run(args)
|
||||
if not data or data.get("status") != "ok":
|
||||
return None
|
||||
|
||||
elem_data = data.get("element")
|
||||
if not elem_data:
|
||||
return None
|
||||
return UiaElement.from_dict(elem_data)
|
||||
|
||||
def find_by_name(
|
||||
self,
|
||||
name: str,
|
||||
control_type: Optional[str] = None,
|
||||
automation_id: Optional[str] = None,
|
||||
window: Optional[str] = None,
|
||||
timeout_ms: int = 2000,
|
||||
) -> Optional[UiaElement]:
|
||||
"""Rechercher un élément par son nom (+ filtres optionnels).
|
||||
|
||||
Args:
|
||||
name: Nom exact de l'élément
|
||||
control_type: Type de contrôle (Button, Edit, MenuItem...)
|
||||
automation_id: ID d'automation
|
||||
window: Restreindre à une fenêtre spécifique
|
||||
timeout_ms: Timeout de recherche en millisecondes
|
||||
"""
|
||||
args = ["find", "--name", name, "--timeout-ms", str(timeout_ms)]
|
||||
if control_type:
|
||||
args.extend(["--control-type", control_type])
|
||||
if automation_id:
|
||||
args.extend(["--automation-id", automation_id])
|
||||
if window:
|
||||
args.extend(["--window", window])
|
||||
|
||||
data = self._run(args)
|
||||
if not data or data.get("status") != "ok":
|
||||
return None
|
||||
|
||||
elem_data = data.get("element")
|
||||
if not elem_data:
|
||||
return None
|
||||
return UiaElement.from_dict(elem_data)
|
||||
|
||||
def capture_focused(self, max_depth: int = 3) -> Optional[UiaElement]:
|
||||
"""Capturer l'élément ayant le focus + son contexte."""
|
||||
data = self._run(["capture", "--max-depth", str(max_depth)])
|
||||
if not data or data.get("status") != "ok":
|
||||
return None
|
||||
|
||||
elem_data = data.get("element")
|
||||
if not elem_data:
|
||||
return None
|
||||
return UiaElement.from_dict(elem_data)
|
||||
|
||||
|
||||
# Instance globale partagée (singleton léger)
|
||||
_SHARED_HELPER: Optional[UIAHelper] = None
|
||||
|
||||
|
||||
def get_shared_helper() -> UIAHelper:
|
||||
"""Retourner une instance partagée de UIAHelper."""
|
||||
global _SHARED_HELPER
|
||||
if _SHARED_HELPER is None:
|
||||
_SHARED_HELPER = UIAHelper()
|
||||
return _SHARED_HELPER
|
||||
@@ -38,8 +38,19 @@ except (ImportError, ValueError):
|
||||
except ImportError:
|
||||
LeaServerClient = None
|
||||
|
||||
# Configuration du logging
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
|
||||
# Configuration du logging — format structuré et lisible pour un TIM
|
||||
# Niveau de détail : INFO par défaut, DEBUG si RPA_AGENT_DEBUG=1
|
||||
_log_level = logging.DEBUG if os.environ.get("RPA_AGENT_DEBUG") == "1" else logging.INFO
|
||||
logging.basicConfig(
|
||||
level=_log_level,
|
||||
format="%(asctime)s %(levelname)-7s %(name)-25s %(message)s",
|
||||
datefmt="%H:%M:%S",
|
||||
)
|
||||
|
||||
# Réduire le bruit de certaines libs
|
||||
for _noisy in ("urllib3", "requests.packages.urllib3", "PIL", "mss"):
|
||||
logging.getLogger(_noisy).setLevel(logging.WARNING)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Intervalle de polling replay (secondes)
|
||||
@@ -371,12 +382,22 @@ class AgentV1:
|
||||
time.sleep(5)
|
||||
|
||||
def stop_session(self):
|
||||
# Arrêter la capture et le streaming de la session d'enregistrement
|
||||
if self.captor: self.captor.stop()
|
||||
if self.streamer: self.streamer.stop()
|
||||
logger.info(f"Session {self.session_id} terminée.")
|
||||
# Sauvegarder le session_id avant de l'annuler (pour les logs)
|
||||
ended_session_id = self.session_id
|
||||
|
||||
# Reset le session_id pour que le poll replay utilise l'ID stable
|
||||
# Arrêter la capture d'abord (plus d'events entrants)
|
||||
if self.captor: self.captor.stop()
|
||||
|
||||
# Attendre que les events en cours de traitement dans _on_event_bridge
|
||||
# aient le temps d'être envoyés au streamer (capture duale + push)
|
||||
import time
|
||||
time.sleep(1.5)
|
||||
|
||||
# Maintenant arrêter le streamer (drain queue + finalize)
|
||||
if self.streamer: self.streamer.stop()
|
||||
logger.info(f"Session {ended_session_id} terminée.")
|
||||
|
||||
# Reset le session_id APRÈS le stop complet du streamer
|
||||
self.session_id = None
|
||||
|
||||
# Reset le backoff de l'executor pour reprendre le polling immédiatement
|
||||
@@ -403,6 +424,7 @@ class AgentV1:
|
||||
"""Capture périodique pour donner du contexte au stagiaire.
|
||||
Déduplication : n'envoie que si l'écran a changé.
|
||||
Tourne tant que session_id est défini (= enregistrement actif).
|
||||
Enrichi avec le titre de la fenêtre active pour contextualisation.
|
||||
"""
|
||||
while self.running and self.session_id:
|
||||
try:
|
||||
@@ -413,7 +435,17 @@ class AgentV1:
|
||||
if img_hash != self._last_heartbeat_hash:
|
||||
self._last_heartbeat_hash = img_hash
|
||||
self.streamer.push_image(full_path, f"heartbeat_{int(time.time())}")
|
||||
self.streamer.push_event({"type": "heartbeat", "image": full_path, "timestamp": time.time(), "machine_id": self.machine_id})
|
||||
heartbeat_event = {
|
||||
"type": "heartbeat",
|
||||
"image": full_path,
|
||||
"timestamp": time.time(),
|
||||
"machine_id": self.machine_id,
|
||||
}
|
||||
# Ajouter le titre de la fenêtre active (léger, pas de crop)
|
||||
window_title = self.vision.get_active_window_title()
|
||||
if window_title:
|
||||
heartbeat_event["active_window_title"] = window_title
|
||||
self.streamer.push_event(heartbeat_event)
|
||||
except Exception as e:
|
||||
logger.error(f"Heartbeat error: {e}")
|
||||
time.sleep(5)
|
||||
@@ -448,7 +480,7 @@ class AgentV1:
|
||||
event["screenshot_context"] = full_path
|
||||
self.streamer.push_image(full_path, f"focus_{int(time.time())}")
|
||||
|
||||
# 🔴 Capture Interactive (Dual)
|
||||
# Capture Interactive (Dual + Fenêtre active)
|
||||
if event["type"] in ["mouse_click", "key_combo"]:
|
||||
self.shot_counter += 1
|
||||
shot_id = f"shot_{self.shot_counter:04d}"
|
||||
@@ -459,9 +491,22 @@ class AgentV1:
|
||||
event["screenshot_id"] = shot_id
|
||||
event["vision_info"] = capture_info
|
||||
|
||||
# Enrichir l'event avec les métadonnées de la fenêtre active
|
||||
# (titre, rect, coordonnées clic relatives, taille fenêtre)
|
||||
window_capture = capture_info.get("window_capture")
|
||||
if window_capture:
|
||||
event["window_capture"] = {
|
||||
"title": window_capture.get("window_title", ""),
|
||||
"app_name": window_capture.get("app_name", ""),
|
||||
"rect": window_capture.get("window_rect"),
|
||||
"click_relative": window_capture.get("click_in_window"),
|
||||
"window_size": window_capture.get("window_size"),
|
||||
"click_inside_window": window_capture.get("click_inside_window", True),
|
||||
}
|
||||
|
||||
self._stream_capture_info(capture_info, shot_id)
|
||||
|
||||
# 🕒 POST-ACTION : Capture du résultat après 1s (pour voir le résultat du clic)
|
||||
# POST-ACTION : Capture du résultat après 1s (pour voir le résultat du clic)
|
||||
threading.Timer(1.0, self._capture_result, args=(shot_id,)).start()
|
||||
|
||||
self.ui.update_stats(self.shot_counter)
|
||||
@@ -481,6 +526,12 @@ class AgentV1:
|
||||
self.streamer.push_image(capture_info["full"], f"{shot_id}_full")
|
||||
if "crop" in capture_info:
|
||||
self.streamer.push_image(capture_info["crop"], f"{shot_id}_crop")
|
||||
# Streamer l'image de la fenêtre active si disponible
|
||||
window_capture = capture_info.get("window_capture")
|
||||
if window_capture and "window_image" in window_capture:
|
||||
self.streamer.push_image(
|
||||
window_capture["window_image"], f"{shot_id}_window"
|
||||
)
|
||||
|
||||
def run(self):
|
||||
self.ui.run()
|
||||
|
||||
418
agent_v0/agent_v1/ui/activity_panel.py
Normal file
418
agent_v0/agent_v1/ui/activity_panel.py
Normal file
@@ -0,0 +1,418 @@
|
||||
# agent_v1/ui/activity_panel.py
|
||||
"""
|
||||
Panel d'activité temps réel de Léa.
|
||||
|
||||
Affiche à l'utilisateur ce que Léa fait *maintenant* :
|
||||
- État courant (Observe / Cherche / Agit / Vérifie / Bloquée)
|
||||
- Action en cours (ex: "Clic sur Rechercher")
|
||||
- Progression (ex: "3/15")
|
||||
- Temps écoulé depuis le début du workflow
|
||||
|
||||
Contraintes :
|
||||
- Fallback silencieux si tkinter absent (ne crash jamais)
|
||||
- Thread-safe (mises à jour depuis les threads de replay)
|
||||
- Pas de dépendance à PyQt5 (seulement tkinter, déjà utilisé par chat_window)
|
||||
|
||||
Utilisation :
|
||||
panel = ActivityPanel()
|
||||
panel.definir_workflow("Saisie patient", nb_etapes=15)
|
||||
panel.mettre_a_jour(etat=EtatLea.AGIT, action="Clic sur Valider", etape=3)
|
||||
panel.masquer()
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from enum import Enum
|
||||
from typing import Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class EtatLea(Enum):
|
||||
"""États macroscopiques de Léa pendant un replay."""
|
||||
|
||||
INACTIVE = ("inactive", "Prête", "#808080") # Gris
|
||||
OBSERVE = ("observe", "Observe", "#4A90E2") # Bleu
|
||||
CHERCHE = ("cherche", "Cherche", "#F5A623") # Orange
|
||||
AGIT = ("agit", "Agit", "#7ED321") # Vert
|
||||
VERIFIE = ("verifie", "Vérifie", "#9013FE") # Violet
|
||||
BLOQUEE = ("bloquee", "Bloquée", "#D0021B") # Rouge
|
||||
TERMINE = ("termine", "Terminé", "#50E3C2") # Turquoise
|
||||
|
||||
def __init__(self, code: str, libelle: str, couleur: str) -> None:
|
||||
self.code = code
|
||||
self.libelle = libelle
|
||||
self.couleur = couleur
|
||||
|
||||
|
||||
@dataclass
|
||||
class EtatActivite:
|
||||
"""Instantané de l'activité courante de Léa.
|
||||
|
||||
Utilisé par le panel et exposé par `ActivityPanel.snapshot()` pour les
|
||||
tests (sans dépendre de tkinter).
|
||||
"""
|
||||
|
||||
etat: EtatLea = EtatLea.INACTIVE
|
||||
action_courante: str = ""
|
||||
nom_workflow: str = ""
|
||||
etape: int = 0
|
||||
nb_etapes: int = 0
|
||||
debut_timestamp: float = 0.0
|
||||
dernier_message: str = ""
|
||||
|
||||
def temps_ecoule_s(self) -> float:
|
||||
"""Temps écoulé depuis le début du workflow (secondes)."""
|
||||
if self.debut_timestamp <= 0:
|
||||
return 0.0
|
||||
return max(0.0, time.time() - self.debut_timestamp)
|
||||
|
||||
def progression_texte(self) -> str:
|
||||
"""Représentation textuelle de la progression (ex: '3/15')."""
|
||||
if self.nb_etapes <= 0:
|
||||
return ""
|
||||
return f"{self.etape}/{self.nb_etapes}"
|
||||
|
||||
def temps_ecoule_texte(self) -> str:
|
||||
"""Représentation humaine du temps écoulé (ex: '12s', '1m24s')."""
|
||||
s = int(self.temps_ecoule_s())
|
||||
if s < 60:
|
||||
return f"{s}s"
|
||||
return f"{s // 60}m{s % 60:02d}s"
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
"""Sérialiser pour le logging et les tests."""
|
||||
return {
|
||||
"etat": self.etat.code,
|
||||
"etat_libelle": self.etat.libelle,
|
||||
"action_courante": self.action_courante,
|
||||
"nom_workflow": self.nom_workflow,
|
||||
"etape": self.etape,
|
||||
"nb_etapes": self.nb_etapes,
|
||||
"progression": self.progression_texte(),
|
||||
"temps_ecoule_s": round(self.temps_ecoule_s(), 1),
|
||||
"dernier_message": self.dernier_message,
|
||||
}
|
||||
|
||||
|
||||
class ActivityPanel:
|
||||
"""Panel d'activité de Léa.
|
||||
|
||||
Thread-safe. Le panel tkinter est créé à la demande (lazy) et uniquement
|
||||
si tkinter est disponible. Toutes les méthodes sont safe à appeler même
|
||||
si l'UI n'est pas dispo (fallback silencieux).
|
||||
"""
|
||||
|
||||
def __init__(self, activer_ui: bool = True) -> None:
|
||||
self._lock = threading.RLock()
|
||||
self._etat = EtatActivite()
|
||||
self._activer_ui = activer_ui
|
||||
# UI tkinter (créée à la demande dans le thread UI)
|
||||
self._tk_root = None
|
||||
self._tk_labels: dict = {}
|
||||
self._ui_disponible = None # Lazy : résolu au premier usage
|
||||
self._listeners = [] # Callbacks pour les changements d'état
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# API publique (thread-safe)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def definir_workflow(self, nom: str, nb_etapes: int = 0) -> None:
|
||||
"""Démarrer le suivi d'un nouveau workflow."""
|
||||
with self._lock:
|
||||
self._etat = EtatActivite(
|
||||
etat=EtatLea.OBSERVE,
|
||||
nom_workflow=nom,
|
||||
nb_etapes=nb_etapes,
|
||||
debut_timestamp=time.time(),
|
||||
)
|
||||
self._notifier_changement()
|
||||
self._rafraichir_ui()
|
||||
logger.info(f"[ACTIVITY] Workflow démarré : {nom} ({nb_etapes} étapes)")
|
||||
|
||||
def mettre_a_jour(
|
||||
self,
|
||||
etat: Optional[EtatLea] = None,
|
||||
action: Optional[str] = None,
|
||||
etape: Optional[int] = None,
|
||||
message: Optional[str] = None,
|
||||
) -> None:
|
||||
"""Mettre à jour l'état affiché.
|
||||
|
||||
Tous les paramètres sont optionnels — on ne met à jour que ce qui est
|
||||
fourni. Les autres champs conservent leur valeur actuelle.
|
||||
"""
|
||||
with self._lock:
|
||||
if etat is not None:
|
||||
self._etat.etat = etat
|
||||
if action is not None:
|
||||
self._etat.action_courante = action
|
||||
if etape is not None:
|
||||
self._etat.etape = etape
|
||||
if message is not None:
|
||||
self._etat.dernier_message = message
|
||||
|
||||
self._notifier_changement()
|
||||
self._rafraichir_ui()
|
||||
|
||||
def terminer(self, succes: bool = True) -> None:
|
||||
"""Marquer le workflow comme terminé."""
|
||||
with self._lock:
|
||||
self._etat.etat = EtatLea.TERMINE if succes else EtatLea.BLOQUEE
|
||||
if not succes:
|
||||
self._etat.dernier_message = (
|
||||
self._etat.dernier_message or "Léa a rendu la main"
|
||||
)
|
||||
self._notifier_changement()
|
||||
self._rafraichir_ui()
|
||||
|
||||
def reinitialiser(self) -> None:
|
||||
"""Remettre le panel en état inactif."""
|
||||
with self._lock:
|
||||
self._etat = EtatActivite()
|
||||
self._notifier_changement()
|
||||
self._rafraichir_ui()
|
||||
|
||||
def snapshot(self) -> EtatActivite:
|
||||
"""Obtenir un instantané immuable de l'état courant (pour les tests)."""
|
||||
with self._lock:
|
||||
return EtatActivite(
|
||||
etat=self._etat.etat,
|
||||
action_courante=self._etat.action_courante,
|
||||
nom_workflow=self._etat.nom_workflow,
|
||||
etape=self._etat.etape,
|
||||
nb_etapes=self._etat.nb_etapes,
|
||||
debut_timestamp=self._etat.debut_timestamp,
|
||||
dernier_message=self._etat.dernier_message,
|
||||
)
|
||||
|
||||
def masquer(self) -> None:
|
||||
"""Masquer le panel UI si affiché."""
|
||||
if self._tk_root is not None:
|
||||
try:
|
||||
self._tk_root.withdraw()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def afficher(self) -> None:
|
||||
"""Afficher le panel UI si disponible."""
|
||||
self._creer_ui_si_besoin()
|
||||
if self._tk_root is not None:
|
||||
try:
|
||||
self._tk_root.deiconify()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def on_change(self, callback) -> None:
|
||||
"""Enregistrer un listener appelé à chaque changement d'état."""
|
||||
with self._lock:
|
||||
self._listeners.append(callback)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Gestion UI tkinter (lazy, fallback silencieux)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _creer_ui_si_besoin(self) -> None:
|
||||
"""Créer la fenêtre tkinter au premier usage (lazy)."""
|
||||
if not self._activer_ui:
|
||||
return
|
||||
if self._tk_root is not None:
|
||||
return
|
||||
if self._ui_disponible is False:
|
||||
return # Déjà testé et indisponible
|
||||
|
||||
try:
|
||||
import tkinter as tk
|
||||
except Exception as e:
|
||||
logger.debug(f"[ACTIVITY] tkinter indisponible : {e}")
|
||||
self._ui_disponible = False
|
||||
return
|
||||
|
||||
try:
|
||||
self._tk_root = tk.Toplevel() if _tk_root_existe() else tk.Tk()
|
||||
self._tk_root.title("Léa — Activité")
|
||||
self._tk_root.geometry("340x180+40+40")
|
||||
self._tk_root.attributes("-topmost", True)
|
||||
self._tk_root.resizable(False, False)
|
||||
self._tk_root.configure(bg="#1E1E1E")
|
||||
|
||||
titre = tk.Label(
|
||||
self._tk_root,
|
||||
text="Léa",
|
||||
font=("Segoe UI", 14, "bold"),
|
||||
fg="#FFFFFF",
|
||||
bg="#1E1E1E",
|
||||
)
|
||||
titre.pack(pady=(10, 2))
|
||||
|
||||
self._tk_labels["etat"] = tk.Label(
|
||||
self._tk_root,
|
||||
text="Prête",
|
||||
font=("Segoe UI", 11),
|
||||
fg="#808080",
|
||||
bg="#1E1E1E",
|
||||
)
|
||||
self._tk_labels["etat"].pack()
|
||||
|
||||
self._tk_labels["action"] = tk.Label(
|
||||
self._tk_root,
|
||||
text="",
|
||||
font=("Segoe UI", 10),
|
||||
fg="#FFFFFF",
|
||||
bg="#1E1E1E",
|
||||
wraplength=300,
|
||||
)
|
||||
self._tk_labels["action"].pack(pady=(8, 2))
|
||||
|
||||
self._tk_labels["progression"] = tk.Label(
|
||||
self._tk_root,
|
||||
text="",
|
||||
font=("Segoe UI", 9),
|
||||
fg="#B0B0B0",
|
||||
bg="#1E1E1E",
|
||||
)
|
||||
self._tk_labels["progression"].pack()
|
||||
|
||||
self._tk_labels["temps"] = tk.Label(
|
||||
self._tk_root,
|
||||
text="",
|
||||
font=("Segoe UI", 9),
|
||||
fg="#808080",
|
||||
bg="#1E1E1E",
|
||||
)
|
||||
self._tk_labels["temps"].pack(pady=(4, 0))
|
||||
|
||||
self._tk_labels["message"] = tk.Label(
|
||||
self._tk_root,
|
||||
text="",
|
||||
font=("Segoe UI", 9, "italic"),
|
||||
fg="#B0B0B0",
|
||||
bg="#1E1E1E",
|
||||
wraplength=300,
|
||||
)
|
||||
self._tk_labels["message"].pack(pady=(6, 10))
|
||||
|
||||
# Masquer par défaut : on affiche seulement pendant un workflow
|
||||
self._tk_root.withdraw()
|
||||
self._ui_disponible = True
|
||||
except Exception as e:
|
||||
logger.debug(f"[ACTIVITY] Impossible de créer l'UI : {e}")
|
||||
self._ui_disponible = False
|
||||
self._tk_root = None
|
||||
|
||||
def _rafraichir_ui(self) -> None:
|
||||
"""Mettre à jour les labels tkinter (safe si l'UI n'existe pas)."""
|
||||
if not self._activer_ui or self._ui_disponible is False:
|
||||
return
|
||||
self._creer_ui_si_besoin()
|
||||
if self._tk_root is None:
|
||||
return
|
||||
|
||||
try:
|
||||
with self._lock:
|
||||
snap = self.snapshot()
|
||||
|
||||
# Utiliser after(0) pour rester dans le thread UI tkinter
|
||||
def _update():
|
||||
try:
|
||||
self._tk_labels["etat"].config(
|
||||
text=snap.etat.libelle,
|
||||
fg=snap.etat.couleur,
|
||||
)
|
||||
if snap.action_courante:
|
||||
self._tk_labels["action"].config(text=snap.action_courante)
|
||||
else:
|
||||
self._tk_labels["action"].config(text="")
|
||||
|
||||
prog = snap.progression_texte()
|
||||
if prog and snap.nom_workflow:
|
||||
self._tk_labels["progression"].config(
|
||||
text=f"« {snap.nom_workflow} » — {prog}"
|
||||
)
|
||||
elif snap.nom_workflow:
|
||||
self._tk_labels["progression"].config(
|
||||
text=f"« {snap.nom_workflow} »"
|
||||
)
|
||||
else:
|
||||
self._tk_labels["progression"].config(text="")
|
||||
|
||||
if snap.debut_timestamp > 0:
|
||||
self._tk_labels["temps"].config(
|
||||
text=f"⏱ {snap.temps_ecoule_texte()}"
|
||||
)
|
||||
else:
|
||||
self._tk_labels["temps"].config(text="")
|
||||
|
||||
self._tk_labels["message"].config(text=snap.dernier_message)
|
||||
|
||||
# Afficher automatiquement si actif
|
||||
if snap.etat != EtatLea.INACTIVE:
|
||||
self._tk_root.deiconify()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
self._tk_root.after(0, _update)
|
||||
except Exception:
|
||||
# Si le root a été détruit
|
||||
self._tk_root = None
|
||||
self._ui_disponible = False
|
||||
except Exception as e:
|
||||
logger.debug(f"[ACTIVITY] Erreur rafraîchissement UI : {e}")
|
||||
|
||||
def _notifier_changement(self) -> None:
|
||||
"""Notifier tous les listeners du changement d'état."""
|
||||
with self._lock:
|
||||
listeners = list(self._listeners)
|
||||
snap = self.snapshot()
|
||||
|
||||
for cb in listeners:
|
||||
try:
|
||||
cb(snap)
|
||||
except Exception as e:
|
||||
logger.debug(f"[ACTIVITY] Listener erreur : {e}")
|
||||
|
||||
|
||||
def _tk_root_existe() -> bool:
|
||||
"""Vérifier si un root tkinter existe déjà (pour créer un Toplevel)."""
|
||||
try:
|
||||
import tkinter as tk
|
||||
|
||||
default_root = getattr(tk, "_default_root", None)
|
||||
return default_root is not None
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Singleton global (optionnel)
|
||||
# ============================================================================
|
||||
|
||||
|
||||
_INSTANCE_GLOBALE: Optional[ActivityPanel] = None
|
||||
_LOCK_SINGLETON = threading.Lock()
|
||||
|
||||
|
||||
def get_activity_panel(activer_ui: bool = True) -> ActivityPanel:
|
||||
"""Obtenir l'instance globale du panel d'activité (lazy)."""
|
||||
global _INSTANCE_GLOBALE
|
||||
with _LOCK_SINGLETON:
|
||||
if _INSTANCE_GLOBALE is None:
|
||||
_INSTANCE_GLOBALE = ActivityPanel(activer_ui=activer_ui)
|
||||
return _INSTANCE_GLOBALE
|
||||
|
||||
|
||||
def reset_activity_panel() -> None:
|
||||
"""Réinitialiser le singleton (utile pour les tests)."""
|
||||
global _INSTANCE_GLOBALE
|
||||
with _LOCK_SINGLETON:
|
||||
if _INSTANCE_GLOBALE is not None:
|
||||
try:
|
||||
_INSTANCE_GLOBALE.masquer()
|
||||
except Exception:
|
||||
pass
|
||||
_INSTANCE_GLOBALE = None
|
||||
583
agent_v0/agent_v1/ui/messages.py
Normal file
583
agent_v0/agent_v1/ui/messages.py
Normal file
@@ -0,0 +1,583 @@
|
||||
# agent_v1/ui/messages.py
|
||||
"""
|
||||
Formatage des messages utilisateur pour Léa.
|
||||
|
||||
Convertit les codes d'erreur techniques (`target_not_found`, `no_screen_change`...)
|
||||
en phrases en français naturel, orientées action, adaptées à un utilisateur non
|
||||
technique (secrétaire médicale, TIM).
|
||||
|
||||
Trois niveaux de sévérité sont définis :
|
||||
- INFO — Léa fait son travail normalement
|
||||
- ATTENTION — Quelque chose de léger (ralentissement, retry)
|
||||
- BLOCAGE — Léa a besoin d'aide, elle rend la main
|
||||
|
||||
Le module est 100% pur (pas d'I/O, pas d'UI) : testable sans mocks lourds.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
from typing import Any, Mapping, Optional
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
# Accès paresseux au DomainContext
|
||||
# ----------------------------------------------------------------------------
|
||||
#
|
||||
# On importe le module à l'appel pour éviter toute dépendance circulaire
|
||||
# avec `agent_v0.server_v1.domain_context` (qui ne doit pas importer l'UI).
|
||||
# Si l'import échoue (contexte client sans server_v1), on retombe sur None
|
||||
# et les formatters gardent leur comportement générique historique.
|
||||
|
||||
|
||||
def _get_domain_ctx(domain_id: Optional[str]):
|
||||
"""Récupérer un DomainContext si possible, sinon None (fallback)."""
|
||||
if not domain_id:
|
||||
return None
|
||||
try:
|
||||
from agent_v0.server_v1.domain_context import get_domain_context # lazy
|
||||
return get_domain_context(domain_id)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _friendly_target(description: str, domain_id: Optional[str] = None) -> str:
|
||||
"""Transformer une description technique en langage métier si possible.
|
||||
|
||||
Ex (tim_codage) : "DP" → "diagnostic principal"
|
||||
Ex (comptabilite) : "TVA" → "montant de TVA"
|
||||
Retombe sur la description nettoyée si aucun domaine ne matche.
|
||||
"""
|
||||
base = _nettoyer_description_cible(description)
|
||||
ctx = _get_domain_ctx(domain_id)
|
||||
if ctx is None or not base:
|
||||
return base
|
||||
try:
|
||||
return ctx._apply_synonyms(base)
|
||||
except Exception:
|
||||
return base
|
||||
|
||||
|
||||
class NiveauMessage(Enum):
|
||||
"""Niveaux hiérarchiques des messages affichés à l'utilisateur."""
|
||||
|
||||
INFO = "info" # Fond vert clair, disparaît tout seul, 3-5s
|
||||
ATTENTION = "attention" # Fond orange clair, disparaît tout seul, 7s
|
||||
BLOCAGE = "blocage" # Fond rouge clair, reste affiché, 15s+
|
||||
|
||||
|
||||
# Durée d'affichage par défaut (secondes), par niveau
|
||||
DUREE_PAR_NIVEAU: dict[NiveauMessage, int] = {
|
||||
NiveauMessage.INFO: 4,
|
||||
NiveauMessage.ATTENTION: 7,
|
||||
NiveauMessage.BLOCAGE: 15,
|
||||
}
|
||||
|
||||
# Icône textuelle par niveau (compatible plyer/Windows/Linux)
|
||||
ICONE_PAR_NIVEAU: dict[NiveauMessage, str] = {
|
||||
NiveauMessage.INFO: "i",
|
||||
NiveauMessage.ATTENTION: "!",
|
||||
NiveauMessage.BLOCAGE: "?",
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class MessageUtilisateur:
|
||||
"""Un message prêt à être affiché à l'utilisateur.
|
||||
|
||||
Attributes:
|
||||
niveau: Hiérarchie (info/attention/blocage)
|
||||
titre: Titre court de la notification (≤60 caractères)
|
||||
corps: Corps du message en français naturel
|
||||
duree_s: Durée d'affichage recommandée (secondes)
|
||||
persistent: Si True, l'utilisateur doit fermer manuellement
|
||||
"""
|
||||
|
||||
niveau: NiveauMessage
|
||||
titre: str
|
||||
corps: str
|
||||
duree_s: int
|
||||
persistent: bool = False
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
"""Sérialiser le message (utile pour les tests et le logging)."""
|
||||
return {
|
||||
"niveau": self.niveau.value,
|
||||
"titre": self.titre,
|
||||
"corps": self.corps,
|
||||
"duree_s": self.duree_s,
|
||||
"persistent": self.persistent,
|
||||
}
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Helpers d'extraction
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def _extraire_nom_application(titre_fenetre: str) -> str:
|
||||
"""Extraire le nom de l'application à partir d'un titre de fenêtre.
|
||||
|
||||
Les titres Windows suivent généralement le format :
|
||||
"Document.txt – Bloc-notes"
|
||||
"Ma Page - Google Chrome"
|
||||
"Sans titre — Paint"
|
||||
|
||||
On retourne la partie après le dernier séparateur, ou le titre entier.
|
||||
"""
|
||||
if not titre_fenetre:
|
||||
return ""
|
||||
titre = titre_fenetre.strip()
|
||||
# Chercher le dernier séparateur parmi " – ", " — ", " - "
|
||||
for sep in (" – ", " — ", " - "):
|
||||
if sep in titre:
|
||||
return titre.rsplit(sep, 1)[-1].strip()
|
||||
return titre
|
||||
|
||||
|
||||
def _nettoyer_description_cible(description: str) -> str:
|
||||
"""Nettoyer la description technique d'une cible pour l'afficher.
|
||||
|
||||
Supprime les caractères techniques (guillemets inutiles, ':').
|
||||
"""
|
||||
if not description:
|
||||
return ""
|
||||
desc = description.strip()
|
||||
# Retirer les guillemets encapsulants
|
||||
desc = desc.strip("'\"`")
|
||||
# Limiter la longueur
|
||||
if len(desc) > 80:
|
||||
desc = desc[:77] + "..."
|
||||
return desc
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Formattage des messages techniques → humains
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def formatter_cible_non_trouvee(
|
||||
description_cible: str,
|
||||
titre_fenetre: Optional[str] = None,
|
||||
domain_id: Optional[str] = None,
|
||||
params: Optional[Mapping[str, Any]] = None,
|
||||
) -> MessageUtilisateur:
|
||||
"""Message quand Léa ne trouve pas un élément à cliquer.
|
||||
|
||||
Si un domaine métier est fourni, la description de la cible est
|
||||
transformée en langage métier via le DomainContext :
|
||||
- tim_codage + "DP" → "diagnostic principal"
|
||||
- comptabilite + "TVA" → "montant de TVA"
|
||||
|
||||
Exemple avant :
|
||||
target_not_found: 'bonjour' dans *bonjour, – Bloc-notes
|
||||
Exemple après :
|
||||
Léa a besoin d'aide
|
||||
Je ne trouve pas "bonjour" dans le Bloc-notes. Peux-tu cliquer
|
||||
dessus toi-même ? Je reprends ensuite.
|
||||
|
||||
Args:
|
||||
description_cible: Description brute de la cible.
|
||||
titre_fenetre: Titre de la fenêtre active (pour extraire l'app).
|
||||
domain_id: Domaine métier pour enrichir la sortie (optionnel).
|
||||
params: Paramètres du workflow (nom_patient, num_facture...)
|
||||
utilisés par les templates de clarification métier.
|
||||
"""
|
||||
cible = _friendly_target(description_cible, domain_id) or "l'élément"
|
||||
app = _extraire_nom_application(titre_fenetre or "")
|
||||
|
||||
# Si un domaine et un template de clarification existent, préférer la
|
||||
# question métier (plus pertinente que le message générique).
|
||||
ctx = _get_domain_ctx(domain_id)
|
||||
if ctx is not None and ctx.clarification_templates:
|
||||
try:
|
||||
corps = ctx.pose_clarification_question(
|
||||
{
|
||||
"blocked_on": "target_not_found",
|
||||
"target": description_cible or "",
|
||||
"app": app,
|
||||
"params": dict(params or {}),
|
||||
}
|
||||
)
|
||||
except Exception:
|
||||
corps = ""
|
||||
if corps:
|
||||
return MessageUtilisateur(
|
||||
niveau=NiveauMessage.BLOCAGE,
|
||||
titre="Léa a besoin d'aide",
|
||||
corps=corps,
|
||||
duree_s=DUREE_PAR_NIVEAU[NiveauMessage.BLOCAGE],
|
||||
persistent=True,
|
||||
)
|
||||
|
||||
if app:
|
||||
corps = (
|
||||
f"Je ne trouve pas « {cible} » dans {app}. "
|
||||
f"Peux-tu cliquer dessus toi-même ? Je reprends ensuite."
|
||||
)
|
||||
else:
|
||||
corps = (
|
||||
f"Je ne trouve pas « {cible} » à l'écran. "
|
||||
f"Peux-tu le faire toi-même ? Je reprends ensuite."
|
||||
)
|
||||
|
||||
return MessageUtilisateur(
|
||||
niveau=NiveauMessage.BLOCAGE,
|
||||
titre="Léa a besoin d'aide",
|
||||
corps=corps,
|
||||
duree_s=DUREE_PAR_NIVEAU[NiveauMessage.BLOCAGE],
|
||||
persistent=True,
|
||||
)
|
||||
|
||||
|
||||
def formatter_fenetre_incorrecte(
|
||||
titre_actuel: str,
|
||||
titre_attendu: str,
|
||||
) -> MessageUtilisateur:
|
||||
"""Message quand la fenêtre active n'est pas celle attendue.
|
||||
|
||||
Exemple avant :
|
||||
Fenêtre incorrecte: 'Program Manager' (attendu: 'Lea : Explorateur de fichiers')
|
||||
Exemple après :
|
||||
Léa attend une fenêtre
|
||||
J'attends « Explorateur de fichiers » mais c'est « Program Manager »
|
||||
qui est affiché. Peux-tu ouvrir la bonne fenêtre ?
|
||||
"""
|
||||
app_actuelle = _extraire_nom_application(titre_actuel) or "une autre fenêtre"
|
||||
app_attendue = _extraire_nom_application(titre_attendu) or titre_attendu
|
||||
|
||||
corps = (
|
||||
f"J'attends « {app_attendue} » mais c'est « {app_actuelle} » "
|
||||
f"qui est affiché. Peux-tu ouvrir la bonne fenêtre ?"
|
||||
)
|
||||
|
||||
return MessageUtilisateur(
|
||||
niveau=NiveauMessage.BLOCAGE,
|
||||
titre="Léa attend une fenêtre",
|
||||
corps=corps,
|
||||
duree_s=DUREE_PAR_NIVEAU[NiveauMessage.BLOCAGE],
|
||||
persistent=True,
|
||||
)
|
||||
|
||||
|
||||
def formatter_ecran_inchange(action_type: str = "") -> MessageUtilisateur:
|
||||
"""Message quand l'action n'a pas eu d'effet visible.
|
||||
|
||||
Exemple avant :
|
||||
Ecran inchange apres l'action
|
||||
Exemple après :
|
||||
Léa vérifie
|
||||
Mon clic n'a pas eu l'air de marcher. Je vais réessayer ou te
|
||||
rendre la main si ça ne passe pas.
|
||||
"""
|
||||
actions_fr = {
|
||||
"click": "Mon clic",
|
||||
"type": "Ma saisie",
|
||||
"key_combo": "Mon raccourci clavier",
|
||||
"scroll": "Mon défilement",
|
||||
}
|
||||
quoi = actions_fr.get(action_type, "Mon action")
|
||||
|
||||
corps = (
|
||||
f"{quoi} n'a pas eu l'air de marcher. Je vais réessayer, "
|
||||
f"ou te rendre la main si ça ne passe pas."
|
||||
)
|
||||
|
||||
return MessageUtilisateur(
|
||||
niveau=NiveauMessage.ATTENTION,
|
||||
titre="Léa vérifie",
|
||||
corps=corps,
|
||||
duree_s=DUREE_PAR_NIVEAU[NiveauMessage.ATTENTION],
|
||||
)
|
||||
|
||||
|
||||
def formatter_connexion_perdue(hote_serveur: str = "") -> MessageUtilisateur:
|
||||
"""Message quand la connexion avec le serveur est perdue.
|
||||
|
||||
Rassurant : on dit qu'on va réessayer automatiquement.
|
||||
"""
|
||||
corps = (
|
||||
"J'ai perdu le lien avec le serveur. Je retente automatiquement, "
|
||||
"pas besoin d'intervenir."
|
||||
)
|
||||
|
||||
return MessageUtilisateur(
|
||||
niveau=NiveauMessage.ATTENTION,
|
||||
titre="Léa est déconnectée",
|
||||
corps=corps,
|
||||
duree_s=DUREE_PAR_NIVEAU[NiveauMessage.ATTENTION],
|
||||
)
|
||||
|
||||
|
||||
def formatter_connexion_retablie() -> MessageUtilisateur:
|
||||
"""Message quand la connexion serveur est rétablie."""
|
||||
return MessageUtilisateur(
|
||||
niveau=NiveauMessage.INFO,
|
||||
titre="Léa",
|
||||
corps="C'est bon, la connexion est revenue. Je continue.",
|
||||
duree_s=DUREE_PAR_NIVEAU[NiveauMessage.INFO],
|
||||
)
|
||||
|
||||
|
||||
def formatter_debut_workflow(nom_workflow: str, nb_etapes: int = 0) -> MessageUtilisateur:
|
||||
"""Message au démarrage d'un workflow de replay."""
|
||||
if nb_etapes > 0:
|
||||
corps = (
|
||||
f"Je démarre « {nom_workflow} » ({nb_etapes} étapes). "
|
||||
f"Je t'indique mon avancement."
|
||||
)
|
||||
else:
|
||||
corps = f"Je démarre « {nom_workflow} ». Je t'indique mon avancement."
|
||||
|
||||
return MessageUtilisateur(
|
||||
niveau=NiveauMessage.INFO,
|
||||
titre="Léa démarre",
|
||||
corps=corps,
|
||||
duree_s=DUREE_PAR_NIVEAU[NiveauMessage.INFO],
|
||||
)
|
||||
|
||||
|
||||
def formatter_etape_workflow(
|
||||
etape_actuelle: int,
|
||||
nb_etapes: int,
|
||||
description: str = "",
|
||||
) -> MessageUtilisateur:
|
||||
"""Message pour la progression d'une étape."""
|
||||
if description:
|
||||
desc = _nettoyer_description_cible(description)
|
||||
corps = f"Étape {etape_actuelle}/{nb_etapes} — {desc}"
|
||||
else:
|
||||
corps = f"Étape {etape_actuelle}/{nb_etapes}"
|
||||
|
||||
return MessageUtilisateur(
|
||||
niveau=NiveauMessage.INFO,
|
||||
titre="Léa avance",
|
||||
corps=corps,
|
||||
duree_s=3,
|
||||
)
|
||||
|
||||
|
||||
def formatter_retry(action_type: str = "", tentative: int = 2) -> MessageUtilisateur:
|
||||
"""Message quand Léa retente une action."""
|
||||
corps = (
|
||||
f"Je retente (tentative {tentative}). Ça arrive parfois, "
|
||||
f"l'écran était peut-être en cours de chargement."
|
||||
)
|
||||
return MessageUtilisateur(
|
||||
niveau=NiveauMessage.ATTENTION,
|
||||
titre="Léa retente",
|
||||
corps=corps,
|
||||
duree_s=DUREE_PAR_NIVEAU[NiveauMessage.ATTENTION],
|
||||
)
|
||||
|
||||
|
||||
def formatter_ralentissement() -> MessageUtilisateur:
|
||||
"""Message quand Léa prend plus de temps que prévu."""
|
||||
return MessageUtilisateur(
|
||||
niveau=NiveauMessage.ATTENTION,
|
||||
titre="Léa prend son temps",
|
||||
corps="Je vais plus lentement que prévu. L'écran met du temps à répondre.",
|
||||
duree_s=DUREE_PAR_NIVEAU[NiveauMessage.ATTENTION],
|
||||
)
|
||||
|
||||
|
||||
def formatter_fin_workflow(
|
||||
succes: bool,
|
||||
nom_workflow: str = "",
|
||||
nb_etapes: int = 0,
|
||||
duree_s: float = 0.0,
|
||||
domain_id: Optional[str] = None,
|
||||
items_count: int = 0,
|
||||
failed_count: int = 0,
|
||||
params: Optional[Mapping[str, Any]] = None,
|
||||
) -> MessageUtilisateur:
|
||||
"""Message à la fin d'un workflow.
|
||||
|
||||
Si un domaine métier est fourni (et qu'il expose des summary_templates),
|
||||
on utilise `DomainContext.describe_workflow_outcome` pour formuler un
|
||||
rapport en langage métier (ex: "J'ai codé 14 dossiers sur 15").
|
||||
|
||||
Args:
|
||||
succes: True si l'ensemble du workflow a réussi.
|
||||
nom_workflow: Nom du workflow.
|
||||
nb_etapes: Nombre d'étapes techniques (pour fallback générique).
|
||||
duree_s: Durée totale en secondes.
|
||||
domain_id: Domaine métier (optionnel).
|
||||
items_count: Nombre d'items métier traités (ex: 15 dossiers).
|
||||
failed_count: Nombre d'items en échec.
|
||||
params: Infos supplémentaires passées aux templates.
|
||||
"""
|
||||
ctx = _get_domain_ctx(domain_id)
|
||||
if ctx is not None and ctx.summary_templates:
|
||||
try:
|
||||
corps = ctx.describe_workflow_outcome(
|
||||
workflow_name=nom_workflow,
|
||||
success=succes,
|
||||
items_count=items_count or max(1, nb_etapes),
|
||||
failed_count=failed_count,
|
||||
elapsed_s=duree_s,
|
||||
extra=dict(params or {}),
|
||||
)
|
||||
except Exception:
|
||||
corps = ""
|
||||
if corps:
|
||||
if succes and failed_count == 0:
|
||||
return MessageUtilisateur(
|
||||
niveau=NiveauMessage.INFO,
|
||||
titre="Léa a terminé",
|
||||
corps=corps,
|
||||
duree_s=6,
|
||||
)
|
||||
if succes and failed_count > 0:
|
||||
return MessageUtilisateur(
|
||||
niveau=NiveauMessage.ATTENTION,
|
||||
titre="Léa a terminé partiellement",
|
||||
corps=corps,
|
||||
duree_s=DUREE_PAR_NIVEAU[NiveauMessage.ATTENTION],
|
||||
)
|
||||
return MessageUtilisateur(
|
||||
niveau=NiveauMessage.BLOCAGE,
|
||||
titre="Léa s'arrête",
|
||||
corps=corps,
|
||||
duree_s=DUREE_PAR_NIVEAU[NiveauMessage.BLOCAGE],
|
||||
persistent=True,
|
||||
)
|
||||
|
||||
if succes:
|
||||
if nom_workflow and nb_etapes > 0:
|
||||
corps = (
|
||||
f"C'est fait ! « {nom_workflow} » est terminé "
|
||||
f"({nb_etapes} étapes en {int(duree_s)}s)."
|
||||
)
|
||||
else:
|
||||
corps = "C'est fait ! Tout s'est bien passé."
|
||||
return MessageUtilisateur(
|
||||
niveau=NiveauMessage.INFO,
|
||||
titre="Léa a terminé",
|
||||
corps=corps,
|
||||
duree_s=6,
|
||||
)
|
||||
else:
|
||||
corps = (
|
||||
"Je n'ai pas pu terminer. Je te rends la main, "
|
||||
"tu peux continuer à partir de là où je me suis arrêtée."
|
||||
)
|
||||
return MessageUtilisateur(
|
||||
niveau=NiveauMessage.BLOCAGE,
|
||||
titre="Léa s'arrête",
|
||||
corps=corps,
|
||||
duree_s=DUREE_PAR_NIVEAU[NiveauMessage.BLOCAGE],
|
||||
persistent=True,
|
||||
)
|
||||
|
||||
|
||||
def formatter_erreur_generique(
|
||||
message_technique: str,
|
||||
domain_id: Optional[str] = None,
|
||||
params: Optional[Mapping[str, Any]] = None,
|
||||
) -> MessageUtilisateur:
|
||||
"""Formater un message d'erreur technique non catégorisé.
|
||||
|
||||
On essaie de détecter les motifs connus dans le message technique pour
|
||||
le router vers le bon formatter spécialisé, sinon on emballe le message.
|
||||
Si `domain_id` est fourni, il est propagé aux formatters spécialisés
|
||||
pour produire un message en langage métier.
|
||||
"""
|
||||
if not message_technique:
|
||||
return MessageUtilisateur(
|
||||
niveau=NiveauMessage.ATTENTION,
|
||||
titre="Léa",
|
||||
corps="J'ai rencontré un petit souci. Je continue.",
|
||||
duree_s=DUREE_PAR_NIVEAU[NiveauMessage.ATTENTION],
|
||||
)
|
||||
|
||||
msg_lower = message_technique.lower()
|
||||
|
||||
# target_not_found[:...]
|
||||
if "target_not_found" in msg_lower:
|
||||
# Essayer d'extraire la description après le ':'
|
||||
match = re.match(r"target_not_found[:\s]*(.*)", message_technique, re.IGNORECASE)
|
||||
desc = match.group(1).strip() if match else ""
|
||||
return formatter_cible_non_trouvee(desc, domain_id=domain_id, params=params)
|
||||
|
||||
# Fenêtre incorrecte: 'X' (attendu: 'Y')
|
||||
if "fenêtre incorrecte" in msg_lower or "fenetre incorrecte" in msg_lower:
|
||||
# Extraire actuel et attendu
|
||||
m_actuel = re.search(r"[:,]\s*['\"]([^'\"]+)['\"]", message_technique)
|
||||
m_attendu = re.search(r"attendu[:\s]*['\"]([^'\"]+)['\"]", message_technique)
|
||||
actuel = m_actuel.group(1) if m_actuel else ""
|
||||
attendu = m_attendu.group(1) if m_attendu else ""
|
||||
return formatter_fenetre_incorrecte(actuel, attendu)
|
||||
|
||||
# Ecran inchangé
|
||||
if "inchang" in msg_lower or "no_screen_change" in msg_lower:
|
||||
return formatter_ecran_inchange()
|
||||
|
||||
# Policy abort / supervise
|
||||
if "policy_abort" in msg_lower or "visual_resolve_failed" in msg_lower:
|
||||
return formatter_cible_non_trouvee(
|
||||
message_technique, domain_id=domain_id, params=params
|
||||
)
|
||||
|
||||
# Fallback : message technique tronqué
|
||||
msg_tronque = message_technique.strip()
|
||||
if len(msg_tronque) > 120:
|
||||
msg_tronque = msg_tronque[:117] + "..."
|
||||
|
||||
return MessageUtilisateur(
|
||||
niveau=NiveauMessage.ATTENTION,
|
||||
titre="Léa",
|
||||
corps=f"J'ai rencontré un souci : {msg_tronque}",
|
||||
duree_s=DUREE_PAR_NIVEAU[NiveauMessage.ATTENTION],
|
||||
)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Détection fenêtre Léa (utilisé par l'executor pour ignorer sa propre UI)
|
||||
# ============================================================================
|
||||
|
||||
|
||||
# Motifs qui identifient une fenêtre appartenant à Léa (l'agent lui-même).
|
||||
# On utilise des regex avec \b pour éviter les faux positifs sur des noms
|
||||
# contenant "lea" (ex: "cléa.txt", "leapfrog", "replay").
|
||||
_MOTIFS_FENETRE_LEA_REGEX = (
|
||||
r"\bléa\b",
|
||||
r"\blea\b(?!p)", # "lea" mot entier, pas "leapfrog"
|
||||
r"lea\s*[—–\-:]", # "Lea —", "Lea -", "Lea :"
|
||||
r"léa\s*[—–\-:]",
|
||||
r"\bassistante ia\b",
|
||||
r"\bléa ia\b",
|
||||
r"\blea ia\b",
|
||||
)
|
||||
|
||||
|
||||
def est_fenetre_lea(titre_fenetre: str) -> bool:
|
||||
"""Détecter si un titre de fenêtre appartient à l'agent Léa lui-même.
|
||||
|
||||
Utilisé pour éviter que Léa ne se considère comme une fenêtre intrusive
|
||||
dans ses propres pré-vérifications.
|
||||
|
||||
Utilise des regex avec des word boundaries pour éviter les faux positifs
|
||||
sur des noms de fichiers contenant "lea" (ex: "cléa.txt", "replay.log").
|
||||
"""
|
||||
if not titre_fenetre:
|
||||
return False
|
||||
titre_lower = titre_fenetre.lower().strip()
|
||||
return any(re.search(motif, titre_lower) for motif in _MOTIFS_FENETRE_LEA_REGEX)
|
||||
|
||||
|
||||
# Conservé pour rétro-compatibilité avec le code qui listait MOTIFS_FENETRE_LEA
|
||||
MOTIFS_FENETRE_LEA = (
|
||||
"léa",
|
||||
"lea —",
|
||||
"léa —",
|
||||
"lea -",
|
||||
"léa -",
|
||||
"lea assistante",
|
||||
"léa assistante",
|
||||
"lea : ",
|
||||
"léa : ",
|
||||
"assistante ia",
|
||||
)
|
||||
@@ -5,6 +5,14 @@ Utilise plyer pour les notifications système, sans dépendance PyQt5.
|
||||
|
||||
Remplace les dialogues Qt par des toasts non-bloquants.
|
||||
Thread-safe avec rate limiting (1 notification / 2 secondes max).
|
||||
|
||||
Les messages utilisateur sont formatés via `agent_v1.ui.messages` qui convertit
|
||||
les codes techniques (target_not_found, etc.) en français naturel.
|
||||
|
||||
Hiérarchie des notifications (cf. messages.NiveauMessage) :
|
||||
- INFO : auto-dismiss en ~4s, rate-limité classique
|
||||
- ATTENTION : auto-dismiss en ~7s, rate-limité classique
|
||||
- BLOCAGE : persistant (15s+), bypass du rate limit
|
||||
"""
|
||||
|
||||
import logging
|
||||
@@ -12,6 +20,22 @@ import threading
|
||||
import time
|
||||
from typing import Optional
|
||||
|
||||
from .messages import (
|
||||
MessageUtilisateur,
|
||||
NiveauMessage,
|
||||
formatter_cible_non_trouvee,
|
||||
formatter_connexion_perdue,
|
||||
formatter_connexion_retablie,
|
||||
formatter_debut_workflow,
|
||||
formatter_ecran_inchange,
|
||||
formatter_erreur_generique,
|
||||
formatter_etape_workflow,
|
||||
formatter_fenetre_incorrecte,
|
||||
formatter_fin_workflow,
|
||||
formatter_ralentissement,
|
||||
formatter_retry,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Import conditionnel de plyer — fallback silencieux si absent
|
||||
@@ -59,7 +83,13 @@ class NotificationManager:
|
||||
# Méthode générique
|
||||
# ------------------------------------------------------------------ #
|
||||
|
||||
def notify(self, title: str, message: str, timeout: int = 5) -> bool:
|
||||
def notify(
|
||||
self,
|
||||
title: str,
|
||||
message: str,
|
||||
timeout: int = 5,
|
||||
bypass_rate_limit: bool = False,
|
||||
) -> bool:
|
||||
"""
|
||||
Affiche une notification toast.
|
||||
|
||||
@@ -67,6 +97,8 @@ class NotificationManager:
|
||||
title: Titre de la notification.
|
||||
message: Corps du message.
|
||||
timeout: Durée d'affichage en secondes.
|
||||
bypass_rate_limit: Si True, ignore le rate limit (pour les blocages
|
||||
importants qui ne doivent pas être écrasés).
|
||||
|
||||
Returns:
|
||||
True si la notification a été envoyée, False sinon
|
||||
@@ -76,6 +108,7 @@ class NotificationManager:
|
||||
logger.debug("Notification ignorée (plyer absent) : %s", title)
|
||||
return False
|
||||
|
||||
if not bypass_rate_limit:
|
||||
with self._lock:
|
||||
now = time.monotonic()
|
||||
elapsed = now - self._last_notification_time
|
||||
@@ -87,6 +120,9 @@ class NotificationManager:
|
||||
)
|
||||
return False
|
||||
self._last_notification_time = now
|
||||
else:
|
||||
with self._lock:
|
||||
self._last_notification_time = time.monotonic()
|
||||
|
||||
# Envoi dans un thread dédié pour ne jamais bloquer l'appelant
|
||||
thread = threading.Thread(
|
||||
@@ -97,6 +133,39 @@ class NotificationManager:
|
||||
thread.start()
|
||||
return True
|
||||
|
||||
def notify_message(self, msg: MessageUtilisateur) -> bool:
|
||||
"""Envoyer un MessageUtilisateur structuré (niveau, titre, corps).
|
||||
|
||||
Les messages BLOCAGE bypass le rate limit pour garantir que
|
||||
l'utilisateur voit qu'on a besoin de lui.
|
||||
"""
|
||||
bypass = msg.niveau == NiveauMessage.BLOCAGE
|
||||
# Log aussi pour tracer dans les logs fichiers
|
||||
self._log_message(msg)
|
||||
return self.notify(
|
||||
title=msg.titre,
|
||||
message=msg.corps,
|
||||
timeout=msg.duree_s,
|
||||
bypass_rate_limit=bypass,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _log_message(msg: MessageUtilisateur) -> None:
|
||||
"""Logger un message utilisateur avec le niveau approprié.
|
||||
|
||||
Les logs agents sont plus lisibles quand on route info → INFO,
|
||||
attention → WARNING, blocage → ERROR, avec un préfixe [LEA].
|
||||
"""
|
||||
prefix = f"[LEA] {msg.titre}: {msg.corps}"
|
||||
if msg.niveau == NiveauMessage.INFO:
|
||||
logger.info(prefix)
|
||||
elif msg.niveau == NiveauMessage.ATTENTION:
|
||||
logger.warning(prefix)
|
||||
elif msg.niveau == NiveauMessage.BLOCAGE:
|
||||
logger.error(prefix)
|
||||
else:
|
||||
logger.info(prefix)
|
||||
|
||||
def _send(self, title: str, message: str, timeout: int) -> None:
|
||||
"""Envoi effectif de la notification (exécuté dans un thread dédié)."""
|
||||
try:
|
||||
@@ -180,40 +249,79 @@ class NotificationManager:
|
||||
timeout=3,
|
||||
)
|
||||
|
||||
def replay_finished(self, success: bool, workflow_name: str) -> bool:
|
||||
"""Notification de fin de replay (succès ou échec)."""
|
||||
if success:
|
||||
return self.notify(
|
||||
title=APP_NAME,
|
||||
message="C'est fait ! Tout s'est bien passé.",
|
||||
timeout=5,
|
||||
)
|
||||
else:
|
||||
return self.notify(
|
||||
title=APP_NAME,
|
||||
message="Hmm, j'ai eu un souci. Vous pouvez me remontrer ?",
|
||||
timeout=7,
|
||||
)
|
||||
def replay_target_not_found(
|
||||
self,
|
||||
target_description: str,
|
||||
window_title: Optional[str] = None,
|
||||
) -> bool:
|
||||
"""Notification quand un élément n'est pas trouvé pendant le replay.
|
||||
|
||||
def connection_changed(self, connected: bool, server_host: str) -> bool:
|
||||
Le replay est mis en pause et attend une intervention humaine.
|
||||
Utilise `messages.formatter_cible_non_trouvee` pour un message en
|
||||
français naturel.
|
||||
"""
|
||||
msg = formatter_cible_non_trouvee(target_description, window_title)
|
||||
return self.notify_message(msg)
|
||||
|
||||
def replay_wrong_window(self, current_title: str, expected_title: str) -> bool:
|
||||
"""Notification quand la fenêtre active n'est pas celle attendue."""
|
||||
msg = formatter_fenetre_incorrecte(current_title, expected_title)
|
||||
return self.notify_message(msg)
|
||||
|
||||
def replay_no_screen_change(self, action_type: str = "") -> bool:
|
||||
"""Notification quand une action n'a pas eu d'effet visible."""
|
||||
msg = formatter_ecran_inchange(action_type)
|
||||
return self.notify_message(msg)
|
||||
|
||||
def replay_retry(self, action_type: str = "", tentative: int = 2) -> bool:
|
||||
"""Notification quand Léa retente une action."""
|
||||
msg = formatter_retry(action_type, tentative)
|
||||
return self.notify_message(msg)
|
||||
|
||||
def replay_slow(self) -> bool:
|
||||
"""Notification quand Léa va plus lentement que prévu."""
|
||||
msg = formatter_ralentissement()
|
||||
return self.notify_message(msg)
|
||||
|
||||
def replay_finished(
|
||||
self,
|
||||
success: bool,
|
||||
workflow_name: str,
|
||||
step_count: int = 0,
|
||||
duration_s: float = 0.0,
|
||||
) -> bool:
|
||||
"""Notification de fin de replay (succès ou échec)."""
|
||||
msg = formatter_fin_workflow(success, workflow_name, step_count, duration_s)
|
||||
return self.notify_message(msg)
|
||||
|
||||
def replay_workflow_started(self, workflow_name: str, step_count: int = 0) -> bool:
|
||||
"""Notification de début de workflow (remplace `replay_started`)."""
|
||||
msg = formatter_debut_workflow(workflow_name, step_count)
|
||||
return self.notify_message(msg)
|
||||
|
||||
def replay_step_progress(
|
||||
self,
|
||||
current: int,
|
||||
total: int,
|
||||
description: str = "",
|
||||
) -> bool:
|
||||
"""Notification de progression d'une étape (niveau INFO)."""
|
||||
msg = formatter_etape_workflow(current, total, description)
|
||||
return self.notify_message(msg)
|
||||
|
||||
def connection_changed(self, connected: bool, server_host: str = "") -> bool:
|
||||
"""Notification de changement d'état de la connexion serveur."""
|
||||
if connected:
|
||||
return self.notify(
|
||||
title=APP_NAME,
|
||||
message="Connectée au serveur.",
|
||||
timeout=5,
|
||||
)
|
||||
msg = formatter_connexion_retablie()
|
||||
else:
|
||||
return self.notify(
|
||||
title=APP_NAME,
|
||||
message="J'ai perdu la connexion avec le serveur.",
|
||||
timeout=7,
|
||||
)
|
||||
msg = formatter_connexion_perdue(server_host)
|
||||
return self.notify_message(msg)
|
||||
|
||||
def error(self, message: str) -> bool:
|
||||
"""Notification d'erreur."""
|
||||
return self.notify(
|
||||
title=APP_NAME,
|
||||
message=f"Oups, un problème : {message}",
|
||||
timeout=10,
|
||||
)
|
||||
"""Notification d'erreur générique.
|
||||
|
||||
Essaie d'abord de détecter un motif technique connu et de formater
|
||||
correctement, sinon fallback sur un message générique aidant.
|
||||
"""
|
||||
msg = formatter_erreur_generique(message)
|
||||
return self.notify_message(msg)
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
622
agent_v0/server_v1/chat_interface.py
Normal file
622
agent_v0/server_v1/chat_interface.py
Normal file
@@ -0,0 +1,622 @@
|
||||
"""
|
||||
ChatInterface — Interface de chat conversationnelle pour Léa.
|
||||
|
||||
Permet au TIM (Technicien Information Médicale) de parler à Léa en langage
|
||||
naturel :
|
||||
- "Ouvre le Bloc-notes et écris bonjour"
|
||||
- Léa comprend (TaskPlanner) et propose un plan
|
||||
- Le TIM confirme (ou refuse)
|
||||
- Léa exécute (replay) et envoie des updates de progression
|
||||
- Historique conversationnel conservé par session
|
||||
|
||||
C'est une couche LÉGÈRE au-dessus du TaskPlanner. Toute la logique de
|
||||
compréhension reste dans TaskPlanner — ChatInterface gère uniquement
|
||||
l'état conversationnel, la confirmation et le suivi d'exécution.
|
||||
|
||||
États de la session :
|
||||
idle → en attente d'un message
|
||||
planning → TaskPlanner.understand() en cours
|
||||
awaiting_confirmation → plan prêt, attend la confirmation du TIM
|
||||
executing → replay en cours
|
||||
done → dernier tour terminé (retour à idle au prochain message)
|
||||
error → erreur interne (instruction non comprise, exception…)
|
||||
|
||||
Langue : 100% français (c'est l'interface utilisateur).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Callable, Dict, List, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# =============================================================================
|
||||
# États
|
||||
# =============================================================================
|
||||
|
||||
STATE_IDLE = "idle"
|
||||
STATE_PLANNING = "planning"
|
||||
STATE_AWAITING_CONFIRMATION = "awaiting_confirmation"
|
||||
STATE_EXECUTING = "executing"
|
||||
STATE_DONE = "done"
|
||||
STATE_ERROR = "error"
|
||||
|
||||
VALID_STATES = {
|
||||
STATE_IDLE,
|
||||
STATE_PLANNING,
|
||||
STATE_AWAITING_CONFIRMATION,
|
||||
STATE_EXECUTING,
|
||||
STATE_DONE,
|
||||
STATE_ERROR,
|
||||
}
|
||||
|
||||
# Rôles de messages
|
||||
ROLE_USER = "user"
|
||||
ROLE_LEA = "lea"
|
||||
ROLE_SYSTEM = "system"
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Message
|
||||
# =============================================================================
|
||||
|
||||
@dataclass
|
||||
class ChatMessage:
|
||||
"""Un message dans l'historique d'une conversation."""
|
||||
role: str # "user", "lea", "system"
|
||||
content: str # Texte du message
|
||||
timestamp: float = field(default_factory=time.time)
|
||||
# Données contextuelles optionnelles (plan, résultat, progression…)
|
||||
meta: Dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"role": self.role,
|
||||
"content": self.content,
|
||||
"timestamp": self.timestamp,
|
||||
"meta": self.meta,
|
||||
}
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# ChatSession
|
||||
# =============================================================================
|
||||
|
||||
class ChatSession:
|
||||
"""Une conversation entre un utilisateur et Léa.
|
||||
|
||||
Maintient l'historique, l'état courant, et le dernier plan en attente
|
||||
de confirmation. Thread-safe (un lock par session).
|
||||
|
||||
Dépendances injectées (pour tester facilement) :
|
||||
- task_planner : instance de TaskPlanner (ou mock)
|
||||
- workflows_provider : callable () -> List[Dict] (liste des workflows)
|
||||
- replay_callback : callable (session_id, machine_id, params) -> replay_id
|
||||
- status_provider : callable (replay_id) -> Dict (pour suivre l'exécution)
|
||||
|
||||
Toutes ces dépendances sont optionnelles : ChatSession dégrade
|
||||
gracieusement (fallback) si gemma4 / replay indisponibles.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
session_id: str = "",
|
||||
task_planner: Any = None,
|
||||
workflows_provider: Optional[Callable[[], List[Dict[str, Any]]]] = None,
|
||||
replay_callback: Optional[Callable[..., str]] = None,
|
||||
status_provider: Optional[Callable[[str], Dict[str, Any]]] = None,
|
||||
machine_id: str = "default",
|
||||
):
|
||||
self.session_id = session_id or f"chat_{uuid.uuid4().hex[:12]}"
|
||||
self.machine_id = machine_id
|
||||
self.created_at = time.time()
|
||||
self.updated_at = self.created_at
|
||||
|
||||
self._task_planner = task_planner
|
||||
self._workflows_provider = workflows_provider
|
||||
self._replay_callback = replay_callback
|
||||
self._status_provider = status_provider
|
||||
|
||||
self._state: str = STATE_IDLE
|
||||
self._messages: List[ChatMessage] = []
|
||||
self._pending_plan: Any = None # TaskPlan en attente de confirmation
|
||||
self._active_replay_id: str = "" # Replay courant (si executing)
|
||||
self._last_progress: Dict[str, Any] = {}
|
||||
|
||||
self._lock = threading.RLock()
|
||||
|
||||
# Message d'accueil
|
||||
self._append(
|
||||
ROLE_LEA,
|
||||
"Bonjour ! Je suis Léa. Dites-moi ce que vous voulez que je fasse.",
|
||||
meta={"welcome": True},
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------
|
||||
# Accesseurs
|
||||
# ---------------------------------------------------------------------
|
||||
|
||||
@property
|
||||
def state(self) -> str:
|
||||
with self._lock:
|
||||
return self._state
|
||||
|
||||
def get_history(self) -> List[Dict[str, Any]]:
|
||||
"""Retourne l'historique complet des messages (sérialisé)."""
|
||||
with self._lock:
|
||||
return [m.to_dict() for m in self._messages]
|
||||
|
||||
def get_snapshot(self) -> Dict[str, Any]:
|
||||
"""État complet pour l'UI (historique + état + progression)."""
|
||||
with self._lock:
|
||||
return {
|
||||
"session_id": self.session_id,
|
||||
"state": self._state,
|
||||
"machine_id": self.machine_id,
|
||||
"created_at": self.created_at,
|
||||
"updated_at": self.updated_at,
|
||||
"messages": [m.to_dict() for m in self._messages],
|
||||
"pending_plan": (
|
||||
self._pending_plan.to_dict()
|
||||
if self._pending_plan is not None
|
||||
else None
|
||||
),
|
||||
"active_replay_id": self._active_replay_id,
|
||||
"progress": dict(self._last_progress),
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------
|
||||
# API publique
|
||||
# ---------------------------------------------------------------------
|
||||
|
||||
def send_message(self, text: str) -> Dict[str, Any]:
|
||||
"""Envoyer un message utilisateur.
|
||||
|
||||
Trois cas possibles selon l'état courant :
|
||||
1. awaiting_confirmation → c'est une réponse OUI/NON
|
||||
2. executing → on rafraîchit la progression
|
||||
3. idle/done/error → nouvelle instruction, on appelle TaskPlanner
|
||||
"""
|
||||
text = (text or "").strip()
|
||||
if not text:
|
||||
return {
|
||||
"ok": False,
|
||||
"error": "Message vide",
|
||||
"state": self._state,
|
||||
}
|
||||
|
||||
with self._lock:
|
||||
# Cas 1 : on attend une confirmation
|
||||
if self._state == STATE_AWAITING_CONFIRMATION:
|
||||
return self._handle_confirmation_reply(text)
|
||||
|
||||
# Cas 2 : en pleine exécution → message ajouté mais pas d'action
|
||||
if self._state == STATE_EXECUTING:
|
||||
self._append(ROLE_USER, text)
|
||||
self._append(
|
||||
ROLE_LEA,
|
||||
"Je suis en train d'exécuter le workflow. Un instant…",
|
||||
)
|
||||
return {"ok": True, "state": self._state}
|
||||
|
||||
# Cas 3 : nouvelle instruction
|
||||
self._append(ROLE_USER, text)
|
||||
self._set_state(STATE_PLANNING)
|
||||
|
||||
# Appel TaskPlanner hors du lock (peut être lent : gemma4)
|
||||
return self._plan_and_reply(text)
|
||||
|
||||
def confirm(self, confirmed: bool = True) -> Dict[str, Any]:
|
||||
"""Confirmer (ou refuser) l'exécution du plan en attente."""
|
||||
with self._lock:
|
||||
if self._state != STATE_AWAITING_CONFIRMATION:
|
||||
return {
|
||||
"ok": False,
|
||||
"error": f"Pas de plan en attente (état={self._state})",
|
||||
"state": self._state,
|
||||
}
|
||||
|
||||
if not confirmed:
|
||||
self._append(
|
||||
ROLE_LEA,
|
||||
"D'accord, j'annule. Dites-moi autre chose quand vous voulez.",
|
||||
)
|
||||
self._pending_plan = None
|
||||
self._set_state(STATE_IDLE)
|
||||
return {"ok": True, "state": self._state, "confirmed": False}
|
||||
|
||||
plan = self._pending_plan
|
||||
if plan is None:
|
||||
self._set_state(STATE_IDLE)
|
||||
return {
|
||||
"ok": False,
|
||||
"error": "Aucun plan à confirmer",
|
||||
"state": self._state,
|
||||
}
|
||||
|
||||
self._set_state(STATE_EXECUTING)
|
||||
|
||||
# Exécution hors du lock
|
||||
return self._execute_plan(plan)
|
||||
|
||||
def refresh_progress(self) -> Dict[str, Any]:
|
||||
"""Rafraîchir la progression du replay en cours.
|
||||
|
||||
Appelé par le client (polling) pour obtenir les updates d'exécution.
|
||||
Si le replay est terminé, passe l'état à done.
|
||||
"""
|
||||
with self._lock:
|
||||
if self._state != STATE_EXECUTING or not self._active_replay_id:
|
||||
return {"ok": True, "state": self._state, "progress": self._last_progress}
|
||||
|
||||
replay_id = self._active_replay_id
|
||||
provider = self._status_provider
|
||||
|
||||
if provider is None:
|
||||
return {"ok": True, "state": self._state, "progress": {}}
|
||||
|
||||
try:
|
||||
status = provider(replay_id) or {}
|
||||
except Exception as e:
|
||||
logger.warning(f"ChatSession: status_provider erreur: {e}")
|
||||
status = {}
|
||||
|
||||
with self._lock:
|
||||
self._last_progress = status
|
||||
self.updated_at = time.time()
|
||||
|
||||
# Détection de fin
|
||||
replay_status = str(status.get("status", "")).lower()
|
||||
completed = status.get("completed_actions", 0)
|
||||
total = status.get("total_actions", 0)
|
||||
|
||||
if replay_status in ("done", "completed", "finished", "success"):
|
||||
summary = (
|
||||
f"Workflow terminé ! {completed}/{total} actions réussies."
|
||||
if total
|
||||
else "Workflow terminé."
|
||||
)
|
||||
self._append(ROLE_LEA, summary, meta={"progress": dict(status)})
|
||||
self._set_state(STATE_DONE)
|
||||
self._active_replay_id = ""
|
||||
elif replay_status in ("failed", "error", "aborted"):
|
||||
err = status.get("error") or status.get("message") or "Erreur inconnue"
|
||||
self._append(
|
||||
ROLE_LEA,
|
||||
f"Le workflow a échoué : {err}",
|
||||
meta={"progress": dict(status)},
|
||||
)
|
||||
self._set_state(STATE_ERROR)
|
||||
self._active_replay_id = ""
|
||||
elif replay_status == "paused_need_help":
|
||||
self._append(
|
||||
ROLE_LEA,
|
||||
"Je suis bloquée sur une action, j'ai besoin d'aide…",
|
||||
meta={"progress": dict(status)},
|
||||
)
|
||||
# on reste en executing pour que le TIM puisse reprendre
|
||||
# else : toujours en cours, pas de message
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
"state": self._state,
|
||||
"progress": dict(self._last_progress),
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------
|
||||
# Logique interne
|
||||
# ---------------------------------------------------------------------
|
||||
|
||||
def _plan_and_reply(self, instruction: str) -> Dict[str, Any]:
|
||||
"""Appeler TaskPlanner.understand() et produire une réponse."""
|
||||
plan = None
|
||||
error_msg = ""
|
||||
|
||||
if self._task_planner is None:
|
||||
error_msg = "Planificateur indisponible"
|
||||
else:
|
||||
try:
|
||||
workflows = []
|
||||
if self._workflows_provider is not None:
|
||||
try:
|
||||
workflows = self._workflows_provider() or []
|
||||
except Exception as e:
|
||||
logger.warning(f"ChatSession: workflows_provider erreur: {e}")
|
||||
workflows = []
|
||||
|
||||
plan = self._task_planner.understand(
|
||||
instruction=instruction,
|
||||
available_workflows=workflows,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"ChatSession: TaskPlanner.understand erreur: {e}")
|
||||
error_msg = f"Erreur de compréhension : {e}"
|
||||
|
||||
# Fallback gracieux si pas de plan / gemma4 indisponible
|
||||
if plan is None:
|
||||
with self._lock:
|
||||
self._append(
|
||||
ROLE_LEA,
|
||||
f"Désolée, je n'arrive pas à comprendre pour l'instant. {error_msg}".strip(),
|
||||
meta={"error": error_msg},
|
||||
)
|
||||
self._set_state(STATE_ERROR)
|
||||
return {
|
||||
"ok": False,
|
||||
"state": self._state,
|
||||
"error": error_msg,
|
||||
}
|
||||
|
||||
# Plan non compris
|
||||
if not plan.understood:
|
||||
reason = plan.error or "je n'ai pas compris votre demande"
|
||||
with self._lock:
|
||||
self._append(
|
||||
ROLE_LEA,
|
||||
(
|
||||
f"Désolée, {reason}. "
|
||||
"Pouvez-vous reformuler ? Je connais les workflows que vous m'avez appris."
|
||||
),
|
||||
meta={"plan": plan.to_dict()},
|
||||
)
|
||||
self._set_state(STATE_ERROR)
|
||||
return {
|
||||
"ok": False,
|
||||
"state": self._state,
|
||||
"plan": plan.to_dict(),
|
||||
"error": reason,
|
||||
}
|
||||
|
||||
# Plan compris → formuler la proposition
|
||||
proposal = self._format_proposal(plan)
|
||||
|
||||
with self._lock:
|
||||
self._pending_plan = plan
|
||||
self._append(ROLE_LEA, proposal, meta={"plan": plan.to_dict()})
|
||||
self._set_state(STATE_AWAITING_CONFIRMATION)
|
||||
return {
|
||||
"ok": True,
|
||||
"state": self._state,
|
||||
"plan": plan.to_dict(),
|
||||
"message": proposal,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _format_proposal(plan: Any) -> str:
|
||||
"""Formuler une proposition en français à partir d'un TaskPlan."""
|
||||
lines = []
|
||||
lines.append(f"J'ai compris : « {plan.instruction} ».")
|
||||
|
||||
if plan.workflow_name:
|
||||
conf_pct = int(round((plan.match_confidence or 0.0) * 100))
|
||||
lines.append(
|
||||
f"Je vais utiliser le workflow « {plan.workflow_name} »"
|
||||
f" (confiance {conf_pct}%)."
|
||||
)
|
||||
elif plan.mode == "free" and plan.steps:
|
||||
lines.append(
|
||||
f"Je n'ai pas de workflow enregistré pour ça, "
|
||||
f"mais j'ai planifié {len(plan.steps)} étape(s) :"
|
||||
)
|
||||
for i, step in enumerate(plan.steps[:5], 1):
|
||||
desc = step.get("description", "") if isinstance(step, dict) else str(step)
|
||||
lines.append(f" {i}. {desc}")
|
||||
if len(plan.steps) > 5:
|
||||
lines.append(f" … et {len(plan.steps) - 5} autre(s) étape(s).")
|
||||
else:
|
||||
lines.append("Je n'ai pas de plan d'action clair pour cette demande.")
|
||||
|
||||
if plan.parameters:
|
||||
params_str = ", ".join(f"{k}={v}" for k, v in plan.parameters.items())
|
||||
lines.append(f"Paramètres détectés : {params_str}.")
|
||||
|
||||
if plan.is_loop:
|
||||
src = plan.loop_source or "éléments à traiter"
|
||||
lines.append(f"Traitement en boucle sur : {src}.")
|
||||
|
||||
lines.append("")
|
||||
lines.append("Est-ce que je peux y aller ? (oui / non)")
|
||||
return "\n".join(lines)
|
||||
|
||||
def _handle_confirmation_reply(self, text: str) -> Dict[str, Any]:
|
||||
"""Interpréter un message utilisateur comme OUI/NON."""
|
||||
self._append(ROLE_USER, text)
|
||||
yes_tokens = {"oui", "yes", "ok", "y", "go", "vas-y", "allez", "allez-y", "confirme", "confirmer", "continue"}
|
||||
no_tokens = {"non", "no", "annule", "annuler", "stop", "arrête", "arrete", "abandonne", "abandonner"}
|
||||
|
||||
t = text.strip().lower().rstrip("!.?")
|
||||
|
||||
if t in yes_tokens or any(t.startswith(tok + " ") for tok in yes_tokens):
|
||||
# Déverrouiller : sortir du lock avant d'exécuter (confirm re-prend le lock)
|
||||
pass
|
||||
elif t in no_tokens or any(t.startswith(tok + " ") for tok in no_tokens):
|
||||
self._append(
|
||||
ROLE_LEA,
|
||||
"D'accord, j'annule. Dites-moi autre chose quand vous voulez.",
|
||||
)
|
||||
self._pending_plan = None
|
||||
self._set_state(STATE_IDLE)
|
||||
return {"ok": True, "state": self._state, "confirmed": False}
|
||||
else:
|
||||
self._append(
|
||||
ROLE_LEA,
|
||||
"Je n'ai pas compris votre réponse. Répondez « oui » pour lancer ou « non » pour annuler.",
|
||||
)
|
||||
return {"ok": True, "state": self._state, "needs_clarification": True}
|
||||
|
||||
# Libérer le lock pour confirm() qui le re-prendra
|
||||
plan = self._pending_plan
|
||||
self._pending_plan = None
|
||||
self._set_state(STATE_EXECUTING)
|
||||
# Exécution hors du lock (sortie du with bloc appelant)
|
||||
# Note : _handle_confirmation_reply est appelé sous lock via send_message
|
||||
# On ne peut pas appeler _execute_plan ici sans risque de double-lock.
|
||||
# On relâche le lock via une astuce : on retourne un marqueur et send_message
|
||||
# orchestrera. Ici on appelle directement _execute_plan qui utilise RLock,
|
||||
# donc c'est safe (re-entrant).
|
||||
return self._execute_plan(plan)
|
||||
|
||||
def _execute_plan(self, plan: Any) -> Dict[str, Any]:
|
||||
"""Lancer le replay correspondant au plan."""
|
||||
if plan is None:
|
||||
with self._lock:
|
||||
self._append(ROLE_LEA, "Rien à exécuter.", meta={})
|
||||
self._set_state(STATE_IDLE)
|
||||
return {"ok": False, "state": self._state, "error": "Aucun plan"}
|
||||
|
||||
if self._replay_callback is None:
|
||||
with self._lock:
|
||||
self._append(
|
||||
ROLE_LEA,
|
||||
"Je ne peux pas exécuter : aucun moteur d'exécution n'est configuré.",
|
||||
)
|
||||
self._set_state(STATE_ERROR)
|
||||
return {
|
||||
"ok": False,
|
||||
"state": self._state,
|
||||
"error": "replay_callback non configuré",
|
||||
}
|
||||
|
||||
# Annoncer le démarrage
|
||||
with self._lock:
|
||||
self._append(
|
||||
ROLE_LEA,
|
||||
"C'est parti ! Je lance le workflow…",
|
||||
meta={"plan": plan.to_dict()},
|
||||
)
|
||||
|
||||
# Appeler le callback
|
||||
try:
|
||||
if plan.workflow_match:
|
||||
replay_id = self._replay_callback(
|
||||
session_id=plan.workflow_match,
|
||||
machine_id=self.machine_id,
|
||||
params=plan.parameters,
|
||||
)
|
||||
else:
|
||||
# Mode libre : pas encore branché côté chat (on refuse proprement)
|
||||
replay_id = ""
|
||||
raise RuntimeError(
|
||||
"Mode libre non supporté pour l'instant — "
|
||||
"entraînez un workflow pour cette tâche"
|
||||
)
|
||||
except Exception as e:
|
||||
with self._lock:
|
||||
self._append(
|
||||
ROLE_LEA,
|
||||
f"Je n'ai pas pu lancer le workflow : {e}",
|
||||
meta={"error": str(e)},
|
||||
)
|
||||
self._set_state(STATE_ERROR)
|
||||
return {"ok": False, "state": self._state, "error": str(e)}
|
||||
|
||||
with self._lock:
|
||||
self._active_replay_id = replay_id or ""
|
||||
return {
|
||||
"ok": True,
|
||||
"state": self._state,
|
||||
"replay_id": self._active_replay_id,
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------
|
||||
|
||||
def _append(self, role: str, content: str, meta: Optional[Dict[str, Any]] = None) -> None:
|
||||
"""Ajouter un message à l'historique (doit être appelé sous lock)."""
|
||||
msg = ChatMessage(role=role, content=content, meta=meta or {})
|
||||
self._messages.append(msg)
|
||||
self.updated_at = msg.timestamp
|
||||
|
||||
def _set_state(self, new_state: str) -> None:
|
||||
"""Changer d'état (doit être appelé sous lock)."""
|
||||
if new_state not in VALID_STATES:
|
||||
raise ValueError(f"État invalide : {new_state}")
|
||||
old = self._state
|
||||
self._state = new_state
|
||||
self.updated_at = time.time()
|
||||
if old != new_state:
|
||||
logger.debug(
|
||||
f"ChatSession {self.session_id}: {old} -> {new_state}"
|
||||
)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# ChatManager — registre en mémoire des sessions
|
||||
# =============================================================================
|
||||
|
||||
class ChatManager:
|
||||
"""Registre en mémoire des sessions de chat.
|
||||
|
||||
Thread-safe. Utilisé par l'API FastAPI pour gérer plusieurs
|
||||
conversations simultanées.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
task_planner: Any = None,
|
||||
workflows_provider: Optional[Callable[[], List[Dict[str, Any]]]] = None,
|
||||
replay_callback: Optional[Callable[..., str]] = None,
|
||||
status_provider: Optional[Callable[[str], Dict[str, Any]]] = None,
|
||||
):
|
||||
self._task_planner = task_planner
|
||||
self._workflows_provider = workflows_provider
|
||||
self._replay_callback = replay_callback
|
||||
self._status_provider = status_provider
|
||||
self._sessions: Dict[str, ChatSession] = {}
|
||||
self._lock = threading.RLock()
|
||||
|
||||
def create_session(self, machine_id: str = "default") -> ChatSession:
|
||||
"""Créer une nouvelle session de chat."""
|
||||
session = ChatSession(
|
||||
task_planner=self._task_planner,
|
||||
workflows_provider=self._workflows_provider,
|
||||
replay_callback=self._replay_callback,
|
||||
status_provider=self._status_provider,
|
||||
machine_id=machine_id,
|
||||
)
|
||||
with self._lock:
|
||||
self._sessions[session.session_id] = session
|
||||
logger.info(f"ChatManager: session créée {session.session_id}")
|
||||
return session
|
||||
|
||||
def get_session(self, session_id: str) -> Optional[ChatSession]:
|
||||
with self._lock:
|
||||
return self._sessions.get(session_id)
|
||||
|
||||
def list_sessions(self) -> List[Dict[str, Any]]:
|
||||
with self._lock:
|
||||
return [
|
||||
{
|
||||
"session_id": s.session_id,
|
||||
"state": s.state,
|
||||
"machine_id": s.machine_id,
|
||||
"created_at": s.created_at,
|
||||
"updated_at": s.updated_at,
|
||||
"message_count": len(s.get_history()),
|
||||
}
|
||||
for s in self._sessions.values()
|
||||
]
|
||||
|
||||
def delete_session(self, session_id: str) -> bool:
|
||||
with self._lock:
|
||||
return self._sessions.pop(session_id, None) is not None
|
||||
|
||||
def cleanup_old(self, max_age_s: float = 3600 * 24) -> int:
|
||||
"""Supprimer les sessions inactives depuis max_age_s secondes."""
|
||||
now = time.time()
|
||||
removed = 0
|
||||
with self._lock:
|
||||
to_delete = [
|
||||
sid for sid, s in self._sessions.items()
|
||||
if (now - s.updated_at) > max_age_s
|
||||
]
|
||||
for sid in to_delete:
|
||||
del self._sessions[sid]
|
||||
removed += 1
|
||||
return removed
|
||||
@@ -3,35 +3,81 @@
|
||||
Contexte métier pour les appels VLM — rend Léa experte du domaine.
|
||||
|
||||
Chaque workflow est associé à un domaine métier (médical, comptable, etc.)
|
||||
qui enrichit TOUS les prompts VLM (Observer, Critic, acteur, enrichissement).
|
||||
qui enrichit TOUS les prompts VLM (Observer, Critic, acteur, enrichissement)
|
||||
ET la personnalité de Léa (résumés, questions de clarification, rapports).
|
||||
|
||||
Un gemma4 qui sait qu'il regarde un DPI et que l'utilisateur fait du codage
|
||||
CIM-10 prend des décisions bien meilleures qu'un VLM générique.
|
||||
CIM-10 prend des décisions bien meilleures qu'un VLM générique. Et Léa qui
|
||||
dit "J'ai codé 14 dossiers sur 15" plutôt que "J'ai exécuté 112 clics" est
|
||||
bien plus utile pour un TIM.
|
||||
|
||||
Premier domaine : TIM (Technicien d'Information Médicale)
|
||||
- Logiciels DPI/DMS (dossier patient informatisé)
|
||||
- Codage CIM-10 / CCAM / GHM
|
||||
- Lecture de comptes rendus médicaux
|
||||
- Validation des séjours / RSS / RSA
|
||||
Domaines pré-configurés :
|
||||
- tim_codage : TIM, codage CIM-10 / CCAM / PMSI, DPI
|
||||
- comptabilite : factures, TVA, OCR, plans comptables
|
||||
- rh_paie : fiches de paie, employés, charges sociales
|
||||
- stocks_logistique : bons, commandes, réceptions, inventaires
|
||||
- generic : fallback bureautique
|
||||
|
||||
Usage :
|
||||
Usage basique :
|
||||
ctx = get_domain_context("tim_codage")
|
||||
prompt = f"{ctx.system_prompt}\n\n{user_prompt}"
|
||||
prompt = ctx.enrich_prompt(user_prompt, role="actor")
|
||||
|
||||
Usage langage métier :
|
||||
ctx = get_domain_context("tim_codage")
|
||||
phrase = ctx.summarize_action("click", {"target": "DP"})
|
||||
# → "saisir le diagnostic principal"
|
||||
|
||||
question = ctx.pose_clarification_question(
|
||||
{"blocked_on": "target_not_found", "target": "Fichier patient",
|
||||
"params": {"nom_patient": "Mme Durand"}}
|
||||
)
|
||||
# → "Je ne trouve pas le dossier de Mme Durand..."
|
||||
|
||||
rapport = ctx.describe_workflow_outcome(
|
||||
workflow_name="Codage séjours janvier",
|
||||
success=True,
|
||||
items_count=15,
|
||||
failed_count=1,
|
||||
)
|
||||
# → "J'ai codé 14 dossiers sur 15..."
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import unicodedata
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Dict, List, Optional
|
||||
from typing import Any, Dict, List, Mapping, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _strip_accents(s: str) -> str:
|
||||
"""Supprimer les accents pour les comparaisons insensibles aux diacritiques."""
|
||||
if not s:
|
||||
return ""
|
||||
nkfd = unicodedata.normalize("NFKD", s)
|
||||
return "".join(c for c in nkfd if not unicodedata.combining(c))
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Data class
|
||||
# =========================================================================
|
||||
|
||||
|
||||
@dataclass
|
||||
class DomainContext:
|
||||
"""Contexte métier pour un domaine spécifique."""
|
||||
domain_id: str # Identifiant unique (tim_codage, comptabilite, etc.)
|
||||
name: str # Nom lisible (Codage médical TIM)
|
||||
description: str # Description courte du métier
|
||||
"""Contexte métier pour un domaine spécifique.
|
||||
|
||||
Contient à la fois les hints pour les prompts VLM et les éléments de
|
||||
personnalité de Léa (langage métier, questions, rapports).
|
||||
"""
|
||||
|
||||
domain_id: str # tim_codage, comptabilite, ...
|
||||
name: str # Nom lisible
|
||||
description: str # Description courte
|
||||
|
||||
# Prompt système injecté dans TOUS les appels VLM
|
||||
system_prompt: str = ""
|
||||
@@ -39,12 +85,41 @@ class DomainContext:
|
||||
# Vocabulaire métier (termes que le VLM doit connaître)
|
||||
vocabulary: List[str] = field(default_factory=list)
|
||||
|
||||
# Applications connues (noms de logiciels que le VLM peut rencontrer)
|
||||
# Applications connues
|
||||
known_apps: List[str] = field(default_factory=list)
|
||||
|
||||
# Écrans types (descriptions des écrans courants du métier)
|
||||
# Écrans types
|
||||
screen_patterns: Dict[str, str] = field(default_factory=dict)
|
||||
|
||||
# --- Personnalité Léa -------------------------------------------------
|
||||
|
||||
# Mapping d'actions techniques (click/type/key_combo) vers description métier,
|
||||
# indexé par un mot-clé lisible trouvé dans la cible/texte.
|
||||
# Format : { (action_type, keyword_lower) : "description métier" }
|
||||
# Exemple : ("click", "dp") → "saisir le diagnostic principal"
|
||||
common_actions: Dict[str, str] = field(default_factory=dict)
|
||||
|
||||
# Synonymes métier : technique → forme lisible
|
||||
# Exemple : {"dp": "diagnostic principal", "das": "diagnostics associés"}
|
||||
vocabulary_synonyms: Dict[str, str] = field(default_factory=dict)
|
||||
|
||||
# Templates de questions de clarification (selon la raison de blocage).
|
||||
# Clé = identifiant de blocage ("target_not_found", "ambiguous_field", ...)
|
||||
# Valeur = template f-string (champs: {target}, {app}, {nom_patient}, ...)
|
||||
clarification_templates: Dict[str, str] = field(default_factory=dict)
|
||||
|
||||
# Templates de résumés de fin de workflow.
|
||||
# Clés attendues :
|
||||
# - "success" : tout a marché
|
||||
# - "partial" : succès partiel (failed_count > 0)
|
||||
# - "failure" : échec complet
|
||||
# - "success_one" : cas 1 élément (pour éviter "1 dossiers")
|
||||
# - "item_singular" : libellé d'un item ("dossier")
|
||||
# - "item_plural" : libellé au pluriel ("dossiers")
|
||||
summary_templates: Dict[str, str] = field(default_factory=dict)
|
||||
|
||||
# ------------------------------------------------------------------ API
|
||||
|
||||
def enrich_prompt(self, prompt: str, role: str = "") -> str:
|
||||
"""Enrichir un prompt avec le contexte métier.
|
||||
|
||||
@@ -65,6 +140,310 @@ class DomainContext:
|
||||
parts.append(prompt)
|
||||
return "\n\n".join(parts)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Personnalité : résumé d'action en langage métier
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def summarize_action(
|
||||
self,
|
||||
action: str,
|
||||
params: Optional[Mapping[str, Any]] = None,
|
||||
) -> str:
|
||||
"""Résumer une action technique en langage métier.
|
||||
|
||||
Args:
|
||||
action: Type d'action ("click", "type", "key_combo", "wait", "scroll")
|
||||
params: Paramètres de l'action (target, text, keys, ...)
|
||||
|
||||
Returns:
|
||||
Phrase en français orientée métier. Fallback générique si aucun
|
||||
mapping ne correspond.
|
||||
|
||||
Exemples (domaine tim_codage) :
|
||||
click sur "DP" → "saisir le diagnostic principal"
|
||||
type "E11.9" → "saisir le code CIM-10 E11.9"
|
||||
click sur "Valider" → "valider le codage"
|
||||
"""
|
||||
params = dict(params or {})
|
||||
target = str(params.get("target") or params.get("description") or "").strip()
|
||||
text = str(params.get("text") or "").strip()
|
||||
keys = params.get("keys") or []
|
||||
|
||||
haystack = _strip_accents(f"{target} {text}".lower())
|
||||
|
||||
# 1) Essayer un match mot-clé dans common_actions.
|
||||
# Clés sous la forme "click:mot" ou "type:mot".
|
||||
# Comparaison insensible à la casse ET aux accents.
|
||||
for key, label in self.common_actions.items():
|
||||
if ":" not in key:
|
||||
continue
|
||||
k_action, k_word = key.split(":", 1)
|
||||
if k_action != action:
|
||||
continue
|
||||
k_word_norm = _strip_accents(k_word.lower())
|
||||
if k_word_norm and k_word_norm in haystack:
|
||||
return label
|
||||
|
||||
# 2) Essayer une substitution via vocabulary_synonyms dans la cible.
|
||||
friendly_target = self._apply_synonyms(target)
|
||||
|
||||
if action == "click":
|
||||
if friendly_target:
|
||||
return f"cliquer sur {friendly_target}"
|
||||
return "cliquer"
|
||||
|
||||
if action == "type":
|
||||
if text and friendly_target:
|
||||
return f"saisir « {text} » dans {friendly_target}"
|
||||
if text:
|
||||
return f"saisir « {text} »"
|
||||
return "saisir du texte"
|
||||
|
||||
if action == "key_combo":
|
||||
if isinstance(keys, (list, tuple)) and keys:
|
||||
return f"utiliser le raccourci {'+'.join(str(k) for k in keys)}"
|
||||
return "utiliser un raccourci clavier"
|
||||
|
||||
if action == "wait":
|
||||
return "attendre le chargement de l'écran"
|
||||
|
||||
if action == "scroll":
|
||||
return "faire défiler l'écran"
|
||||
|
||||
# Fallback ultime
|
||||
return f"effectuer l'action {action}"
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Personnalité : question de clarification
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def pose_clarification_question(
|
||||
self,
|
||||
context: Optional[Mapping[str, Any]] = None,
|
||||
) -> str:
|
||||
"""Générer une question pertinente quand Léa bloque.
|
||||
|
||||
Cherche un template dans clarification_templates selon :
|
||||
- context["blocked_on"] (ex: "target_not_found", "ambiguous_field")
|
||||
- context["target"] (la cible visée)
|
||||
- paramètres du workflow (params) disponibles pour substitution
|
||||
|
||||
Args:
|
||||
context: Dictionnaire libre contenant au minimum 'blocked_on' ou
|
||||
'target'. Peut contenir 'params' pour la substitution.
|
||||
|
||||
Returns:
|
||||
Question en français. Fallback générique si aucun template ne
|
||||
correspond.
|
||||
"""
|
||||
ctx = dict(context or {})
|
||||
blocked_on = str(ctx.get("blocked_on") or "").strip()
|
||||
target = str(ctx.get("target") or "").strip()
|
||||
params = dict(ctx.get("params") or {})
|
||||
|
||||
# Dictionnaire de substitution : champs du context + params + target
|
||||
subs: Dict[str, Any] = {
|
||||
"target": target,
|
||||
"target_friendly": self._apply_synonyms(target) or target or "cet élément",
|
||||
"app": ctx.get("app", ""),
|
||||
}
|
||||
subs.update(params)
|
||||
|
||||
# 1) Essai par clé exacte de blocage
|
||||
template = self.clarification_templates.get(blocked_on, "")
|
||||
|
||||
# 2) Essai par cible (si la cible matche un mot-clé connu)
|
||||
if not template and target:
|
||||
low = target.lower()
|
||||
for key, tpl in self.clarification_templates.items():
|
||||
if key.startswith("target:") and key.split(":", 1)[1].lower() in low:
|
||||
template = tpl
|
||||
break
|
||||
|
||||
# 3) Template générique du domaine
|
||||
if not template:
|
||||
template = self.clarification_templates.get("default", "")
|
||||
|
||||
if template:
|
||||
try:
|
||||
return template.format_map(_SafeDict(subs))
|
||||
except Exception as e: # pragma: no cover - format inattendu
|
||||
logger.warning("clarification template format error: %s", e)
|
||||
|
||||
# 4) Fallback ultime cross-domaine
|
||||
friendly = subs["target_friendly"]
|
||||
return (
|
||||
f"Je ne trouve pas {friendly}. "
|
||||
f"Peux-tu me le montrer ou me confirmer que c'est le bon écran ?"
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Personnalité : rapport final
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def describe_workflow_outcome(
|
||||
self,
|
||||
workflow_name: str = "",
|
||||
success: bool = True,
|
||||
items_count: int = 1,
|
||||
failed_count: int = 0,
|
||||
elapsed_s: float = 0.0,
|
||||
extra: Optional[Mapping[str, Any]] = None,
|
||||
use_llm: bool = False,
|
||||
) -> str:
|
||||
"""Générer un rapport de fin de workflow en langage métier.
|
||||
|
||||
Args:
|
||||
workflow_name: Nom du workflow ("Codage janvier").
|
||||
success: True si le workflow a globalement réussi.
|
||||
items_count: Nombre d'items traités (ex: 15 dossiers). 1 par défaut.
|
||||
failed_count: Nombre d'items en échec.
|
||||
elapsed_s: Durée totale (secondes).
|
||||
extra: Infos supplémentaires (hint pour le LLM).
|
||||
use_llm: Si True, tenter un appel à gemma4 pour produire le
|
||||
rapport. Fallback sur les templates en cas d'échec.
|
||||
|
||||
Returns:
|
||||
Rapport en français. Toujours une chaîne, jamais None.
|
||||
"""
|
||||
extra = dict(extra or {})
|
||||
done = max(0, items_count - failed_count)
|
||||
|
||||
item_sg = self.summary_templates.get("item_singular", "élément")
|
||||
item_pl = self.summary_templates.get("item_plural", "éléments")
|
||||
item_word = item_sg if done <= 1 else item_pl
|
||||
|
||||
# Données disponibles pour les templates
|
||||
subs = {
|
||||
"workflow_name": workflow_name or "le workflow",
|
||||
"items_count": items_count,
|
||||
"done": done,
|
||||
"failed": failed_count,
|
||||
"item_singular": item_sg,
|
||||
"item_plural": item_pl,
|
||||
"item_word": item_word,
|
||||
"elapsed_s": int(elapsed_s),
|
||||
}
|
||||
subs.update(extra)
|
||||
|
||||
# Choisir le template adéquat
|
||||
if not success and failed_count >= items_count:
|
||||
key = "failure"
|
||||
elif failed_count > 0:
|
||||
key = "partial"
|
||||
elif items_count == 1:
|
||||
key = "success_one" if "success_one" in self.summary_templates else "success"
|
||||
else:
|
||||
key = "success"
|
||||
|
||||
template = self.summary_templates.get(key, "")
|
||||
|
||||
# Optionnel : raffiner via gemma4
|
||||
if use_llm:
|
||||
llm_text = self._llm_refine_summary(template, subs, success)
|
||||
if llm_text:
|
||||
return llm_text
|
||||
|
||||
if template:
|
||||
try:
|
||||
return template.format_map(_SafeDict(subs))
|
||||
except Exception as e: # pragma: no cover
|
||||
logger.warning("summary template format error: %s", e)
|
||||
|
||||
# Fallback générique
|
||||
if success:
|
||||
if items_count <= 1:
|
||||
return f"C'est fait, j'ai terminé « {workflow_name or 'le workflow'} »."
|
||||
return (
|
||||
f"J'ai traité {done} {item_word} sur {items_count}"
|
||||
+ (f", {failed_count} en échec." if failed_count else ".")
|
||||
)
|
||||
return (
|
||||
f"Je n'ai pas pu terminer « {workflow_name or 'le workflow'} ». "
|
||||
f"Je te rends la main."
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Helpers internes
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _apply_synonyms(self, text: str) -> str:
|
||||
"""Remplacer les sigles/termes techniques par leur forme métier.
|
||||
|
||||
Cherche mots entiers (word boundaries) en insensible à la casse.
|
||||
"""
|
||||
if not text or not self.vocabulary_synonyms:
|
||||
return text
|
||||
result = text
|
||||
for short, full in self.vocabulary_synonyms.items():
|
||||
if not short:
|
||||
continue
|
||||
pattern = r"\b" + re.escape(short) + r"\b"
|
||||
result = re.sub(pattern, full, result, flags=re.IGNORECASE)
|
||||
return result
|
||||
|
||||
def _llm_refine_summary(
|
||||
self,
|
||||
template: str,
|
||||
subs: Dict[str, Any],
|
||||
success: bool,
|
||||
) -> str:
|
||||
"""Tenter un raffinement du rapport via gemma4.
|
||||
|
||||
Appel best-effort : toute erreur retourne "" et le caller retombe sur
|
||||
le template brut. Isolé dans une méthode pour pouvoir le monkey-patcher
|
||||
dans les tests.
|
||||
"""
|
||||
try:
|
||||
import requests as _requests
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
port = os.environ.get("GEMMA4_PORT", "11435")
|
||||
url = f"http://localhost:{port}/api/chat"
|
||||
|
||||
base = ""
|
||||
if template:
|
||||
try:
|
||||
base = template.format_map(_SafeDict(subs))
|
||||
except Exception:
|
||||
base = ""
|
||||
|
||||
prompt = (
|
||||
f"Tu es Léa, une assistante RPA dans le domaine : {self.name}.\n"
|
||||
f"Tu viens de terminer un workflow. Résume en UNE à DEUX phrases "
|
||||
f"en langage métier, chaleureux mais professionnel, en français.\n\n"
|
||||
f"Données :\n"
|
||||
f"- workflow : {subs.get('workflow_name', '')}\n"
|
||||
f"- items traités : {subs.get('done', 0)} / {subs.get('items_count', 0)}\n"
|
||||
f"- échecs : {subs.get('failed', 0)}\n"
|
||||
f"- succès global : {'oui' if success else 'non'}\n"
|
||||
f"- durée : {subs.get('elapsed_s', 0)}s\n\n"
|
||||
f"Base suggérée (tu peux la reformuler) : {base or '(aucune)'}\n\n"
|
||||
f"Ta phrase :"
|
||||
)
|
||||
|
||||
try:
|
||||
resp = _requests.post(
|
||||
url,
|
||||
json={
|
||||
"model": "gemma4:e4b",
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
"stream": False,
|
||||
"options": {"temperature": 0.3, "num_predict": 200},
|
||||
},
|
||||
timeout=30,
|
||||
)
|
||||
if not resp.ok:
|
||||
return ""
|
||||
content = resp.json().get("message", {}).get("content", "").strip()
|
||||
# Nettoyage basique : supprimer guillemets typographiques en bord
|
||||
content = content.strip("\"' \n")
|
||||
return content
|
||||
except Exception as e:
|
||||
logger.debug("gemma4 refine summary failed: %s", e)
|
||||
return ""
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"domain_id": self.domain_id,
|
||||
@@ -72,9 +451,24 @@ class DomainContext:
|
||||
"description": self.description,
|
||||
"known_apps": self.known_apps,
|
||||
"vocabulary_count": len(self.vocabulary),
|
||||
"common_actions_count": len(self.common_actions),
|
||||
"has_clarification_templates": bool(self.clarification_templates),
|
||||
"has_summary_templates": bool(self.summary_templates),
|
||||
}
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Utilitaires
|
||||
# =========================================================================
|
||||
|
||||
|
||||
class _SafeDict(dict):
|
||||
"""dict pour str.format_map qui retourne "" pour les clés manquantes."""
|
||||
|
||||
def __missing__(self, key): # type: ignore[override]
|
||||
return ""
|
||||
|
||||
|
||||
# Hints par rôle VLM — adaptés au contexte métier
|
||||
_ROLE_HINTS = {
|
||||
"observer": (
|
||||
@@ -100,6 +494,7 @@ _ROLE_HINTS = {
|
||||
# Domaines pré-configurés
|
||||
# =========================================================================
|
||||
|
||||
|
||||
_TIM_CODAGE = DomainContext(
|
||||
domain_id="tim_codage",
|
||||
name="Codage médical TIM",
|
||||
@@ -156,8 +551,405 @@ _TIM_CODAGE = DomainContext(
|
||||
"recherche_code": "Recherche de code CIM-10 ou CCAM (champ de recherche + arborescence)",
|
||||
"validation_ghm": "Écran de validation du groupage avec GHM calculé et valorisation",
|
||||
},
|
||||
vocabulary_synonyms={
|
||||
"DP": "diagnostic principal",
|
||||
"DAS": "diagnostics associés",
|
||||
"CMA": "complication associée",
|
||||
"UM": "unité médicale",
|
||||
"CR": "compte rendu",
|
||||
"RSS": "résumé de sortie",
|
||||
"RSA": "résumé anonymisé",
|
||||
"GHM": "groupe homogène de malades",
|
||||
"IPP": "identifiant patient",
|
||||
},
|
||||
common_actions={
|
||||
"click:dp": "saisir le diagnostic principal",
|
||||
"click:diagnostic principal": "saisir le diagnostic principal",
|
||||
"click:das": "ajouter un diagnostic associé",
|
||||
"click:ccam": "saisir un acte CCAM",
|
||||
"click:valider": "valider le codage",
|
||||
"click:valider le codage": "valider le codage",
|
||||
"click:grouper": "calculer le GHM",
|
||||
"click:ghm": "consulter le groupage GHM",
|
||||
"click:dossier patient": "ouvrir le dossier patient",
|
||||
"click:fiche patient": "ouvrir la fiche patient",
|
||||
"click:compte rendu": "consulter le compte rendu",
|
||||
"click:cr": "consulter le compte rendu",
|
||||
"click:rechercher": "rechercher un code CIM-10",
|
||||
"type:cim": "saisir un code CIM-10",
|
||||
},
|
||||
clarification_templates={
|
||||
"default": (
|
||||
"Je ne trouve pas {target_friendly}. "
|
||||
"Tu peux me montrer où il se trouve dans le dossier ?"
|
||||
),
|
||||
"target_not_found": (
|
||||
"Je ne trouve pas {target_friendly}. "
|
||||
"Le dossier de {nom_patient} est peut-être déjà codé ou archivé ?"
|
||||
),
|
||||
"target:fichier patient": (
|
||||
"Je ne trouve pas le dossier de {nom_patient}. "
|
||||
"Il est peut-être archivé ? Tu peux me le montrer ?"
|
||||
),
|
||||
"target:dossier": (
|
||||
"Je ne trouve pas le dossier de {nom_patient}. "
|
||||
"Il est peut-être archivé ? Tu peux me le montrer ?"
|
||||
),
|
||||
"ambiguous_code": (
|
||||
"Le compte rendu mentionne plusieurs codes possibles. "
|
||||
"Est-ce le code CIM-10 {code_a} ou {code_b} que tu préfères ?"
|
||||
),
|
||||
"no_cr": (
|
||||
"Je ne trouve pas de compte rendu pour {nom_patient}. "
|
||||
"Tu veux que je saute ce dossier ou que je continue sans ?"
|
||||
),
|
||||
},
|
||||
summary_templates={
|
||||
"item_singular": "dossier",
|
||||
"item_plural": "dossiers",
|
||||
"success_one": (
|
||||
"J'ai codé le dossier de {nom_patient} en {elapsed_s}s. "
|
||||
"Tu peux vérifier le groupage GHM."
|
||||
),
|
||||
"success": (
|
||||
"J'ai codé {done} dossiers sur {items_count}. "
|
||||
"Tout est passé sans erreur, tu peux valider le groupage."
|
||||
),
|
||||
"partial": (
|
||||
"J'ai codé {done} dossiers sur {items_count}. "
|
||||
"{failed} sont en attente — codes CIM-10 ambigus, à valider manuellement."
|
||||
),
|
||||
"failure": (
|
||||
"Je n'ai pas pu coder les dossiers de {workflow_name}. "
|
||||
"Je te rends la main, les comptes rendus sont peut-être inaccessibles."
|
||||
),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
_COMPTABILITE = DomainContext(
|
||||
domain_id="comptabilite",
|
||||
name="Comptabilité",
|
||||
description=(
|
||||
"Comptable : saisie de factures fournisseurs et clients, lettrage, "
|
||||
"rapprochement bancaire, déclarations de TVA, bilans, immobilisations."
|
||||
),
|
||||
system_prompt=(
|
||||
"Tu es un assistant expert en comptabilité d'entreprise. "
|
||||
"L'utilisateur est un comptable qui utilise un logiciel de saisie comptable "
|
||||
"(Sage, Cegid, EBP, Quadra, Isacompta) pour saisir des factures, faire "
|
||||
"les rapprochements bancaires, préparer la TVA et les bilans.\n\n"
|
||||
"Vocabulaire du métier :\n"
|
||||
"- Facture : justificatif de vente ou d'achat (numéro, date, HT, TVA, TTC)\n"
|
||||
"- HT/TVA/TTC : montants hors taxes, taxe, toutes taxes\n"
|
||||
"- Compte comptable : numéro du plan comptable général (PCG), ex 401 (fournisseurs), 411 (clients)\n"
|
||||
"- Journal : journal de saisie (achats, ventes, banque, OD)\n"
|
||||
"- Lettrage : association d'une facture avec son paiement\n"
|
||||
"- Rapprochement : comparaison compte comptable / relevé bancaire\n"
|
||||
"- OCR / LAD : reconnaissance automatique des factures scannées\n"
|
||||
"- Écriture : ligne comptable (débit/crédit)\n"
|
||||
"- Exercice : période comptable annuelle\n"
|
||||
"- Bilan / compte de résultat : états financiers\n"
|
||||
"- CA : chiffre d'affaires\n\n"
|
||||
"Écrans courants :\n"
|
||||
"- Saisie d'écritures (numéro de compte, libellé, débit, crédit)\n"
|
||||
"- Import OCR de factures fournisseurs\n"
|
||||
"- Lettrage / rapprochement\n"
|
||||
"- Brouillard / journal\n"
|
||||
"- Balance / grand livre"
|
||||
),
|
||||
vocabulary=[
|
||||
"facture", "HT", "TVA", "TTC", "compte", "journal", "lettrage",
|
||||
"rapprochement", "OCR", "LAD", "écriture", "débit", "crédit",
|
||||
"exercice", "bilan", "compte de résultat", "CA", "immobilisation",
|
||||
"fournisseur", "client", "PCG", "plan comptable",
|
||||
],
|
||||
known_apps=[
|
||||
"Sage", "Cegid", "EBP", "Quadra", "Isacompta", "Ciel Compta",
|
||||
"Odoo", "Pennylane", "Dext", "Agicap",
|
||||
],
|
||||
screen_patterns={
|
||||
"saisie_ecriture": "Saisie d'écriture comptable (compte, libellé, débit, crédit)",
|
||||
"ocr_facture": "Import OCR : zone image + champs extraits (numéro, date, HT, TVA, TTC, fournisseur)",
|
||||
"lettrage": "Liste d'écritures à lettrer (débit vs crédit)",
|
||||
"rapprochement": "Comparaison compte banque / relevé",
|
||||
"balance": "Balance comptable (comptes agrégés avec soldes)",
|
||||
},
|
||||
vocabulary_synonyms={
|
||||
"HT": "montant hors taxes",
|
||||
"TVA": "montant de TVA",
|
||||
"TTC": "montant toutes taxes",
|
||||
"CA": "chiffre d'affaires",
|
||||
"PCG": "plan comptable général",
|
||||
"OD": "opération diverse",
|
||||
},
|
||||
common_actions={
|
||||
"click:valider": "valider l'écriture",
|
||||
"click:enregistrer": "enregistrer la saisie",
|
||||
"click:lettrer": "lettrer les écritures",
|
||||
"click:rapprocher": "rapprocher avec la banque",
|
||||
"click:ocr": "lancer la reconnaissance OCR",
|
||||
"click:facture": "ouvrir la facture",
|
||||
"click:compte": "sélectionner le compte comptable",
|
||||
"type:ht": "saisir le montant hors taxes",
|
||||
"type:tva": "saisir le montant de TVA",
|
||||
"type:ttc": "saisir le montant toutes taxes",
|
||||
},
|
||||
clarification_templates={
|
||||
"default": (
|
||||
"Je ne trouve pas {target_friendly}. "
|
||||
"C'est bien la facture {num_facture} que tu veux saisir ?"
|
||||
),
|
||||
"target_not_found": (
|
||||
"Je ne trouve pas le champ {target_friendly}. "
|
||||
"C'est bien la facture {num_facture} qui doit être saisie ?"
|
||||
),
|
||||
"target:montant": (
|
||||
"Je ne trouve pas le champ « Montant HT ». "
|
||||
"C'est bien la facture {num_facture} que tu veux saisir ?"
|
||||
),
|
||||
"target:tva": (
|
||||
"Je ne trouve pas le champ TVA. Est-ce une facture à taux {taux_tva} % ?"
|
||||
),
|
||||
"ambiguous_account": (
|
||||
"Je ne sais pas sur quel compte imputer : {compte_a} ou {compte_b} ?"
|
||||
),
|
||||
},
|
||||
summary_templates={
|
||||
"item_singular": "facture",
|
||||
"item_plural": "factures",
|
||||
"success_one": (
|
||||
"J'ai saisi la facture {num_facture} en {elapsed_s}s."
|
||||
),
|
||||
"success": (
|
||||
"J'ai saisi {done} factures sur {items_count}. "
|
||||
"Tout est en brouillard, tu peux valider."
|
||||
),
|
||||
"partial": (
|
||||
"J'ai saisi {done} factures sur {items_count}. "
|
||||
"{failed} factures sont en attente — imputation comptable à vérifier."
|
||||
),
|
||||
"failure": (
|
||||
"Je n'ai pas pu saisir les factures de {workflow_name}. "
|
||||
"L'OCR n'a peut-être pas fonctionné, je te rends la main."
|
||||
),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
_RH_PAIE = DomainContext(
|
||||
domain_id="rh_paie",
|
||||
name="Ressources humaines et paie",
|
||||
description=(
|
||||
"Gestionnaire RH / paie : fiches employés, contrats, bulletins de salaire, "
|
||||
"déclarations sociales (DSN), charges, congés, absences."
|
||||
),
|
||||
system_prompt=(
|
||||
"Tu es un assistant expert en gestion RH et paie française. "
|
||||
"L'utilisateur est un gestionnaire RH ou de paie qui utilise un logiciel "
|
||||
"(Silae, Sage Paie, Cegid, ADP, PayFit) pour éditer des bulletins de salaire, "
|
||||
"gérer les contrats, les absences, et envoyer les DSN.\n\n"
|
||||
"Vocabulaire du métier :\n"
|
||||
"- Bulletin de paie : fiche de salaire mensuelle\n"
|
||||
"- DSN : Déclaration Sociale Nominative (mensuelle, transmise à l'URSSAF)\n"
|
||||
"- Brut / Net : salaire avant et après charges\n"
|
||||
"- Charges sociales / patronales : cotisations employeur et salarié\n"
|
||||
"- CDI / CDD : types de contrats\n"
|
||||
"- Période de paie : mois concerné par le bulletin\n"
|
||||
"- SMIC : salaire minimum\n"
|
||||
"- IJSS : indemnités journalières sécurité sociale\n"
|
||||
"- Congés payés : solde de congés\n"
|
||||
"- RTT : réduction du temps de travail\n"
|
||||
"- Saisie sur salaire : retenue judiciaire\n"
|
||||
"- Solde de tout compte : dernier bulletin d'un salarié qui part\n\n"
|
||||
"Écrans courants :\n"
|
||||
"- Fiche employé (identité, contrat, poste, salaire)\n"
|
||||
"- Saisie des variables (heures, absences, primes)\n"
|
||||
"- Bulletin de paie (aperçu avant validation)\n"
|
||||
"- Déclaration DSN\n"
|
||||
"- Gestion des absences / congés"
|
||||
),
|
||||
vocabulary=[
|
||||
"bulletin", "salaire", "brut", "net", "charges sociales", "DSN",
|
||||
"CDI", "CDD", "congés", "RTT", "SMIC", "IJSS", "URSSAF",
|
||||
"employé", "salarié", "contrat", "prime", "heures supplémentaires",
|
||||
"absence", "solde de tout compte", "STC",
|
||||
],
|
||||
known_apps=[
|
||||
"Silae", "Sage Paie", "Cegid Paie", "ADP", "PayFit", "Nibelis",
|
||||
"Cegedim SRH", "Lucca", "HR Access",
|
||||
],
|
||||
screen_patterns={
|
||||
"fiche_employe": "Fiche employé avec identité, contrat, poste",
|
||||
"saisie_variables": "Saisie des variables de paie (heures, absences, primes)",
|
||||
"apercu_bulletin": "Aperçu du bulletin de paie avant validation",
|
||||
"dsn": "Écran DSN (déclaration sociale nominative)",
|
||||
"conges": "Gestion des absences et congés",
|
||||
},
|
||||
vocabulary_synonyms={
|
||||
"DSN": "déclaration sociale",
|
||||
"RTT": "réduction du temps de travail",
|
||||
"STC": "solde de tout compte",
|
||||
"IJSS": "indemnités journalières",
|
||||
"CP": "congés payés",
|
||||
},
|
||||
common_actions={
|
||||
"click:valider": "valider le bulletin",
|
||||
"click:editer": "éditer le bulletin",
|
||||
"click:bulletin": "ouvrir le bulletin de paie",
|
||||
"click:employe": "ouvrir la fiche employé",
|
||||
"click:dsn": "lancer la DSN",
|
||||
"click:conges": "gérer les congés",
|
||||
"click:absence": "saisir une absence",
|
||||
"type:heures": "saisir les heures travaillées",
|
||||
"type:prime": "saisir une prime",
|
||||
},
|
||||
clarification_templates={
|
||||
"default": (
|
||||
"Je ne trouve pas {target_friendly} pour {nom_employe}. "
|
||||
"Tu peux me confirmer la période de paie ?"
|
||||
),
|
||||
"target_not_found": (
|
||||
"Je ne trouve pas {target_friendly} dans la fiche de {nom_employe}. "
|
||||
"Le contrat est peut-être clôturé ?"
|
||||
),
|
||||
"target:employe": (
|
||||
"Je ne trouve pas {nom_employe} dans la liste. "
|
||||
"Est-il encore actif dans l'entreprise ?"
|
||||
),
|
||||
"ambiguous_period": (
|
||||
"Est-ce la période {periode_a} ou {periode_b} que tu veux traiter ?"
|
||||
),
|
||||
},
|
||||
summary_templates={
|
||||
"item_singular": "bulletin",
|
||||
"item_plural": "bulletins",
|
||||
"success_one": (
|
||||
"J'ai édité le bulletin de {nom_employe} en {elapsed_s}s."
|
||||
),
|
||||
"success": (
|
||||
"J'ai édité {done} bulletins sur {items_count}. "
|
||||
"La paie est prête pour validation."
|
||||
),
|
||||
"partial": (
|
||||
"J'ai édité {done} bulletins sur {items_count}. "
|
||||
"{failed} sont en attente — variables de paie à compléter."
|
||||
),
|
||||
"failure": (
|
||||
"Je n'ai pas pu éditer les bulletins de {workflow_name}. "
|
||||
"Il y a peut-être un blocage côté logiciel de paie."
|
||||
),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
_STOCKS_LOGISTIQUE = DomainContext(
|
||||
domain_id="stocks_logistique",
|
||||
name="Stocks et logistique",
|
||||
description=(
|
||||
"Gestionnaire de stocks / logistique : bons de commande, bons de livraison, "
|
||||
"réceptions, inventaires, mouvements de stock, expéditions."
|
||||
),
|
||||
system_prompt=(
|
||||
"Tu es un assistant expert en gestion de stocks et logistique. "
|
||||
"L'utilisateur utilise un ERP ou WMS (SAP, Dynamics, Odoo, Sage, Divalto) "
|
||||
"pour gérer les commandes, les réceptions, les expéditions et les inventaires.\n\n"
|
||||
"Vocabulaire du métier :\n"
|
||||
"- BC : Bon de Commande (achat ou vente)\n"
|
||||
"- BL : Bon de Livraison\n"
|
||||
"- BR : Bon de Réception\n"
|
||||
"- Article / Référence / SKU : produit en stock\n"
|
||||
"- Emplacement : localisation physique (allée, rayon, emplacement)\n"
|
||||
"- Mouvement de stock : entrée, sortie, transfert\n"
|
||||
"- Inventaire : comptage physique pour recaler le stock théorique\n"
|
||||
"- FIFO / LIFO : ordre de sortie des stocks\n"
|
||||
"- ERP : progiciel de gestion intégré\n"
|
||||
"- WMS : Warehouse Management System\n"
|
||||
"- Picking : préparation de commande\n"
|
||||
"- Quantité en stock / disponible / réservée\n\n"
|
||||
"Écrans courants :\n"
|
||||
"- Saisie de bon de commande / réception\n"
|
||||
"- Liste des articles (avec photo, quantité, emplacement)\n"
|
||||
"- Inventaire (comptage)\n"
|
||||
"- Mouvements de stock\n"
|
||||
"- Picking list (liste de préparation)"
|
||||
),
|
||||
vocabulary=[
|
||||
"bon de commande", "BC", "bon de livraison", "BL", "bon de réception", "BR",
|
||||
"article", "référence", "SKU", "emplacement", "stock", "inventaire",
|
||||
"mouvement", "entrée", "sortie", "picking", "FIFO", "LIFO", "ERP", "WMS",
|
||||
"fournisseur", "client", "quantité", "disponible", "réservé",
|
||||
],
|
||||
known_apps=[
|
||||
"SAP", "Dynamics", "Odoo", "Sage X3", "Divalto", "Cegid",
|
||||
"Oracle NetSuite", "Reflex WMS", "Infolog",
|
||||
],
|
||||
screen_patterns={
|
||||
"bon_commande": "Saisie de bon de commande (fournisseur, lignes d'articles, quantités)",
|
||||
"reception": "Bon de réception (rapprochement avec la commande)",
|
||||
"inventaire": "Saisie d'inventaire (article, emplacement, quantité comptée)",
|
||||
"picking": "Liste de préparation avec articles et emplacements",
|
||||
"mouvement": "Mouvement de stock (entrée/sortie/transfert)",
|
||||
},
|
||||
vocabulary_synonyms={
|
||||
"BC": "bon de commande",
|
||||
"BL": "bon de livraison",
|
||||
"BR": "bon de réception",
|
||||
"SKU": "référence produit",
|
||||
"WMS": "gestion d'entrepôt",
|
||||
"ERP": "progiciel de gestion",
|
||||
},
|
||||
common_actions={
|
||||
"click:valider": "valider le bon",
|
||||
"click:commande": "ouvrir le bon de commande",
|
||||
"click:livraison": "ouvrir le bon de livraison",
|
||||
"click:reception": "saisir la réception",
|
||||
"click:inventaire": "démarrer l'inventaire",
|
||||
"click:article": "sélectionner un article",
|
||||
"click:picking": "démarrer la préparation",
|
||||
"type:quantite": "saisir la quantité",
|
||||
"type:reference": "saisir la référence article",
|
||||
},
|
||||
clarification_templates={
|
||||
"default": (
|
||||
"Je ne trouve pas {target_friendly}. "
|
||||
"C'est bien la commande {num_bc} qu'on traite ?"
|
||||
),
|
||||
"target_not_found": (
|
||||
"Je ne trouve pas {target_friendly}. "
|
||||
"La commande {num_bc} est peut-être déjà clôturée ?"
|
||||
),
|
||||
"target:article": (
|
||||
"Je ne trouve pas l'article {ref_article}. "
|
||||
"Il est peut-être archivé ou mal référencé ?"
|
||||
),
|
||||
"quantity_mismatch": (
|
||||
"La quantité reçue ({qte_recue}) ne correspond pas à la commande "
|
||||
"({qte_commandee}). Je saisis un écart ou tu vérifies ?"
|
||||
),
|
||||
},
|
||||
summary_templates={
|
||||
"item_singular": "bon",
|
||||
"item_plural": "bons",
|
||||
"success_one": (
|
||||
"J'ai traité le bon {num_bc} en {elapsed_s}s."
|
||||
),
|
||||
"success": (
|
||||
"J'ai traité {done} bons sur {items_count}. "
|
||||
"Les mouvements de stock sont validés."
|
||||
),
|
||||
"partial": (
|
||||
"J'ai traité {done} bons sur {items_count}. "
|
||||
"{failed} bons sont en attente — écarts de quantité à vérifier."
|
||||
),
|
||||
"failure": (
|
||||
"Je n'ai pas pu traiter les bons de {workflow_name}. "
|
||||
"L'ERP a peut-être refusé une ligne, je te rends la main."
|
||||
),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
_GENERIC = DomainContext(
|
||||
domain_id="generic",
|
||||
name="Bureautique générale",
|
||||
@@ -166,11 +958,37 @@ _GENERIC = DomainContext(
|
||||
"Tu es un assistant RPA qui observe des applications bureautiques. "
|
||||
"Décris précisément ce que tu vois à l'écran."
|
||||
),
|
||||
summary_templates={
|
||||
"item_singular": "action",
|
||||
"item_plural": "actions",
|
||||
"success_one": "C'est fait, j'ai terminé « {workflow_name} » en {elapsed_s}s.",
|
||||
"success": (
|
||||
"J'ai terminé « {workflow_name} » : {done} {item_word} exécutées "
|
||||
"sur {items_count}."
|
||||
),
|
||||
"partial": (
|
||||
"J'ai terminé « {workflow_name} » partiellement : "
|
||||
"{done} {item_word} sur {items_count} ({failed} en échec)."
|
||||
),
|
||||
"failure": (
|
||||
"Je n'ai pas pu terminer « {workflow_name} ». Je te rends la main."
|
||||
),
|
||||
},
|
||||
clarification_templates={
|
||||
"default": (
|
||||
"Je ne trouve pas {target_friendly} à l'écran. "
|
||||
"Tu peux me le montrer ?"
|
||||
),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
# Registre des domaines disponibles
|
||||
_DOMAINS: Dict[str, DomainContext] = {
|
||||
"tim_codage": _TIM_CODAGE,
|
||||
"comptabilite": _COMPTABILITE,
|
||||
"rh_paie": _RH_PAIE,
|
||||
"stocks_logistique": _STOCKS_LOGISTIQUE,
|
||||
"generic": _GENERIC,
|
||||
}
|
||||
|
||||
@@ -179,7 +997,8 @@ def get_domain_context(domain_id: str = "generic") -> DomainContext:
|
||||
"""Récupérer le contexte métier par ID.
|
||||
|
||||
Args:
|
||||
domain_id: Identifiant du domaine (tim_codage, generic, etc.)
|
||||
domain_id: Identifiant du domaine (tim_codage, comptabilite, rh_paie,
|
||||
stocks_logistique, generic, etc.)
|
||||
|
||||
Returns:
|
||||
DomainContext correspondant, ou generic si non trouvé.
|
||||
|
||||
373
agent_v0/server_v1/execution_plan_runner.py
Normal file
373
agent_v0/server_v1/execution_plan_runner.py
Normal file
@@ -0,0 +1,373 @@
|
||||
# agent_v0/server_v1/execution_plan_runner.py
|
||||
"""
|
||||
ExecutionPlanRunner — Adaptateur ExecutionPlan → actions replay.
|
||||
|
||||
Pièce d'intégration du pipeline V4 :
|
||||
RawTrace → IRBuilder → WorkflowIR → ExecutionCompiler → ExecutionPlan → Runtime
|
||||
|
||||
Ce module convertit un `ExecutionPlan` (plan pré-compilé, déterministe) en
|
||||
liste d'actions au format attendu par l'executor replay actuel (clé x_pct,
|
||||
y_pct, target_spec, etc.), puis les injecte dans `_replay_queues`.
|
||||
|
||||
L'ancien chemin `build_replay_from_raw_events()` dans stream_processor.py
|
||||
reste inchangé — les deux chemins coexistent pendant la transition.
|
||||
|
||||
Format d'action produit (compatible executor existant) :
|
||||
{
|
||||
"action_id": "act_...",
|
||||
"type": "click",
|
||||
"x_pct": 0.5,
|
||||
"y_pct": 0.3,
|
||||
"visual_mode": True,
|
||||
"target_spec": {
|
||||
"by_text": "...",
|
||||
"window_title": "...",
|
||||
"vlm_description": "...",
|
||||
"anchor_image_base64": "...",
|
||||
},
|
||||
"expected_window_title": "...",
|
||||
}
|
||||
|
||||
Auteur: Dom, Alice - Avril 2026
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
import threading
|
||||
import uuid
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from core.workflow.execution_plan import (
|
||||
ExecutionNode,
|
||||
ExecutionPlan,
|
||||
ResolutionStrategy,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Substitution de variables
|
||||
# =========================================================================
|
||||
# Le WorkflowIR utilise la syntaxe `{var}` dans les champs texte.
|
||||
# Ici on supporte les deux : `{var}` (IR natif) et `${var}` (replay legacy).
|
||||
_VARIABLE_RE_CURLY = re.compile(r"\{(\w+)\}")
|
||||
_VARIABLE_RE_DOLLAR = re.compile(r"\$\{(\w+)\}")
|
||||
|
||||
|
||||
def substitute_variables(text: str, variables: Dict[str, Any]) -> str:
|
||||
"""Remplacer `{var}` et `${var}` par leurs valeurs.
|
||||
|
||||
Priorité : variables fournies > placeholder brut (inchangé si inconnu).
|
||||
"""
|
||||
if not text or not variables:
|
||||
return text
|
||||
|
||||
def replacer(match: "re.Match[str]") -> str:
|
||||
var_name = match.group(1)
|
||||
if var_name in variables:
|
||||
return str(variables[var_name])
|
||||
return match.group(0)
|
||||
|
||||
text = _VARIABLE_RE_DOLLAR.sub(replacer, text)
|
||||
text = _VARIABLE_RE_CURLY.sub(replacer, text)
|
||||
return text
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Conversion ExecutionNode → action replay
|
||||
# =========================================================================
|
||||
|
||||
|
||||
def _strategy_to_target_spec(
|
||||
strategy: Optional[ResolutionStrategy],
|
||||
fallbacks: Optional[List[ResolutionStrategy]] = None,
|
||||
intent: str = "",
|
||||
) -> Dict[str, Any]:
|
||||
"""Construire un `target_spec` depuis les stratégies de résolution.
|
||||
|
||||
Fusionne la primaire et les fallbacks pour donner un maximum d'indices
|
||||
au resolve_engine :
|
||||
- OCR → by_text
|
||||
- template → anchor_image_base64 (depuis anchor_b64)
|
||||
- VLM → vlm_description
|
||||
|
||||
Règle V4 : la stratégie primaire dicte la méthode préférée.
|
||||
Le champ `resolve_order` liste les méthodes dans l'ordre à essayer.
|
||||
Le resolve_engine honore cet ordre au lieu de sa cascade par défaut.
|
||||
|
||||
resolve_order est la clé du "zéro VLM au runtime" :
|
||||
- ["ocr", "template", "vlm"] → V4 typique (OCR rapide)
|
||||
- ["template", "ocr", "vlm"] → apprentissage : template marche mieux
|
||||
- ["vlm"] → éléments sans texte (icônes)
|
||||
"""
|
||||
spec: Dict[str, Any] = {}
|
||||
|
||||
all_strategies: List[ResolutionStrategy] = []
|
||||
if strategy is not None:
|
||||
all_strategies.append(strategy)
|
||||
if fallbacks:
|
||||
all_strategies.extend(fallbacks)
|
||||
|
||||
by_text_candidate = ""
|
||||
anchor_candidate = ""
|
||||
vlm_candidate = ""
|
||||
uia_data: Dict[str, Any] = {}
|
||||
dom_data: Dict[str, Any] = {}
|
||||
resolve_order: List[str] = []
|
||||
seen_methods: set = set()
|
||||
|
||||
for strat in all_strategies:
|
||||
if not strat:
|
||||
continue
|
||||
if strat.method == "ocr" and strat.target_text and not by_text_candidate:
|
||||
by_text_candidate = strat.target_text
|
||||
elif strat.method == "template":
|
||||
if strat.anchor_b64 and not anchor_candidate:
|
||||
anchor_candidate = strat.anchor_b64
|
||||
if strat.target_text and not by_text_candidate:
|
||||
by_text_candidate = strat.target_text
|
||||
elif strat.method == "vlm" and strat.vlm_description and not vlm_candidate:
|
||||
vlm_candidate = strat.vlm_description
|
||||
elif strat.method == "uia" and strat.uia_name and not uia_data:
|
||||
uia_data = {
|
||||
"name": strat.uia_name,
|
||||
"control_type": strat.uia_control_type,
|
||||
"automation_id": strat.uia_automation_id,
|
||||
"parent_path": strat.uia_parent_path,
|
||||
}
|
||||
elif strat.method == "dom" and strat.dom_selector and not dom_data:
|
||||
dom_data = {
|
||||
"selector": strat.dom_selector,
|
||||
"xpath": strat.dom_xpath,
|
||||
"url_pattern": strat.dom_url_pattern,
|
||||
}
|
||||
|
||||
# Construire l'ordre des méthodes (dans l'ordre primaire → fallbacks)
|
||||
if strat.method and strat.method not in seen_methods:
|
||||
resolve_order.append(strat.method)
|
||||
seen_methods.add(strat.method)
|
||||
|
||||
if by_text_candidate:
|
||||
spec["by_text"] = by_text_candidate
|
||||
if anchor_candidate:
|
||||
spec["anchor_image_base64"] = anchor_candidate
|
||||
if vlm_candidate:
|
||||
spec["vlm_description"] = vlm_candidate
|
||||
elif intent and "vlm_description" not in spec:
|
||||
# L'intention métier devient le prompt VLM de dernier recours
|
||||
spec["vlm_description"] = intent
|
||||
|
||||
# Données UIA — consommées par l'agent Windows via lea_uia.exe
|
||||
if uia_data:
|
||||
spec["uia_target"] = uia_data
|
||||
|
||||
# Données DOM — consommées par l'agent Windows via CDP (futur)
|
||||
if dom_data:
|
||||
spec["dom_target"] = dom_data
|
||||
|
||||
# Ordre de résolution pré-compilé — c'est LA pièce centrale du V4
|
||||
if resolve_order:
|
||||
spec["resolve_order"] = resolve_order
|
||||
|
||||
return spec
|
||||
|
||||
|
||||
def execution_node_to_action(
|
||||
node: ExecutionNode,
|
||||
variables: Optional[Dict[str, Any]] = None,
|
||||
id_prefix: str = "act_plan",
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Convertir un `ExecutionNode` en action replay.
|
||||
|
||||
Retourne `None` si le nœud n'est pas exécutable (type inconnu).
|
||||
|
||||
Args:
|
||||
node: Le nœud à convertir.
|
||||
variables: Dictionnaire de variables pour substituer les {var}.
|
||||
id_prefix: Préfixe pour l'action_id générée.
|
||||
"""
|
||||
variables = variables or {}
|
||||
|
||||
action: Dict[str, Any] = {
|
||||
"action_id": f"{id_prefix}_{uuid.uuid4().hex[:8]}",
|
||||
"plan_node_id": node.node_id,
|
||||
}
|
||||
|
||||
if node.intent:
|
||||
action["intention"] = node.intent
|
||||
if node.step_id:
|
||||
action["plan_step_id"] = node.step_id
|
||||
if node.is_optional:
|
||||
action["is_optional"] = True
|
||||
|
||||
# Métadonnées d'exécution utiles au runtime
|
||||
if node.timeout_ms:
|
||||
action["timeout_ms"] = node.timeout_ms
|
||||
if node.max_retries:
|
||||
action["max_retries"] = node.max_retries
|
||||
if node.recovery_action:
|
||||
action["recovery_action"] = node.recovery_action
|
||||
if node.success_condition:
|
||||
action["success_condition"] = node.success_condition.to_dict()
|
||||
|
||||
action_type = node.action_type
|
||||
|
||||
if action_type == "click":
|
||||
action["type"] = "click"
|
||||
|
||||
strategy = node.strategy_primary
|
||||
fallbacks = node.strategy_fallbacks or []
|
||||
|
||||
# ── Déduction des coordonnées depuis la stratégie primaire ──
|
||||
# - OCR : pas de coordonnées (le runtime trouve via OCR)
|
||||
# - template : l'anchor sera utilisé au runtime
|
||||
# - VLM : la description sera utilisée au runtime
|
||||
# Dans tous les cas le resolve_engine retrouve les pixels au replay.
|
||||
# On expose néanmoins un centre (0.5, 0.5) neutre pour rester
|
||||
# compatible avec les validations de queue existantes.
|
||||
action["x_pct"] = 0.5
|
||||
action["y_pct"] = 0.5
|
||||
action["visual_mode"] = True
|
||||
|
||||
target_spec = _strategy_to_target_spec(
|
||||
strategy=strategy,
|
||||
fallbacks=fallbacks,
|
||||
intent=node.intent,
|
||||
)
|
||||
|
||||
# Titre fenêtre attendu AVANT (pré-vérif stricte)
|
||||
# Si absent, aucune pré-vérif → l'action s'exécute quel que soit l'écran
|
||||
if node.expected_window_before:
|
||||
action["expected_window_before"] = node.expected_window_before
|
||||
target_spec["window_title"] = node.expected_window_before
|
||||
|
||||
# Titre fenêtre attendu APRÈS (post-vérif stricte)
|
||||
# C'est la garantie de passage à l'action suivante
|
||||
if node.success_condition and node.success_condition.expected_title:
|
||||
action["expected_window_title"] = node.success_condition.expected_title
|
||||
action["success_strict"] = (
|
||||
node.success_condition.method == "title_match"
|
||||
)
|
||||
if "window_title" not in target_spec:
|
||||
target_spec["window_title"] = node.success_condition.expected_title
|
||||
|
||||
if target_spec:
|
||||
action["target_spec"] = target_spec
|
||||
|
||||
elif action_type == "type":
|
||||
action["type"] = "type"
|
||||
text = node.text or ""
|
||||
# Substituer les variables avant d'envoyer (ex: {patient} → "DUPONT")
|
||||
action["text"] = substitute_variables(text, variables)
|
||||
if node.variable_name:
|
||||
action["variable_name"] = node.variable_name
|
||||
|
||||
elif action_type in ("key_combo", "key_press"):
|
||||
action["type"] = "key_combo"
|
||||
keys = list(node.keys or [])
|
||||
if not keys:
|
||||
return None
|
||||
action["keys"] = keys
|
||||
|
||||
elif action_type == "wait":
|
||||
action["type"] = "wait"
|
||||
duration = node.duration_ms or 1000
|
||||
action["duration_ms"] = int(duration)
|
||||
|
||||
elif action_type == "scroll":
|
||||
action["type"] = "scroll"
|
||||
# Les stratégies peuvent contenir une zone — pas exploitée ici,
|
||||
# le scroll est implicitement sur la fenêtre active.
|
||||
action["delta"] = -3
|
||||
|
||||
else:
|
||||
logger.debug("execution_node_to_action: type inconnu '%s' ignoré", action_type)
|
||||
return None
|
||||
|
||||
return action
|
||||
|
||||
|
||||
def execution_plan_to_actions(
|
||||
plan: ExecutionPlan,
|
||||
variables: Optional[Dict[str, Any]] = None,
|
||||
id_prefix: str = "act_plan",
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Convertir un `ExecutionPlan` complet en liste d'actions replay.
|
||||
|
||||
Les variables passées en argument écrasent celles du plan.
|
||||
"""
|
||||
merged_vars: Dict[str, Any] = dict(plan.variables or {})
|
||||
if variables:
|
||||
merged_vars.update(variables)
|
||||
|
||||
actions: List[Dict[str, Any]] = []
|
||||
for node in plan.nodes:
|
||||
action = execution_node_to_action(
|
||||
node=node,
|
||||
variables=merged_vars,
|
||||
id_prefix=id_prefix,
|
||||
)
|
||||
if action is not None:
|
||||
actions.append(action)
|
||||
|
||||
logger.info(
|
||||
"execution_plan_to_actions(%s) : %d nœuds → %d actions replay "
|
||||
"(vars=%d)",
|
||||
plan.plan_id, plan.total_nodes, len(actions), len(merged_vars),
|
||||
)
|
||||
return actions
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Injection dans la queue de replay
|
||||
# =========================================================================
|
||||
|
||||
|
||||
def inject_plan_into_queue(
|
||||
plan: ExecutionPlan,
|
||||
session_id: str,
|
||||
replay_queues: Dict[str, List[Dict[str, Any]]],
|
||||
variables: Optional[Dict[str, Any]] = None,
|
||||
lock: Optional[threading.Lock] = None,
|
||||
replace: bool = True,
|
||||
id_prefix: str = "act_plan",
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Injecter un `ExecutionPlan` dans la queue de replay d'une session.
|
||||
|
||||
Args:
|
||||
plan: Le plan à exécuter.
|
||||
session_id: La session Agent V1 cible.
|
||||
replay_queues: Le dict global `_replay_queues` partagé par le serveur.
|
||||
variables: Variables à substituer dans les actions.
|
||||
lock: Verrou optionnel à acquérir avant d'écrire (threadsafe).
|
||||
replace: Si True (défaut), remplace la queue existante. Sinon, append.
|
||||
id_prefix: Préfixe pour les action_id générés.
|
||||
|
||||
Returns:
|
||||
La liste des actions injectées (après substitution).
|
||||
"""
|
||||
actions = execution_plan_to_actions(
|
||||
plan=plan, variables=variables, id_prefix=id_prefix,
|
||||
)
|
||||
|
||||
def _write() -> None:
|
||||
if replace:
|
||||
replay_queues[session_id] = list(actions)
|
||||
else:
|
||||
replay_queues[session_id].extend(actions)
|
||||
|
||||
if lock is not None:
|
||||
with lock:
|
||||
_write()
|
||||
else:
|
||||
_write()
|
||||
|
||||
logger.info(
|
||||
"inject_plan_into_queue(%s) : %d actions injectées dans la queue "
|
||||
"de la session '%s' (replace=%s)",
|
||||
plan.plan_id, len(actions), session_id, replace,
|
||||
)
|
||||
return actions
|
||||
1284
agent_v0/server_v1/replay_engine.py
Normal file
1284
agent_v0/server_v1/replay_engine.py
Normal file
File diff suppressed because it is too large
Load Diff
2183
agent_v0/server_v1/resolve_engine.py
Normal file
2183
agent_v0/server_v1/resolve_engine.py
Normal file
File diff suppressed because it is too large
Load Diff
369
core/workflow/execution_compiler.py
Normal file
369
core/workflow/execution_compiler.py
Normal file
@@ -0,0 +1,369 @@
|
||||
# core/workflow/execution_compiler.py
|
||||
"""
|
||||
ExecutionCompiler — Compile un WorkflowIR en ExecutionPlan.
|
||||
|
||||
Pièce maîtresse de l'architecture V4.
|
||||
"Le LLM prépare et compile. Le runtime exécute."
|
||||
|
||||
Le compilateur :
|
||||
1. Prend chaque étape du WorkflowIR
|
||||
2. Compile une stratégie de résolution pour chaque action (OCR > template > VLM)
|
||||
3. Définit les timeouts, retries, fallbacks et recovery
|
||||
4. Produit un ExecutionPlan déterministe et borné
|
||||
|
||||
L'objectif : zéro VLM au runtime pour les cas normaux.
|
||||
Le VLM est un exception handler, pas le chemin principal.
|
||||
|
||||
Le compilateur utilise :
|
||||
- Les données de l'enregistrement (crops, textes OCR) pour pré-compiler
|
||||
- L'historique d'apprentissage (ReplayLearner) pour choisir la meilleure stratégie
|
||||
- Le contexte métier (DomainContext) pour adapter les paramètres
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
import uuid
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from .workflow_ir import WorkflowIR, Step, Action
|
||||
from .execution_plan import (
|
||||
ExecutionPlan, ExecutionNode, ResolutionStrategy, SuccessCondition,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Temps estimé par type d'action (ms)
|
||||
_ACTION_TIME_ESTIMATES = {
|
||||
"click": 200, # OCR lookup + clic
|
||||
"type": 500, # Frappe char-by-char
|
||||
"key_combo": 100,
|
||||
"wait": 0, # Le duration_ms est dans l'action
|
||||
"scroll": 200,
|
||||
}
|
||||
|
||||
|
||||
class ExecutionCompiler:
|
||||
"""Compile un WorkflowIR en ExecutionPlan.
|
||||
|
||||
Usage :
|
||||
compiler = ExecutionCompiler()
|
||||
plan = compiler.compile(workflow_ir, target_machine="VM_Win11")
|
||||
plan.save("data/plans/")
|
||||
"""
|
||||
|
||||
def __init__(self, learning_dir: str = ""):
|
||||
self._learning_dir = learning_dir or "data/learning/replay_results"
|
||||
|
||||
def compile(
|
||||
self,
|
||||
ir: WorkflowIR,
|
||||
target_machine: str = "",
|
||||
target_resolution: str = "1280x800",
|
||||
params: Optional[Dict[str, str]] = None,
|
||||
surface_profile=None,
|
||||
) -> ExecutionPlan:
|
||||
"""Compiler un WorkflowIR en ExecutionPlan.
|
||||
|
||||
Args:
|
||||
ir: Le WorkflowIR à compiler
|
||||
target_machine: Machine cible (pour adapter les stratégies)
|
||||
target_resolution: Résolution de la machine cible
|
||||
params: Variables à substituer
|
||||
surface_profile: SurfaceProfile optionnel pour adapter les paramètres.
|
||||
Si fourni, timeouts/seuils/retries sont tirés du profil.
|
||||
"""
|
||||
t_start = time.time()
|
||||
|
||||
plan = ExecutionPlan(
|
||||
plan_id=f"plan_{uuid.uuid4().hex[:8]}",
|
||||
workflow_id=ir.workflow_id,
|
||||
version=ir.version,
|
||||
created_at=time.time(),
|
||||
domain=ir.domain,
|
||||
target_machine=target_machine,
|
||||
target_resolution=target_resolution,
|
||||
variables=params or {v.name: v.default for v in ir.variables},
|
||||
)
|
||||
|
||||
# Consulter l'historique d'apprentissage
|
||||
learned_strategies = self._load_learned_strategies()
|
||||
|
||||
# Compiler chaque étape
|
||||
for step in ir.steps:
|
||||
nodes = self._compile_step(step, ir, learned_strategies, surface_profile)
|
||||
plan.nodes.extend(nodes)
|
||||
|
||||
# Statistiques de compilation
|
||||
plan.total_nodes = len(plan.nodes)
|
||||
plan.nodes_with_ocr = sum(
|
||||
1 for n in plan.nodes
|
||||
if n.strategy_primary and n.strategy_primary.method == "ocr"
|
||||
)
|
||||
plan.nodes_with_template = sum(
|
||||
1 for n in plan.nodes
|
||||
if n.strategy_primary and n.strategy_primary.method == "template"
|
||||
)
|
||||
plan.nodes_with_vlm = sum(
|
||||
1 for n in plan.nodes
|
||||
if n.strategy_primary and n.strategy_primary.method == "vlm"
|
||||
)
|
||||
plan.estimated_duration_s = sum(
|
||||
_ACTION_TIME_ESTIMATES.get(n.action_type, 200) + n.duration_ms
|
||||
for n in plan.nodes
|
||||
) / 1000.0
|
||||
|
||||
elapsed = time.time() - t_start
|
||||
logger.info(
|
||||
f"Compilation: {plan.total_nodes} nœuds en {elapsed:.1f}s — "
|
||||
f"OCR={plan.nodes_with_ocr}, template={plan.nodes_with_template}, "
|
||||
f"VLM={plan.nodes_with_vlm} (exception handler)"
|
||||
)
|
||||
|
||||
return plan
|
||||
|
||||
def _compile_step(
|
||||
self,
|
||||
step: Step,
|
||||
ir: WorkflowIR,
|
||||
learned: Dict[str, str],
|
||||
surface_profile=None,
|
||||
) -> List[ExecutionNode]:
|
||||
"""Compiler une étape en nœuds d'exécution."""
|
||||
nodes = []
|
||||
|
||||
for i, action in enumerate(step.actions):
|
||||
node = self._compile_action(
|
||||
action=action,
|
||||
step=step,
|
||||
action_index=i,
|
||||
ir=ir,
|
||||
learned=learned,
|
||||
surface_profile=surface_profile,
|
||||
)
|
||||
nodes.append(node)
|
||||
|
||||
return nodes
|
||||
|
||||
def _compile_action(
|
||||
self,
|
||||
action: Action,
|
||||
step: Step,
|
||||
action_index: int,
|
||||
ir: WorkflowIR,
|
||||
learned: Dict[str, str],
|
||||
surface_profile=None,
|
||||
) -> ExecutionNode:
|
||||
"""Compiler une action en nœud d'exécution avec stratégie de résolution."""
|
||||
|
||||
node = ExecutionNode(
|
||||
node_id=f"n_{step.step_id}_{action_index}",
|
||||
action_type=action.type,
|
||||
intent=step.intent,
|
||||
step_id=step.step_id,
|
||||
is_optional=step.is_optional,
|
||||
)
|
||||
|
||||
# Paramètres par défaut, surchargés par le surface_profile si fourni
|
||||
default_click_timeout = 10000
|
||||
default_click_retries = 2
|
||||
if surface_profile is not None:
|
||||
default_click_timeout = getattr(surface_profile, "timeout_click_ms", 10000)
|
||||
default_click_retries = getattr(surface_profile, "max_retries", 2)
|
||||
|
||||
if action.type == "click":
|
||||
# Compiler les stratégies de résolution pour ce clic
|
||||
node.strategy_primary, node.strategy_fallbacks = self._compile_click_resolution(
|
||||
action, step, learned, surface_profile,
|
||||
)
|
||||
node.timeout_ms = default_click_timeout
|
||||
node.max_retries = default_click_retries
|
||||
node.recovery_action = "escape"
|
||||
|
||||
# Condition de succès STRICTE basée sur le titre de fenêtre attendu.
|
||||
# Si expected_window_after est défini, on fait du title_match (strict).
|
||||
# Sinon on retombe sur screen_changed (faible).
|
||||
expected_after = getattr(action, "expected_window_after", "")
|
||||
if expected_after and expected_after != "unknown_window":
|
||||
node.success_condition = SuccessCondition(
|
||||
method="title_match",
|
||||
expected_title=expected_after,
|
||||
description=step.postcondition or f"Fenêtre attendue: {expected_after}",
|
||||
)
|
||||
elif step.postcondition:
|
||||
node.success_condition = SuccessCondition(
|
||||
method="screen_changed",
|
||||
description=step.postcondition,
|
||||
)
|
||||
|
||||
# Pré-condition stricte : la fenêtre AVANT le clic doit matcher
|
||||
# Stockée en tant que champ dédié sur le nœud pour l'exécuteur
|
||||
expected_before = getattr(action, "expected_window_before", "")
|
||||
if expected_before and expected_before != "unknown_window":
|
||||
# On l'injecte dans la condition de succès (cas "avant")
|
||||
# Le nœud portera les deux via des champs séparés
|
||||
node.expected_window_before = expected_before
|
||||
|
||||
elif action.type == "type":
|
||||
node.text = action.text
|
||||
node.variable_name = action.text.strip("{}") if action.variable else ""
|
||||
node.timeout_ms = 5000
|
||||
node.max_retries = 0 # Pas de retry sur la frappe
|
||||
node.recovery_action = "undo"
|
||||
|
||||
elif action.type == "key_combo":
|
||||
node.keys = action.keys
|
||||
node.timeout_ms = 3000
|
||||
node.max_retries = 0
|
||||
node.recovery_action = "undo"
|
||||
|
||||
elif action.type == "wait":
|
||||
node.duration_ms = action.duration_ms or 1000
|
||||
node.timeout_ms = action.duration_ms + 2000
|
||||
node.max_retries = 0
|
||||
node.recovery_action = "none"
|
||||
|
||||
elif action.type == "scroll":
|
||||
node.timeout_ms = 3000
|
||||
node.max_retries = 0
|
||||
node.recovery_action = "none"
|
||||
|
||||
return node
|
||||
|
||||
def _compile_click_resolution(
|
||||
self,
|
||||
action: Action,
|
||||
step: Step,
|
||||
learned: Dict[str, str],
|
||||
surface_profile=None,
|
||||
) -> tuple:
|
||||
"""Compiler les stratégies de résolution pour un clic.
|
||||
|
||||
Utilise les données d'enrichissement visuel (action._enrichment) si
|
||||
disponibles :
|
||||
- by_text (OCR)
|
||||
- anchor_image_base64 (template)
|
||||
- vlm_description (VLM)
|
||||
- uia_snapshot (UIA sur Windows natif)
|
||||
|
||||
Ordre de priorité (variable selon la surface) :
|
||||
1. UIA (si snapshot dispo ET surface native ET helper dispo) — 10-20ms
|
||||
2. OCR exact (si texte visible) — 100-200ms
|
||||
3. Template matching (si crop) — 10ms
|
||||
4. VLM — exception handler
|
||||
|
||||
Le learning peut réordonner si une stratégie a mieux marché avant.
|
||||
"""
|
||||
primary = None
|
||||
fallbacks = []
|
||||
|
||||
# Lire l'enrichissement visuel si dispo
|
||||
enrichment = getattr(action, "_enrichment", None) or {}
|
||||
by_text_from_enrich = enrichment.get("by_text", "")
|
||||
anchor_b64 = enrichment.get("anchor_image_base64", "")
|
||||
vlm_desc_from_enrich = enrichment.get("vlm_description", "")
|
||||
window_title = enrichment.get("window_title", "")
|
||||
uia_snapshot = enrichment.get("uia_snapshot") or {}
|
||||
|
||||
# Source de texte : enrichissement > anchor_hint > target
|
||||
target_text = by_text_from_enrich or action.anchor_hint or action.target
|
||||
# Ne pas utiliser "unknown_window" comme texte OCR
|
||||
if target_text == "unknown_window":
|
||||
target_text = ""
|
||||
|
||||
learned_method = learned.get(target_text, "")
|
||||
|
||||
# Est-ce qu'on est sur une surface où UIA est activable ?
|
||||
uia_eligible = False
|
||||
if surface_profile is not None:
|
||||
from .surface_classifier import SurfaceType
|
||||
surface_type = getattr(surface_profile, "surface_type", None)
|
||||
uia_available = getattr(surface_profile, "uia_available", False)
|
||||
uia_eligible = (
|
||||
uia_available
|
||||
and surface_type == SurfaceType.WINDOWS_NATIVE
|
||||
)
|
||||
else:
|
||||
# Sans profil explicite, on active UIA si le snapshot est présent
|
||||
# (l'agent décidera au runtime s'il peut l'utiliser)
|
||||
uia_eligible = bool(uia_snapshot)
|
||||
|
||||
# Stratégie UIA — la plus rapide et la plus précise sur Windows natif
|
||||
if uia_snapshot and uia_snapshot.get("name") and uia_eligible:
|
||||
uia_strategy = ResolutionStrategy(
|
||||
method="uia",
|
||||
uia_name=uia_snapshot.get("name", ""),
|
||||
uia_control_type=uia_snapshot.get("control_type", ""),
|
||||
uia_automation_id=uia_snapshot.get("automation_id", ""),
|
||||
uia_parent_path=uia_snapshot.get("parent_path", []),
|
||||
threshold=0.95,
|
||||
)
|
||||
primary = uia_strategy
|
||||
|
||||
# Stratégie OCR — le texte visible est la meilleure ancre
|
||||
if target_text:
|
||||
ocr_strategy = ResolutionStrategy(
|
||||
method="ocr",
|
||||
target_text=target_text,
|
||||
threshold=0.7,
|
||||
)
|
||||
if primary is None and (
|
||||
not learned_method
|
||||
or learned_method in ("ocr", "som_text_match", "hybrid_text_direct", "v4_ocr")
|
||||
):
|
||||
primary = ocr_strategy
|
||||
else:
|
||||
fallbacks.append(ocr_strategy)
|
||||
|
||||
# Stratégie template — le crop visuel de l'enregistrement
|
||||
if anchor_b64:
|
||||
template_strategy = ResolutionStrategy(
|
||||
method="template",
|
||||
target_text=target_text,
|
||||
anchor_b64=anchor_b64,
|
||||
threshold=0.85,
|
||||
)
|
||||
if primary is None and learned_method in (
|
||||
"anchor_template", "template_matching", "v4_template"
|
||||
):
|
||||
primary = template_strategy
|
||||
else:
|
||||
fallbacks.append(template_strategy)
|
||||
|
||||
# Stratégie VLM — exception handler (dernier recours)
|
||||
vlm_description = vlm_desc_from_enrich or action.target or step.intent
|
||||
if vlm_description and vlm_description != "unknown_window":
|
||||
vlm_strategy = ResolutionStrategy(
|
||||
method="vlm",
|
||||
vlm_description=vlm_description,
|
||||
threshold=0.6,
|
||||
)
|
||||
fallbacks.append(vlm_strategy)
|
||||
|
||||
# Si aucune primaire trouvée, prendre le premier fallback
|
||||
if primary is None:
|
||||
if fallbacks:
|
||||
primary = fallbacks.pop(0)
|
||||
else:
|
||||
# Dernier recours : VLM avec l'intention métier
|
||||
primary = ResolutionStrategy(
|
||||
method="vlm",
|
||||
vlm_description=step.intent or "élément UI",
|
||||
threshold=0.5,
|
||||
)
|
||||
|
||||
return primary, fallbacks
|
||||
|
||||
def _load_learned_strategies(self) -> Dict[str, str]:
|
||||
"""Charger les stratégies apprises (ReplayLearner)."""
|
||||
try:
|
||||
from agent_v0.server_v1.replay_learner import ReplayLearner
|
||||
learner = ReplayLearner(learning_dir=self._learning_dir)
|
||||
# Construire un mapping target → best_method depuis l'historique
|
||||
strategies = {}
|
||||
for outcome in learner._recent:
|
||||
if outcome.success and outcome.resolution_method and outcome.target_description:
|
||||
strategies[outcome.target_description] = outcome.resolution_method
|
||||
return strategies
|
||||
except Exception:
|
||||
return {}
|
||||
285
core/workflow/execution_plan.py
Normal file
285
core/workflow/execution_plan.py
Normal file
@@ -0,0 +1,285 @@
|
||||
# core/workflow/execution_plan.py
|
||||
"""
|
||||
ExecutionPlan — Plan d'exécution strict, borné et versionné.
|
||||
|
||||
C'est ce que le runtime exécute. Pas d'improvisation — tout est pré-compilé :
|
||||
- chaque nœud a une stratégie de résolution primaire + fallbacks
|
||||
- chaque nœud a un timeout, un retry policy, une condition de succès
|
||||
- le VLM n'intervient qu'en exception handler (pas en chemin principal)
|
||||
|
||||
Le runtime ne fait que : exécuter → observer → vérifier → suite ou fallback.
|
||||
|
||||
Cycle : WorkflowIR → ExecutionCompiler → ExecutionPlan → Runtime
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
import uuid
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ResolutionStrategy:
|
||||
"""Stratégie de résolution visuelle pour un élément UI.
|
||||
|
||||
Pré-compilée — le runtime n'a pas besoin du VLM pour résoudre.
|
||||
"""
|
||||
method: str # "uia", "ocr", "template", "position", "vlm", "dom"
|
||||
target_text: str = "" # Texte à chercher (pour OCR)
|
||||
anchor_b64: str = "" # Crop de référence (pour template matching)
|
||||
zone: Dict[str, float] = field(default_factory=dict) # Zone de recherche {x_min, y_min, x_max, y_max}
|
||||
position_hint: str = "" # "en haut à droite", "dans la barre des tâches"
|
||||
vlm_description: str = "" # Description VLM (dernier recours)
|
||||
threshold: float = 0.8 # Seuil de confiance
|
||||
|
||||
# Stratégie UIA (Windows UI Automation)
|
||||
# Utilisée quand l'enregistrement a capturé un snapshot UIA au moment du clic.
|
||||
# Au replay, l'agent Windows appelle lea_uia.exe find --name ... pour retrouver
|
||||
# l'élément par son chemin logique (100% fiable sur Windows natif).
|
||||
uia_name: str = "" # Name property de l'élément
|
||||
uia_control_type: str = "" # ControlType (Button, Edit, MenuItem, ...)
|
||||
uia_automation_id: str = "" # AutomationId (optionnel)
|
||||
uia_parent_path: List[Dict[str, str]] = field(default_factory=list)
|
||||
|
||||
# Stratégie DOM (web avec CDP activé) — préparation pour plus tard
|
||||
dom_selector: str = "" # CSS selector
|
||||
dom_xpath: str = "" # XPath
|
||||
dom_url_pattern: str = "" # Pattern URL à matcher
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
d = {"method": self.method}
|
||||
if self.target_text:
|
||||
d["target_text"] = self.target_text
|
||||
if self.anchor_b64:
|
||||
d["anchor_b64"] = self.anchor_b64[:50] + "..." # Tronqué pour la lisibilité
|
||||
if self.zone:
|
||||
d["zone"] = self.zone
|
||||
if self.position_hint:
|
||||
d["position_hint"] = self.position_hint
|
||||
if self.vlm_description:
|
||||
d["vlm_description"] = self.vlm_description
|
||||
if self.uia_name:
|
||||
d["uia_name"] = self.uia_name
|
||||
if self.uia_control_type:
|
||||
d["uia_control_type"] = self.uia_control_type
|
||||
if self.uia_automation_id:
|
||||
d["uia_automation_id"] = self.uia_automation_id
|
||||
if self.uia_parent_path:
|
||||
d["uia_parent_path"] = self.uia_parent_path
|
||||
if self.dom_selector:
|
||||
d["dom_selector"] = self.dom_selector
|
||||
if self.dom_xpath:
|
||||
d["dom_xpath"] = self.dom_xpath
|
||||
if self.dom_url_pattern:
|
||||
d["dom_url_pattern"] = self.dom_url_pattern
|
||||
d["threshold"] = self.threshold
|
||||
return d
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, d: Dict) -> "ResolutionStrategy":
|
||||
return cls(**{k: v for k, v in d.items() if k in cls.__dataclass_fields__})
|
||||
|
||||
|
||||
@dataclass
|
||||
class SuccessCondition:
|
||||
"""Condition de succès d'un nœud — comment vérifier que l'action a marché."""
|
||||
method: str = "screen_changed" # "screen_changed", "title_match", "text_visible", "none"
|
||||
expected_title: str = "" # Titre fenêtre attendu après l'action
|
||||
expected_text: str = "" # Texte qui doit apparaître
|
||||
description: str = "" # Description pour le Critic VLM (exception handler)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
d = {"method": self.method}
|
||||
if self.expected_title:
|
||||
d["expected_title"] = self.expected_title
|
||||
if self.expected_text:
|
||||
d["expected_text"] = self.expected_text
|
||||
if self.description:
|
||||
d["description"] = self.description
|
||||
return d
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, d: Dict) -> "SuccessCondition":
|
||||
return cls(**{k: v for k, v in d.items() if k in cls.__dataclass_fields__})
|
||||
|
||||
|
||||
@dataclass
|
||||
class ExecutionNode:
|
||||
"""Nœud d'exécution — une action à exécuter avec sa stratégie complète."""
|
||||
node_id: str
|
||||
action_type: str # click, type, key_combo, wait, scroll
|
||||
intent: str = "" # Intention métier (pour le logging/audit)
|
||||
|
||||
# Résolution visuelle pré-compilée
|
||||
strategy_primary: Optional[ResolutionStrategy] = None
|
||||
strategy_fallbacks: List[ResolutionStrategy] = field(default_factory=list)
|
||||
|
||||
# Données de l'action
|
||||
text: str = "" # Texte à taper
|
||||
keys: List[str] = field(default_factory=list)
|
||||
duration_ms: int = 0
|
||||
variable_name: str = "" # Si le texte est une variable
|
||||
|
||||
# Bornes d'exécution
|
||||
timeout_ms: int = 10000 # Timeout pour cette action
|
||||
max_retries: int = 1 # Nombre de retries autorisés
|
||||
retry_delay_ms: int = 2000 # Délai entre retries
|
||||
|
||||
# Vérification
|
||||
success_condition: Optional[SuccessCondition] = None
|
||||
|
||||
# Contrôle strict de fenêtre (pré-condition)
|
||||
expected_window_before: str = "" # La fenêtre active doit matcher AVANT l'action
|
||||
|
||||
# Recovery
|
||||
recovery_action: str = "escape" # "escape", "undo", "close", "none"
|
||||
|
||||
# Contexte
|
||||
step_id: str = "" # Référence vers l'étape WorkflowIR
|
||||
is_optional: bool = False
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
d = {
|
||||
"node_id": self.node_id,
|
||||
"action_type": self.action_type,
|
||||
}
|
||||
if self.intent:
|
||||
d["intent"] = self.intent
|
||||
if self.strategy_primary:
|
||||
d["strategy_primary"] = self.strategy_primary.to_dict()
|
||||
if self.strategy_fallbacks:
|
||||
d["strategy_fallbacks"] = [s.to_dict() for s in self.strategy_fallbacks]
|
||||
if self.text:
|
||||
d["text"] = self.text
|
||||
if self.keys:
|
||||
d["keys"] = self.keys
|
||||
if self.duration_ms:
|
||||
d["duration_ms"] = self.duration_ms
|
||||
if self.variable_name:
|
||||
d["variable_name"] = self.variable_name
|
||||
d["timeout_ms"] = self.timeout_ms
|
||||
d["max_retries"] = self.max_retries
|
||||
if self.success_condition:
|
||||
d["success_condition"] = self.success_condition.to_dict()
|
||||
if self.expected_window_before:
|
||||
d["expected_window_before"] = self.expected_window_before
|
||||
d["recovery_action"] = self.recovery_action
|
||||
if self.is_optional:
|
||||
d["is_optional"] = True
|
||||
return d
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, d: Dict) -> "ExecutionNode":
|
||||
primary = ResolutionStrategy.from_dict(d["strategy_primary"]) if d.get("strategy_primary") else None
|
||||
fallbacks = [ResolutionStrategy.from_dict(f) for f in d.get("strategy_fallbacks", [])]
|
||||
success = SuccessCondition.from_dict(d["success_condition"]) if d.get("success_condition") else None
|
||||
return cls(
|
||||
node_id=d["node_id"],
|
||||
action_type=d["action_type"],
|
||||
intent=d.get("intent", ""),
|
||||
strategy_primary=primary,
|
||||
strategy_fallbacks=fallbacks,
|
||||
text=d.get("text", ""),
|
||||
keys=d.get("keys", []),
|
||||
duration_ms=d.get("duration_ms", 0),
|
||||
variable_name=d.get("variable_name", ""),
|
||||
timeout_ms=d.get("timeout_ms", 10000),
|
||||
max_retries=d.get("max_retries", 1),
|
||||
retry_delay_ms=d.get("retry_delay_ms", 2000),
|
||||
success_condition=success,
|
||||
expected_window_before=d.get("expected_window_before", ""),
|
||||
recovery_action=d.get("recovery_action", "escape"),
|
||||
step_id=d.get("step_id", ""),
|
||||
is_optional=d.get("is_optional", False),
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ExecutionPlan:
|
||||
"""Plan d'exécution versionné — ce que le runtime exécute."""
|
||||
plan_id: str
|
||||
workflow_id: str # Référence vers le WorkflowIR source
|
||||
version: int = 1
|
||||
created_at: float = 0.0
|
||||
|
||||
# Nœuds d'exécution (séquence ordonnée)
|
||||
nodes: List[ExecutionNode] = field(default_factory=list)
|
||||
|
||||
# Variables à substituer avant exécution
|
||||
variables: Dict[str, str] = field(default_factory=dict)
|
||||
|
||||
# Configuration globale
|
||||
domain: str = "generic"
|
||||
target_machine: str = "" # Machine cible
|
||||
target_resolution: str = "" # "1280x800", "1920x1080"
|
||||
|
||||
# Métriques de compilation
|
||||
total_nodes: int = 0
|
||||
nodes_with_ocr: int = 0 # Résolution OCR (rapide, précis)
|
||||
nodes_with_template: int = 0 # Résolution template (rapide)
|
||||
nodes_with_vlm: int = 0 # Résolution VLM (lent, dernier recours)
|
||||
estimated_duration_s: float = 0.0
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"plan_id": self.plan_id,
|
||||
"workflow_id": self.workflow_id,
|
||||
"version": self.version,
|
||||
"created_at": self.created_at,
|
||||
"domain": self.domain,
|
||||
"target_machine": self.target_machine,
|
||||
"target_resolution": self.target_resolution,
|
||||
"variables": self.variables,
|
||||
"nodes": [n.to_dict() for n in self.nodes],
|
||||
"stats": {
|
||||
"total_nodes": self.total_nodes,
|
||||
"nodes_with_ocr": self.nodes_with_ocr,
|
||||
"nodes_with_template": self.nodes_with_template,
|
||||
"nodes_with_vlm": self.nodes_with_vlm,
|
||||
"estimated_duration_s": round(self.estimated_duration_s, 1),
|
||||
},
|
||||
}
|
||||
|
||||
def to_json(self, indent: int = 2) -> str:
|
||||
return json.dumps(self.to_dict(), ensure_ascii=False, indent=indent)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, d: Dict) -> "ExecutionPlan":
|
||||
nodes = [ExecutionNode.from_dict(n) for n in d.get("nodes", [])]
|
||||
stats = d.get("stats", {})
|
||||
return cls(
|
||||
plan_id=d["plan_id"],
|
||||
workflow_id=d.get("workflow_id", ""),
|
||||
version=d.get("version", 1),
|
||||
created_at=d.get("created_at", 0),
|
||||
domain=d.get("domain", "generic"),
|
||||
target_machine=d.get("target_machine", ""),
|
||||
target_resolution=d.get("target_resolution", ""),
|
||||
variables=d.get("variables", {}),
|
||||
nodes=nodes,
|
||||
total_nodes=stats.get("total_nodes", len(nodes)),
|
||||
nodes_with_ocr=stats.get("nodes_with_ocr", 0),
|
||||
nodes_with_template=stats.get("nodes_with_template", 0),
|
||||
nodes_with_vlm=stats.get("nodes_with_vlm", 0),
|
||||
estimated_duration_s=stats.get("estimated_duration_s", 0),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_json(cls, json_str: str) -> "ExecutionPlan":
|
||||
return cls.from_dict(json.loads(json_str))
|
||||
|
||||
def save(self, directory: str) -> Path:
|
||||
dir_path = Path(directory)
|
||||
dir_path.mkdir(parents=True, exist_ok=True)
|
||||
file_path = dir_path / f"{self.plan_id}.json"
|
||||
file_path.write_text(self.to_json(), encoding="utf-8")
|
||||
return file_path
|
||||
|
||||
@classmethod
|
||||
def load(cls, file_path: str) -> "ExecutionPlan":
|
||||
return cls.from_json(Path(file_path).read_text(encoding="utf-8"))
|
||||
627
core/workflow/ir_builder.py
Normal file
627
core/workflow/ir_builder.py
Normal file
@@ -0,0 +1,627 @@
|
||||
# core/workflow/ir_builder.py
|
||||
"""
|
||||
IRBuilder — Transforme une RawTrace en WorkflowIR.
|
||||
|
||||
C'est le "compilateur de savoir-faire" :
|
||||
RawTrace (clics bruts) → WorkflowIR (connaissance structurée)
|
||||
|
||||
Le builder utilise gemma4 pour COMPRENDRE ce que l'utilisateur a fait :
|
||||
- Segmenter les actions en étapes logiques
|
||||
- Identifier l'intention de chaque étape
|
||||
- Détecter les variables (données qui changent entre les exécutions)
|
||||
- Définir les pré/postconditions
|
||||
|
||||
Le builder est appelé UNE SEULE FOIS après l'enregistrement.
|
||||
Le WorkflowIR produit est ensuite réutilisé pour chaque replay.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from .workflow_ir import WorkflowIR, Step, Action, Variable
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class IRBuilder:
|
||||
"""Construit un WorkflowIR depuis une RawTrace (événements bruts).
|
||||
|
||||
Usage :
|
||||
builder = IRBuilder()
|
||||
ir = builder.build(
|
||||
events=raw_events,
|
||||
session_id="sess_xxx",
|
||||
domain="tim_codage",
|
||||
)
|
||||
ir.save("data/workflows/")
|
||||
"""
|
||||
|
||||
def __init__(self, gemma4_port: str = ""):
|
||||
self._gemma4_port = gemma4_port or os.environ.get("GEMMA4_PORT", "11435")
|
||||
self._gemma4_url = f"http://localhost:{self._gemma4_port}/api/chat"
|
||||
|
||||
def build(
|
||||
self,
|
||||
events: List[Dict[str, Any]],
|
||||
session_id: str = "",
|
||||
session_dir: str = "",
|
||||
domain: str = "generic",
|
||||
name: str = "",
|
||||
) -> WorkflowIR:
|
||||
"""Construire un WorkflowIR depuis des événements bruts.
|
||||
|
||||
Étapes :
|
||||
1. Filtrer les événements parasites
|
||||
2. Segmenter en étapes logiques (par changement de fenêtre/intention)
|
||||
3. Pour chaque étape, identifier l'intention via gemma4
|
||||
4. Détecter les variables
|
||||
5. Définir pré/postconditions
|
||||
"""
|
||||
t_start = time.time()
|
||||
|
||||
# Résoudre le session_dir_path pour l'enrichissement visuel
|
||||
session_dir_path = Path(session_dir) if session_dir else None
|
||||
if session_dir_path and not session_dir_path.is_dir():
|
||||
logger.warning(
|
||||
f"IRBuilder: session_dir '{session_dir}' introuvable — "
|
||||
f"enrichissement visuel désactivé"
|
||||
)
|
||||
session_dir_path = None
|
||||
|
||||
# Créer le WorkflowIR vide
|
||||
ir = WorkflowIR.new(
|
||||
name=name or f"Workflow du {time.strftime('%d/%m/%Y %H:%M')}",
|
||||
domain=domain,
|
||||
learned_from=session_id,
|
||||
)
|
||||
|
||||
# 1. Filtrer les événements utiles
|
||||
actionable = self._filter_events(events)
|
||||
if not actionable:
|
||||
logger.warning("IRBuilder: aucun événement actionable")
|
||||
return ir
|
||||
|
||||
# 2. Détecter les applications utilisées
|
||||
ir.applications = self._detect_applications(actionable)
|
||||
|
||||
# 3. Segmenter en étapes logiques
|
||||
segments = self._segment_into_steps(actionable)
|
||||
|
||||
# 4. Pour chaque segment, construire une Step
|
||||
for i, segment in enumerate(segments):
|
||||
step = self._build_step(
|
||||
segment=segment,
|
||||
step_index=i,
|
||||
total_steps=len(segments),
|
||||
workflow_name=ir.name,
|
||||
domain=domain,
|
||||
session_dir_path=session_dir_path,
|
||||
)
|
||||
ir.steps.append(step)
|
||||
|
||||
# 5. Contrôle strict : remplir expected_window_before/after pour chaque action
|
||||
# C'est la clé de la robustesse : chaque action sait dans quelle fenêtre
|
||||
# elle doit s'exécuter ET dans quelle fenêtre elle doit aboutir.
|
||||
self._attach_window_expectations(ir, actionable)
|
||||
|
||||
# 6. Détecter les variables
|
||||
ir.variables = self._detect_variables(ir.steps, actionable)
|
||||
|
||||
elapsed = time.time() - t_start
|
||||
logger.info(
|
||||
f"IRBuilder: WorkflowIR construit en {elapsed:.1f}s — "
|
||||
f"{len(ir.steps)} étapes, {len(ir.variables)} variables, "
|
||||
f"{len(ir.applications)} applications"
|
||||
)
|
||||
|
||||
return ir
|
||||
|
||||
def _filter_events(self, events: List[Dict]) -> List[Dict]:
|
||||
"""Filtrer les événements parasites.
|
||||
|
||||
Exclusions :
|
||||
1. Types d'événements de bruit (heartbeat, focus_change, action_result)
|
||||
2. Clics dont la CIBLE UIA est dans Léa elle-même
|
||||
(via uia_snapshot.parent_path — on vérifie où va le clic, pas d'où
|
||||
il vient). Un clic "sur la taskbar" peut avoir window.title="Léa"
|
||||
si Léa avait le focus, mais sa cible UIA est la taskbar.
|
||||
"""
|
||||
ignored_types = {"heartbeat", "focus_change", "action_result", "window_focus_change"}
|
||||
lea_markers = (
|
||||
"léa", "lea -", "léa -", "lea —", "léa —",
|
||||
"lea assistante", "léa assistante",
|
||||
"agent v1",
|
||||
)
|
||||
|
||||
def _uia_target_is_lea(uia_snapshot: dict) -> bool:
|
||||
"""L'élément UIA cliqué est-il dans la fenêtre de Léa ?"""
|
||||
if not uia_snapshot:
|
||||
return False
|
||||
# Vérifier le nom de l'élément lui-même
|
||||
name = (uia_snapshot.get("name", "") or "").lower()
|
||||
if any(m in name for m in lea_markers):
|
||||
return True
|
||||
# Vérifier les parents
|
||||
for parent in uia_snapshot.get("parent_path", []):
|
||||
p_name = (parent.get("name", "") or "").lower()
|
||||
if any(m in p_name for m in lea_markers):
|
||||
return True
|
||||
return False
|
||||
|
||||
result = []
|
||||
filtered_lea = 0
|
||||
for raw_evt in events:
|
||||
evt = raw_evt.get("event", raw_evt)
|
||||
evt_type = evt.get("type", "")
|
||||
if evt_type in ignored_types:
|
||||
continue
|
||||
|
||||
# Filtrer uniquement les clics dont la CIBLE est dans Léa
|
||||
# (pas les clics depuis Léa vers l'extérieur)
|
||||
if evt_type == "mouse_click":
|
||||
uia = evt.get("uia_snapshot") or {}
|
||||
if _uia_target_is_lea(uia):
|
||||
filtered_lea += 1
|
||||
continue
|
||||
|
||||
result.append(evt)
|
||||
|
||||
if filtered_lea > 0:
|
||||
logger.info(
|
||||
f"IRBuilder: {filtered_lea} clic(s) filtré(s) "
|
||||
f"(cible UIA dans la fenêtre Léa)"
|
||||
)
|
||||
return result
|
||||
|
||||
def _attach_window_expectations(self, ir: WorkflowIR, events: List[Dict]) -> None:
|
||||
"""Remplir expected_window_before/after pour chaque action du workflow.
|
||||
|
||||
C'est LA clé du contrôle strict : chaque action connaît la fenêtre
|
||||
dans laquelle elle doit s'exécuter ET celle qui doit apparaître
|
||||
après. Toute divergence au replay → STOP immédiat.
|
||||
|
||||
On reconstruit la séquence d'événements "actionables" (clicks, type,
|
||||
key_combo) et on aligne chaque Action du workflow sur son événement
|
||||
source pour récupérer :
|
||||
- expected_window_before : titre de la fenêtre AU MOMENT du clic
|
||||
- expected_window_after : titre de la fenêtre du PROCHAIN click
|
||||
|
||||
Filtre critique : la fenêtre de Léa elle-même n'est JAMAIS une
|
||||
fenêtre cible valide (c'est l'overlay agent, pas l'app métier).
|
||||
Les fenêtres "unknown_window" et les titres vides sont ignorés.
|
||||
"""
|
||||
def _is_valid_target_window(title: str) -> bool:
|
||||
"""Un titre de fenêtre est valide comme expected_window_* si :
|
||||
- non vide, non "unknown_window"
|
||||
- pas la fenêtre de Léa elle-même
|
||||
"""
|
||||
if not title or title == "unknown_window":
|
||||
return False
|
||||
title_lower = title.lower()
|
||||
lea_markers = (
|
||||
"léa", "lea -", "léa -", "lea —", "léa —",
|
||||
"lea assistante", "léa assistante",
|
||||
"agent v1",
|
||||
)
|
||||
for marker in lea_markers:
|
||||
if marker in title_lower:
|
||||
return False
|
||||
return True
|
||||
|
||||
def _extract_uia_root_window(uia_snapshot: dict) -> str:
|
||||
"""Extraire le nom de la fenêtre racine depuis un snapshot UIA.
|
||||
|
||||
Le parent_path contient la hiérarchie de l'élément cliqué.
|
||||
La première entrée avec control_type="fenêtre" est la fenêtre
|
||||
qui CONTIENT l'élément cliqué — c'est la vraie cible.
|
||||
"""
|
||||
if not uia_snapshot:
|
||||
return ""
|
||||
for parent in uia_snapshot.get("parent_path", []):
|
||||
ct = (parent.get("control_type", "") or "").lower()
|
||||
if ct in ("fenêtre", "window"):
|
||||
name = (parent.get("name", "") or "").strip()
|
||||
if name:
|
||||
return name
|
||||
return ""
|
||||
|
||||
# Extraire la séquence des événements actionables avec leurs titres
|
||||
# Source de vérité pour les clics : parent_path UIA (où va vraiment
|
||||
# le clic), sinon window.title (fallback).
|
||||
# Pour les type/key_combo : window.title uniquement.
|
||||
event_sequence: List[Dict[str, Any]] = []
|
||||
for evt in events:
|
||||
t = evt.get("type", "")
|
||||
if t not in ("mouse_click", "text_input", "key_combo", "key_press", "scroll"):
|
||||
continue
|
||||
|
||||
# Titre de référence : priorité à la cible UIA pour les clics
|
||||
effective_title = ""
|
||||
if t == "mouse_click":
|
||||
uia = evt.get("uia_snapshot") or {}
|
||||
uia_root = _extract_uia_root_window(uia)
|
||||
if uia_root and _is_valid_target_window(uia_root):
|
||||
effective_title = uia_root
|
||||
|
||||
# Fallback sur window.title
|
||||
if not effective_title:
|
||||
raw_title = evt.get("window", {}).get("title", "") or ""
|
||||
if _is_valid_target_window(raw_title):
|
||||
effective_title = raw_title
|
||||
|
||||
event_sequence.append({"type": t, "title": effective_title})
|
||||
|
||||
# Aligner avec les actions du workflow
|
||||
flat_actions: List[tuple] = []
|
||||
for si, step in enumerate(ir.steps):
|
||||
for ai, action in enumerate(step.actions):
|
||||
if action.type in ("click", "type", "key_combo"):
|
||||
flat_actions.append((si, ai, action))
|
||||
|
||||
# Limite : on prend le min entre les 2 listes
|
||||
n = min(len(flat_actions), len(event_sequence))
|
||||
|
||||
for i in range(n):
|
||||
si, ai, action = flat_actions[i]
|
||||
title_now = event_sequence[i]["title"]
|
||||
if title_now:
|
||||
action.expected_window_before = title_now
|
||||
|
||||
# Chercher le prochain événement avec un titre valide
|
||||
# Et qui est DIFFÉRENT du titre actuel (sinon pas de transition à vérifier)
|
||||
for j in range(i + 1, len(event_sequence)):
|
||||
next_title = event_sequence[j]["title"]
|
||||
if next_title and next_title != title_now:
|
||||
action.expected_window_after = next_title
|
||||
break
|
||||
|
||||
def _detect_applications(self, events: List[Dict]) -> List[str]:
|
||||
"""Détecter les applications utilisées."""
|
||||
apps = set()
|
||||
for evt in events:
|
||||
title = evt.get("window", {}).get("title", "")
|
||||
if title and title != "unknown_window":
|
||||
for sep in [" – ", " - ", " — "]:
|
||||
if sep in title:
|
||||
apps.add(title.split(sep)[-1].strip())
|
||||
break
|
||||
return sorted(apps)
|
||||
|
||||
def _segment_into_steps(self, events: List[Dict]) -> List[List[Dict]]:
|
||||
"""Segmenter les événements en étapes logiques.
|
||||
|
||||
Critères de coupure :
|
||||
- Changement d'application (fenêtre différente)
|
||||
- Pause longue (> 5s entre deux événements)
|
||||
- Transition logique (clic → frappe → clic = étapes différentes)
|
||||
"""
|
||||
if not events:
|
||||
return []
|
||||
|
||||
segments = []
|
||||
current_segment = [events[0]]
|
||||
current_app = self._get_app_name(events[0])
|
||||
|
||||
for evt in events[1:]:
|
||||
app = self._get_app_name(evt)
|
||||
evt_type = evt.get("type", "")
|
||||
|
||||
# Coupure par changement d'application
|
||||
app_changed = app and current_app and app != current_app
|
||||
|
||||
# Coupure par pause longue
|
||||
prev_ts = float(current_segment[-1].get("timestamp", 0))
|
||||
curr_ts = float(evt.get("timestamp", 0))
|
||||
long_pause = (curr_ts - prev_ts) > 5.0 if prev_ts > 0 and curr_ts > 0 else False
|
||||
|
||||
# Coupure par transition clic → nouveau clic (nouvelle intention)
|
||||
transition = (
|
||||
evt_type == "mouse_click"
|
||||
and len(current_segment) >= 2
|
||||
and current_segment[-1].get("type") not in ("mouse_click",)
|
||||
)
|
||||
|
||||
if app_changed or long_pause:
|
||||
if current_segment:
|
||||
segments.append(current_segment)
|
||||
current_segment = [evt]
|
||||
current_app = app
|
||||
else:
|
||||
current_segment.append(evt)
|
||||
|
||||
if current_segment:
|
||||
segments.append(current_segment)
|
||||
|
||||
return segments
|
||||
|
||||
def _get_app_name(self, evt: Dict) -> str:
|
||||
"""Extraire le nom d'application depuis un événement."""
|
||||
title = evt.get("window", {}).get("title", "")
|
||||
for sep in [" – ", " - ", " — "]:
|
||||
if sep in title:
|
||||
return title.split(sep)[-1].strip()
|
||||
return title
|
||||
|
||||
def _build_step(
|
||||
self,
|
||||
segment: List[Dict],
|
||||
step_index: int,
|
||||
total_steps: int,
|
||||
workflow_name: str,
|
||||
domain: str,
|
||||
session_dir_path: Optional[Path] = None,
|
||||
) -> Step:
|
||||
"""Construire une Step depuis un segment d'événements.
|
||||
|
||||
Utilise gemma4 pour comprendre l'intention du segment.
|
||||
"""
|
||||
# Construire la description du segment pour gemma4
|
||||
actions = []
|
||||
for evt in segment:
|
||||
action = self._event_to_action(evt, session_dir_path=session_dir_path)
|
||||
if action:
|
||||
actions.append(action)
|
||||
|
||||
# Description textuelle du segment
|
||||
segment_desc = self._describe_segment(segment)
|
||||
|
||||
# Demander à gemma4 l'intention
|
||||
intent, precondition, postcondition = self._analyze_intent(
|
||||
segment_desc, step_index, total_steps, workflow_name, domain,
|
||||
)
|
||||
|
||||
return Step(
|
||||
step_id=f"s{step_index + 1}",
|
||||
intent=intent or segment_desc,
|
||||
precondition=precondition,
|
||||
postcondition=postcondition,
|
||||
actions=actions,
|
||||
)
|
||||
|
||||
def _event_to_action(self, evt: Dict, session_dir_path: Optional[Path] = None) -> Optional[Action]:
|
||||
"""Convertir un événement brut en Action enrichie.
|
||||
|
||||
Pour les clics : appelle enrich_click_from_screenshot() si le session_dir
|
||||
est disponible pour obtenir :
|
||||
- by_text (texte OCR exact de l'élément cliqué)
|
||||
- anchor_image_base64 (crop 80x80 pour template matching)
|
||||
- vlm_description (description positionnelle)
|
||||
- window_capture (rect pour le grounding ciblé)
|
||||
|
||||
Cet enrichissement est LA clé pour que l'ExecutionCompiler produise
|
||||
des plans V4 complets avec toutes les stratégies (OCR + template + VLM).
|
||||
"""
|
||||
evt_type = evt.get("type", "")
|
||||
|
||||
if evt_type == "mouse_click":
|
||||
window = evt.get("window", {}).get("title", "")
|
||||
pos = evt.get("pos", [0, 0])
|
||||
|
||||
# Action de base (fallback sans enrichissement)
|
||||
action = Action(
|
||||
type="click",
|
||||
target=window,
|
||||
anchor_hint=evt.get("vision_info", {}).get("text", "") if isinstance(evt.get("vision_info"), dict) else "",
|
||||
)
|
||||
|
||||
# Enrichissement visuel via enrich_click_from_screenshot
|
||||
# Accès direct au crop OCR + anchor pour l'ExecutionCompiler
|
||||
if session_dir_path and isinstance(pos, list) and len(pos) == 2:
|
||||
enrichment = self._enrich_click(
|
||||
evt, session_dir_path, window, int(pos[0]), int(pos[1]),
|
||||
)
|
||||
if enrichment:
|
||||
# Le texte OCR devient l'anchor_hint pour l'OCR primaire
|
||||
by_text = enrichment.get("by_text", "")
|
||||
if by_text:
|
||||
action.anchor_hint = by_text
|
||||
# Stocker les métadonnées d'enrichissement dans l'action
|
||||
# (utilisé par l'ExecutionCompiler pour construire les stratégies)
|
||||
action._enrichment = enrichment
|
||||
|
||||
# Lire le snapshot UIA si l'agent Windows l'a capturé.
|
||||
# Format attendu dans l'événement :
|
||||
# evt["uia_snapshot"] = {
|
||||
# "name": "Enregistrer",
|
||||
# "control_type": "bouton",
|
||||
# "automation_id": "btnSave",
|
||||
# "parent_path": [{"name": "...", "control_type": "..."}],
|
||||
# }
|
||||
# Si présent, il est fusionné dans _enrichment pour que
|
||||
# l'ExecutionCompiler puisse créer une stratégie UIA prioritaire.
|
||||
uia_snapshot = evt.get("uia_snapshot")
|
||||
if uia_snapshot and isinstance(uia_snapshot, dict):
|
||||
if not hasattr(action, "_enrichment") or action._enrichment is None:
|
||||
action._enrichment = {}
|
||||
action._enrichment["uia_snapshot"] = uia_snapshot
|
||||
|
||||
return action
|
||||
|
||||
elif evt_type == "text_input":
|
||||
text = evt.get("text", "")
|
||||
if text:
|
||||
return Action(type="type", text=text)
|
||||
elif evt_type in ("key_combo", "key_press"):
|
||||
keys = evt.get("keys", [])
|
||||
if keys:
|
||||
return Action(type="key_combo", keys=keys)
|
||||
elif evt_type == "scroll":
|
||||
return Action(type="scroll")
|
||||
|
||||
return None
|
||||
|
||||
def _enrich_click(
|
||||
self,
|
||||
evt: Dict,
|
||||
session_dir_path: Path,
|
||||
window_title: str,
|
||||
click_x: int,
|
||||
click_y: int,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Enrichir un clic avec OCR + crop + description.
|
||||
|
||||
Réutilise enrich_click_from_screenshot du stream_processor (éprouvé).
|
||||
Retourne un dict avec by_text, anchor_image_base64, vlm_description, etc.
|
||||
"""
|
||||
try:
|
||||
from agent_v0.server_v1.stream_processor import enrich_click_from_screenshot
|
||||
|
||||
# Trouver le screenshot full
|
||||
screenshot_id = evt.get("screenshot_id", "")
|
||||
if not screenshot_id:
|
||||
return None
|
||||
|
||||
full_path = session_dir_path / "shots" / f"{screenshot_id}_full.png"
|
||||
if not full_path.is_file():
|
||||
return None
|
||||
|
||||
# Résolution écran
|
||||
screen_w = 1280
|
||||
screen_h = 800
|
||||
window_capture = evt.get("window_capture", {})
|
||||
if window_capture.get("window_rect"):
|
||||
rect = window_capture["window_rect"]
|
||||
screen_w = max(screen_w, rect[2])
|
||||
screen_h = max(screen_h, rect[3])
|
||||
|
||||
return enrich_click_from_screenshot(
|
||||
screenshot_path=full_path,
|
||||
click_x=click_x,
|
||||
click_y=click_y,
|
||||
screen_w=screen_w,
|
||||
screen_h=screen_h,
|
||||
window_title=window_title,
|
||||
vision_info=evt.get("vision_info") if isinstance(evt.get("vision_info"), dict) else None,
|
||||
session_dir=session_dir_path,
|
||||
screenshot_id=screenshot_id,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug(f"IRBuilder._enrich_click: {e}")
|
||||
return None
|
||||
|
||||
def _describe_segment(self, segment: List[Dict]) -> str:
|
||||
"""Décrire un segment en langage naturel (pour gemma4)."""
|
||||
parts = []
|
||||
window = ""
|
||||
for evt in segment:
|
||||
evt_type = evt.get("type", "")
|
||||
w = evt.get("window", {}).get("title", "")
|
||||
if w and w != window:
|
||||
window = w
|
||||
parts.append(f"[{w}]")
|
||||
if evt_type == "mouse_click":
|
||||
text = evt.get("vision_info", {}).get("text", "")
|
||||
parts.append(f"clic sur '{text}'" if text else "clic")
|
||||
elif evt_type == "text_input":
|
||||
text = evt.get("text", "")
|
||||
parts.append(f"saisie '{text[:30]}'")
|
||||
elif evt_type in ("key_combo", "key_press"):
|
||||
keys = evt.get("keys", [])
|
||||
parts.append(f"touche {'+'.join(keys)}")
|
||||
return " → ".join(parts) if parts else "action"
|
||||
|
||||
def _analyze_intent(
|
||||
self,
|
||||
segment_desc: str,
|
||||
step_index: int,
|
||||
total_steps: int,
|
||||
workflow_name: str,
|
||||
domain: str,
|
||||
) -> tuple:
|
||||
"""Demander à gemma4 de comprendre l'intention d'un segment.
|
||||
|
||||
Returns:
|
||||
(intent, precondition, postcondition)
|
||||
"""
|
||||
import requests as _requests
|
||||
|
||||
# Charger le contexte métier
|
||||
domain_prompt = ""
|
||||
try:
|
||||
from agent_v0.server_v1.domain_context import get_domain_context
|
||||
ctx = get_domain_context(domain)
|
||||
if ctx.system_prompt:
|
||||
domain_prompt = f"\nContexte métier : {ctx.name}\n"
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
prompt = (
|
||||
f"{domain_prompt}"
|
||||
f"Workflow : {workflow_name} (étape {step_index + 1}/{total_steps})\n"
|
||||
f"Actions observées : {segment_desc}\n\n"
|
||||
f"Réponds en 3 lignes :\n"
|
||||
f"INTENTION: que veut faire l'utilisateur avec ces actions (1 phrase)\n"
|
||||
f"AVANT: état attendu de l'écran avant cette étape (1 phrase)\n"
|
||||
f"APRÈS: état attendu de l'écran après cette étape (1 phrase)"
|
||||
)
|
||||
|
||||
try:
|
||||
resp = _requests.post(
|
||||
self._gemma4_url,
|
||||
json={
|
||||
"model": "gemma4:e4b",
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
"stream": False,
|
||||
"think": True,
|
||||
"options": {"temperature": 0.1, "num_predict": 800},
|
||||
},
|
||||
timeout=30,
|
||||
)
|
||||
if resp.ok:
|
||||
content = resp.json().get("message", {}).get("content", "")
|
||||
return self._parse_intent_response(content)
|
||||
except Exception as e:
|
||||
logger.debug(f"IRBuilder: gemma4 indisponible ({e})")
|
||||
|
||||
return (segment_desc, "", "")
|
||||
|
||||
def _parse_intent_response(self, content: str) -> tuple:
|
||||
"""Parser la réponse gemma4 (INTENTION/AVANT/APRÈS)."""
|
||||
intent = ""
|
||||
precondition = ""
|
||||
postcondition = ""
|
||||
|
||||
for line in content.split("\n"):
|
||||
clean = line.strip()
|
||||
upper = clean.upper()
|
||||
if upper.startswith("INTENTION:"):
|
||||
intent = clean.split(":", 1)[1].strip()
|
||||
elif upper.startswith("AVANT:"):
|
||||
precondition = clean.split(":", 1)[1].strip()
|
||||
elif upper.startswith(("APRÈS:", "APRES:")):
|
||||
postcondition = clean.split(":", 1)[1].strip()
|
||||
|
||||
return (intent, precondition, postcondition)
|
||||
|
||||
def _detect_variables(self, steps: List[Step], events: List[Dict]) -> List[Variable]:
|
||||
"""Détecter les variables dans le workflow.
|
||||
|
||||
Une variable est une donnée qui change entre les exécutions :
|
||||
- Texte saisi par l'utilisateur (noms, codes, dates)
|
||||
- Données lues à l'écran (résultats de recherche)
|
||||
"""
|
||||
variables = []
|
||||
seen_texts = set()
|
||||
|
||||
for step in steps:
|
||||
for action in step.actions:
|
||||
if action.type == "type" and action.text:
|
||||
text = action.text.strip()
|
||||
if text and text not in seen_texts and len(text) > 2:
|
||||
seen_texts.add(text)
|
||||
var_name = f"texte_{len(variables) + 1}"
|
||||
variables.append(Variable(
|
||||
name=var_name,
|
||||
description=f"Texte saisi : '{text[:50]}'",
|
||||
source="user",
|
||||
default=text,
|
||||
))
|
||||
# Marquer l'action comme variable
|
||||
action.variable = True
|
||||
action.text = f"{{{var_name}}}"
|
||||
|
||||
return variables
|
||||
693
core/workflow/shadow_observer.py
Normal file
693
core/workflow/shadow_observer.py
Normal file
@@ -0,0 +1,693 @@
|
||||
# core/workflow/shadow_observer.py
|
||||
"""
|
||||
ShadowObserver — Observation en temps réel de ce que Léa comprend.
|
||||
|
||||
C'est le "mode Shadow amélioré" : pendant que l'utilisateur enregistre
|
||||
une démonstration, Léa lui dit ce qu'elle comprend au fur et à mesure.
|
||||
|
||||
Contrairement à l'IRBuilder (qui analyse TOUT à la fin en appelant gemma4),
|
||||
le ShadowObserver travaille en incrémental :
|
||||
- À chaque événement reçu, il met à jour sa compréhension locale.
|
||||
- Il segmente dès qu'un critère de coupure est détecté.
|
||||
- Il émet des notifications légères ("Léa a compris : tu viens d'ouvrir le
|
||||
Bloc-notes") via un callback.
|
||||
- Il détecte les variables (texte saisi) pendant la frappe.
|
||||
|
||||
Le ShadowObserver n'est pas la source de vérité — c'est une couche
|
||||
d'observation. La source de vérité reste `live_events.jsonl`.
|
||||
Le WorkflowIR final est toujours reconstruit par l'IRBuilder après
|
||||
validation, mais la compréhension temps réel accélère la boucle de
|
||||
rétroaction avec l'utilisateur.
|
||||
|
||||
Usage :
|
||||
|
||||
def on_notify(event):
|
||||
print(f"[{event.niveau}] {event.message}")
|
||||
|
||||
observer = ShadowObserver(notify_callback=on_notify)
|
||||
observer.start("sess_abc")
|
||||
observer.observe_event(event1)
|
||||
observer.observe_event(event2)
|
||||
...
|
||||
comprehension = observer.get_understanding()
|
||||
# → [{"step": 1, "intent": "Ouvrir le Bloc-notes", "confidence": 0.8}, ...]
|
||||
observer.stop()
|
||||
|
||||
Contraintes :
|
||||
- 100% asynchrone côté performance : la méthode observe_event() ne doit
|
||||
jamais bloquer la capture (pas d'appel réseau synchrone).
|
||||
- Optionnel : activable via paramètre, ne modifie pas la capture existante.
|
||||
- 100% français dans les messages utilisateur.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from dataclasses import dataclass, field
|
||||
from enum import Enum
|
||||
from typing import Any, Callable, Dict, List, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Types d'événements observationnels
|
||||
# =========================================================================
|
||||
|
||||
|
||||
class NiveauNotification(str, Enum):
|
||||
"""Niveau d'importance d'une notification.
|
||||
|
||||
- INFO : information passive ("Léa observe...")
|
||||
- DECOUVERTE : Léa vient de comprendre quelque chose de nouveau
|
||||
- QUESTION : Léa aimerait une confirmation (non bloquant)
|
||||
- VARIABLE : une variable a été détectée
|
||||
"""
|
||||
|
||||
INFO = "info"
|
||||
DECOUVERTE = "decouverte"
|
||||
QUESTION = "question"
|
||||
VARIABLE = "variable"
|
||||
|
||||
|
||||
@dataclass
|
||||
class NotificationShadow:
|
||||
"""Notification émise par le ShadowObserver vers la GUI utilisateur."""
|
||||
|
||||
notif_id: str
|
||||
niveau: NiveauNotification
|
||||
message: str # Texte affichable à l'utilisateur (français)
|
||||
session_id: str
|
||||
step_index: int = -1 # Index de l'étape concernée, -1 si global
|
||||
data: Dict[str, Any] = field(default_factory=dict)
|
||||
timestamp: float = 0.0
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"notif_id": self.notif_id,
|
||||
"niveau": self.niveau.value,
|
||||
"message": self.message,
|
||||
"session_id": self.session_id,
|
||||
"step_index": self.step_index,
|
||||
"data": self.data,
|
||||
"timestamp": self.timestamp,
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class UnderstoodStep:
|
||||
"""Étape logique comprise en temps réel par le ShadowObserver.
|
||||
|
||||
C'est une version simplifiée de `Step` (core.workflow.workflow_ir),
|
||||
optimisée pour la construction incrémentale. Elle sera convertie
|
||||
en `Step` final par le ShadowValidator après validation.
|
||||
"""
|
||||
|
||||
step_index: int
|
||||
intent: str # Intention humaine (ex: "Ouvrir le Bloc-notes")
|
||||
intent_provisoire: bool = True # True tant que gemma4 n'a pas confirmé
|
||||
confidence: float = 0.5 # Score de confiance (0..1)
|
||||
app_name: str = "" # Application principale
|
||||
window_title: str = "" # Titre de la fenêtre au début du segment
|
||||
events: List[Dict[str, Any]] = field(default_factory=list)
|
||||
variables_detectees: List[str] = field(default_factory=list)
|
||||
started_at: float = 0.0
|
||||
ended_at: float = 0.0
|
||||
validated: bool = False # L'utilisateur a validé l'étape
|
||||
corrected: bool = False # L'utilisateur a corrigé l'intention
|
||||
cancelled: bool = False # L'utilisateur a annulé l'étape
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"step_index": self.step_index,
|
||||
"intent": self.intent,
|
||||
"intent_provisoire": self.intent_provisoire,
|
||||
"confidence": round(self.confidence, 3),
|
||||
"app_name": self.app_name,
|
||||
"window_title": self.window_title,
|
||||
"events_count": len(self.events),
|
||||
"variables_detectees": list(self.variables_detectees),
|
||||
"started_at": self.started_at,
|
||||
"ended_at": self.ended_at,
|
||||
"validated": self.validated,
|
||||
"corrected": self.corrected,
|
||||
"cancelled": self.cancelled,
|
||||
}
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Observer
|
||||
# =========================================================================
|
||||
|
||||
|
||||
# Constantes de segmentation (en secondes). On évite de re-déclarer les
|
||||
# constantes de l'IRBuilder car l'observation est incrémentale — on peut
|
||||
# se permettre des seuils plus courts pour plus de réactivité.
|
||||
_SEUIL_PAUSE_LONGUE_S = 4.0
|
||||
_SEUIL_CONFIANCE_BASE = 0.5
|
||||
_SEUIL_CONFIANCE_APP_CHANGE = 0.8
|
||||
|
||||
# Types d'événements ignorés
|
||||
_EVENT_TYPES_IGNORES = {
|
||||
"heartbeat",
|
||||
"focus_change",
|
||||
"action_result",
|
||||
"window_focus_change",
|
||||
}
|
||||
|
||||
|
||||
class ShadowObserver:
|
||||
"""Observe les événements en temps réel et met à jour la compréhension.
|
||||
|
||||
Thread-safe : peut être appelé depuis plusieurs threads (capture,
|
||||
API, worker).
|
||||
|
||||
Le callback `notify_callback` est appelé de manière synchrone mais les
|
||||
notifications sont extrêmement légères (juste un dataclass) — elles
|
||||
sont destinées à être envoyées via WebSocket/HTTP long-poll depuis la
|
||||
couche API.
|
||||
"""
|
||||
|
||||
NotifyCallback = Callable[[NotificationShadow], None]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
notify_callback: Optional[NotifyCallback] = None,
|
||||
*,
|
||||
enable_gemma4: bool = False,
|
||||
gemma4_callback: Optional[Callable[[UnderstoodStep], None]] = None,
|
||||
):
|
||||
"""
|
||||
Args:
|
||||
notify_callback: Fonction appelée à chaque notification
|
||||
(doit être rapide, pas d'IO bloquant).
|
||||
enable_gemma4: Si True, une tâche asynchrone peut enrichir
|
||||
les intentions via gemma4 (non bloquant). En pratique,
|
||||
on laisse le caller le brancher via `gemma4_callback`.
|
||||
gemma4_callback: Fonction appelée en arrière-plan pour
|
||||
enrichir une étape (via gemma4 ou autre LLM). Non bloquant.
|
||||
"""
|
||||
self._notify_callback = notify_callback
|
||||
self._enable_gemma4 = enable_gemma4
|
||||
self._gemma4_callback = gemma4_callback
|
||||
|
||||
self._lock = threading.RLock()
|
||||
self._sessions: Dict[str, Dict[str, Any]] = {}
|
||||
|
||||
# ----- Cycle de vie --------------------------------------------------
|
||||
|
||||
def start(self, session_id: str) -> None:
|
||||
"""Démarrer l'observation d'une session."""
|
||||
with self._lock:
|
||||
self._sessions[session_id] = {
|
||||
"steps": [], # List[UnderstoodStep]
|
||||
"current_step": None, # Optional[UnderstoodStep]
|
||||
"last_event_ts": 0.0,
|
||||
"last_notif_ts": 0.0,
|
||||
"total_events": 0,
|
||||
"notifications": [], # Historique des notifications
|
||||
"started_at": time.time(),
|
||||
"stopped_at": 0.0,
|
||||
}
|
||||
self._notifier(
|
||||
session_id,
|
||||
NiveauNotification.INFO,
|
||||
"Léa t'observe. Fais ta tâche normalement, je vais apprendre.",
|
||||
)
|
||||
|
||||
def stop(self, session_id: str) -> None:
|
||||
"""Arrêter l'observation et finaliser le segment en cours."""
|
||||
with self._lock:
|
||||
state = self._sessions.get(session_id)
|
||||
if not state:
|
||||
return
|
||||
current = state.get("current_step")
|
||||
if current is not None and current.events:
|
||||
current.ended_at = state["last_event_ts"] or time.time()
|
||||
state["steps"].append(current)
|
||||
state["current_step"] = None
|
||||
state["stopped_at"] = time.time()
|
||||
|
||||
nb_steps = len(self.get_understanding(session_id))
|
||||
if nb_steps > 0:
|
||||
self._notifier(
|
||||
session_id,
|
||||
NiveauNotification.DECOUVERTE,
|
||||
f"J'ai observé {nb_steps} étape(s). Tu veux que je te les "
|
||||
f"montre pour validation ?",
|
||||
)
|
||||
|
||||
def reset(self, session_id: str) -> None:
|
||||
"""Supprimer l'état d'une session (après finalisation)."""
|
||||
with self._lock:
|
||||
self._sessions.pop(session_id, None)
|
||||
|
||||
# ----- Observation ---------------------------------------------------
|
||||
|
||||
def observe_event(self, session_id: str, event: Dict[str, Any]) -> None:
|
||||
"""Observer un nouvel événement pendant la capture.
|
||||
|
||||
Cette méthode est appelée à chaque événement reçu par le serveur.
|
||||
Elle doit être RAPIDE (pas d'IO réseau synchrone).
|
||||
"""
|
||||
evt_type = event.get("type", "")
|
||||
if evt_type in _EVENT_TYPES_IGNORES:
|
||||
return
|
||||
|
||||
with self._lock:
|
||||
state = self._sessions.get(session_id)
|
||||
if not state:
|
||||
# Auto-start si pas encore démarré (robustesse)
|
||||
self.start(session_id)
|
||||
state = self._sessions[session_id]
|
||||
|
||||
state["total_events"] += 1
|
||||
|
||||
# 1. Décider si on démarre un nouveau segment
|
||||
current = state.get("current_step")
|
||||
should_cut, cut_reason = self._should_cut(state, event)
|
||||
|
||||
if should_cut and current is not None:
|
||||
current.ended_at = state["last_event_ts"] or time.time()
|
||||
state["steps"].append(current)
|
||||
self._emit_step_closed(session_id, current, cut_reason)
|
||||
current = None
|
||||
state["current_step"] = None
|
||||
|
||||
if current is None:
|
||||
step_index = len(state["steps"]) + 1
|
||||
current = UnderstoodStep(
|
||||
step_index=step_index,
|
||||
intent=self._initial_intent(event),
|
||||
intent_provisoire=True,
|
||||
confidence=_SEUIL_CONFIANCE_BASE,
|
||||
app_name=self._get_app_name(event),
|
||||
window_title=self._get_window_title(event),
|
||||
started_at=float(event.get("timestamp", 0)) or time.time(),
|
||||
)
|
||||
state["current_step"] = current
|
||||
|
||||
# 2. Ajouter l'événement au segment courant
|
||||
current.events.append(event)
|
||||
ts = float(event.get("timestamp", 0)) or time.time()
|
||||
state["last_event_ts"] = ts
|
||||
|
||||
# 3. Rafraîchir l'intent provisoire à partir du contexte accumulé
|
||||
current.intent = self._refine_intent(current, event)
|
||||
|
||||
# 4. Détection de variable pendant la frappe
|
||||
if evt_type == "text_input":
|
||||
self._handle_text_input(session_id, current, event)
|
||||
|
||||
# 5. Émission périodique d'un résumé (toutes les 5s)
|
||||
self._maybe_emit_heartbeat(session_id, state)
|
||||
|
||||
# ----- API publique --------------------------------------------------
|
||||
|
||||
def get_understanding(
|
||||
self, session_id: str, include_current: bool = True
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Récupérer ce que Léa a compris jusqu'ici.
|
||||
|
||||
Returns:
|
||||
Liste de dicts au format :
|
||||
[{"step": 1, "intent": "Ouvrir le Bloc-notes",
|
||||
"confidence": 0.9, "app": "Bloc-notes",
|
||||
"events_count": 4, ...}, ...]
|
||||
"""
|
||||
with self._lock:
|
||||
state = self._sessions.get(session_id)
|
||||
if not state:
|
||||
return []
|
||||
steps = list(state["steps"])
|
||||
if include_current and state.get("current_step") is not None:
|
||||
steps = steps + [state["current_step"]]
|
||||
|
||||
out = []
|
||||
for step in steps:
|
||||
d = step.to_dict()
|
||||
d["step"] = d.pop("step_index")
|
||||
out.append(d)
|
||||
return out
|
||||
|
||||
def get_notifications(
|
||||
self, session_id: str, since_ts: float = 0.0
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Récupérer les notifications émises depuis un timestamp."""
|
||||
with self._lock:
|
||||
state = self._sessions.get(session_id)
|
||||
if not state:
|
||||
return []
|
||||
return [
|
||||
n.to_dict() for n in state["notifications"]
|
||||
if n.timestamp >= since_ts
|
||||
]
|
||||
|
||||
def get_current_step(
|
||||
self, session_id: str
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Retourner l'étape en cours de construction."""
|
||||
with self._lock:
|
||||
state = self._sessions.get(session_id)
|
||||
if not state:
|
||||
return None
|
||||
current = state.get("current_step")
|
||||
if current is None:
|
||||
return None
|
||||
return current.to_dict()
|
||||
|
||||
def get_steps_internal(
|
||||
self, session_id: str, include_current: bool = True
|
||||
) -> List[UnderstoodStep]:
|
||||
"""Version interne : retourne les objets `UnderstoodStep`.
|
||||
|
||||
Utilisé par le ShadowValidator pour reconstruire un WorkflowIR.
|
||||
"""
|
||||
with self._lock:
|
||||
state = self._sessions.get(session_id)
|
||||
if not state:
|
||||
return []
|
||||
steps = list(state["steps"])
|
||||
if include_current and state.get("current_step") is not None:
|
||||
steps = steps + [state["current_step"]]
|
||||
# Retourner des copies pour éviter les mutations externes
|
||||
return [self._copy_step(s) for s in steps]
|
||||
|
||||
def has_session(self, session_id: str) -> bool:
|
||||
with self._lock:
|
||||
return session_id in self._sessions
|
||||
|
||||
# ----- Internals : segmentation --------------------------------------
|
||||
|
||||
def _should_cut(
|
||||
self, state: Dict[str, Any], event: Dict[str, Any]
|
||||
) -> tuple:
|
||||
"""Décider si l'événement doit démarrer un nouveau segment.
|
||||
|
||||
Returns:
|
||||
(should_cut, reason)
|
||||
"""
|
||||
current = state.get("current_step")
|
||||
if current is None or not current.events:
|
||||
return (False, "")
|
||||
|
||||
# Coupure : changement d'application
|
||||
new_app = self._get_app_name(event)
|
||||
if new_app and current.app_name and new_app != current.app_name:
|
||||
return (True, "changement_application")
|
||||
|
||||
# Coupure : pause longue entre deux événements
|
||||
prev_ts = float(current.events[-1].get("timestamp", 0))
|
||||
curr_ts = float(event.get("timestamp", 0))
|
||||
if prev_ts > 0 and curr_ts > 0:
|
||||
if (curr_ts - prev_ts) > _SEUIL_PAUSE_LONGUE_S:
|
||||
return (True, "pause_longue")
|
||||
|
||||
# Coupure : key_combo « lourd » type ctrl+s (sauvegarde) → fin logique
|
||||
evt_type = event.get("type", "")
|
||||
if evt_type in ("key_combo", "key_press"):
|
||||
keys = [str(k).lower() for k in event.get("keys", [])]
|
||||
if "ctrl" in keys and any(k in keys for k in ("s", "enter")):
|
||||
# On accroche le key_combo à l'étape courante, puis on coupe
|
||||
# APRÈS — retourner False ici, la coupure se fera au prochain
|
||||
# événement. C'est voulu.
|
||||
return (False, "")
|
||||
|
||||
return (False, "")
|
||||
|
||||
def _initial_intent(self, event: Dict[str, Any]) -> str:
|
||||
"""Intention provisoire d'un tout nouveau segment."""
|
||||
app = self._get_app_name(event) or self._get_window_title(event)
|
||||
evt_type = event.get("type", "")
|
||||
if evt_type == "mouse_click":
|
||||
hint = event.get("vision_info", {}).get("text", "")
|
||||
if hint:
|
||||
return f"Cliquer sur « {hint} »"
|
||||
if app:
|
||||
return f"Interagir avec {app}"
|
||||
return "Cliquer quelque part"
|
||||
if evt_type == "text_input":
|
||||
text = event.get("text", "")[:40]
|
||||
return f"Saisir du texte" + (f" « {text} »" if text else "")
|
||||
if evt_type in ("key_combo", "key_press"):
|
||||
keys = event.get("keys", [])
|
||||
return f"Appuyer sur {'+'.join(keys)}" if keys else "Raccourci clavier"
|
||||
return f"Action dans {app}" if app else "Action"
|
||||
|
||||
def _refine_intent(
|
||||
self, step: UnderstoodStep, event: Dict[str, Any]
|
||||
) -> str:
|
||||
"""Raffiner l'intention au fur et à mesure qu'on voit plus d'événements.
|
||||
|
||||
Heuristiques simples — pas de gemma4 ici pour rester rapide.
|
||||
"""
|
||||
types = [e.get("type", "") for e in step.events]
|
||||
has_click = "mouse_click" in types
|
||||
has_type = "text_input" in types
|
||||
has_key = any(t in ("key_combo", "key_press") for t in types)
|
||||
app = step.app_name or self._get_window_title(event)
|
||||
|
||||
# Cas 1 : clic + saisie + entrée → "Rechercher X"
|
||||
if has_click and has_type:
|
||||
texts = [e.get("text", "") for e in step.events if e.get("type") == "text_input"]
|
||||
if texts and any("enter" in [k.lower() for k in e.get("keys", [])]
|
||||
for e in step.events if e.get("type") in ("key_combo", "key_press")):
|
||||
premier_texte = next((t for t in texts if t), "")
|
||||
if premier_texte:
|
||||
step.confidence = min(0.85, step.confidence + 0.05)
|
||||
return f"Rechercher « {premier_texte[:30]} »"
|
||||
|
||||
# Cas 2 : saisie seule → "Écrire du texte"
|
||||
if has_type and not has_click:
|
||||
texts = [e.get("text", "") for e in step.events if e.get("type") == "text_input"]
|
||||
premier_texte = next((t for t in texts if t), "")
|
||||
if premier_texte:
|
||||
return f"Écrire « {premier_texte[:40]} »"
|
||||
return "Écrire du texte"
|
||||
|
||||
# Cas 3 : ctrl+s → "Sauvegarder"
|
||||
if has_key:
|
||||
for e in step.events:
|
||||
if e.get("type") in ("key_combo", "key_press"):
|
||||
keys = [str(k).lower() for k in e.get("keys", [])]
|
||||
if "ctrl" in keys and "s" in keys:
|
||||
step.confidence = min(0.9, step.confidence + 0.1)
|
||||
return f"Sauvegarder{' dans ' + app if app else ''}"
|
||||
if "ctrl" in keys and "c" in keys:
|
||||
return f"Copier{' depuis ' + app if app else ''}"
|
||||
if "ctrl" in keys and "v" in keys:
|
||||
return f"Coller{' dans ' + app if app else ''}"
|
||||
|
||||
# Cas 4 : clic seul + app identifiable
|
||||
if has_click and app:
|
||||
hint = ""
|
||||
for e in step.events:
|
||||
if e.get("type") == "mouse_click":
|
||||
hint = e.get("vision_info", {}).get("text", "")
|
||||
if hint:
|
||||
break
|
||||
if hint:
|
||||
return f"Cliquer sur « {hint} » dans {app}"
|
||||
return f"Interagir avec {app}"
|
||||
|
||||
return step.intent
|
||||
|
||||
def _handle_text_input(
|
||||
self,
|
||||
session_id: str,
|
||||
step: UnderstoodStep,
|
||||
event: Dict[str, Any],
|
||||
) -> None:
|
||||
"""Détecter et notifier une variable lors d'une saisie texte."""
|
||||
text = (event.get("text") or "").strip()
|
||||
if not text or len(text) < 3:
|
||||
return
|
||||
|
||||
# Déduire un nom de variable provisoire
|
||||
var_name = f"texte_{len(step.variables_detectees) + 1}"
|
||||
step.variables_detectees.append(var_name)
|
||||
|
||||
# Heuristique : détecter le type plausible
|
||||
var_type = self._guess_variable_type(text)
|
||||
|
||||
self._notifier(
|
||||
session_id,
|
||||
NiveauNotification.VARIABLE,
|
||||
f"Variable détectée : tu as tapé « {text[:40]} » — c'est {var_type} ?",
|
||||
step_index=step.step_index,
|
||||
data={
|
||||
"variable_name": var_name,
|
||||
"value": text,
|
||||
"variable_type": var_type,
|
||||
},
|
||||
)
|
||||
|
||||
def _guess_variable_type(self, text: str) -> str:
|
||||
"""Deviner le type d'une variable à partir de sa valeur."""
|
||||
t = text.strip()
|
||||
# Date (basique)
|
||||
if len(t) == 10 and t[2] in "/-" and t[5] in "/-":
|
||||
return "une date"
|
||||
if t.isdigit():
|
||||
return "un numéro"
|
||||
if "@" in t and "." in t:
|
||||
return "une adresse e-mail"
|
||||
if len(t) <= 10 and t.replace(" ", "").replace("-", "").isalnum() and not any(c.islower() for c in t):
|
||||
return "un code"
|
||||
if " " in t and len(t) > 10:
|
||||
return "un texte libre"
|
||||
return "un texte"
|
||||
|
||||
# ----- Internals : notifications -------------------------------------
|
||||
|
||||
def _notifier(
|
||||
self,
|
||||
session_id: str,
|
||||
niveau: NiveauNotification,
|
||||
message: str,
|
||||
*,
|
||||
step_index: int = -1,
|
||||
data: Optional[Dict[str, Any]] = None,
|
||||
) -> None:
|
||||
"""Créer et émettre une notification."""
|
||||
notif = NotificationShadow(
|
||||
notif_id=uuid.uuid4().hex[:12],
|
||||
niveau=niveau,
|
||||
message=message,
|
||||
session_id=session_id,
|
||||
step_index=step_index,
|
||||
data=data or {},
|
||||
timestamp=time.time(),
|
||||
)
|
||||
|
||||
with self._lock:
|
||||
state = self._sessions.get(session_id)
|
||||
if state is not None:
|
||||
state["notifications"].append(notif)
|
||||
state["last_notif_ts"] = notif.timestamp
|
||||
|
||||
if self._notify_callback is not None:
|
||||
try:
|
||||
self._notify_callback(notif)
|
||||
except Exception as e:
|
||||
logger.debug(f"ShadowObserver: callback a échoué : {e}")
|
||||
|
||||
def _emit_step_closed(
|
||||
self,
|
||||
session_id: str,
|
||||
step: UnderstoodStep,
|
||||
reason: str,
|
||||
) -> None:
|
||||
"""Émettre une notification quand une étape est fermée."""
|
||||
raison_humaine = {
|
||||
"changement_application": "tu es passé à une autre application",
|
||||
"pause_longue": "tu as fait une pause",
|
||||
}.get(reason, "")
|
||||
|
||||
suffixe = f" ({raison_humaine})" if raison_humaine else ""
|
||||
self._notifier(
|
||||
session_id,
|
||||
NiveauNotification.DECOUVERTE,
|
||||
f"Nouvelle étape comprise : {step.intent}{suffixe}",
|
||||
step_index=step.step_index,
|
||||
data={"step": step.to_dict()},
|
||||
)
|
||||
|
||||
if self._enable_gemma4 and self._gemma4_callback is not None:
|
||||
# Non bloquant : on délègue au caller (qui peut utiliser un thread)
|
||||
try:
|
||||
self._gemma4_callback(self._copy_step(step))
|
||||
except Exception as e:
|
||||
logger.debug(f"ShadowObserver: gemma4_callback a échoué : {e}")
|
||||
|
||||
def _maybe_emit_heartbeat(
|
||||
self,
|
||||
session_id: str,
|
||||
state: Dict[str, Any],
|
||||
) -> None:
|
||||
"""Émettre un résumé périodique (toutes les 5s env.)."""
|
||||
now = time.time()
|
||||
last = state.get("last_notif_ts", 0)
|
||||
if now - last < 5.0:
|
||||
return
|
||||
nb_steps = len(state["steps"])
|
||||
if state.get("current_step") is not None:
|
||||
nb_steps += 1
|
||||
if nb_steps == 0:
|
||||
return
|
||||
self._notifier(
|
||||
session_id,
|
||||
NiveauNotification.INFO,
|
||||
f"J'ai compris {nb_steps} étape(s) jusqu'ici.",
|
||||
data={"steps_count": nb_steps},
|
||||
)
|
||||
|
||||
# ----- Utilitaires ---------------------------------------------------
|
||||
|
||||
@staticmethod
|
||||
def _get_app_name(event: Dict[str, Any]) -> str:
|
||||
"""Extraire le nom d'application depuis un événement."""
|
||||
window = event.get("window") or {}
|
||||
if isinstance(window, dict):
|
||||
title = window.get("title", "")
|
||||
app_name = window.get("app_name", "")
|
||||
else:
|
||||
title = event.get("window_title", "")
|
||||
app_name = ""
|
||||
|
||||
# Préférer app_name si disponible
|
||||
if app_name and app_name != "unknown":
|
||||
return app_name
|
||||
|
||||
# Sinon, extraire depuis le titre
|
||||
for sep in [" – ", " - ", " — "]:
|
||||
if sep in title:
|
||||
return title.split(sep)[-1].strip()
|
||||
return title.strip() if title else ""
|
||||
|
||||
@staticmethod
|
||||
def _get_window_title(event: Dict[str, Any]) -> str:
|
||||
window = event.get("window") or {}
|
||||
if isinstance(window, dict):
|
||||
return window.get("title", "") or ""
|
||||
return event.get("window_title", "") or ""
|
||||
|
||||
@staticmethod
|
||||
def _copy_step(step: UnderstoodStep) -> UnderstoodStep:
|
||||
"""Copie superficielle pour éviter les fuites de mutation."""
|
||||
return UnderstoodStep(
|
||||
step_index=step.step_index,
|
||||
intent=step.intent,
|
||||
intent_provisoire=step.intent_provisoire,
|
||||
confidence=step.confidence,
|
||||
app_name=step.app_name,
|
||||
window_title=step.window_title,
|
||||
events=list(step.events),
|
||||
variables_detectees=list(step.variables_detectees),
|
||||
started_at=step.started_at,
|
||||
ended_at=step.ended_at,
|
||||
validated=step.validated,
|
||||
corrected=step.corrected,
|
||||
cancelled=step.cancelled,
|
||||
)
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Singleton partagé (optionnel)
|
||||
# =========================================================================
|
||||
|
||||
|
||||
_shared_observer: Optional[ShadowObserver] = None
|
||||
_shared_lock = threading.Lock()
|
||||
|
||||
|
||||
def get_shared_observer() -> ShadowObserver:
|
||||
"""Observer partagé pour l'API (lazy init)."""
|
||||
global _shared_observer
|
||||
with _shared_lock:
|
||||
if _shared_observer is None:
|
||||
_shared_observer = ShadowObserver()
|
||||
return _shared_observer
|
||||
468
core/workflow/shadow_validator.py
Normal file
468
core/workflow/shadow_validator.py
Normal file
@@ -0,0 +1,468 @@
|
||||
# core/workflow/shadow_validator.py
|
||||
"""
|
||||
ShadowValidator — Applique les feedbacks utilisateur et reconstruit un WorkflowIR.
|
||||
|
||||
Le ShadowObserver observe et comprend en temps réel. Le ShadowValidator,
|
||||
lui, prend les décisions de l'utilisateur (valider, corriger, annuler,
|
||||
combiner) et reconstruit un WorkflowIR final « propre » qui sera
|
||||
persisté et exécutable par le runtime.
|
||||
|
||||
Opérations supportées :
|
||||
- validate(step_index) : marquer l'étape comme validée
|
||||
- correct(step_index, new_intent) : corriger l'intention
|
||||
- undo(step_index) : annuler l'étape (elle sera exclue du WorkflowIR)
|
||||
- merge_with_next(step_index) : fusionner avec l'étape suivante
|
||||
- cancel() : annuler tout le workflow
|
||||
- split(step_index, at_event_index) : couper une étape en deux (bonus)
|
||||
|
||||
Le validator ne touche PAS aux événements bruts (events.jsonl) — il
|
||||
travaille sur la liste des `UnderstoodStep` fournie par le ShadowObserver.
|
||||
|
||||
Une fois toutes les actions appliquées, `build_workflow_ir()` produit
|
||||
un WorkflowIR exécutable à partir des étapes validées/corrigées.
|
||||
|
||||
Usage :
|
||||
|
||||
validator = ShadowValidator()
|
||||
validator.set_steps(observer.get_steps_internal(session_id))
|
||||
|
||||
validator.apply_feedback({"action": "validate", "step_index": 1})
|
||||
validator.apply_feedback({
|
||||
"action": "correct",
|
||||
"step_index": 2,
|
||||
"new_intent": "Sauvegarder le document",
|
||||
})
|
||||
validator.apply_feedback({"action": "undo", "step_index": 3})
|
||||
|
||||
ir = validator.build_workflow_ir(
|
||||
session_id="sess_abc",
|
||||
name="Mon workflow",
|
||||
domain="generic",
|
||||
)
|
||||
ir.save("data/workflows/")
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from .shadow_observer import UnderstoodStep
|
||||
from .workflow_ir import Action, Step, Variable, WorkflowIR
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# Actions supportées par le feedback
|
||||
FEEDBACK_ACTIONS = {
|
||||
"validate",
|
||||
"correct",
|
||||
"undo",
|
||||
"cancel",
|
||||
"merge_next",
|
||||
"split",
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class FeedbackResult:
|
||||
"""Résultat d'une opération de feedback."""
|
||||
|
||||
ok: bool
|
||||
action: str
|
||||
step_index: int
|
||||
message: str
|
||||
data: Dict[str, Any]
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"ok": self.ok,
|
||||
"action": self.action,
|
||||
"step_index": self.step_index,
|
||||
"message": self.message,
|
||||
"data": dict(self.data),
|
||||
}
|
||||
|
||||
|
||||
class ShadowValidator:
|
||||
"""Applique les feedbacks utilisateur et produit un WorkflowIR."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._steps: List[UnderstoodStep] = []
|
||||
self._cancelled_workflow: bool = False
|
||||
self._history: List[FeedbackResult] = []
|
||||
|
||||
# ----- API -----------------------------------------------------------
|
||||
|
||||
def set_steps(self, steps: List[UnderstoodStep]) -> None:
|
||||
"""Initialiser le validator avec la liste des étapes observées."""
|
||||
self._steps = [self._clone(s) for s in steps]
|
||||
self._cancelled_workflow = False
|
||||
self._history = []
|
||||
|
||||
@property
|
||||
def steps(self) -> List[UnderstoodStep]:
|
||||
"""Vue en lecture des étapes courantes."""
|
||||
return list(self._steps)
|
||||
|
||||
@property
|
||||
def history(self) -> List[FeedbackResult]:
|
||||
"""Historique des feedbacks appliqués."""
|
||||
return list(self._history)
|
||||
|
||||
@property
|
||||
def is_cancelled(self) -> bool:
|
||||
return self._cancelled_workflow
|
||||
|
||||
def apply_feedback(self, feedback: Dict[str, Any]) -> FeedbackResult:
|
||||
"""Appliquer un feedback utilisateur.
|
||||
|
||||
Le `feedback` est un dict au format :
|
||||
{
|
||||
"action": "validate" | "correct" | "undo" | "cancel" | "merge_next" | "split",
|
||||
"step_index": 1, # Index 1-based (comme dans get_understanding)
|
||||
"new_intent": "...", # Pour correct
|
||||
"at_event_index": 3, # Pour split
|
||||
}
|
||||
|
||||
Returns:
|
||||
FeedbackResult
|
||||
"""
|
||||
action = (feedback.get("action") or "").strip()
|
||||
if action not in FEEDBACK_ACTIONS:
|
||||
return self._record(FeedbackResult(
|
||||
ok=False, action=action, step_index=-1,
|
||||
message=f"Action inconnue : « {action} »",
|
||||
data={"supported": sorted(FEEDBACK_ACTIONS)},
|
||||
))
|
||||
|
||||
if action == "cancel":
|
||||
return self._do_cancel()
|
||||
|
||||
step_index = int(feedback.get("step_index", -1))
|
||||
if not self._is_valid_step_index(step_index):
|
||||
return self._record(FeedbackResult(
|
||||
ok=False, action=action, step_index=step_index,
|
||||
message=f"Index d'étape invalide : {step_index}",
|
||||
data={"nb_steps": len(self._steps)},
|
||||
))
|
||||
|
||||
if action == "validate":
|
||||
return self._do_validate(step_index)
|
||||
if action == "correct":
|
||||
return self._do_correct(step_index, feedback.get("new_intent", ""))
|
||||
if action == "undo":
|
||||
return self._do_undo(step_index)
|
||||
if action == "merge_next":
|
||||
return self._do_merge_next(step_index)
|
||||
if action == "split":
|
||||
return self._do_split(
|
||||
step_index, int(feedback.get("at_event_index", -1))
|
||||
)
|
||||
|
||||
return self._record(FeedbackResult(
|
||||
ok=False, action=action, step_index=step_index,
|
||||
message="Action non implémentée", data={},
|
||||
))
|
||||
|
||||
def apply_feedbacks(
|
||||
self, feedbacks: List[Dict[str, Any]]
|
||||
) -> List[FeedbackResult]:
|
||||
"""Appliquer plusieurs feedbacks dans l'ordre."""
|
||||
return [self.apply_feedback(f) for f in feedbacks]
|
||||
|
||||
# ----- Opérations ---------------------------------------------------
|
||||
|
||||
def _do_validate(self, step_index: int) -> FeedbackResult:
|
||||
step = self._get_step(step_index)
|
||||
step.validated = True
|
||||
step.intent_provisoire = False
|
||||
step.confidence = max(step.confidence, 0.95)
|
||||
return self._record(FeedbackResult(
|
||||
ok=True, action="validate", step_index=step_index,
|
||||
message=f"Étape {step_index} validée : {step.intent}",
|
||||
data={"intent": step.intent},
|
||||
))
|
||||
|
||||
def _do_correct(
|
||||
self, step_index: int, new_intent: str
|
||||
) -> FeedbackResult:
|
||||
new_intent = (new_intent or "").strip()
|
||||
if not new_intent:
|
||||
return self._record(FeedbackResult(
|
||||
ok=False, action="correct", step_index=step_index,
|
||||
message="Nouvelle intention vide",
|
||||
data={},
|
||||
))
|
||||
step = self._get_step(step_index)
|
||||
old_intent = step.intent
|
||||
step.intent = new_intent
|
||||
step.corrected = True
|
||||
step.validated = True # Corriger = implicitement valider
|
||||
step.intent_provisoire = False
|
||||
step.confidence = 1.0
|
||||
return self._record(FeedbackResult(
|
||||
ok=True, action="correct", step_index=step_index,
|
||||
message=f"Étape {step_index} corrigée : « {old_intent} » → « {new_intent} »",
|
||||
data={"old_intent": old_intent, "new_intent": new_intent},
|
||||
))
|
||||
|
||||
def _do_undo(self, step_index: int) -> FeedbackResult:
|
||||
step = self._get_step(step_index)
|
||||
step.cancelled = True
|
||||
return self._record(FeedbackResult(
|
||||
ok=True, action="undo", step_index=step_index,
|
||||
message=f"Étape {step_index} annulée : {step.intent}",
|
||||
data={"intent": step.intent},
|
||||
))
|
||||
|
||||
def _do_merge_next(self, step_index: int) -> FeedbackResult:
|
||||
"""Fusionner l'étape avec la suivante."""
|
||||
if step_index >= len(self._steps):
|
||||
return self._record(FeedbackResult(
|
||||
ok=False, action="merge_next", step_index=step_index,
|
||||
message="Aucune étape suivante à fusionner",
|
||||
data={},
|
||||
))
|
||||
step = self._get_step(step_index)
|
||||
next_step = self._get_step(step_index + 1)
|
||||
|
||||
merged = UnderstoodStep(
|
||||
step_index=step.step_index,
|
||||
intent=step.intent if len(step.intent) >= len(next_step.intent) else next_step.intent,
|
||||
intent_provisoire=False,
|
||||
confidence=max(step.confidence, next_step.confidence),
|
||||
app_name=step.app_name or next_step.app_name,
|
||||
window_title=step.window_title or next_step.window_title,
|
||||
events=list(step.events) + list(next_step.events),
|
||||
variables_detectees=list(step.variables_detectees)
|
||||
+ list(next_step.variables_detectees),
|
||||
started_at=step.started_at or next_step.started_at,
|
||||
ended_at=next_step.ended_at or step.ended_at,
|
||||
validated=True,
|
||||
corrected=step.corrected or next_step.corrected,
|
||||
cancelled=False,
|
||||
)
|
||||
|
||||
# Remplacer [step, next_step] par [merged]
|
||||
idx0 = step_index - 1 # 1-based → 0-based
|
||||
self._steps.pop(idx0 + 1) # next_step
|
||||
self._steps[idx0] = merged
|
||||
self._renumber()
|
||||
|
||||
return self._record(FeedbackResult(
|
||||
ok=True, action="merge_next", step_index=step_index,
|
||||
message=f"Étapes {step_index} et {step_index + 1} fusionnées",
|
||||
data={"intent": merged.intent},
|
||||
))
|
||||
|
||||
def _do_split(
|
||||
self, step_index: int, at_event_index: int
|
||||
) -> FeedbackResult:
|
||||
"""Couper une étape en deux au niveau de l'événement at_event_index.
|
||||
|
||||
`at_event_index` est 0-based parmi les events de l'étape.
|
||||
"""
|
||||
step = self._get_step(step_index)
|
||||
if at_event_index <= 0 or at_event_index >= len(step.events):
|
||||
return self._record(FeedbackResult(
|
||||
ok=False, action="split", step_index=step_index,
|
||||
message=f"Index de coupe invalide : {at_event_index}",
|
||||
data={"nb_events": len(step.events)},
|
||||
))
|
||||
|
||||
left_events = step.events[:at_event_index]
|
||||
right_events = step.events[at_event_index:]
|
||||
|
||||
left = UnderstoodStep(
|
||||
step_index=step.step_index,
|
||||
intent=step.intent + " (1/2)",
|
||||
intent_provisoire=True,
|
||||
confidence=step.confidence * 0.9,
|
||||
app_name=step.app_name,
|
||||
window_title=step.window_title,
|
||||
events=left_events,
|
||||
started_at=step.started_at,
|
||||
)
|
||||
right = UnderstoodStep(
|
||||
step_index=step.step_index + 1,
|
||||
intent=step.intent + " (2/2)",
|
||||
intent_provisoire=True,
|
||||
confidence=step.confidence * 0.9,
|
||||
app_name=step.app_name,
|
||||
window_title=step.window_title,
|
||||
events=right_events,
|
||||
started_at=float(right_events[0].get("timestamp", 0))
|
||||
if right_events else step.started_at,
|
||||
ended_at=step.ended_at,
|
||||
)
|
||||
|
||||
idx0 = step_index - 1
|
||||
self._steps[idx0] = left
|
||||
self._steps.insert(idx0 + 1, right)
|
||||
self._renumber()
|
||||
|
||||
return self._record(FeedbackResult(
|
||||
ok=True, action="split", step_index=step_index,
|
||||
message=f"Étape {step_index} coupée en 2",
|
||||
data={"nb_steps": len(self._steps)},
|
||||
))
|
||||
|
||||
def _do_cancel(self) -> FeedbackResult:
|
||||
self._cancelled_workflow = True
|
||||
return self._record(FeedbackResult(
|
||||
ok=True, action="cancel", step_index=-1,
|
||||
message="Workflow annulé",
|
||||
data={},
|
||||
))
|
||||
|
||||
# ----- Construction du WorkflowIR -----------------------------------
|
||||
|
||||
def build_workflow_ir(
|
||||
self,
|
||||
session_id: str = "",
|
||||
name: str = "",
|
||||
domain: str = "generic",
|
||||
*,
|
||||
require_all_validated: bool = False,
|
||||
) -> Optional[WorkflowIR]:
|
||||
"""Construire un WorkflowIR à partir des étapes corrigées.
|
||||
|
||||
Args:
|
||||
session_id: Identifiant de la session source.
|
||||
name: Nom du workflow.
|
||||
domain: Domaine métier.
|
||||
require_all_validated: Si True, lève une erreur si au moins
|
||||
une étape n'a pas été validée explicitement.
|
||||
|
||||
Returns:
|
||||
WorkflowIR ou None si le workflow a été annulé.
|
||||
"""
|
||||
if self._cancelled_workflow:
|
||||
logger.info("ShadowValidator: workflow annulé, pas de build")
|
||||
return None
|
||||
|
||||
ir = WorkflowIR.new(
|
||||
name=name or f"Workflow du {time.strftime('%d/%m/%Y %H:%M')}",
|
||||
domain=domain,
|
||||
learned_from=session_id,
|
||||
)
|
||||
|
||||
variables: List[Variable] = []
|
||||
seen_texts = set()
|
||||
applications: set = set()
|
||||
|
||||
for step in self._steps:
|
||||
if step.cancelled:
|
||||
continue
|
||||
if require_all_validated and not step.validated:
|
||||
raise ValueError(
|
||||
f"Étape {step.step_index} non validée : {step.intent}"
|
||||
)
|
||||
|
||||
if step.app_name:
|
||||
applications.add(step.app_name)
|
||||
|
||||
actions = []
|
||||
for evt in step.events:
|
||||
action = self._event_to_action(evt)
|
||||
if action is None:
|
||||
continue
|
||||
|
||||
# Détection de variable (texte saisi)
|
||||
if action.type == "type" and action.text:
|
||||
text = action.text.strip()
|
||||
if text and text not in seen_texts and len(text) > 2:
|
||||
seen_texts.add(text)
|
||||
var_name = f"texte_{len(variables) + 1}"
|
||||
variables.append(Variable(
|
||||
name=var_name,
|
||||
description=f"Texte saisi : « {text[:50]} »",
|
||||
source="user",
|
||||
default=text,
|
||||
))
|
||||
action.variable = True
|
||||
action.text = "{" + var_name + "}"
|
||||
|
||||
actions.append(action)
|
||||
|
||||
ir_step = Step(
|
||||
step_id=f"s{len(ir.steps) + 1}",
|
||||
intent=step.intent,
|
||||
actions=actions,
|
||||
)
|
||||
ir.steps.append(ir_step)
|
||||
|
||||
ir.variables = variables
|
||||
ir.applications = sorted(applications)
|
||||
ir.updated_at = time.time()
|
||||
|
||||
logger.info(
|
||||
f"ShadowValidator: WorkflowIR construit — {len(ir.steps)} étapes, "
|
||||
f"{len(ir.variables)} variables"
|
||||
)
|
||||
return ir
|
||||
|
||||
# ----- Utilitaires --------------------------------------------------
|
||||
|
||||
def _is_valid_step_index(self, step_index: int) -> bool:
|
||||
return 1 <= step_index <= len(self._steps)
|
||||
|
||||
def _get_step(self, step_index: int) -> UnderstoodStep:
|
||||
return self._steps[step_index - 1]
|
||||
|
||||
def _renumber(self) -> None:
|
||||
for i, s in enumerate(self._steps, start=1):
|
||||
s.step_index = i
|
||||
|
||||
def _record(self, result: FeedbackResult) -> FeedbackResult:
|
||||
self._history.append(result)
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def _clone(step: UnderstoodStep) -> UnderstoodStep:
|
||||
return UnderstoodStep(
|
||||
step_index=step.step_index,
|
||||
intent=step.intent,
|
||||
intent_provisoire=step.intent_provisoire,
|
||||
confidence=step.confidence,
|
||||
app_name=step.app_name,
|
||||
window_title=step.window_title,
|
||||
events=list(step.events),
|
||||
variables_detectees=list(step.variables_detectees),
|
||||
started_at=step.started_at,
|
||||
ended_at=step.ended_at,
|
||||
validated=step.validated,
|
||||
corrected=step.corrected,
|
||||
cancelled=step.cancelled,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _event_to_action(evt: Dict[str, Any]) -> Optional[Action]:
|
||||
"""Convertir un événement brut en Action (miroir de IRBuilder)."""
|
||||
evt_type = evt.get("type", "")
|
||||
|
||||
if evt_type == "mouse_click":
|
||||
window = evt.get("window") or {}
|
||||
if isinstance(window, dict):
|
||||
target = window.get("title", "")
|
||||
else:
|
||||
target = evt.get("window_title", "")
|
||||
return Action(
|
||||
type="click",
|
||||
target=target or "",
|
||||
anchor_hint=(evt.get("vision_info") or {}).get("text", ""),
|
||||
)
|
||||
if evt_type == "text_input":
|
||||
text = evt.get("text", "")
|
||||
if text:
|
||||
return Action(type="type", text=text)
|
||||
if evt_type in ("key_combo", "key_press"):
|
||||
keys = evt.get("keys", [])
|
||||
if keys:
|
||||
return Action(type="key_combo", keys=list(keys))
|
||||
if evt_type == "scroll":
|
||||
return Action(type="scroll")
|
||||
return None
|
||||
337
core/workflow/surface_classifier.py
Normal file
337
core/workflow/surface_classifier.py
Normal file
@@ -0,0 +1,337 @@
|
||||
# core/workflow/surface_classifier.py
|
||||
"""
|
||||
SurfaceClassifier — détecte le type de surface applicative au moment de l'exécution.
|
||||
|
||||
4 types de surfaces reconnus :
|
||||
- citrix : session Citrix/RDP/TSE (wfica32.exe, mstsc.exe, CDViewer.exe)
|
||||
→ vision pure obligatoire, paramètres tolérants
|
||||
- windows_native : application Windows native (notepad.exe, explorer.exe, DPI...)
|
||||
→ vision + UIA bonus, paramètres standards
|
||||
- web_local : navigateur local (chrome.exe, firefox.exe, msedge.exe)
|
||||
→ vision + DOM/CDP bonus (si activé), paramètres rapides
|
||||
- unknown : fallback → vision pure, paramètres par défaut
|
||||
|
||||
Le classifier s'exécute UNE SEULE FOIS au début d'une session ou d'un replay.
|
||||
Son résultat détermine :
|
||||
1. Quels helpers sont activés (UIA ? CDP ?)
|
||||
2. Les paramètres de résolution (timeouts, seuils OCR)
|
||||
3. La stratégie de recovery
|
||||
|
||||
Principe : la vision reste le fondement. Le classifier décide juste
|
||||
des bonus à activer et des paramètres à tuner.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import platform
|
||||
import subprocess
|
||||
from dataclasses import dataclass, field
|
||||
from enum import Enum
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SurfaceType(str, Enum):
|
||||
"""Types de surfaces applicatives."""
|
||||
CITRIX = "citrix"
|
||||
WINDOWS_NATIVE = "windows_native"
|
||||
WEB_LOCAL = "web_local"
|
||||
UNKNOWN = "unknown"
|
||||
|
||||
|
||||
# Processus connus par type de surface
|
||||
_CITRIX_PROCESSES = {
|
||||
"wfica32.exe", # Citrix Workspace (Windows 10+)
|
||||
"cdviewer.exe", # Citrix Desktop Viewer
|
||||
"cdviewer.exe",
|
||||
"mstsc.exe", # Microsoft Remote Desktop
|
||||
"vmware-vmx.exe", # VMware (cas RDS)
|
||||
"xen.exe", # Citrix XenApp
|
||||
"receiver.exe", # Citrix Receiver (ancien)
|
||||
"selfservice.exe", # Citrix Self-Service Plug-in
|
||||
}
|
||||
|
||||
_BROWSER_PROCESSES = {
|
||||
"chrome.exe",
|
||||
"msedge.exe",
|
||||
"firefox.exe",
|
||||
"brave.exe",
|
||||
"opera.exe",
|
||||
"vivaldi.exe",
|
||||
}
|
||||
|
||||
# Processus système Windows qui ne sont PAS des surfaces applicatives
|
||||
_SYSTEM_PROCESSES = {
|
||||
"explorer.exe", # Shell Windows (cas spécial — on le compte comme natif)
|
||||
"searchhost.exe", # Recherche Windows
|
||||
"startmenuexperiencehost.exe",
|
||||
"shellexperiencehost.exe",
|
||||
"applicationframehost.exe",
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class SurfaceProfile:
|
||||
"""Profil complet d'une surface détectée."""
|
||||
surface_type: SurfaceType
|
||||
process_name: str = "" # Processus de la fenêtre active
|
||||
window_title: str = "" # Titre de la fenêtre active
|
||||
confidence: float = 1.0 # Confiance de la détection (0-1)
|
||||
|
||||
# Capacités disponibles
|
||||
uia_available: bool = False # Le helper UIA peut être utilisé
|
||||
cdp_available: bool = False # Chrome DevTools Protocol accessible
|
||||
ocr_available: bool = True # OCR toujours dispo (docTR)
|
||||
vlm_available: bool = True # VLM toujours dispo (qwen2.5vl)
|
||||
|
||||
# Paramètres adaptés à la surface
|
||||
timeout_click_ms: int = 10000
|
||||
timeout_resolve_ms: int = 5000
|
||||
ocr_threshold: float = 0.75
|
||||
template_threshold: float = 0.85
|
||||
max_retries: int = 2
|
||||
retry_delay_ms: int = 2000
|
||||
|
||||
# Métadonnées
|
||||
detected_at: float = 0.0
|
||||
details: Dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"surface_type": self.surface_type.value,
|
||||
"process_name": self.process_name,
|
||||
"window_title": self.window_title,
|
||||
"confidence": round(self.confidence, 3),
|
||||
"capabilities": {
|
||||
"uia": self.uia_available,
|
||||
"cdp": self.cdp_available,
|
||||
"ocr": self.ocr_available,
|
||||
"vlm": self.vlm_available,
|
||||
},
|
||||
"parameters": {
|
||||
"timeout_click_ms": self.timeout_click_ms,
|
||||
"timeout_resolve_ms": self.timeout_resolve_ms,
|
||||
"ocr_threshold": self.ocr_threshold,
|
||||
"template_threshold": self.template_threshold,
|
||||
"max_retries": self.max_retries,
|
||||
"retry_delay_ms": self.retry_delay_ms,
|
||||
},
|
||||
"details": self.details,
|
||||
}
|
||||
|
||||
def resolve_order(self) -> List[str]:
|
||||
"""Construire l'ordre de résolution selon la surface et les capacités."""
|
||||
order = []
|
||||
if self.uia_available and self.surface_type == SurfaceType.WINDOWS_NATIVE:
|
||||
order.append("uia")
|
||||
if self.cdp_available and self.surface_type == SurfaceType.WEB_LOCAL:
|
||||
order.append("dom")
|
||||
order.extend(["ocr", "template", "vlm"])
|
||||
return order
|
||||
|
||||
|
||||
class SurfaceClassifier:
|
||||
"""Détecte la surface et configure les paramètres adaptés.
|
||||
|
||||
Usage :
|
||||
classifier = SurfaceClassifier()
|
||||
profile = classifier.classify(process="notepad.exe", title="Sans titre – Bloc-notes")
|
||||
if profile.uia_available:
|
||||
# Utiliser lea_uia.exe
|
||||
"""
|
||||
|
||||
def __init__(self, uia_helper_path: str = ""):
|
||||
"""
|
||||
Args:
|
||||
uia_helper_path: Chemin vers lea_uia.exe (optionnel, auto-détection sinon)
|
||||
"""
|
||||
self._uia_helper_path = uia_helper_path or self._find_uia_helper()
|
||||
|
||||
def _find_uia_helper(self) -> str:
|
||||
"""Trouver lea_uia.exe dans les emplacements standards."""
|
||||
candidates = [
|
||||
r"C:\Lea\helpers\lea_uia.exe",
|
||||
r".\helpers\lea_uia.exe",
|
||||
os.path.join(os.path.dirname(__file__), "..", "..", "agent_rust", "lea_uia",
|
||||
"target", "x86_64-pc-windows-gnu", "release", "lea_uia.exe"),
|
||||
]
|
||||
for path in candidates:
|
||||
if os.path.isfile(path):
|
||||
return os.path.abspath(path)
|
||||
return ""
|
||||
|
||||
def classify(
|
||||
self,
|
||||
process_name: str = "",
|
||||
window_title: str = "",
|
||||
screen_info: Optional[Dict] = None,
|
||||
) -> SurfaceProfile:
|
||||
"""Classifier une surface depuis le contexte fenêtre.
|
||||
|
||||
Args:
|
||||
process_name: Nom du processus (ex: "notepad.exe")
|
||||
window_title: Titre de la fenêtre active
|
||||
screen_info: Infos écran (résolution, DPI, compression détectée)
|
||||
"""
|
||||
import time
|
||||
|
||||
process_lower = process_name.lower().strip()
|
||||
title_lower = window_title.lower()
|
||||
|
||||
# Détection Citrix — priorité absolue
|
||||
if process_lower in _CITRIX_PROCESSES:
|
||||
return self._build_citrix_profile(process_name, window_title, time.time())
|
||||
|
||||
# Titre Citrix (ex: "Session Citrix", "Citrix Receiver")
|
||||
if any(marker in title_lower for marker in ["citrix", "ica session", "rdp session"]):
|
||||
return self._build_citrix_profile(process_name, window_title, time.time())
|
||||
|
||||
# Navigateur
|
||||
if process_lower in _BROWSER_PROCESSES:
|
||||
# Cas particulier : navigateur qui contient du Citrix embedded
|
||||
if "citrix" in title_lower:
|
||||
return self._build_citrix_profile(process_name, window_title, time.time())
|
||||
return self._build_web_profile(process_name, window_title, time.time())
|
||||
|
||||
# Application Windows native
|
||||
if process_lower.endswith(".exe") and process_lower not in _SYSTEM_PROCESSES:
|
||||
return self._build_windows_profile(process_name, window_title, time.time())
|
||||
|
||||
# Shell Windows (explorer.exe) — compté comme natif
|
||||
if process_lower == "explorer.exe":
|
||||
return self._build_windows_profile(process_name, window_title, time.time())
|
||||
|
||||
# Unknown — fallback sûr
|
||||
return self._build_unknown_profile(process_name, window_title, time.time())
|
||||
|
||||
def _build_citrix_profile(self, process: str, title: str, ts: float) -> SurfaceProfile:
|
||||
"""Profil Citrix — vision pure, paramètres tolérants."""
|
||||
return SurfaceProfile(
|
||||
surface_type=SurfaceType.CITRIX,
|
||||
process_name=process,
|
||||
window_title=title,
|
||||
confidence=0.95,
|
||||
uia_available=False, # UIA n'est pas dispo dans Citrix
|
||||
cdp_available=False,
|
||||
ocr_available=True,
|
||||
vlm_available=True,
|
||||
# Citrix : compression JPEG, latence, retries agressifs
|
||||
timeout_click_ms=15000,
|
||||
timeout_resolve_ms=10000,
|
||||
ocr_threshold=0.65, # Plus tolérant (compression)
|
||||
template_threshold=0.75, # Plus tolérant
|
||||
max_retries=3,
|
||||
retry_delay_ms=3000,
|
||||
detected_at=ts,
|
||||
details={"reason": "citrix_process_or_title"},
|
||||
)
|
||||
|
||||
def _build_windows_profile(self, process: str, title: str, ts: float) -> SurfaceProfile:
|
||||
"""Profil Windows natif — vision + UIA bonus."""
|
||||
uia_ok = self._check_uia_available()
|
||||
return SurfaceProfile(
|
||||
surface_type=SurfaceType.WINDOWS_NATIVE,
|
||||
process_name=process,
|
||||
window_title=title,
|
||||
confidence=0.9,
|
||||
uia_available=uia_ok,
|
||||
cdp_available=False,
|
||||
ocr_available=True,
|
||||
vlm_available=True,
|
||||
timeout_click_ms=8000,
|
||||
timeout_resolve_ms=5000,
|
||||
ocr_threshold=0.75,
|
||||
template_threshold=0.85,
|
||||
max_retries=2,
|
||||
retry_delay_ms=2000,
|
||||
detected_at=ts,
|
||||
details={
|
||||
"reason": "native_windows_process",
|
||||
"uia_helper": self._uia_helper_path if uia_ok else "",
|
||||
},
|
||||
)
|
||||
|
||||
def _build_web_profile(self, process: str, title: str, ts: float) -> SurfaceProfile:
|
||||
"""Profil web local — vision (+ CDP plus tard)."""
|
||||
cdp_ok = self._check_cdp_available()
|
||||
return SurfaceProfile(
|
||||
surface_type=SurfaceType.WEB_LOCAL,
|
||||
process_name=process,
|
||||
window_title=title,
|
||||
confidence=0.9,
|
||||
uia_available=False, # UIA limité pour les navigateurs
|
||||
cdp_available=cdp_ok,
|
||||
ocr_available=True,
|
||||
vlm_available=True,
|
||||
# Web local : rapide, texte bien rendu
|
||||
timeout_click_ms=5000,
|
||||
timeout_resolve_ms=3000,
|
||||
ocr_threshold=0.80,
|
||||
template_threshold=0.88,
|
||||
max_retries=1,
|
||||
retry_delay_ms=1000,
|
||||
detected_at=ts,
|
||||
details={"reason": "browser_process"},
|
||||
)
|
||||
|
||||
def _build_unknown_profile(self, process: str, title: str, ts: float) -> SurfaceProfile:
|
||||
"""Profil inconnu — paramètres sûrs par défaut."""
|
||||
return SurfaceProfile(
|
||||
surface_type=SurfaceType.UNKNOWN,
|
||||
process_name=process,
|
||||
window_title=title,
|
||||
confidence=0.5,
|
||||
uia_available=False,
|
||||
cdp_available=False,
|
||||
ocr_available=True,
|
||||
vlm_available=True,
|
||||
timeout_click_ms=10000,
|
||||
timeout_resolve_ms=5000,
|
||||
ocr_threshold=0.70,
|
||||
template_threshold=0.80,
|
||||
max_retries=2,
|
||||
retry_delay_ms=2000,
|
||||
detected_at=ts,
|
||||
details={"reason": "fallback"},
|
||||
)
|
||||
|
||||
def _check_uia_available(self) -> bool:
|
||||
"""Vérifier que lea_uia.exe est dispo et fonctionnel.
|
||||
|
||||
Sur Windows : appelle `lea_uia.exe health`.
|
||||
Sur Linux : toujours False (stub).
|
||||
"""
|
||||
if platform.system() != "Windows":
|
||||
return False
|
||||
if not self._uia_helper_path or not os.path.isfile(self._uia_helper_path):
|
||||
return False
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[self._uia_helper_path, "health"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=5,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
return False
|
||||
import json
|
||||
data = json.loads(result.stdout.strip())
|
||||
return data.get("status") == "ok"
|
||||
except Exception as e:
|
||||
logger.debug(f"UIA health check failed: {e}")
|
||||
return False
|
||||
|
||||
def _check_cdp_available(self) -> bool:
|
||||
"""Vérifier que Chrome DevTools Protocol est accessible.
|
||||
|
||||
Teste la présence d'un endpoint CDP sur localhost:9222.
|
||||
"""
|
||||
try:
|
||||
import urllib.request
|
||||
with urllib.request.urlopen(
|
||||
"http://localhost:9222/json/version", timeout=1
|
||||
) as resp:
|
||||
return resp.status == 200
|
||||
except Exception:
|
||||
return False
|
||||
278
core/workflow/uia_helper.py
Normal file
278
core/workflow/uia_helper.py
Normal file
@@ -0,0 +1,278 @@
|
||||
# core/workflow/uia_helper.py
|
||||
"""
|
||||
UIAHelper — Wrapper Python pour lea_uia.exe (helper Rust UI Automation).
|
||||
|
||||
Expose une API Python simple pour interroger UIA via le binaire Rust.
|
||||
Communique via subprocess + stdin/stdout JSON.
|
||||
|
||||
Pourquoi un helper Rust ?
|
||||
- 5-10x plus rapide que pywinauto (10-20ms vs 50-200ms)
|
||||
- Binaire standalone ~500 Ko, aucune dépendance runtime
|
||||
- Pas de problèmes de threading COM en Python
|
||||
- Crash-safe (le crash du helper n'affecte pas l'agent Python)
|
||||
|
||||
Architecture :
|
||||
Python executor
|
||||
↓ subprocess.run
|
||||
lea_uia.exe query --x 812 --y 436
|
||||
↓ UIA API Windows
|
||||
JSON response
|
||||
↓ stdout
|
||||
Python executor parse JSON
|
||||
|
||||
Si lea_uia.exe n'est pas disponible (Linux, binaire absent, crash) :
|
||||
toutes les méthodes retournent None → fallback vision automatique.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import platform
|
||||
import subprocess
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Timeout par défaut pour les appels UIA (en secondes)
|
||||
_DEFAULT_TIMEOUT = 5.0
|
||||
|
||||
|
||||
@dataclass
|
||||
class UiaElement:
|
||||
"""Représentation Python d'un élément UIA."""
|
||||
name: str = ""
|
||||
control_type: str = ""
|
||||
class_name: str = ""
|
||||
automation_id: str = ""
|
||||
bounding_rect: Tuple[int, int, int, int] = (0, 0, 0, 0)
|
||||
is_enabled: bool = False
|
||||
is_offscreen: bool = True
|
||||
parent_path: List[Dict[str, str]] = field(default_factory=list)
|
||||
process_name: str = ""
|
||||
|
||||
def center(self) -> Tuple[int, int]:
|
||||
"""Retourner le centre du rectangle (pixels)."""
|
||||
x1, y1, x2, y2 = self.bounding_rect
|
||||
return ((x1 + x2) // 2, (y1 + y2) // 2)
|
||||
|
||||
def width(self) -> int:
|
||||
return self.bounding_rect[2] - self.bounding_rect[0]
|
||||
|
||||
def height(self) -> int:
|
||||
return self.bounding_rect[3] - self.bounding_rect[1]
|
||||
|
||||
def is_clickable(self) -> bool:
|
||||
"""Peut-on cliquer dessus ?"""
|
||||
return (
|
||||
self.is_enabled
|
||||
and not self.is_offscreen
|
||||
and self.width() > 0
|
||||
and self.height() > 0
|
||||
)
|
||||
|
||||
def path_signature(self) -> str:
|
||||
"""Signature du chemin parent (pour retrouver l'élément)."""
|
||||
parts = [f"{p['control_type']}[{p['name']}]" for p in self.parent_path if p.get("name")]
|
||||
parts.append(f"{self.control_type}[{self.name}]")
|
||||
return " > ".join(parts)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"name": self.name,
|
||||
"control_type": self.control_type,
|
||||
"class_name": self.class_name,
|
||||
"automation_id": self.automation_id,
|
||||
"bounding_rect": list(self.bounding_rect),
|
||||
"is_enabled": self.is_enabled,
|
||||
"is_offscreen": self.is_offscreen,
|
||||
"parent_path": self.parent_path,
|
||||
"process_name": self.process_name,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, d: Dict[str, Any]) -> "UiaElement":
|
||||
rect = d.get("bounding_rect", [0, 0, 0, 0])
|
||||
if isinstance(rect, list) and len(rect) >= 4:
|
||||
rect = tuple(rect[:4])
|
||||
else:
|
||||
rect = (0, 0, 0, 0)
|
||||
return cls(
|
||||
name=d.get("name", ""),
|
||||
control_type=d.get("control_type", ""),
|
||||
class_name=d.get("class_name", ""),
|
||||
automation_id=d.get("automation_id", ""),
|
||||
bounding_rect=rect,
|
||||
is_enabled=d.get("is_enabled", False),
|
||||
is_offscreen=d.get("is_offscreen", True),
|
||||
parent_path=d.get("parent_path", []),
|
||||
process_name=d.get("process_name", ""),
|
||||
)
|
||||
|
||||
|
||||
class UIAHelper:
|
||||
"""Wrapper Python pour lea_uia.exe."""
|
||||
|
||||
def __init__(self, helper_path: str = "", timeout: float = _DEFAULT_TIMEOUT):
|
||||
self._helper_path = helper_path or self._find_helper()
|
||||
self._timeout = timeout
|
||||
self._available = self._check_available()
|
||||
|
||||
def _find_helper(self) -> str:
|
||||
"""Trouver lea_uia.exe dans les emplacements standards."""
|
||||
candidates = [
|
||||
r"C:\Lea\helpers\lea_uia.exe",
|
||||
os.path.join(os.path.dirname(__file__), "..", "..",
|
||||
"agent_rust", "lea_uia", "target",
|
||||
"x86_64-pc-windows-gnu", "release", "lea_uia.exe"),
|
||||
"./helpers/lea_uia.exe",
|
||||
"lea_uia.exe",
|
||||
]
|
||||
for path in candidates:
|
||||
if os.path.isfile(path):
|
||||
return os.path.abspath(path)
|
||||
return ""
|
||||
|
||||
def _check_available(self) -> bool:
|
||||
"""Vérifier que le helper est utilisable (Windows + binaire + health OK)."""
|
||||
if platform.system() != "Windows":
|
||||
logger.debug("UIAHelper: Linux/Mac — helper désactivé")
|
||||
return False
|
||||
if not self._helper_path:
|
||||
logger.debug("UIAHelper: lea_uia.exe introuvable")
|
||||
return False
|
||||
if not os.path.isfile(self._helper_path):
|
||||
logger.debug(f"UIAHelper: chemin invalide {self._helper_path}")
|
||||
return False
|
||||
return True
|
||||
|
||||
@property
|
||||
def available(self) -> bool:
|
||||
return self._available
|
||||
|
||||
@property
|
||||
def helper_path(self) -> str:
|
||||
return self._helper_path
|
||||
|
||||
def _run(self, args: List[str]) -> Optional[Dict[str, Any]]:
|
||||
"""Exécuter lea_uia.exe avec les arguments et parser le JSON."""
|
||||
if not self._available:
|
||||
return None
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[self._helper_path] + args,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=self._timeout,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
)
|
||||
if result.returncode != 0:
|
||||
logger.debug(
|
||||
f"UIAHelper: exit code {result.returncode}, "
|
||||
f"stderr: {result.stderr[:200]}"
|
||||
)
|
||||
return None
|
||||
output = result.stdout.strip()
|
||||
if not output:
|
||||
return None
|
||||
return json.loads(output)
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.debug(f"UIAHelper: timeout ({self._timeout}s) sur {args}")
|
||||
return None
|
||||
except json.JSONDecodeError as e:
|
||||
logger.debug(f"UIAHelper: JSON invalide — {e}")
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.debug(f"UIAHelper: erreur {e}")
|
||||
return None
|
||||
|
||||
def health(self) -> bool:
|
||||
"""Vérifier que UIA répond."""
|
||||
data = self._run(["health"])
|
||||
return data is not None and data.get("status") == "ok"
|
||||
|
||||
def query_at(
|
||||
self,
|
||||
x: int,
|
||||
y: int,
|
||||
with_parents: bool = True,
|
||||
) -> Optional[UiaElement]:
|
||||
"""Récupérer l'élément UIA à une position écran.
|
||||
|
||||
Args:
|
||||
x, y: Coordonnées pixel absolues
|
||||
with_parents: Inclure la hiérarchie des parents
|
||||
|
||||
Returns:
|
||||
UiaElement si trouvé, None sinon (pas d'élément ou UIA indispo)
|
||||
"""
|
||||
args = ["query", "--x", str(x), "--y", str(y)]
|
||||
if not with_parents:
|
||||
args.append("--with-parents=false")
|
||||
|
||||
data = self._run(args)
|
||||
if not data or data.get("status") != "ok":
|
||||
return None
|
||||
|
||||
elem_data = data.get("element")
|
||||
if not elem_data:
|
||||
return None
|
||||
return UiaElement.from_dict(elem_data)
|
||||
|
||||
def find_by_name(
|
||||
self,
|
||||
name: str,
|
||||
control_type: Optional[str] = None,
|
||||
automation_id: Optional[str] = None,
|
||||
window: Optional[str] = None,
|
||||
timeout_ms: int = 2000,
|
||||
) -> Optional[UiaElement]:
|
||||
"""Rechercher un élément par son nom (+ filtres optionnels).
|
||||
|
||||
Args:
|
||||
name: Nom exact de l'élément
|
||||
control_type: Type de contrôle (Button, Edit, MenuItem...)
|
||||
automation_id: ID d'automation
|
||||
window: Restreindre à une fenêtre spécifique
|
||||
timeout_ms: Timeout de recherche en millisecondes
|
||||
"""
|
||||
args = ["find", "--name", name, "--timeout-ms", str(timeout_ms)]
|
||||
if control_type:
|
||||
args.extend(["--control-type", control_type])
|
||||
if automation_id:
|
||||
args.extend(["--automation-id", automation_id])
|
||||
if window:
|
||||
args.extend(["--window", window])
|
||||
|
||||
data = self._run(args)
|
||||
if not data or data.get("status") != "ok":
|
||||
return None
|
||||
|
||||
elem_data = data.get("element")
|
||||
if not elem_data:
|
||||
return None
|
||||
return UiaElement.from_dict(elem_data)
|
||||
|
||||
def capture_focused(self, max_depth: int = 3) -> Optional[UiaElement]:
|
||||
"""Capturer l'élément ayant le focus + son contexte."""
|
||||
data = self._run(["capture", "--max-depth", str(max_depth)])
|
||||
if not data or data.get("status") != "ok":
|
||||
return None
|
||||
|
||||
elem_data = data.get("element")
|
||||
if not elem_data:
|
||||
return None
|
||||
return UiaElement.from_dict(elem_data)
|
||||
|
||||
|
||||
# Instance globale partagée (singleton léger)
|
||||
_SHARED_HELPER: Optional[UIAHelper] = None
|
||||
|
||||
|
||||
def get_shared_helper() -> UIAHelper:
|
||||
"""Retourner une instance partagée de UIAHelper."""
|
||||
global _SHARED_HELPER
|
||||
if _SHARED_HELPER is None:
|
||||
_SHARED_HELPER = UIAHelper()
|
||||
return _SHARED_HELPER
|
||||
278
core/workflow/workflow_ir.py
Normal file
278
core/workflow/workflow_ir.py
Normal file
@@ -0,0 +1,278 @@
|
||||
# core/workflow/workflow_ir.py
|
||||
"""
|
||||
WorkflowIR — Représentation Intermédiaire d'un workflow.
|
||||
|
||||
C'est la CONNAISSANCE que Léa a acquise en observant un utilisateur.
|
||||
Pas les clics bruts (RawTrace), pas le plan d'exécution (ExecutionPlan).
|
||||
C'est ce que Léa a COMPRIS.
|
||||
|
||||
Format générique — fonctionne pour n'importe quel métier :
|
||||
- TIM qui code des dossiers patients
|
||||
- Comptable qui saisit des factures
|
||||
- RH qui édite des fiches de paie
|
||||
- Logisticien qui gère des stocks
|
||||
|
||||
Le domaine métier est une couche par-dessus (domain_context),
|
||||
pas dans le WorkflowIR lui-même.
|
||||
|
||||
Cycle de vie :
|
||||
RawTrace (capture) → WorkflowIR (compréhension) → ExecutionPlan (exécution)
|
||||
|
||||
Le WorkflowIR est :
|
||||
- versionné (chaque recompilation incrémente la version)
|
||||
- indépendant de la résolution d'écran
|
||||
- indépendant du poste cible
|
||||
- paramétrable (variables substituables)
|
||||
- enrichi par l'apprentissage (chaque replay améliore le IR)
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
import uuid
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Structures de données
|
||||
# =========================================================================
|
||||
|
||||
|
||||
@dataclass
|
||||
class Variable:
|
||||
"""Variable substituable dans un workflow."""
|
||||
name: str # Identifiant (ex: "patient", "facture_num")
|
||||
description: str = "" # Description humaine
|
||||
source: str = "user" # Origine : "user", "screen", "file", "previous_step"
|
||||
default: str = "" # Valeur par défaut
|
||||
required: bool = True
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"name": self.name,
|
||||
"description": self.description,
|
||||
"source": self.source,
|
||||
"default": self.default,
|
||||
"required": self.required,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, d: Dict) -> "Variable":
|
||||
return cls(**{k: v for k, v in d.items() if k in cls.__dataclass_fields__})
|
||||
|
||||
|
||||
@dataclass
|
||||
class Action:
|
||||
"""Action élémentaire dans une étape."""
|
||||
type: str # click, type, key_combo, wait, scroll
|
||||
target: str = "" # Description de la cible ("bouton Enregistrer")
|
||||
text: str = "" # Texte à taper (pour type)
|
||||
keys: List[str] = field(default_factory=list) # Touches (pour key_combo)
|
||||
duration_ms: int = 0 # Durée (pour wait)
|
||||
variable: bool = False # True si le texte contient une variable {var}
|
||||
anchor_hint: str = "" # Indice visuel pour aider la résolution
|
||||
# Contrôle strict des étapes — l'action ne peut s'exécuter que si la fenêtre
|
||||
# active correspond à `expected_window_before`, et ne peut passer à la
|
||||
# suivante que si la fenêtre résultante correspond à `expected_window_after`.
|
||||
# Ces champs sont extraits par l'IRBuilder depuis les événements bruts.
|
||||
expected_window_before: str = ""
|
||||
expected_window_after: str = ""
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
d = {"type": self.type}
|
||||
if self.target:
|
||||
d["target"] = self.target
|
||||
if self.text:
|
||||
d["text"] = self.text
|
||||
if self.keys:
|
||||
d["keys"] = self.keys
|
||||
if self.duration_ms:
|
||||
d["duration_ms"] = self.duration_ms
|
||||
if self.variable:
|
||||
d["variable"] = True
|
||||
if self.anchor_hint:
|
||||
d["anchor_hint"] = self.anchor_hint
|
||||
if self.expected_window_before:
|
||||
d["expected_window_before"] = self.expected_window_before
|
||||
if self.expected_window_after:
|
||||
d["expected_window_after"] = self.expected_window_after
|
||||
return d
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, d: Dict) -> "Action":
|
||||
return cls(**{k: v for k, v in d.items() if k in cls.__dataclass_fields__})
|
||||
|
||||
|
||||
@dataclass
|
||||
class Step:
|
||||
"""Étape logique d'un workflow — une intention métier."""
|
||||
step_id: str
|
||||
intent: str # "Ouvrir le dossier", "Saisir le code"
|
||||
precondition: str = "" # "L'application est sur l'écran de liste"
|
||||
postcondition: str = "" # "Le dossier est affiché"
|
||||
actions: List[Action] = field(default_factory=list)
|
||||
is_optional: bool = False # Étape optionnelle (peut être sautée)
|
||||
is_loop: bool = False # Étape répétée (pour chaque élément)
|
||||
loop_variable: str = "" # Variable de boucle
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
d = {
|
||||
"step_id": self.step_id,
|
||||
"intent": self.intent,
|
||||
"actions": [a.to_dict() for a in self.actions],
|
||||
}
|
||||
if self.precondition:
|
||||
d["precondition"] = self.precondition
|
||||
if self.postcondition:
|
||||
d["postcondition"] = self.postcondition
|
||||
if self.is_optional:
|
||||
d["is_optional"] = True
|
||||
if self.is_loop:
|
||||
d["is_loop"] = True
|
||||
d["loop_variable"] = self.loop_variable
|
||||
return d
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, d: Dict) -> "Step":
|
||||
actions = [Action.from_dict(a) for a in d.get("actions", [])]
|
||||
return cls(
|
||||
step_id=d["step_id"],
|
||||
intent=d.get("intent", ""),
|
||||
precondition=d.get("precondition", ""),
|
||||
postcondition=d.get("postcondition", ""),
|
||||
actions=actions,
|
||||
is_optional=d.get("is_optional", False),
|
||||
is_loop=d.get("is_loop", False),
|
||||
loop_variable=d.get("loop_variable", ""),
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class WorkflowIR:
|
||||
"""Représentation Intermédiaire d'un workflow — la connaissance compilée.
|
||||
|
||||
C'est ce que Léa a compris en observant l'utilisateur.
|
||||
Indépendant du poste, de la résolution, du runtime.
|
||||
"""
|
||||
workflow_id: str
|
||||
version: int = 1
|
||||
name: str = ""
|
||||
description: str = ""
|
||||
domain: str = "generic" # Domaine métier (tim_codage, compta, rh, stocks...)
|
||||
learned_from: str = "" # session_id source
|
||||
created_at: float = 0.0
|
||||
updated_at: float = 0.0
|
||||
|
||||
# Contenu
|
||||
variables: List[Variable] = field(default_factory=list)
|
||||
steps: List[Step] = field(default_factory=list)
|
||||
|
||||
# Métadonnées d'apprentissage
|
||||
replay_count: int = 0 # Nombre de replays effectués
|
||||
success_rate: float = 0.0 # Taux de succès moyen
|
||||
last_replay_at: float = 0.0
|
||||
|
||||
# Applications utilisées (détectées lors de l'apprentissage)
|
||||
applications: List[str] = field(default_factory=list)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"workflow_id": self.workflow_id,
|
||||
"version": self.version,
|
||||
"name": self.name,
|
||||
"description": self.description,
|
||||
"domain": self.domain,
|
||||
"learned_from": self.learned_from,
|
||||
"created_at": self.created_at,
|
||||
"updated_at": self.updated_at,
|
||||
"variables": [v.to_dict() for v in self.variables],
|
||||
"steps": [s.to_dict() for s in self.steps],
|
||||
"replay_count": self.replay_count,
|
||||
"success_rate": round(self.success_rate, 3),
|
||||
"last_replay_at": self.last_replay_at,
|
||||
"applications": self.applications,
|
||||
}
|
||||
|
||||
def to_json(self, indent: int = 2) -> str:
|
||||
return json.dumps(self.to_dict(), ensure_ascii=False, indent=indent)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, d: Dict) -> "WorkflowIR":
|
||||
variables = [Variable.from_dict(v) for v in d.get("variables", [])]
|
||||
steps = [Step.from_dict(s) for s in d.get("steps", [])]
|
||||
return cls(
|
||||
workflow_id=d["workflow_id"],
|
||||
version=d.get("version", 1),
|
||||
name=d.get("name", ""),
|
||||
description=d.get("description", ""),
|
||||
domain=d.get("domain", "generic"),
|
||||
learned_from=d.get("learned_from", ""),
|
||||
created_at=d.get("created_at", 0),
|
||||
updated_at=d.get("updated_at", 0),
|
||||
variables=variables,
|
||||
steps=steps,
|
||||
replay_count=d.get("replay_count", 0),
|
||||
success_rate=d.get("success_rate", 0),
|
||||
last_replay_at=d.get("last_replay_at", 0),
|
||||
applications=d.get("applications", []),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_json(cls, json_str: str) -> "WorkflowIR":
|
||||
return cls.from_dict(json.loads(json_str))
|
||||
|
||||
def save(self, directory: str) -> Path:
|
||||
"""Sauvegarder le WorkflowIR dans un fichier JSON."""
|
||||
dir_path = Path(directory)
|
||||
dir_path.mkdir(parents=True, exist_ok=True)
|
||||
file_path = dir_path / f"{self.workflow_id}_v{self.version}.json"
|
||||
file_path.write_text(self.to_json(), encoding="utf-8")
|
||||
logger.info(f"WorkflowIR sauvegardé : {file_path}")
|
||||
return file_path
|
||||
|
||||
@classmethod
|
||||
def load(cls, file_path: str) -> "WorkflowIR":
|
||||
"""Charger un WorkflowIR depuis un fichier JSON."""
|
||||
return cls.from_json(Path(file_path).read_text(encoding="utf-8"))
|
||||
|
||||
def increment_version(self) -> "WorkflowIR":
|
||||
"""Créer une nouvelle version du workflow (après recompilation)."""
|
||||
import copy
|
||||
new = copy.deepcopy(self)
|
||||
new.version += 1
|
||||
new.updated_at = time.time()
|
||||
return new
|
||||
|
||||
def add_step(self, intent: str, actions: List[Dict] = None, **kwargs) -> Step:
|
||||
"""Ajouter une étape au workflow."""
|
||||
step = Step(
|
||||
step_id=f"s{len(self.steps) + 1}",
|
||||
intent=intent,
|
||||
actions=[Action.from_dict(a) for a in (actions or [])],
|
||||
**kwargs,
|
||||
)
|
||||
self.steps.append(step)
|
||||
return step
|
||||
|
||||
def add_variable(self, name: str, **kwargs) -> Variable:
|
||||
"""Ajouter une variable au workflow."""
|
||||
var = Variable(name=name, **kwargs)
|
||||
self.variables.append(var)
|
||||
return var
|
||||
|
||||
@staticmethod
|
||||
def new(name: str, domain: str = "generic", learned_from: str = "") -> "WorkflowIR":
|
||||
"""Créer un nouveau WorkflowIR vide."""
|
||||
return WorkflowIR(
|
||||
workflow_id=f"wf_{uuid.uuid4().hex[:12]}",
|
||||
version=1,
|
||||
name=name,
|
||||
domain=domain,
|
||||
learned_from=learned_from,
|
||||
created_at=time.time(),
|
||||
updated_at=time.time(),
|
||||
)
|
||||
441
tests/unit/test_chat_interface.py
Normal file
441
tests/unit/test_chat_interface.py
Normal file
@@ -0,0 +1,441 @@
|
||||
# tests/unit/test_chat_interface.py
|
||||
"""
|
||||
Tests unitaires du module chat_interface (Léa conversationnelle).
|
||||
|
||||
Vérifie :
|
||||
1. Création de session (état initial, message d'accueil)
|
||||
2. Envoi de message → appel TaskPlanner mocké
|
||||
3. Historique (get_history)
|
||||
4. Transitions d'états idle → planning → awaiting_confirmation → executing → done
|
||||
5. Abandon (utilisateur répond "non")
|
||||
6. Fallback gracieux quand gemma4/TaskPlanner indisponible
|
||||
7. ChatManager (création, listing, cleanup)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
_ROOT = str(Path(__file__).resolve().parents[2])
|
||||
if _ROOT not in sys.path:
|
||||
sys.path.insert(0, _ROOT)
|
||||
|
||||
from agent_v0.server_v1.chat_interface import (
|
||||
ChatSession,
|
||||
ChatManager,
|
||||
STATE_IDLE,
|
||||
STATE_PLANNING,
|
||||
STATE_AWAITING_CONFIRMATION,
|
||||
STATE_EXECUTING,
|
||||
STATE_DONE,
|
||||
STATE_ERROR,
|
||||
ROLE_USER,
|
||||
ROLE_LEA,
|
||||
)
|
||||
from agent_v0.server_v1.task_planner import TaskPlan
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Fixtures
|
||||
# =============================================================================
|
||||
|
||||
@pytest.fixture
|
||||
def sample_workflows():
|
||||
return [
|
||||
{
|
||||
"session_id": "sess_bloc_notes",
|
||||
"name": "Bloc-notes",
|
||||
"description": "Ouvrir Bloc-notes via Exécuter (Win+R) et écrire du texte",
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def understood_plan():
|
||||
"""TaskPlan qui a compris l'ordre et matche un workflow."""
|
||||
return TaskPlan(
|
||||
instruction="ouvre le bloc-notes et écris bonjour",
|
||||
understood=True,
|
||||
workflow_match="sess_bloc_notes",
|
||||
workflow_name="Bloc-notes",
|
||||
match_confidence=0.9,
|
||||
parameters={"texte": "bonjour"},
|
||||
is_loop=False,
|
||||
mode="replay",
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def unknown_plan():
|
||||
"""TaskPlan qui n'a pas compris."""
|
||||
return TaskPlan(
|
||||
instruction="fais le café",
|
||||
understood=False,
|
||||
error="aucun workflow ne correspond",
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_task_planner(understood_plan):
|
||||
planner = MagicMock()
|
||||
planner.understand.return_value = understood_plan
|
||||
return planner
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_replay_callback():
|
||||
return MagicMock(return_value="replay_abc123")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_status_provider():
|
||||
"""Retourne un dict par défaut 'running' — peut être modifié dans les tests."""
|
||||
return MagicMock(return_value={
|
||||
"status": "running",
|
||||
"completed_actions": 1,
|
||||
"total_actions": 5,
|
||||
})
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def session(mock_task_planner, sample_workflows, mock_replay_callback, mock_status_provider):
|
||||
return ChatSession(
|
||||
task_planner=mock_task_planner,
|
||||
workflows_provider=lambda: sample_workflows,
|
||||
replay_callback=mock_replay_callback,
|
||||
status_provider=mock_status_provider,
|
||||
)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Tests création session
|
||||
# =============================================================================
|
||||
|
||||
class TestSessionCreation:
|
||||
def test_session_id_generated(self):
|
||||
s = ChatSession()
|
||||
assert s.session_id.startswith("chat_")
|
||||
|
||||
def test_initial_state_is_idle(self):
|
||||
s = ChatSession()
|
||||
assert s.state == STATE_IDLE
|
||||
|
||||
def test_welcome_message_present(self):
|
||||
s = ChatSession()
|
||||
history = s.get_history()
|
||||
assert len(history) == 1
|
||||
assert history[0]["role"] == ROLE_LEA
|
||||
assert "Bonjour" in history[0]["content"] or "Léa" in history[0]["content"]
|
||||
|
||||
def test_session_id_custom(self):
|
||||
s = ChatSession(session_id="custom_42")
|
||||
assert s.session_id == "custom_42"
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Tests send_message
|
||||
# =============================================================================
|
||||
|
||||
class TestSendMessage:
|
||||
def test_empty_message_rejected(self, session):
|
||||
result = session.send_message("")
|
||||
assert result["ok"] is False
|
||||
|
||||
def test_send_message_calls_planner(self, session, mock_task_planner):
|
||||
session.send_message("ouvre le bloc-notes")
|
||||
mock_task_planner.understand.assert_called_once()
|
||||
call = mock_task_planner.understand.call_args
|
||||
assert call.kwargs["instruction"] == "ouvre le bloc-notes"
|
||||
# workflows_provider a été appelé et passé
|
||||
assert "available_workflows" in call.kwargs
|
||||
assert len(call.kwargs["available_workflows"]) == 1
|
||||
|
||||
def test_send_message_transitions_to_awaiting_confirmation(self, session):
|
||||
result = session.send_message("ouvre le bloc-notes")
|
||||
assert result["ok"] is True
|
||||
assert session.state == STATE_AWAITING_CONFIRMATION
|
||||
assert result["state"] == STATE_AWAITING_CONFIRMATION
|
||||
|
||||
def test_user_message_added_to_history(self, session):
|
||||
session.send_message("ouvre le bloc-notes")
|
||||
history = session.get_history()
|
||||
user_msgs = [m for m in history if m["role"] == ROLE_USER]
|
||||
assert len(user_msgs) == 1
|
||||
assert user_msgs[0]["content"] == "ouvre le bloc-notes"
|
||||
|
||||
def test_lea_proposal_added_to_history(self, session):
|
||||
session.send_message("ouvre le bloc-notes")
|
||||
history = session.get_history()
|
||||
lea_msgs = [m for m in history if m["role"] == ROLE_LEA]
|
||||
# Bienvenue + proposition
|
||||
assert len(lea_msgs) == 2
|
||||
proposal = lea_msgs[-1]["content"]
|
||||
assert "Bloc-notes" in proposal
|
||||
assert "oui" in proposal.lower() or "y aller" in proposal.lower()
|
||||
|
||||
def test_proposal_contains_confidence(self, session):
|
||||
session.send_message("ouvre le bloc-notes")
|
||||
history = session.get_history()
|
||||
proposal = history[-1]["content"]
|
||||
# 0.9 → 90%
|
||||
assert "90" in proposal
|
||||
|
||||
def test_proposal_contains_parameters(self, session):
|
||||
session.send_message("ouvre le bloc-notes")
|
||||
history = session.get_history()
|
||||
proposal = history[-1]["content"]
|
||||
assert "texte" in proposal
|
||||
assert "bonjour" in proposal
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Tests get_history
|
||||
# =============================================================================
|
||||
|
||||
class TestGetHistory:
|
||||
def test_history_returns_list_of_dicts(self, session):
|
||||
history = session.get_history()
|
||||
assert isinstance(history, list)
|
||||
assert all(isinstance(m, dict) for m in history)
|
||||
|
||||
def test_history_message_structure(self, session):
|
||||
history = session.get_history()
|
||||
msg = history[0]
|
||||
assert "role" in msg
|
||||
assert "content" in msg
|
||||
assert "timestamp" in msg
|
||||
assert "meta" in msg
|
||||
|
||||
def test_history_grows_with_messages(self, session):
|
||||
initial = len(session.get_history())
|
||||
session.send_message("ouvre le bloc-notes")
|
||||
assert len(session.get_history()) > initial
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Tests transitions d'états
|
||||
# =============================================================================
|
||||
|
||||
class TestStateTransitions:
|
||||
def test_full_happy_path(self, session, mock_task_planner, mock_replay_callback):
|
||||
"""idle → planning → awaiting_confirmation → executing → done."""
|
||||
# Départ : idle
|
||||
assert session.state == STATE_IDLE
|
||||
|
||||
# Envoi message → planning → awaiting_confirmation
|
||||
session.send_message("ouvre le bloc-notes")
|
||||
assert session.state == STATE_AWAITING_CONFIRMATION
|
||||
|
||||
# Confirmation → executing
|
||||
result = session.confirm(confirmed=True)
|
||||
assert result["ok"] is True
|
||||
assert session.state == STATE_EXECUTING
|
||||
mock_replay_callback.assert_called_once()
|
||||
call = mock_replay_callback.call_args
|
||||
assert call.kwargs["session_id"] == "sess_bloc_notes"
|
||||
|
||||
# Simulation : replay terminé → done
|
||||
session._status_provider.return_value = {
|
||||
"status": "done",
|
||||
"completed_actions": 5,
|
||||
"total_actions": 5,
|
||||
}
|
||||
session.refresh_progress()
|
||||
assert session.state == STATE_DONE
|
||||
|
||||
def test_confirm_via_message_oui(self, session, mock_replay_callback):
|
||||
"""Le TIM peut répondre 'oui' en message au lieu d'un bouton."""
|
||||
session.send_message("ouvre le bloc-notes")
|
||||
assert session.state == STATE_AWAITING_CONFIRMATION
|
||||
|
||||
session.send_message("oui")
|
||||
assert session.state == STATE_EXECUTING
|
||||
mock_replay_callback.assert_called_once()
|
||||
|
||||
def test_refusal_via_confirm_false(self, session, mock_replay_callback):
|
||||
"""confirm(False) → retour à idle, pas d'exécution."""
|
||||
session.send_message("ouvre le bloc-notes")
|
||||
result = session.confirm(confirmed=False)
|
||||
assert result["ok"] is True
|
||||
assert result["confirmed"] is False
|
||||
assert session.state == STATE_IDLE
|
||||
mock_replay_callback.assert_not_called()
|
||||
|
||||
def test_refusal_via_message_non(self, session, mock_replay_callback):
|
||||
"""Le TIM répond 'non' → annulation."""
|
||||
session.send_message("ouvre le bloc-notes")
|
||||
session.send_message("non")
|
||||
assert session.state == STATE_IDLE
|
||||
mock_replay_callback.assert_not_called()
|
||||
# Le message d'annulation doit être dans l'historique
|
||||
history = session.get_history()
|
||||
assert any("annule" in m["content"].lower() for m in history)
|
||||
|
||||
def test_ambiguous_confirmation_reply(self, session):
|
||||
"""Réponse ambiguë pendant awaiting_confirmation → demande de clarification."""
|
||||
session.send_message("ouvre le bloc-notes")
|
||||
result = session.send_message("peut-être")
|
||||
assert session.state == STATE_AWAITING_CONFIRMATION
|
||||
assert result.get("needs_clarification") is True
|
||||
|
||||
def test_failed_replay_transitions_to_error(self, session):
|
||||
"""replay_callback lève une exception → état error."""
|
||||
session._replay_callback = MagicMock(side_effect=RuntimeError("boom"))
|
||||
session.send_message("ouvre le bloc-notes")
|
||||
result = session.confirm(confirmed=True)
|
||||
assert result["ok"] is False
|
||||
assert session.state == STATE_ERROR
|
||||
|
||||
def test_replay_failure_from_status(self, session):
|
||||
"""Le replay rapporte 'failed' → état error."""
|
||||
session.send_message("ouvre le bloc-notes")
|
||||
session.confirm(confirmed=True)
|
||||
assert session.state == STATE_EXECUTING
|
||||
|
||||
session._status_provider.return_value = {
|
||||
"status": "failed",
|
||||
"error": "element introuvable",
|
||||
}
|
||||
session.refresh_progress()
|
||||
assert session.state == STATE_ERROR
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Tests fallback / résilience
|
||||
# =============================================================================
|
||||
|
||||
class TestResilience:
|
||||
def test_no_task_planner_graceful(self):
|
||||
"""Sans TaskPlanner, on reste gracieux."""
|
||||
s = ChatSession(task_planner=None)
|
||||
result = s.send_message("test")
|
||||
assert result["ok"] is False
|
||||
assert s.state == STATE_ERROR
|
||||
# Message d'erreur présent dans l'historique
|
||||
history = s.get_history()
|
||||
assert any("désolée" in m["content"].lower() or "indisponible" in m["content"].lower()
|
||||
for m in history)
|
||||
|
||||
def test_task_planner_exception_graceful(self, mock_replay_callback):
|
||||
"""TaskPlanner lève une exception (gemma4 down) → état error propre."""
|
||||
planner = MagicMock()
|
||||
planner.understand.side_effect = RuntimeError("gemma4 offline")
|
||||
|
||||
s = ChatSession(
|
||||
task_planner=planner,
|
||||
workflows_provider=lambda: [],
|
||||
replay_callback=mock_replay_callback,
|
||||
)
|
||||
result = s.send_message("test")
|
||||
assert result["ok"] is False
|
||||
assert s.state == STATE_ERROR
|
||||
|
||||
def test_instruction_not_understood(self, unknown_plan, mock_replay_callback):
|
||||
"""Plan.understood = False → message d'erreur explicite."""
|
||||
planner = MagicMock()
|
||||
planner.understand.return_value = unknown_plan
|
||||
|
||||
s = ChatSession(
|
||||
task_planner=planner,
|
||||
workflows_provider=lambda: [],
|
||||
replay_callback=mock_replay_callback,
|
||||
)
|
||||
result = s.send_message("fais le café")
|
||||
assert result["ok"] is False
|
||||
assert s.state == STATE_ERROR
|
||||
history = s.get_history()
|
||||
assert any("reformuler" in m["content"].lower() for m in history)
|
||||
|
||||
def test_no_replay_callback(self, mock_task_planner, sample_workflows):
|
||||
"""Sans replay_callback, on refuse l'exécution proprement."""
|
||||
s = ChatSession(
|
||||
task_planner=mock_task_planner,
|
||||
workflows_provider=lambda: sample_workflows,
|
||||
replay_callback=None,
|
||||
)
|
||||
s.send_message("ouvre le bloc-notes")
|
||||
result = s.confirm(confirmed=True)
|
||||
assert result["ok"] is False
|
||||
assert s.state == STATE_ERROR
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Tests snapshot
|
||||
# =============================================================================
|
||||
|
||||
class TestSnapshot:
|
||||
def test_snapshot_structure(self, session):
|
||||
snap = session.get_snapshot()
|
||||
assert "session_id" in snap
|
||||
assert "state" in snap
|
||||
assert "messages" in snap
|
||||
assert "pending_plan" in snap
|
||||
assert "active_replay_id" in snap
|
||||
assert "progress" in snap
|
||||
|
||||
def test_snapshot_includes_pending_plan_when_awaiting(self, session):
|
||||
session.send_message("ouvre le bloc-notes")
|
||||
snap = session.get_snapshot()
|
||||
assert snap["state"] == STATE_AWAITING_CONFIRMATION
|
||||
assert snap["pending_plan"] is not None
|
||||
assert snap["pending_plan"]["workflow_name"] == "Bloc-notes"
|
||||
|
||||
def test_snapshot_no_pending_plan_in_idle(self, session):
|
||||
snap = session.get_snapshot()
|
||||
assert snap["pending_plan"] is None
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Tests ChatManager
|
||||
# =============================================================================
|
||||
|
||||
class TestChatManager:
|
||||
def test_create_session(self, mock_task_planner, sample_workflows):
|
||||
mgr = ChatManager(
|
||||
task_planner=mock_task_planner,
|
||||
workflows_provider=lambda: sample_workflows,
|
||||
)
|
||||
s = mgr.create_session()
|
||||
assert s is not None
|
||||
assert s.session_id in [x["session_id"] for x in mgr.list_sessions()]
|
||||
|
||||
def test_get_session(self, mock_task_planner):
|
||||
mgr = ChatManager(task_planner=mock_task_planner)
|
||||
s = mgr.create_session()
|
||||
retrieved = mgr.get_session(s.session_id)
|
||||
assert retrieved is s
|
||||
|
||||
def test_get_session_not_found(self):
|
||||
mgr = ChatManager()
|
||||
assert mgr.get_session("unknown") is None
|
||||
|
||||
def test_delete_session(self, mock_task_planner):
|
||||
mgr = ChatManager(task_planner=mock_task_planner)
|
||||
s = mgr.create_session()
|
||||
assert mgr.delete_session(s.session_id) is True
|
||||
assert mgr.get_session(s.session_id) is None
|
||||
|
||||
def test_cleanup_old_sessions(self, mock_task_planner):
|
||||
mgr = ChatManager(task_planner=mock_task_planner)
|
||||
s = mgr.create_session()
|
||||
# Simuler une session très ancienne
|
||||
s.updated_at = time.time() - 100000
|
||||
removed = mgr.cleanup_old(max_age_s=3600)
|
||||
assert removed == 1
|
||||
assert mgr.get_session(s.session_id) is None
|
||||
|
||||
def test_list_sessions_structure(self, mock_task_planner):
|
||||
mgr = ChatManager(task_planner=mock_task_planner)
|
||||
mgr.create_session(machine_id="pc-01")
|
||||
sessions = mgr.list_sessions()
|
||||
assert len(sessions) == 1
|
||||
s = sessions[0]
|
||||
assert "session_id" in s
|
||||
assert "state" in s
|
||||
assert "machine_id" in s
|
||||
assert s["machine_id"] == "pc-01"
|
||||
543
tests/unit/test_domain_personality.py
Normal file
543
tests/unit/test_domain_personality.py
Normal file
@@ -0,0 +1,543 @@
|
||||
"""Tests unitaires pour la personnalité métier de Léa.
|
||||
|
||||
Couvre :
|
||||
- summarize_action : résumé d'actions en langage métier par domaine
|
||||
- pose_clarification_question : questions contextuelles quand Léa bloque
|
||||
- describe_workflow_outcome : rapports de fin en langage métier
|
||||
- Fallback domaine inconnu / vocabulaire synonyme
|
||||
- Intégration avec agent_v0.agent_v1.ui.messages (formatters enrichis)
|
||||
- Appel gemma4 mocké pour le raffinement de résumé
|
||||
|
||||
Tous les tests sont 100% offline : aucun appel réseau réel.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
# Assurer que la racine du projet est dans le path (comme les autres tests unit)
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from agent_v0.server_v1.domain_context import (
|
||||
DomainContext,
|
||||
get_domain_context,
|
||||
list_domains,
|
||||
register_domain,
|
||||
)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Domaines pré-configurés
|
||||
# ============================================================================
|
||||
|
||||
|
||||
class TestDomainesDisponibles:
|
||||
"""Tous les domaines prévus doivent être enregistrés."""
|
||||
|
||||
def test_tim_codage_present(self):
|
||||
ctx = get_domain_context("tim_codage")
|
||||
assert ctx.domain_id == "tim_codage"
|
||||
assert "CIM-10" in ctx.vocabulary
|
||||
assert ctx.common_actions # non vide
|
||||
assert ctx.clarification_templates
|
||||
assert ctx.summary_templates
|
||||
|
||||
def test_comptabilite_present(self):
|
||||
ctx = get_domain_context("comptabilite")
|
||||
assert ctx.domain_id == "comptabilite"
|
||||
assert "facture" in ctx.vocabulary
|
||||
assert ctx.summary_templates["item_plural"] == "factures"
|
||||
|
||||
def test_rh_paie_present(self):
|
||||
ctx = get_domain_context("rh_paie")
|
||||
assert ctx.domain_id == "rh_paie"
|
||||
assert "bulletin" in ctx.vocabulary
|
||||
assert ctx.summary_templates["item_plural"] == "bulletins"
|
||||
|
||||
def test_stocks_logistique_present(self):
|
||||
ctx = get_domain_context("stocks_logistique")
|
||||
assert ctx.domain_id == "stocks_logistique"
|
||||
assert "BC" in ctx.vocabulary or "bon de commande" in ctx.vocabulary
|
||||
assert ctx.summary_templates["item_plural"] == "bons"
|
||||
|
||||
def test_generic_fallback(self):
|
||||
"""Un domaine inconnu retourne le contexte générique."""
|
||||
ctx = get_domain_context("n_existe_pas_42")
|
||||
assert ctx.domain_id == "generic"
|
||||
|
||||
def test_list_domains_contains_all(self):
|
||||
ids = {d["domain_id"] for d in list_domains()}
|
||||
assert {
|
||||
"tim_codage",
|
||||
"comptabilite",
|
||||
"rh_paie",
|
||||
"stocks_logistique",
|
||||
"generic",
|
||||
}.issubset(ids)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# summarize_action — résumé d'actions
|
||||
# ============================================================================
|
||||
|
||||
|
||||
class TestSummarizeAction:
|
||||
"""Résumés en langage métier par domaine."""
|
||||
|
||||
def test_tim_click_dp_saisir_diagnostic_principal(self):
|
||||
ctx = get_domain_context("tim_codage")
|
||||
phrase = ctx.summarize_action("click", {"target": "DP"})
|
||||
assert phrase == "saisir le diagnostic principal"
|
||||
|
||||
def test_tim_click_valider_codage(self):
|
||||
ctx = get_domain_context("tim_codage")
|
||||
phrase = ctx.summarize_action("click", {"target": "Valider le codage"})
|
||||
assert phrase == "valider le codage"
|
||||
|
||||
def test_tim_click_dossier_patient(self):
|
||||
ctx = get_domain_context("tim_codage")
|
||||
phrase = ctx.summarize_action(
|
||||
"click", {"target": "Ouvrir le dossier patient"}
|
||||
)
|
||||
assert phrase == "ouvrir le dossier patient"
|
||||
|
||||
def test_compta_type_ht(self):
|
||||
ctx = get_domain_context("comptabilite")
|
||||
phrase = ctx.summarize_action(
|
||||
"type", {"target": "Montant HT", "text": "1500"}
|
||||
)
|
||||
# La mention "ht" dans la cible déclenche le mapping
|
||||
assert phrase == "saisir le montant hors taxes"
|
||||
|
||||
def test_compta_click_lettrer(self):
|
||||
ctx = get_domain_context("comptabilite")
|
||||
phrase = ctx.summarize_action("click", {"target": "Lettrer"})
|
||||
assert phrase == "lettrer les écritures"
|
||||
|
||||
def test_rh_click_bulletin(self):
|
||||
ctx = get_domain_context("rh_paie")
|
||||
phrase = ctx.summarize_action("click", {"target": "Bulletin de paie"})
|
||||
assert phrase == "ouvrir le bulletin de paie"
|
||||
|
||||
def test_stocks_type_quantite(self):
|
||||
ctx = get_domain_context("stocks_logistique")
|
||||
phrase = ctx.summarize_action(
|
||||
"type", {"target": "Quantité reçue", "text": "42"}
|
||||
)
|
||||
assert phrase == "saisir la quantité"
|
||||
|
||||
def test_generic_click_fallback(self):
|
||||
ctx = get_domain_context("generic")
|
||||
phrase = ctx.summarize_action("click", {"target": "Bouton quelconque"})
|
||||
# Pas de mapping mais une description → "cliquer sur ..."
|
||||
assert "cliquer sur" in phrase
|
||||
|
||||
def test_unknown_domain_click(self):
|
||||
"""Un domaine inconnu ne plante pas."""
|
||||
ctx = get_domain_context("inconnu")
|
||||
phrase = ctx.summarize_action("click", {"target": "Quelque chose"})
|
||||
assert phrase # non vide
|
||||
assert "cliquer" in phrase
|
||||
|
||||
def test_tim_synonymes_dp_dans_cible_longue(self):
|
||||
"""Si aucun mapping exact mais la cible contient DP → substitution synonyme."""
|
||||
ctx = get_domain_context("tim_codage")
|
||||
# Aucun mapping direct "saisir le" mais "DP" est dans les synonymes
|
||||
phrase = ctx.summarize_action("click", {"target": "Saisir le DP"})
|
||||
assert phrase == "saisir le diagnostic principal"
|
||||
|
||||
def test_key_combo_generic(self):
|
||||
ctx = get_domain_context("generic")
|
||||
phrase = ctx.summarize_action("key_combo", {"keys": ["ctrl", "s"]})
|
||||
assert "ctrl+s" in phrase
|
||||
|
||||
def test_wait_and_scroll(self):
|
||||
ctx = get_domain_context("tim_codage")
|
||||
assert "attendre" in ctx.summarize_action("wait", {})
|
||||
assert "défiler" in ctx.summarize_action("scroll", {})
|
||||
|
||||
def test_type_no_target(self):
|
||||
ctx = get_domain_context("generic")
|
||||
phrase = ctx.summarize_action("type", {"text": "hello"})
|
||||
assert "hello" in phrase
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# pose_clarification_question — questions de blocage
|
||||
# ============================================================================
|
||||
|
||||
|
||||
class TestClarification:
|
||||
"""Questions posées par Léa en cas de blocage."""
|
||||
|
||||
def test_tim_fichier_patient_avec_nom(self):
|
||||
ctx = get_domain_context("tim_codage")
|
||||
question = ctx.pose_clarification_question(
|
||||
{
|
||||
"blocked_on": "target_not_found",
|
||||
"target": "Fichier patient",
|
||||
"params": {"nom_patient": "Mme Durand"},
|
||||
}
|
||||
)
|
||||
assert "Mme Durand" in question
|
||||
# Langage métier : mention "dossier" (pas juste "fichier")
|
||||
assert "dossier" in question.lower()
|
||||
|
||||
def test_compta_montant_avec_num_facture(self):
|
||||
ctx = get_domain_context("comptabilite")
|
||||
question = ctx.pose_clarification_question(
|
||||
{
|
||||
"blocked_on": "target_not_found",
|
||||
"target": "Montant HT",
|
||||
"params": {"num_facture": "F2026-0145"},
|
||||
}
|
||||
)
|
||||
assert "F2026-0145" in question
|
||||
assert "Montant HT" in question or "Montant" in question
|
||||
|
||||
def test_rh_employe_non_trouve(self):
|
||||
ctx = get_domain_context("rh_paie")
|
||||
question = ctx.pose_clarification_question(
|
||||
{
|
||||
"blocked_on": "target_not_found",
|
||||
"target": "Fiche employé",
|
||||
"params": {"nom_employe": "Jean Martin"},
|
||||
}
|
||||
)
|
||||
assert "Jean Martin" in question
|
||||
|
||||
def test_stocks_article_non_trouve(self):
|
||||
ctx = get_domain_context("stocks_logistique")
|
||||
question = ctx.pose_clarification_question(
|
||||
{
|
||||
"blocked_on": "target_not_found",
|
||||
"target": "Article",
|
||||
"params": {"ref_article": "REF-4242", "num_bc": "BC-2026-042"},
|
||||
}
|
||||
)
|
||||
# Un des deux identifiants au moins apparaît
|
||||
assert "REF-4242" in question or "BC-2026-042" in question
|
||||
|
||||
def test_ambiguous_code_tim(self):
|
||||
ctx = get_domain_context("tim_codage")
|
||||
question = ctx.pose_clarification_question(
|
||||
{
|
||||
"blocked_on": "ambiguous_code",
|
||||
"params": {"code_a": "E11.9", "code_b": "E11.8"},
|
||||
}
|
||||
)
|
||||
assert "E11.9" in question
|
||||
assert "E11.8" in question
|
||||
|
||||
def test_clarification_unknown_domain_fallback(self):
|
||||
"""Domaine inconnu → message générique, jamais de crash."""
|
||||
ctx = get_domain_context("inconnu")
|
||||
question = ctx.pose_clarification_question(
|
||||
{"blocked_on": "target_not_found", "target": "Un champ"}
|
||||
)
|
||||
assert question
|
||||
assert "trouve pas" in question.lower()
|
||||
|
||||
def test_clarification_empty_context(self):
|
||||
"""Pas de contexte du tout → fallback."""
|
||||
ctx = get_domain_context("tim_codage")
|
||||
question = ctx.pose_clarification_question(None)
|
||||
assert question # non vide
|
||||
assert isinstance(question, str)
|
||||
|
||||
def test_clarification_missing_params_no_crash(self):
|
||||
"""Si un template mentionne {nom_patient} mais qu'il n'est pas fourni,
|
||||
on ne plante pas — les champs manquants sont vides."""
|
||||
ctx = get_domain_context("tim_codage")
|
||||
question = ctx.pose_clarification_question(
|
||||
{
|
||||
"blocked_on": "target_not_found",
|
||||
"target": "Fichier patient",
|
||||
# pas de nom_patient
|
||||
}
|
||||
)
|
||||
assert isinstance(question, str)
|
||||
assert question
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# describe_workflow_outcome — rapports finaux
|
||||
# ============================================================================
|
||||
|
||||
|
||||
class TestWorkflowOutcome:
|
||||
"""Rapports de fin de workflow en langage métier."""
|
||||
|
||||
def test_tim_succes_complet(self):
|
||||
ctx = get_domain_context("tim_codage")
|
||||
rapport = ctx.describe_workflow_outcome(
|
||||
workflow_name="Codage janvier",
|
||||
success=True,
|
||||
items_count=15,
|
||||
failed_count=0,
|
||||
)
|
||||
assert "15 dossiers" in rapport
|
||||
assert "codé" in rapport
|
||||
|
||||
def test_tim_succes_partiel(self):
|
||||
ctx = get_domain_context("tim_codage")
|
||||
rapport = ctx.describe_workflow_outcome(
|
||||
workflow_name="Codage janvier",
|
||||
success=True,
|
||||
items_count=15,
|
||||
failed_count=1,
|
||||
)
|
||||
assert "14 dossiers" in rapport
|
||||
assert "15" in rapport
|
||||
assert "1" in rapport # nombre en attente
|
||||
|
||||
def test_tim_echec_complet(self):
|
||||
ctx = get_domain_context("tim_codage")
|
||||
rapport = ctx.describe_workflow_outcome(
|
||||
workflow_name="Codage janvier",
|
||||
success=False,
|
||||
items_count=15,
|
||||
failed_count=15,
|
||||
)
|
||||
assert "Codage janvier" in rapport
|
||||
assert "pas" in rapport.lower() or "rends la main" in rapport.lower()
|
||||
|
||||
def test_compta_success_factures(self):
|
||||
ctx = get_domain_context("comptabilite")
|
||||
rapport = ctx.describe_workflow_outcome(
|
||||
workflow_name="Saisie factures mars",
|
||||
success=True,
|
||||
items_count=30,
|
||||
failed_count=0,
|
||||
)
|
||||
assert "30 factures" in rapport
|
||||
|
||||
def test_rh_success_bulletins(self):
|
||||
ctx = get_domain_context("rh_paie")
|
||||
rapport = ctx.describe_workflow_outcome(
|
||||
workflow_name="Paie avril",
|
||||
success=True,
|
||||
items_count=50,
|
||||
failed_count=2,
|
||||
)
|
||||
assert "48" in rapport
|
||||
assert "50" in rapport
|
||||
assert "bulletins" in rapport
|
||||
|
||||
def test_stocks_success_bons(self):
|
||||
ctx = get_domain_context("stocks_logistique")
|
||||
rapport = ctx.describe_workflow_outcome(
|
||||
workflow_name="Réceptions semaine 14",
|
||||
success=True,
|
||||
items_count=12,
|
||||
failed_count=0,
|
||||
)
|
||||
assert "12 bons" in rapport
|
||||
|
||||
def test_generic_fallback(self):
|
||||
"""Domaine inconnu → rapport générique cohérent."""
|
||||
ctx = get_domain_context("inconnu")
|
||||
rapport = ctx.describe_workflow_outcome(
|
||||
workflow_name="Mon workflow",
|
||||
success=True,
|
||||
items_count=5,
|
||||
failed_count=0,
|
||||
)
|
||||
assert rapport
|
||||
assert "Mon workflow" in rapport or "5" in rapport
|
||||
|
||||
def test_tim_success_one_avec_nom_patient(self):
|
||||
"""Cas 1 item : utilise success_one avec un paramètre métier."""
|
||||
ctx = get_domain_context("tim_codage")
|
||||
rapport = ctx.describe_workflow_outcome(
|
||||
workflow_name="Codage urgent",
|
||||
success=True,
|
||||
items_count=1,
|
||||
failed_count=0,
|
||||
elapsed_s=42,
|
||||
extra={"nom_patient": "M. Dupont"},
|
||||
)
|
||||
assert "M. Dupont" in rapport
|
||||
assert "42" in rapport
|
||||
|
||||
|
||||
class TestWorkflowOutcomeLLM:
|
||||
"""Tests du raffinement LLM (gemma4) pour le rapport final."""
|
||||
|
||||
def test_use_llm_success_mocked(self):
|
||||
"""Quand use_llm=True et gemma4 répond, on utilise sa réponse."""
|
||||
ctx = get_domain_context("tim_codage")
|
||||
|
||||
def fake_refine(self, template, subs, success):
|
||||
return "Voilà, j'ai codé tous tes dossiers, bon café !"
|
||||
|
||||
with patch.object(DomainContext, "_llm_refine_summary", fake_refine):
|
||||
rapport = ctx.describe_workflow_outcome(
|
||||
workflow_name="Codage", success=True,
|
||||
items_count=10, use_llm=True,
|
||||
)
|
||||
assert "bon café" in rapport
|
||||
|
||||
def test_use_llm_failure_falls_back_to_template(self):
|
||||
"""Si l'appel LLM retourne "" → on retombe sur le template."""
|
||||
ctx = get_domain_context("tim_codage")
|
||||
|
||||
def fake_refine(self, template, subs, success):
|
||||
return "" # simulate failure
|
||||
|
||||
with patch.object(DomainContext, "_llm_refine_summary", fake_refine):
|
||||
rapport = ctx.describe_workflow_outcome(
|
||||
workflow_name="Codage", success=True,
|
||||
items_count=10, failed_count=0, use_llm=True,
|
||||
)
|
||||
assert "10 dossiers" in rapport
|
||||
|
||||
def test_llm_refine_network_error_safe(self):
|
||||
"""_llm_refine_summary ne doit jamais lever, même si requests échoue."""
|
||||
ctx = get_domain_context("tim_codage")
|
||||
|
||||
fake_requests = MagicMock()
|
||||
fake_requests.post.side_effect = RuntimeError("boom")
|
||||
|
||||
with patch.dict("sys.modules", {"requests": fake_requests}):
|
||||
out = ctx._llm_refine_summary(
|
||||
template="ok", subs={"workflow_name": "x"}, success=True
|
||||
)
|
||||
assert out == ""
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Domaine custom enregistré dynamiquement
|
||||
# ============================================================================
|
||||
|
||||
|
||||
class TestRegisterDomain:
|
||||
def test_register_custom_domain(self):
|
||||
custom = DomainContext(
|
||||
domain_id="test_custom_xyz",
|
||||
name="Test",
|
||||
description="test",
|
||||
common_actions={"click:foo": "faire foo"},
|
||||
summary_templates={
|
||||
"item_singular": "truc",
|
||||
"item_plural": "trucs",
|
||||
"success": "J'ai fait {done} trucs sur {items_count}.",
|
||||
"partial": "Partiel : {done}/{items_count}.",
|
||||
"failure": "Echec.",
|
||||
},
|
||||
)
|
||||
register_domain(custom)
|
||||
fetched = get_domain_context("test_custom_xyz")
|
||||
assert fetched.name == "Test"
|
||||
assert fetched.summarize_action("click", {"target": "FOO"}) == "faire foo"
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Intégration avec ui.messages
|
||||
# ============================================================================
|
||||
|
||||
|
||||
class TestMessagesIntegration:
|
||||
"""Les formatters de messages utilisent le domaine quand fourni."""
|
||||
|
||||
def test_cible_non_trouvee_domain_tim(self):
|
||||
from agent_v0.agent_v1.ui.messages import formatter_cible_non_trouvee
|
||||
|
||||
msg = formatter_cible_non_trouvee(
|
||||
description_cible="Fichier patient",
|
||||
titre_fenetre="DxCare",
|
||||
domain_id="tim_codage",
|
||||
params={"nom_patient": "Mme Durand"},
|
||||
)
|
||||
assert "Mme Durand" in msg.corps
|
||||
|
||||
def test_cible_non_trouvee_domain_comptabilite(self):
|
||||
from agent_v0.agent_v1.ui.messages import formatter_cible_non_trouvee
|
||||
|
||||
msg = formatter_cible_non_trouvee(
|
||||
description_cible="Montant HT",
|
||||
titre_fenetre="Sage",
|
||||
domain_id="comptabilite",
|
||||
params={"num_facture": "F2026-007"},
|
||||
)
|
||||
assert "F2026-007" in msg.corps
|
||||
|
||||
def test_cible_non_trouvee_sans_domain_retrocompat(self):
|
||||
"""Sans domain_id, comportement historique conservé."""
|
||||
from agent_v0.agent_v1.ui.messages import formatter_cible_non_trouvee
|
||||
|
||||
msg = formatter_cible_non_trouvee(
|
||||
description_cible="bonjour",
|
||||
titre_fenetre="Test – Bloc-notes",
|
||||
)
|
||||
assert "bonjour" in msg.corps
|
||||
assert "Bloc-notes" in msg.corps
|
||||
|
||||
def test_fin_workflow_tim_partiel(self):
|
||||
from agent_v0.agent_v1.ui.messages import (
|
||||
NiveauMessage,
|
||||
formatter_fin_workflow,
|
||||
)
|
||||
|
||||
msg = formatter_fin_workflow(
|
||||
succes=True,
|
||||
nom_workflow="Codage janvier",
|
||||
nb_etapes=120,
|
||||
duree_s=900,
|
||||
domain_id="tim_codage",
|
||||
items_count=15,
|
||||
failed_count=1,
|
||||
)
|
||||
# Langage métier, pas "120 étapes"
|
||||
assert "14 dossiers" in msg.corps
|
||||
assert msg.niveau == NiveauMessage.ATTENTION # succès partiel
|
||||
|
||||
def test_fin_workflow_tim_complet(self):
|
||||
from agent_v0.agent_v1.ui.messages import (
|
||||
NiveauMessage,
|
||||
formatter_fin_workflow,
|
||||
)
|
||||
|
||||
msg = formatter_fin_workflow(
|
||||
succes=True,
|
||||
nom_workflow="Codage janvier",
|
||||
nb_etapes=120,
|
||||
duree_s=900,
|
||||
domain_id="tim_codage",
|
||||
items_count=15,
|
||||
failed_count=0,
|
||||
)
|
||||
assert "15 dossiers" in msg.corps
|
||||
assert msg.niveau == NiveauMessage.INFO
|
||||
|
||||
def test_fin_workflow_sans_domain_retrocompat(self):
|
||||
from agent_v0.agent_v1.ui.messages import formatter_fin_workflow
|
||||
|
||||
msg = formatter_fin_workflow(
|
||||
succes=True, nom_workflow="Demo", nb_etapes=5, duree_s=10
|
||||
)
|
||||
assert "Demo" in msg.corps
|
||||
assert "5 étapes" in msg.corps
|
||||
|
||||
def test_erreur_generique_propagate_domain(self):
|
||||
from agent_v0.agent_v1.ui.messages import formatter_erreur_generique
|
||||
|
||||
msg = formatter_erreur_generique(
|
||||
"target_not_found: Montant HT",
|
||||
domain_id="comptabilite",
|
||||
params={"num_facture": "F-001"},
|
||||
)
|
||||
assert "F-001" in msg.corps
|
||||
|
||||
def test_friendly_target_tim_synonyme(self):
|
||||
from agent_v0.agent_v1.ui.messages import _friendly_target
|
||||
|
||||
assert _friendly_target("DP", "tim_codage") == "diagnostic principal"
|
||||
assert _friendly_target("DP", None) == "DP" # pas de domaine → identique
|
||||
assert _friendly_target("DP", "domaine_inexistant") == "DP"
|
||||
264
tests/unit/test_execution_compiler.py
Normal file
264
tests/unit/test_execution_compiler.py
Normal file
@@ -0,0 +1,264 @@
|
||||
"""
|
||||
Tests de l'ExecutionCompiler et de l'ExecutionPlan.
|
||||
|
||||
Vérifie que :
|
||||
- Le compilateur produit un plan déterministe depuis un WorkflowIR
|
||||
- Les stratégies de résolution sont correctement compilées (OCR > template > VLM)
|
||||
- Les timeouts, retries et recovery sont définis
|
||||
- Le plan est sérialisable et versionné
|
||||
"""
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
_ROOT = str(Path(__file__).resolve().parents[2])
|
||||
if _ROOT not in sys.path:
|
||||
sys.path.insert(0, _ROOT)
|
||||
|
||||
from core.workflow.workflow_ir import WorkflowIR
|
||||
from core.workflow.execution_plan import ExecutionPlan, ExecutionNode, ResolutionStrategy, SuccessCondition
|
||||
from core.workflow.execution_compiler import ExecutionCompiler
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# ExecutionPlan — format et sérialisation
|
||||
# =========================================================================
|
||||
|
||||
|
||||
class TestExecutionPlan:
|
||||
|
||||
def test_serialisation_roundtrip(self):
|
||||
plan = ExecutionPlan(
|
||||
plan_id="plan_test",
|
||||
workflow_id="wf_123",
|
||||
version=1,
|
||||
)
|
||||
plan.nodes.append(ExecutionNode(
|
||||
node_id="n1",
|
||||
action_type="click",
|
||||
intent="Cliquer sur Enregistrer",
|
||||
strategy_primary=ResolutionStrategy(method="ocr", target_text="Enregistrer"),
|
||||
strategy_fallbacks=[ResolutionStrategy(method="vlm", vlm_description="bouton Enregistrer")],
|
||||
success_condition=SuccessCondition(method="title_match", expected_title="Fichier sauvegardé"),
|
||||
))
|
||||
|
||||
json_str = plan.to_json()
|
||||
plan2 = ExecutionPlan.from_json(json_str)
|
||||
|
||||
assert plan2.plan_id == "plan_test"
|
||||
assert len(plan2.nodes) == 1
|
||||
assert plan2.nodes[0].strategy_primary.method == "ocr"
|
||||
assert len(plan2.nodes[0].strategy_fallbacks) == 1
|
||||
assert plan2.nodes[0].success_condition.method == "title_match"
|
||||
|
||||
def test_save_load(self):
|
||||
tmpdir = tempfile.mkdtemp()
|
||||
try:
|
||||
plan = ExecutionPlan(plan_id="plan_save", workflow_id="wf_1")
|
||||
plan.nodes.append(ExecutionNode(node_id="n1", action_type="click"))
|
||||
path = plan.save(tmpdir)
|
||||
|
||||
plan2 = ExecutionPlan.load(str(path))
|
||||
assert plan2.plan_id == "plan_save"
|
||||
assert len(plan2.nodes) == 1
|
||||
finally:
|
||||
shutil.rmtree(tmpdir)
|
||||
|
||||
def test_node_avec_variable(self):
|
||||
node = ExecutionNode(
|
||||
node_id="n1",
|
||||
action_type="type",
|
||||
text="{patient}",
|
||||
variable_name="patient",
|
||||
)
|
||||
d = node.to_dict()
|
||||
assert d["variable_name"] == "patient"
|
||||
assert d["text"] == "{patient}"
|
||||
|
||||
def test_node_timeout_et_retry(self):
|
||||
node = ExecutionNode(
|
||||
node_id="n1",
|
||||
action_type="click",
|
||||
timeout_ms=5000,
|
||||
max_retries=3,
|
||||
recovery_action="undo",
|
||||
)
|
||||
assert node.timeout_ms == 5000
|
||||
assert node.max_retries == 3
|
||||
assert node.recovery_action == "undo"
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# ExecutionCompiler — compilation WorkflowIR → ExecutionPlan
|
||||
# =========================================================================
|
||||
|
||||
|
||||
class TestExecutionCompiler:
|
||||
|
||||
def _make_ir(self):
|
||||
"""Créer un WorkflowIR de test."""
|
||||
ir = WorkflowIR.new("Test workflow", domain="generic")
|
||||
ir.add_step(
|
||||
"Ouvrir le fichier",
|
||||
actions=[
|
||||
{"type": "click", "target": "bouton Ouvrir", "anchor_hint": "Ouvrir"},
|
||||
{"type": "wait", "duration_ms": 2000},
|
||||
],
|
||||
precondition="L'application est ouverte",
|
||||
postcondition="La fenêtre Ouvrir est affichée",
|
||||
)
|
||||
ir.add_step(
|
||||
"Saisir le nom",
|
||||
actions=[
|
||||
{"type": "type", "text": "{nom_fichier}", "variable": True},
|
||||
{"type": "key_combo", "keys": ["enter"]},
|
||||
],
|
||||
)
|
||||
ir.add_variable("nom_fichier", description="Nom du fichier", default="rapport.pdf")
|
||||
return ir
|
||||
|
||||
def test_compilation_basique(self):
|
||||
compiler = ExecutionCompiler()
|
||||
ir = self._make_ir()
|
||||
plan = compiler.compile(ir)
|
||||
|
||||
assert plan.workflow_id == ir.workflow_id
|
||||
assert plan.total_nodes == 4 # click + wait + type + key_combo
|
||||
assert plan.domain == "generic"
|
||||
|
||||
def test_click_a_strategie_resolution(self):
|
||||
"""Un clic doit avoir une stratégie primaire et des fallbacks."""
|
||||
compiler = ExecutionCompiler()
|
||||
ir = self._make_ir()
|
||||
plan = compiler.compile(ir)
|
||||
|
||||
click_nodes = [n for n in plan.nodes if n.action_type == "click"]
|
||||
assert len(click_nodes) == 1
|
||||
|
||||
click = click_nodes[0]
|
||||
assert click.strategy_primary is not None
|
||||
assert click.strategy_primary.method in ("ocr", "template", "vlm")
|
||||
assert len(click.strategy_fallbacks) >= 1
|
||||
|
||||
def test_ocr_est_prioritaire(self):
|
||||
"""Quand du texte est disponible, OCR est la stratégie primaire."""
|
||||
compiler = ExecutionCompiler()
|
||||
ir = self._make_ir()
|
||||
plan = compiler.compile(ir)
|
||||
|
||||
click = [n for n in plan.nodes if n.action_type == "click"][0]
|
||||
assert click.strategy_primary.method == "ocr"
|
||||
assert click.strategy_primary.target_text == "Ouvrir"
|
||||
|
||||
def test_vlm_est_fallback(self):
|
||||
"""Le VLM est toujours en dernier fallback (exception handler)."""
|
||||
compiler = ExecutionCompiler()
|
||||
ir = self._make_ir()
|
||||
plan = compiler.compile(ir)
|
||||
|
||||
click = [n for n in plan.nodes if n.action_type == "click"][0]
|
||||
vlm_fallbacks = [f for f in click.strategy_fallbacks if f.method == "vlm"]
|
||||
assert len(vlm_fallbacks) >= 1
|
||||
|
||||
def test_type_a_variable(self):
|
||||
"""Une action type avec variable a le bon variable_name."""
|
||||
compiler = ExecutionCompiler()
|
||||
ir = self._make_ir()
|
||||
plan = compiler.compile(ir)
|
||||
|
||||
type_nodes = [n for n in plan.nodes if n.action_type == "type"]
|
||||
assert len(type_nodes) == 1
|
||||
assert type_nodes[0].variable_name == "nom_fichier"
|
||||
|
||||
def test_wait_a_duration(self):
|
||||
"""Un wait a la bonne durée."""
|
||||
compiler = ExecutionCompiler()
|
||||
ir = self._make_ir()
|
||||
plan = compiler.compile(ir)
|
||||
|
||||
wait_nodes = [n for n in plan.nodes if n.action_type == "wait"]
|
||||
assert len(wait_nodes) == 1
|
||||
assert wait_nodes[0].duration_ms == 2000
|
||||
|
||||
def test_click_a_recovery(self):
|
||||
"""Un clic a une politique de recovery."""
|
||||
compiler = ExecutionCompiler()
|
||||
ir = self._make_ir()
|
||||
plan = compiler.compile(ir)
|
||||
|
||||
click = [n for n in plan.nodes if n.action_type == "click"][0]
|
||||
assert click.recovery_action in ("escape", "undo", "close", "none")
|
||||
assert click.max_retries >= 1
|
||||
|
||||
def test_postcondition_devient_success_condition(self):
|
||||
"""La postcondition du WorkflowIR devient la condition de succès du nœud."""
|
||||
compiler = ExecutionCompiler()
|
||||
ir = self._make_ir()
|
||||
plan = compiler.compile(ir)
|
||||
|
||||
click = [n for n in plan.nodes if n.action_type == "click"][0]
|
||||
assert click.success_condition is not None
|
||||
assert "Ouvrir" in click.success_condition.description
|
||||
|
||||
def test_statistiques_compilation(self):
|
||||
"""Les statistiques de compilation sont correctes."""
|
||||
compiler = ExecutionCompiler()
|
||||
ir = self._make_ir()
|
||||
plan = compiler.compile(ir)
|
||||
|
||||
assert plan.total_nodes == 4
|
||||
assert plan.nodes_with_ocr >= 1
|
||||
assert plan.estimated_duration_s > 0
|
||||
|
||||
def test_variables_dans_le_plan(self):
|
||||
"""Les variables du WorkflowIR sont dans le plan avec leurs valeurs par défaut."""
|
||||
compiler = ExecutionCompiler()
|
||||
ir = self._make_ir()
|
||||
plan = compiler.compile(ir)
|
||||
|
||||
assert "nom_fichier" in plan.variables
|
||||
assert plan.variables["nom_fichier"] == "rapport.pdf"
|
||||
|
||||
def test_params_override_defaults(self):
|
||||
"""Les params passés au compile écrasent les valeurs par défaut."""
|
||||
compiler = ExecutionCompiler()
|
||||
ir = self._make_ir()
|
||||
plan = compiler.compile(ir, params={"nom_fichier": "facture_mars.pdf"})
|
||||
|
||||
assert plan.variables["nom_fichier"] == "facture_mars.pdf"
|
||||
|
||||
def test_plan_json_roundtrip(self):
|
||||
"""Compiler → JSON → recharger → même plan."""
|
||||
compiler = ExecutionCompiler()
|
||||
ir = self._make_ir()
|
||||
plan = compiler.compile(ir)
|
||||
|
||||
json_str = plan.to_json()
|
||||
plan2 = ExecutionPlan.from_json(json_str)
|
||||
|
||||
assert plan2.total_nodes == plan.total_nodes
|
||||
assert plan2.workflow_id == plan.workflow_id
|
||||
assert len(plan2.nodes) == len(plan.nodes)
|
||||
|
||||
def test_compilation_workflow_vide(self):
|
||||
"""Un workflow vide produit un plan vide."""
|
||||
compiler = ExecutionCompiler()
|
||||
ir = WorkflowIR.new("Vide")
|
||||
plan = compiler.compile(ir)
|
||||
|
||||
assert plan.total_nodes == 0
|
||||
assert plan.nodes == []
|
||||
|
||||
def test_plusieurs_domaines(self):
|
||||
"""Le compilateur fonctionne pour différents domaines."""
|
||||
compiler = ExecutionCompiler()
|
||||
for domain in ["tim_codage", "comptabilite", "rh_paie", "generic"]:
|
||||
ir = WorkflowIR.new("Test", domain=domain)
|
||||
ir.add_step("Action", actions=[{"type": "click", "target": "bouton"}])
|
||||
plan = compiler.compile(ir)
|
||||
assert plan.domain == domain
|
||||
565
tests/unit/test_execution_plan_runner.py
Normal file
565
tests/unit/test_execution_plan_runner.py
Normal file
@@ -0,0 +1,565 @@
|
||||
"""
|
||||
Tests de execution_plan_runner — adaptateur ExecutionPlan → queue de replay.
|
||||
|
||||
Vérifie que :
|
||||
- Un ExecutionNode est correctement converti en action replay
|
||||
- Les stratégies de résolution (OCR / template / VLM) produisent le bon target_spec
|
||||
- Les variables {var} et ${var} sont substituées dans les textes
|
||||
- L'injection dans la queue _replay_queues est correcte (avec et sans lock)
|
||||
- La conversion d'un plan complet respecte l'ordre et les limites
|
||||
- Les types d'actions non exécutables sont ignorés
|
||||
|
||||
Ces tests sont isolés et ne dépendent pas du serveur FastAPI (on importe
|
||||
uniquement execution_plan_runner et les dataclasses du core).
|
||||
"""
|
||||
|
||||
import sys
|
||||
import threading
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
_ROOT = str(Path(__file__).resolve().parents[2])
|
||||
if _ROOT not in sys.path:
|
||||
sys.path.insert(0, _ROOT)
|
||||
|
||||
from core.workflow.execution_plan import (
|
||||
ExecutionNode,
|
||||
ExecutionPlan,
|
||||
ResolutionStrategy,
|
||||
SuccessCondition,
|
||||
)
|
||||
from core.workflow.execution_compiler import ExecutionCompiler
|
||||
from core.workflow.workflow_ir import WorkflowIR
|
||||
|
||||
from agent_v0.server_v1.execution_plan_runner import (
|
||||
execution_node_to_action,
|
||||
execution_plan_to_actions,
|
||||
inject_plan_into_queue,
|
||||
substitute_variables,
|
||||
)
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Substitution de variables
|
||||
# =========================================================================
|
||||
|
||||
|
||||
class TestSubstituteVariables:
|
||||
|
||||
def test_substitution_curly(self):
|
||||
assert substitute_variables("{nom}", {"nom": "Dupont"}) == "Dupont"
|
||||
|
||||
def test_substitution_dollar(self):
|
||||
assert substitute_variables("${nom}", {"nom": "Dupont"}) == "Dupont"
|
||||
|
||||
def test_substitution_dans_phrase(self):
|
||||
assert (
|
||||
substitute_variables("Bonjour {nom}, votre code est ${code}",
|
||||
{"nom": "Alice", "code": "A42"})
|
||||
== "Bonjour Alice, votre code est A42"
|
||||
)
|
||||
|
||||
def test_variable_inconnue_inchangee(self):
|
||||
# Une variable inconnue reste dans le texte (pas de KeyError)
|
||||
assert substitute_variables("{inconnu}", {"autre": "val"}) == "{inconnu}"
|
||||
|
||||
def test_texte_sans_variable(self):
|
||||
assert substitute_variables("texte simple", {"x": "1"}) == "texte simple"
|
||||
|
||||
def test_texte_vide(self):
|
||||
assert substitute_variables("", {"x": "1"}) == ""
|
||||
|
||||
def test_variables_vides(self):
|
||||
assert substitute_variables("{x}", {}) == "{x}"
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Conversion ExecutionNode → action replay
|
||||
# =========================================================================
|
||||
|
||||
|
||||
class TestExecutionNodeToAction:
|
||||
|
||||
def test_click_avec_strategie_ocr(self):
|
||||
"""Un clic avec stratégie OCR produit une action click visuelle avec by_text."""
|
||||
node = ExecutionNode(
|
||||
node_id="n1",
|
||||
action_type="click",
|
||||
intent="Cliquer sur Enregistrer",
|
||||
strategy_primary=ResolutionStrategy(
|
||||
method="ocr",
|
||||
target_text="Enregistrer",
|
||||
threshold=0.8,
|
||||
),
|
||||
)
|
||||
action = execution_node_to_action(node)
|
||||
|
||||
assert action is not None
|
||||
assert action["type"] == "click"
|
||||
assert action["action_id"].startswith("act_plan_")
|
||||
assert action["plan_node_id"] == "n1"
|
||||
assert action["intention"] == "Cliquer sur Enregistrer"
|
||||
assert action["visual_mode"] is True
|
||||
assert "x_pct" in action and "y_pct" in action
|
||||
assert action["target_spec"]["by_text"] == "Enregistrer"
|
||||
|
||||
def test_click_avec_strategie_template(self):
|
||||
"""Un clic avec stratégie template expose l'anchor_image_base64."""
|
||||
node = ExecutionNode(
|
||||
node_id="n2",
|
||||
action_type="click",
|
||||
strategy_primary=ResolutionStrategy(
|
||||
method="template",
|
||||
anchor_b64="AAABBBCCCDDD",
|
||||
target_text="Ouvrir",
|
||||
),
|
||||
)
|
||||
action = execution_node_to_action(node)
|
||||
|
||||
assert action is not None
|
||||
assert action["type"] == "click"
|
||||
assert action["target_spec"]["anchor_image_base64"] == "AAABBBCCCDDD"
|
||||
assert action["target_spec"]["by_text"] == "Ouvrir"
|
||||
assert action["visual_mode"] is True
|
||||
|
||||
def test_click_avec_strategie_vlm(self):
|
||||
"""Un clic avec stratégie VLM expose vlm_description."""
|
||||
node = ExecutionNode(
|
||||
node_id="n3",
|
||||
action_type="click",
|
||||
strategy_primary=ResolutionStrategy(
|
||||
method="vlm",
|
||||
vlm_description="bouton rouge en haut à droite",
|
||||
),
|
||||
)
|
||||
action = execution_node_to_action(node)
|
||||
|
||||
assert action is not None
|
||||
assert action["target_spec"]["vlm_description"] == "bouton rouge en haut à droite"
|
||||
assert action["visual_mode"] is True
|
||||
|
||||
def test_click_avec_fallbacks_ajoute_hints(self):
|
||||
"""Les fallbacks enrichissent le target_spec avec toutes les ancres disponibles."""
|
||||
node = ExecutionNode(
|
||||
node_id="n4",
|
||||
action_type="click",
|
||||
intent="Ouvrir le menu",
|
||||
strategy_primary=ResolutionStrategy(method="ocr", target_text="Menu"),
|
||||
strategy_fallbacks=[
|
||||
ResolutionStrategy(
|
||||
method="template", anchor_b64="XYZ", target_text="Menu",
|
||||
),
|
||||
ResolutionStrategy(
|
||||
method="vlm", vlm_description="menu déroulant",
|
||||
),
|
||||
],
|
||||
)
|
||||
action = execution_node_to_action(node)
|
||||
|
||||
spec = action["target_spec"]
|
||||
assert spec["by_text"] == "Menu"
|
||||
assert spec["anchor_image_base64"] == "XYZ"
|
||||
assert spec["vlm_description"] == "menu déroulant"
|
||||
|
||||
def test_click_avec_success_condition_expected_title(self):
|
||||
"""La success_condition avec expected_title passe dans expected_window_title."""
|
||||
node = ExecutionNode(
|
||||
node_id="n5",
|
||||
action_type="click",
|
||||
strategy_primary=ResolutionStrategy(method="ocr", target_text="OK"),
|
||||
success_condition=SuccessCondition(
|
||||
method="title_match",
|
||||
expected_title="Document sauvegardé",
|
||||
),
|
||||
)
|
||||
action = execution_node_to_action(node)
|
||||
|
||||
assert action["expected_window_title"] == "Document sauvegardé"
|
||||
assert action["target_spec"]["window_title"] == "Document sauvegardé"
|
||||
|
||||
def test_type_avec_variable_substitution(self):
|
||||
"""Un node type avec variable {patient} est substitué."""
|
||||
node = ExecutionNode(
|
||||
node_id="n6",
|
||||
action_type="type",
|
||||
text="{patient}",
|
||||
variable_name="patient",
|
||||
)
|
||||
action = execution_node_to_action(node, variables={"patient": "DUPONT"})
|
||||
|
||||
assert action["type"] == "type"
|
||||
assert action["text"] == "DUPONT"
|
||||
assert action["variable_name"] == "patient"
|
||||
|
||||
def test_type_sans_variable(self):
|
||||
"""Un texte sans variable est inchangé."""
|
||||
node = ExecutionNode(
|
||||
node_id="n7",
|
||||
action_type="type",
|
||||
text="Bonjour",
|
||||
)
|
||||
action = execution_node_to_action(node)
|
||||
assert action["text"] == "Bonjour"
|
||||
|
||||
def test_key_combo(self):
|
||||
"""Un key_combo expose les touches."""
|
||||
node = ExecutionNode(
|
||||
node_id="n8",
|
||||
action_type="key_combo",
|
||||
keys=["ctrl", "s"],
|
||||
)
|
||||
action = execution_node_to_action(node)
|
||||
|
||||
assert action["type"] == "key_combo"
|
||||
assert action["keys"] == ["ctrl", "s"]
|
||||
|
||||
def test_key_combo_vide_retourne_none(self):
|
||||
"""Un key_combo sans touches est ignoré."""
|
||||
node = ExecutionNode(
|
||||
node_id="n9",
|
||||
action_type="key_combo",
|
||||
keys=[],
|
||||
)
|
||||
assert execution_node_to_action(node) is None
|
||||
|
||||
def test_wait(self):
|
||||
"""Un wait expose duration_ms."""
|
||||
node = ExecutionNode(
|
||||
node_id="n10",
|
||||
action_type="wait",
|
||||
duration_ms=2500,
|
||||
)
|
||||
action = execution_node_to_action(node)
|
||||
|
||||
assert action["type"] == "wait"
|
||||
assert action["duration_ms"] == 2500
|
||||
|
||||
def test_wait_sans_duration_default(self):
|
||||
"""Un wait sans duration a un défaut de 1000ms."""
|
||||
node = ExecutionNode(node_id="n11", action_type="wait")
|
||||
action = execution_node_to_action(node)
|
||||
assert action["duration_ms"] == 1000
|
||||
|
||||
def test_scroll(self):
|
||||
"""Un scroll produit une action scroll."""
|
||||
node = ExecutionNode(node_id="n12", action_type="scroll")
|
||||
action = execution_node_to_action(node)
|
||||
|
||||
assert action["type"] == "scroll"
|
||||
assert "delta" in action
|
||||
|
||||
def test_type_inconnu_retourne_none(self):
|
||||
"""Un type d'action inconnu est ignoré (retourne None)."""
|
||||
node = ExecutionNode(node_id="n13", action_type="unknown_thing")
|
||||
assert execution_node_to_action(node) is None
|
||||
|
||||
def test_metadonnees_execution_propagees(self):
|
||||
"""timeout_ms, max_retries, recovery_action passent dans l'action."""
|
||||
node = ExecutionNode(
|
||||
node_id="n14",
|
||||
action_type="click",
|
||||
strategy_primary=ResolutionStrategy(method="ocr", target_text="X"),
|
||||
timeout_ms=15000,
|
||||
max_retries=3,
|
||||
recovery_action="undo",
|
||||
)
|
||||
action = execution_node_to_action(node)
|
||||
|
||||
assert action["timeout_ms"] == 15000
|
||||
assert action["max_retries"] == 3
|
||||
assert action["recovery_action"] == "undo"
|
||||
|
||||
def test_node_optionnel(self):
|
||||
"""is_optional est propagé."""
|
||||
node = ExecutionNode(
|
||||
node_id="n15",
|
||||
action_type="click",
|
||||
strategy_primary=ResolutionStrategy(method="ocr", target_text="X"),
|
||||
is_optional=True,
|
||||
)
|
||||
action = execution_node_to_action(node)
|
||||
assert action["is_optional"] is True
|
||||
|
||||
def test_id_prefix_custom(self):
|
||||
"""Le préfixe d'id peut être personnalisé."""
|
||||
node = ExecutionNode(
|
||||
node_id="n16",
|
||||
action_type="click",
|
||||
strategy_primary=ResolutionStrategy(method="ocr", target_text="X"),
|
||||
)
|
||||
action = execution_node_to_action(node, id_prefix="act_custom")
|
||||
assert action["action_id"].startswith("act_custom_")
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Conversion ExecutionPlan → liste d'actions
|
||||
# =========================================================================
|
||||
|
||||
|
||||
class TestExecutionPlanToActions:
|
||||
|
||||
def _make_plan(self) -> ExecutionPlan:
|
||||
plan = ExecutionPlan(
|
||||
plan_id="plan_test",
|
||||
workflow_id="wf_test",
|
||||
version=1,
|
||||
variables={"nom_fichier": "rapport.pdf"},
|
||||
)
|
||||
plan.nodes = [
|
||||
ExecutionNode(
|
||||
node_id="n1",
|
||||
action_type="click",
|
||||
strategy_primary=ResolutionStrategy(method="ocr", target_text="Ouvrir"),
|
||||
),
|
||||
ExecutionNode(
|
||||
node_id="n2",
|
||||
action_type="type",
|
||||
text="{nom_fichier}",
|
||||
variable_name="nom_fichier",
|
||||
),
|
||||
ExecutionNode(
|
||||
node_id="n3",
|
||||
action_type="key_combo",
|
||||
keys=["enter"],
|
||||
),
|
||||
ExecutionNode(
|
||||
node_id="n4",
|
||||
action_type="wait",
|
||||
duration_ms=1500,
|
||||
),
|
||||
]
|
||||
plan.total_nodes = 4
|
||||
return plan
|
||||
|
||||
def test_conversion_ordre_respecte(self):
|
||||
plan = self._make_plan()
|
||||
actions = execution_plan_to_actions(plan)
|
||||
|
||||
assert len(actions) == 4
|
||||
assert actions[0]["type"] == "click"
|
||||
assert actions[1]["type"] == "type"
|
||||
assert actions[2]["type"] == "key_combo"
|
||||
assert actions[3]["type"] == "wait"
|
||||
|
||||
def test_variables_du_plan_appliquees(self):
|
||||
plan = self._make_plan()
|
||||
actions = execution_plan_to_actions(plan)
|
||||
type_action = next(a for a in actions if a["type"] == "type")
|
||||
assert type_action["text"] == "rapport.pdf"
|
||||
|
||||
def test_variables_override(self):
|
||||
"""Les variables passées en argument écrasent celles du plan."""
|
||||
plan = self._make_plan()
|
||||
actions = execution_plan_to_actions(
|
||||
plan, variables={"nom_fichier": "facture.pdf"},
|
||||
)
|
||||
type_action = next(a for a in actions if a["type"] == "type")
|
||||
assert type_action["text"] == "facture.pdf"
|
||||
|
||||
def test_plan_vide(self):
|
||||
plan = ExecutionPlan(plan_id="empty", workflow_id="wf_empty")
|
||||
actions = execution_plan_to_actions(plan)
|
||||
assert actions == []
|
||||
|
||||
def test_noeud_non_convertible_ignore(self):
|
||||
"""Un nœud inconnu ne bloque pas la conversion."""
|
||||
plan = ExecutionPlan(plan_id="p", workflow_id="wf")
|
||||
plan.nodes = [
|
||||
ExecutionNode(
|
||||
node_id="n1",
|
||||
action_type="click",
|
||||
strategy_primary=ResolutionStrategy(method="ocr", target_text="OK"),
|
||||
),
|
||||
ExecutionNode(node_id="n2", action_type="unknown_type"),
|
||||
ExecutionNode(
|
||||
node_id="n3",
|
||||
action_type="type",
|
||||
text="hello",
|
||||
),
|
||||
]
|
||||
actions = execution_plan_to_actions(plan)
|
||||
assert len(actions) == 2
|
||||
assert actions[0]["type"] == "click"
|
||||
assert actions[1]["type"] == "type"
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Injection dans la queue de replay
|
||||
# =========================================================================
|
||||
|
||||
|
||||
class TestInjectPlanIntoQueue:
|
||||
|
||||
def _make_simple_plan(self) -> ExecutionPlan:
|
||||
plan = ExecutionPlan(plan_id="p_inj", workflow_id="wf_inj")
|
||||
plan.nodes = [
|
||||
ExecutionNode(
|
||||
node_id="n1",
|
||||
action_type="click",
|
||||
strategy_primary=ResolutionStrategy(method="ocr", target_text="Go"),
|
||||
),
|
||||
ExecutionNode(node_id="n2", action_type="wait", duration_ms=500),
|
||||
]
|
||||
return plan
|
||||
|
||||
def test_injection_replace(self):
|
||||
"""Par défaut, la queue est remplacée."""
|
||||
plan = self._make_simple_plan()
|
||||
queues: dict = defaultdict(list)
|
||||
queues["sess_abc"] = [{"type": "click", "action_id": "old"}]
|
||||
|
||||
actions = inject_plan_into_queue(
|
||||
plan=plan,
|
||||
session_id="sess_abc",
|
||||
replay_queues=queues,
|
||||
)
|
||||
|
||||
assert len(actions) == 2
|
||||
assert len(queues["sess_abc"]) == 2
|
||||
# L'ancienne action a été remplacée
|
||||
assert all(a["action_id"] != "old" for a in queues["sess_abc"])
|
||||
|
||||
def test_injection_append(self):
|
||||
"""Avec replace=False, on ajoute aux actions existantes."""
|
||||
plan = self._make_simple_plan()
|
||||
queues: dict = defaultdict(list)
|
||||
queues["sess_abc"] = [{"type": "click", "action_id": "existing"}]
|
||||
|
||||
inject_plan_into_queue(
|
||||
plan=plan,
|
||||
session_id="sess_abc",
|
||||
replay_queues=queues,
|
||||
replace=False,
|
||||
)
|
||||
|
||||
assert len(queues["sess_abc"]) == 3
|
||||
assert queues["sess_abc"][0]["action_id"] == "existing"
|
||||
|
||||
def test_injection_avec_lock(self):
|
||||
"""Le lock est respecté pendant l'injection."""
|
||||
plan = self._make_simple_plan()
|
||||
queues: dict = defaultdict(list)
|
||||
lock = threading.Lock()
|
||||
|
||||
actions = inject_plan_into_queue(
|
||||
plan=plan,
|
||||
session_id="sess_x",
|
||||
replay_queues=queues,
|
||||
lock=lock,
|
||||
)
|
||||
|
||||
assert len(actions) == 2
|
||||
assert len(queues["sess_x"]) == 2
|
||||
# Le lock est bien libéré après l'injection
|
||||
assert lock.acquire(blocking=False) is True
|
||||
lock.release()
|
||||
|
||||
def test_injection_avec_variables(self):
|
||||
"""Les variables sont substituées lors de l'injection."""
|
||||
plan = ExecutionPlan(plan_id="p_var", workflow_id="wf_var")
|
||||
plan.nodes = [
|
||||
ExecutionNode(
|
||||
node_id="n1",
|
||||
action_type="type",
|
||||
text="{patient}",
|
||||
variable_name="patient",
|
||||
),
|
||||
]
|
||||
|
||||
queues: dict = defaultdict(list)
|
||||
actions = inject_plan_into_queue(
|
||||
plan=plan,
|
||||
session_id="sess_v",
|
||||
replay_queues=queues,
|
||||
variables={"patient": "MARTIN"},
|
||||
)
|
||||
|
||||
assert actions[0]["text"] == "MARTIN"
|
||||
assert queues["sess_v"][0]["text"] == "MARTIN"
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Intégration : pipeline complet IR → Plan → Actions
|
||||
# =========================================================================
|
||||
|
||||
|
||||
class TestFullPipelineV4:
|
||||
"""Teste le pipeline complet : WorkflowIR → ExecutionPlan → actions replay."""
|
||||
|
||||
def test_pipeline_complet_ir_vers_actions(self):
|
||||
# 1. Construire un WorkflowIR
|
||||
ir = WorkflowIR.new("Test pipeline V4", domain="generic")
|
||||
ir.add_step(
|
||||
"Ouvrir le fichier",
|
||||
actions=[
|
||||
{"type": "click", "target": "bouton Ouvrir", "anchor_hint": "Ouvrir"},
|
||||
{"type": "wait", "duration_ms": 1000},
|
||||
],
|
||||
postcondition="La fenêtre Ouvrir est visible",
|
||||
)
|
||||
ir.add_step(
|
||||
"Saisir le nom",
|
||||
actions=[
|
||||
{"type": "type", "text": "{nom_fichier}", "variable": True},
|
||||
{"type": "key_combo", "keys": ["enter"]},
|
||||
],
|
||||
)
|
||||
ir.add_variable("nom_fichier", description="Fichier", default="doc.pdf")
|
||||
|
||||
# 2. Compiler → ExecutionPlan
|
||||
compiler = ExecutionCompiler()
|
||||
plan = compiler.compile(ir)
|
||||
assert plan.total_nodes == 4
|
||||
|
||||
# 3. Convertir → actions replay
|
||||
actions = execution_plan_to_actions(plan)
|
||||
assert len(actions) == 4
|
||||
|
||||
types = [a["type"] for a in actions]
|
||||
assert types == ["click", "wait", "type", "key_combo"]
|
||||
|
||||
# Le clic a une stratégie OCR → by_text
|
||||
click = actions[0]
|
||||
assert click["visual_mode"] is True
|
||||
assert click["target_spec"].get("by_text") == "Ouvrir"
|
||||
|
||||
# Le type a substitué la variable depuis le plan
|
||||
type_action = actions[2]
|
||||
assert type_action["text"] == "doc.pdf"
|
||||
|
||||
# Le key_combo a les touches
|
||||
assert actions[3]["keys"] == ["enter"]
|
||||
|
||||
def test_pipeline_avec_params_override(self):
|
||||
"""Les params passés à l'injection prévalent sur le plan."""
|
||||
ir = WorkflowIR.new("Variables override")
|
||||
ir.add_step("Saisie", actions=[
|
||||
{"type": "type", "text": "{code}", "variable": True},
|
||||
])
|
||||
ir.add_variable("code", default="DEFAULT")
|
||||
|
||||
compiler = ExecutionCompiler()
|
||||
plan = compiler.compile(ir)
|
||||
|
||||
actions = execution_plan_to_actions(
|
||||
plan, variables={"code": "RUNTIME"},
|
||||
)
|
||||
assert actions[0]["text"] == "RUNTIME"
|
||||
|
||||
def test_pipeline_plan_serialise_et_recharge(self):
|
||||
"""Le plan peut être sérialisé/rechargé puis converti en actions."""
|
||||
ir = WorkflowIR.new("Roundtrip")
|
||||
ir.add_step("X", actions=[
|
||||
{"type": "click", "target": "btn", "anchor_hint": "Valider"},
|
||||
])
|
||||
compiler = ExecutionCompiler()
|
||||
plan = compiler.compile(ir)
|
||||
|
||||
json_str = plan.to_json()
|
||||
plan2 = ExecutionPlan.from_json(json_str)
|
||||
|
||||
actions = execution_plan_to_actions(plan2)
|
||||
assert len(actions) == 1
|
||||
assert actions[0]["type"] == "click"
|
||||
727
tests/unit/test_lea_notifications.py
Normal file
727
tests/unit/test_lea_notifications.py
Normal file
@@ -0,0 +1,727 @@
|
||||
"""Tests unitaires pour l'UX de Léa (notifications, messages, activity panel).
|
||||
|
||||
Couvre :
|
||||
- Formatage des messages techniques → français naturel (module messages.py)
|
||||
- Hiérarchie info/attention/blocage
|
||||
- Détection de la fenêtre Léa
|
||||
- NotificationManager avec plyer mocké
|
||||
- ActivityPanel sans tkinter (fallback silencieux)
|
||||
|
||||
Ces tests ne nécessitent ni tkinter ni plyer : tout est mocké ou géré en
|
||||
fallback silencieux. Ils doivent passer sur toutes les plateformes.
|
||||
|
||||
Auteur: Dom, avril 2026
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
# Assurer que la racine du projet est dans le path (comme conftest)
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from agent_v0.agent_v1.ui import activity_panel, messages, notifications
|
||||
from agent_v0.agent_v1.ui.activity_panel import ActivityPanel, EtatLea, reset_activity_panel
|
||||
from agent_v0.agent_v1.ui.messages import (
|
||||
MessageUtilisateur,
|
||||
NiveauMessage,
|
||||
_extraire_nom_application,
|
||||
_nettoyer_description_cible,
|
||||
est_fenetre_lea,
|
||||
formatter_cible_non_trouvee,
|
||||
formatter_connexion_perdue,
|
||||
formatter_connexion_retablie,
|
||||
formatter_debut_workflow,
|
||||
formatter_ecran_inchange,
|
||||
formatter_erreur_generique,
|
||||
formatter_etape_workflow,
|
||||
formatter_fenetre_incorrecte,
|
||||
formatter_fin_workflow,
|
||||
formatter_ralentissement,
|
||||
formatter_retry,
|
||||
)
|
||||
from agent_v0.agent_v1.ui.notifications import NotificationManager
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Tests : helpers d'extraction
|
||||
# ============================================================================
|
||||
|
||||
|
||||
class TestExtraction:
|
||||
"""Tests des helpers _extraire_nom_application et _nettoyer_description_cible."""
|
||||
|
||||
def test_extraire_app_avec_em_dash(self):
|
||||
assert _extraire_nom_application("Document.txt – Bloc-notes") == "Bloc-notes"
|
||||
|
||||
def test_extraire_app_avec_em_dash_long(self):
|
||||
assert _extraire_nom_application("Ma Page — Google Chrome") == "Google Chrome"
|
||||
|
||||
def test_extraire_app_avec_dash_simple(self):
|
||||
assert _extraire_nom_application("Session 1 - Firefox") == "Firefox"
|
||||
|
||||
def test_extraire_app_sans_separateur(self):
|
||||
assert _extraire_nom_application("Bloc-notes") == "Bloc-notes"
|
||||
|
||||
def test_extraire_app_vide(self):
|
||||
assert _extraire_nom_application("") == ""
|
||||
assert _extraire_nom_application(None) == ""
|
||||
|
||||
def test_extraire_app_garde_dernier_separateur(self):
|
||||
# Cas multi-séparateurs : on garde la dernière partie
|
||||
assert _extraire_nom_application("A - B - C") == "C"
|
||||
|
||||
def test_nettoyer_description_retire_guillemets(self):
|
||||
assert _nettoyer_description_cible("'bonjour'") == "bonjour"
|
||||
assert _nettoyer_description_cible('"bonjour"') == "bonjour"
|
||||
assert _nettoyer_description_cible("`code`") == "code"
|
||||
|
||||
def test_nettoyer_description_vide(self):
|
||||
assert _nettoyer_description_cible("") == ""
|
||||
assert _nettoyer_description_cible(None) == ""
|
||||
|
||||
def test_nettoyer_description_tronque(self):
|
||||
longue = "x" * 200
|
||||
resultat = _nettoyer_description_cible(longue)
|
||||
assert len(resultat) <= 80
|
||||
assert resultat.endswith("...")
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Tests : détection fenêtre Léa
|
||||
# ============================================================================
|
||||
|
||||
|
||||
class TestFenetreLea:
|
||||
"""Tests de est_fenetre_lea — crucial pour la robustesse."""
|
||||
|
||||
@pytest.mark.parametrize("titre", [
|
||||
"Léa",
|
||||
"Léa — Assistante IA",
|
||||
"Lea - Assistante",
|
||||
"Léa — Activité",
|
||||
"Lea : Explorateur de fichiers",
|
||||
"LÉA — ASSISTANTE IA", # casse mixte
|
||||
"Léa assistante",
|
||||
"Assistante IA",
|
||||
])
|
||||
def test_detecte_fenetres_lea(self, titre):
|
||||
assert est_fenetre_lea(titre), f"Devrait détecter : {titre!r}"
|
||||
|
||||
@pytest.mark.parametrize("titre", [
|
||||
"Bloc-notes",
|
||||
"Google Chrome",
|
||||
"Program Manager",
|
||||
"Microsoft Word - Document1",
|
||||
"Sans titre - Paint",
|
||||
"",
|
||||
"cléa.txt", # contient "léa" mais c'est un fichier
|
||||
"replay.log", # contient "lea"
|
||||
"leapfrog.exe", # contient "lea"
|
||||
"nucleaire.pdf", # contient "lea"
|
||||
])
|
||||
def test_ignore_fenetres_non_lea(self, titre):
|
||||
"""Les faux positifs sur des noms contenant 'lea' doivent être évités
|
||||
grâce aux word boundaries regex."""
|
||||
assert not est_fenetre_lea(titre), f"Ne devrait pas détecter : {titre!r}"
|
||||
|
||||
def test_titre_none(self):
|
||||
assert est_fenetre_lea(None) is False
|
||||
|
||||
def test_espaces_en_trop(self):
|
||||
assert est_fenetre_lea(" Léa — Assistante IA ") is True
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Tests : formatage des messages techniques → humains
|
||||
# ============================================================================
|
||||
|
||||
|
||||
class TestFormatterCibleNonTrouvee:
|
||||
"""Tests du formatage quand un élément n'est pas trouvé."""
|
||||
|
||||
def test_message_blocage(self):
|
||||
msg = formatter_cible_non_trouvee("bonjour", "Document – Bloc-notes")
|
||||
assert msg.niveau == NiveauMessage.BLOCAGE
|
||||
assert msg.persistent is True
|
||||
assert "besoin d'aide" in msg.titre.lower()
|
||||
|
||||
def test_message_contient_nom_element(self):
|
||||
msg = formatter_cible_non_trouvee("Rechercher", "Chrome")
|
||||
assert "rechercher" in msg.corps.lower()
|
||||
|
||||
def test_message_contient_nom_application(self):
|
||||
msg = formatter_cible_non_trouvee("bonjour", "Doc – Bloc-notes")
|
||||
assert "bloc-notes" in msg.corps.lower()
|
||||
|
||||
def test_message_action_orientee(self):
|
||||
"""Le message doit proposer une action à l'utilisateur."""
|
||||
msg = formatter_cible_non_trouvee("bouton", "App")
|
||||
corps_lower = msg.corps.lower()
|
||||
# Doit contenir un verbe d'action type "cliquer", "faire"
|
||||
assert any(verb in corps_lower for verb in ["cliqu", "faire", "peux-tu"])
|
||||
|
||||
def test_sans_fenetre(self):
|
||||
msg = formatter_cible_non_trouvee("Submit", None)
|
||||
assert msg.niveau == NiveauMessage.BLOCAGE
|
||||
assert "submit" in msg.corps.lower()
|
||||
|
||||
def test_description_vide(self):
|
||||
msg = formatter_cible_non_trouvee("", "App")
|
||||
# Doit quand même produire un message utilisable
|
||||
assert msg.corps
|
||||
assert msg.niveau == NiveauMessage.BLOCAGE
|
||||
|
||||
def test_message_techniques_nettoyes(self):
|
||||
"""Pas de '__target_not_found__' ni code technique visible."""
|
||||
msg = formatter_cible_non_trouvee("'bonjour'", "Bloc-notes")
|
||||
assert "target_not_found" not in msg.corps
|
||||
# Les guillemets techniques sont nettoyés, mais on en ajoute des français
|
||||
assert "bonjour" in msg.corps
|
||||
|
||||
|
||||
class TestFormatterFenetreIncorrecte:
|
||||
"""Tests du formatage quand la mauvaise fenêtre est active."""
|
||||
|
||||
def test_message_blocage_persistent(self):
|
||||
msg = formatter_fenetre_incorrecte(
|
||||
"Program Manager",
|
||||
"Lea : Explorateur de fichiers",
|
||||
)
|
||||
assert msg.niveau == NiveauMessage.BLOCAGE
|
||||
assert msg.persistent is True
|
||||
|
||||
def test_mentionne_fenetre_attendue(self):
|
||||
msg = formatter_fenetre_incorrecte("Program Manager", "Chrome")
|
||||
assert "chrome" in msg.corps.lower()
|
||||
|
||||
def test_mentionne_fenetre_actuelle(self):
|
||||
msg = formatter_fenetre_incorrecte("Program Manager", "Chrome")
|
||||
assert "program manager" in msg.corps.lower()
|
||||
|
||||
def test_suggere_action(self):
|
||||
msg = formatter_fenetre_incorrecte("A", "B")
|
||||
# Propose d'ouvrir la bonne fenêtre
|
||||
assert "ouvr" in msg.corps.lower() or "fenêtre" in msg.corps.lower()
|
||||
|
||||
|
||||
class TestFormatterEcranInchange:
|
||||
"""Tests du formatage quand l'écran ne change pas après une action."""
|
||||
|
||||
def test_niveau_attention(self):
|
||||
"""L'écran inchangé est de niveau ATTENTION, pas BLOCAGE."""
|
||||
msg = formatter_ecran_inchange("click")
|
||||
assert msg.niveau == NiveauMessage.ATTENTION
|
||||
|
||||
def test_message_pour_click(self):
|
||||
msg = formatter_ecran_inchange("click")
|
||||
assert "clic" in msg.corps.lower()
|
||||
|
||||
def test_message_pour_type(self):
|
||||
msg = formatter_ecran_inchange("type")
|
||||
assert "saisie" in msg.corps.lower()
|
||||
|
||||
def test_message_pour_key_combo(self):
|
||||
msg = formatter_ecran_inchange("key_combo")
|
||||
assert "raccourci" in msg.corps.lower()
|
||||
|
||||
def test_sans_type_action(self):
|
||||
msg = formatter_ecran_inchange("")
|
||||
assert msg.corps # Doit quand même produire quelque chose
|
||||
|
||||
def test_pas_persistent(self):
|
||||
msg = formatter_ecran_inchange("click")
|
||||
assert msg.persistent is False
|
||||
|
||||
|
||||
class TestFormatterConnexion:
|
||||
"""Tests des messages de connexion serveur."""
|
||||
|
||||
def test_connexion_perdue_attention(self):
|
||||
msg = formatter_connexion_perdue("localhost")
|
||||
assert msg.niveau == NiveauMessage.ATTENTION
|
||||
|
||||
def test_connexion_perdue_rassurante(self):
|
||||
"""Le message doit rassurer (reconnexion automatique)."""
|
||||
msg = formatter_connexion_perdue()
|
||||
assert "automatique" in msg.corps.lower() or "retent" in msg.corps.lower()
|
||||
|
||||
def test_connexion_retablie_info(self):
|
||||
msg = formatter_connexion_retablie()
|
||||
assert msg.niveau == NiveauMessage.INFO
|
||||
|
||||
def test_connexion_retablie_positive(self):
|
||||
msg = formatter_connexion_retablie()
|
||||
assert "bon" in msg.corps.lower() or "revenue" in msg.corps.lower()
|
||||
|
||||
|
||||
class TestFormatterWorkflow:
|
||||
"""Tests des messages de workflow (début, étape, fin)."""
|
||||
|
||||
def test_debut_avec_etapes(self):
|
||||
msg = formatter_debut_workflow("Saisie patient", 15)
|
||||
assert msg.niveau == NiveauMessage.INFO
|
||||
assert "saisie patient" in msg.corps.lower()
|
||||
assert "15" in msg.corps
|
||||
|
||||
def test_debut_sans_etapes(self):
|
||||
msg = formatter_debut_workflow("Backup")
|
||||
assert msg.niveau == NiveauMessage.INFO
|
||||
assert "backup" in msg.corps.lower()
|
||||
|
||||
def test_etape_progression(self):
|
||||
msg = formatter_etape_workflow(3, 15, "Clic sur Valider")
|
||||
assert "3" in msg.corps
|
||||
assert "15" in msg.corps
|
||||
assert "valider" in msg.corps.lower()
|
||||
|
||||
def test_etape_sans_description(self):
|
||||
msg = formatter_etape_workflow(5, 20)
|
||||
assert "5" in msg.corps
|
||||
assert "20" in msg.corps
|
||||
|
||||
def test_fin_succes(self):
|
||||
msg = formatter_fin_workflow(True, "Ma tâche", 10, 45.0)
|
||||
assert msg.niveau == NiveauMessage.INFO
|
||||
assert "terminé" in msg.corps.lower() or "fait" in msg.corps.lower()
|
||||
|
||||
def test_fin_echec_blocage(self):
|
||||
msg = formatter_fin_workflow(False, "Ma tâche")
|
||||
assert msg.niveau == NiveauMessage.BLOCAGE
|
||||
assert msg.persistent is True
|
||||
|
||||
|
||||
class TestFormatterRetryRalentissement:
|
||||
"""Tests des messages de retry et ralentissement."""
|
||||
|
||||
def test_retry_attention(self):
|
||||
msg = formatter_retry("click", 2)
|
||||
assert msg.niveau == NiveauMessage.ATTENTION
|
||||
assert "2" in msg.corps # numéro de tentative
|
||||
|
||||
def test_ralentissement_attention(self):
|
||||
msg = formatter_ralentissement()
|
||||
assert msg.niveau == NiveauMessage.ATTENTION
|
||||
assert "lent" in msg.corps.lower()
|
||||
|
||||
|
||||
class TestFormatterErreurGenerique:
|
||||
"""Tests du router formatter_erreur_generique → spécialisé."""
|
||||
|
||||
def test_detecte_target_not_found(self):
|
||||
msg = formatter_erreur_generique("target_not_found: 'bouton'")
|
||||
assert msg.niveau == NiveauMessage.BLOCAGE
|
||||
assert "bouton" in msg.corps.lower()
|
||||
|
||||
def test_detecte_fenetre_incorrecte(self):
|
||||
msg = formatter_erreur_generique(
|
||||
"Fenêtre incorrecte: 'Program Manager' (attendu: 'Chrome')"
|
||||
)
|
||||
assert msg.niveau == NiveauMessage.BLOCAGE
|
||||
assert "chrome" in msg.corps.lower() or "program manager" in msg.corps.lower()
|
||||
|
||||
def test_detecte_ecran_inchange(self):
|
||||
msg = formatter_erreur_generique("Ecran inchange apres l'action")
|
||||
assert msg.niveau == NiveauMessage.ATTENTION
|
||||
|
||||
def test_detecte_no_screen_change(self):
|
||||
msg = formatter_erreur_generique("no_screen_change after click")
|
||||
assert msg.niveau == NiveauMessage.ATTENTION
|
||||
|
||||
def test_detecte_policy_abort(self):
|
||||
msg = formatter_erreur_generique("policy_abort:target_desc_x")
|
||||
assert msg.niveau == NiveauMessage.BLOCAGE
|
||||
|
||||
def test_message_vide(self):
|
||||
msg = formatter_erreur_generique("")
|
||||
assert msg.corps
|
||||
assert msg.niveau == NiveauMessage.ATTENTION
|
||||
|
||||
def test_message_inconnu_tronque(self):
|
||||
long_msg = "erreur très longue " * 20
|
||||
msg = formatter_erreur_generique(long_msg)
|
||||
assert len(msg.corps) <= 200 # tronqué avec "..."
|
||||
|
||||
def test_pas_de_code_technique_dans_message_utilisateur(self):
|
||||
"""Les messages présentés à l'utilisateur ne doivent pas contenir de
|
||||
noms de variables, de fonctions, ou de types Python."""
|
||||
msg = formatter_erreur_generique("target_not_found: 'bouton'")
|
||||
# Le code technique ne doit pas apparaître tel quel dans le corps
|
||||
assert "target_not_found" not in msg.corps
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Tests : hiérarchie NiveauMessage
|
||||
# ============================================================================
|
||||
|
||||
|
||||
class TestHierarchieNiveau:
|
||||
"""Tests de la hiérarchie info/attention/blocage."""
|
||||
|
||||
def test_niveau_info_duree_courte(self):
|
||||
msg = formatter_connexion_retablie()
|
||||
assert msg.niveau == NiveauMessage.INFO
|
||||
assert msg.duree_s <= 6
|
||||
|
||||
def test_niveau_attention_duree_moyenne(self):
|
||||
msg = formatter_ecran_inchange("click")
|
||||
assert msg.niveau == NiveauMessage.ATTENTION
|
||||
assert 5 <= msg.duree_s <= 10
|
||||
|
||||
def test_niveau_blocage_duree_longue_persistent(self):
|
||||
msg = formatter_cible_non_trouvee("x", "y")
|
||||
assert msg.niveau == NiveauMessage.BLOCAGE
|
||||
assert msg.duree_s >= 10
|
||||
assert msg.persistent is True
|
||||
|
||||
def test_niveau_info_non_persistent(self):
|
||||
msg = formatter_debut_workflow("test")
|
||||
assert msg.persistent is False
|
||||
|
||||
def test_to_dict_serialisation(self):
|
||||
msg = MessageUtilisateur(
|
||||
niveau=NiveauMessage.INFO,
|
||||
titre="Test",
|
||||
corps="Corps",
|
||||
duree_s=5,
|
||||
)
|
||||
d = msg.to_dict()
|
||||
assert d["niveau"] == "info"
|
||||
assert d["titre"] == "Test"
|
||||
assert d["corps"] == "Corps"
|
||||
assert d["duree_s"] == 5
|
||||
assert d["persistent"] is False
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Tests : NotificationManager (avec plyer mocké)
|
||||
# ============================================================================
|
||||
|
||||
|
||||
class TestNotificationManager:
|
||||
"""Tests du NotificationManager avec plyer mocké.
|
||||
|
||||
Ces tests ne dépendent pas de l'environnement : plyer est patché pour
|
||||
qu'on puisse vérifier les appels sans afficher de vraies notifications.
|
||||
"""
|
||||
|
||||
def test_instanciation(self):
|
||||
mgr = NotificationManager()
|
||||
assert mgr is not None
|
||||
|
||||
def test_notify_sans_plyer(self, monkeypatch):
|
||||
"""Si plyer n'est pas dispo, notify() retourne False sans crasher."""
|
||||
monkeypatch.setattr(notifications, "_PLYER_AVAILABLE", False)
|
||||
mgr = NotificationManager()
|
||||
assert mgr.notify("titre", "message") is False
|
||||
|
||||
def test_notify_avec_plyer_mocke(self, monkeypatch):
|
||||
mock_plyer = MagicMock()
|
||||
monkeypatch.setattr(notifications, "_PLYER_AVAILABLE", True)
|
||||
monkeypatch.setattr(notifications, "_plyer_notification", mock_plyer)
|
||||
|
||||
mgr = NotificationManager()
|
||||
result = mgr.notify("Titre", "Message", timeout=5)
|
||||
assert result is True
|
||||
# L'envoi est asynchrone, laissons le thread démarrer
|
||||
time.sleep(0.1)
|
||||
mock_plyer.notify.assert_called_once()
|
||||
|
||||
def test_rate_limit(self, monkeypatch):
|
||||
"""Le rate limit bloque les notifications trop rapprochées."""
|
||||
mock_plyer = MagicMock()
|
||||
monkeypatch.setattr(notifications, "_PLYER_AVAILABLE", True)
|
||||
monkeypatch.setattr(notifications, "_plyer_notification", mock_plyer)
|
||||
|
||||
mgr = NotificationManager()
|
||||
assert mgr.notify("T1", "M1") is True
|
||||
# Immédiatement après → bloqué
|
||||
assert mgr.notify("T2", "M2") is False
|
||||
|
||||
def test_bypass_rate_limit_pour_blocage(self, monkeypatch):
|
||||
"""Les messages BLOCAGE bypass le rate limit."""
|
||||
mock_plyer = MagicMock()
|
||||
monkeypatch.setattr(notifications, "_PLYER_AVAILABLE", True)
|
||||
monkeypatch.setattr(notifications, "_plyer_notification", mock_plyer)
|
||||
|
||||
mgr = NotificationManager()
|
||||
assert mgr.notify("T1", "M1") is True
|
||||
# Sans bypass → bloqué
|
||||
assert mgr.notify("T2", "M2") is False
|
||||
# Avec bypass → passe
|
||||
assert mgr.notify("T3", "M3", bypass_rate_limit=True) is True
|
||||
|
||||
def test_notify_message_niveau_blocage_bypass(self, monkeypatch):
|
||||
mock_plyer = MagicMock()
|
||||
monkeypatch.setattr(notifications, "_PLYER_AVAILABLE", True)
|
||||
monkeypatch.setattr(notifications, "_plyer_notification", mock_plyer)
|
||||
|
||||
mgr = NotificationManager()
|
||||
# Occuper le rate limit
|
||||
mgr.notify("T0", "M0")
|
||||
# Message BLOCAGE doit passer même pendant le rate limit
|
||||
msg_blocage = formatter_cible_non_trouvee("x", "y")
|
||||
assert mgr.notify_message(msg_blocage) is True
|
||||
|
||||
def test_replay_target_not_found_avec_titre(self, monkeypatch):
|
||||
"""L'API spécialisée produit un message contenant le nom d'app."""
|
||||
mock_plyer = MagicMock()
|
||||
monkeypatch.setattr(notifications, "_PLYER_AVAILABLE", True)
|
||||
monkeypatch.setattr(notifications, "_plyer_notification", mock_plyer)
|
||||
|
||||
mgr = NotificationManager()
|
||||
mgr.replay_target_not_found("Rechercher", "Document – Bloc-notes")
|
||||
time.sleep(0.1)
|
||||
# Vérifier qu'on a bien envoyé un message qui mentionne l'app
|
||||
args, kwargs = mock_plyer.notify.call_args
|
||||
message_envoye = kwargs.get("message", "")
|
||||
assert "bloc-notes" in message_envoye.lower()
|
||||
assert "rechercher" in message_envoye.lower()
|
||||
|
||||
def test_replay_wrong_window(self, monkeypatch):
|
||||
mock_plyer = MagicMock()
|
||||
monkeypatch.setattr(notifications, "_PLYER_AVAILABLE", True)
|
||||
monkeypatch.setattr(notifications, "_plyer_notification", mock_plyer)
|
||||
|
||||
mgr = NotificationManager()
|
||||
mgr.replay_wrong_window("Program Manager", "Chrome")
|
||||
time.sleep(0.1)
|
||||
args, kwargs = mock_plyer.notify.call_args
|
||||
titre = kwargs.get("title", "")
|
||||
# Le titre doit indiquer l'attente d'une fenêtre
|
||||
assert "fenêtre" in titre.lower() or "attend" in titre.lower()
|
||||
|
||||
def test_error_route_vers_formatter_specialise(self, monkeypatch):
|
||||
"""error() détecte 'target_not_found' et produit un message de blocage."""
|
||||
mock_plyer = MagicMock()
|
||||
monkeypatch.setattr(notifications, "_PLYER_AVAILABLE", True)
|
||||
monkeypatch.setattr(notifications, "_plyer_notification", mock_plyer)
|
||||
|
||||
mgr = NotificationManager()
|
||||
mgr.error("target_not_found: 'bonjour'")
|
||||
time.sleep(0.1)
|
||||
mock_plyer.notify.assert_called_once()
|
||||
args, kwargs = mock_plyer.notify.call_args
|
||||
# Le message envoyé doit être en français naturel, pas le code brut
|
||||
message_envoye = kwargs.get("message", "")
|
||||
assert "target_not_found" not in message_envoye
|
||||
|
||||
def test_backward_compat_connection_changed(self, monkeypatch):
|
||||
"""L'API existante connection_changed reste fonctionnelle."""
|
||||
mock_plyer = MagicMock()
|
||||
monkeypatch.setattr(notifications, "_PLYER_AVAILABLE", True)
|
||||
monkeypatch.setattr(notifications, "_plyer_notification", mock_plyer)
|
||||
|
||||
mgr = NotificationManager()
|
||||
# Déconnexion
|
||||
mgr.connection_changed(False, "localhost")
|
||||
time.sleep(0.1)
|
||||
assert mock_plyer.notify.called
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Tests : ActivityPanel (sans tkinter)
|
||||
# ============================================================================
|
||||
|
||||
|
||||
class TestActivityPanelFallback:
|
||||
"""Tests du panel d'activité en mode fallback (sans tkinter)."""
|
||||
|
||||
def setup_method(self):
|
||||
reset_activity_panel()
|
||||
|
||||
def teardown_method(self):
|
||||
reset_activity_panel()
|
||||
|
||||
def test_creation_sans_ui(self):
|
||||
"""Le panel peut être créé sans UI (activer_ui=False)."""
|
||||
panel = ActivityPanel(activer_ui=False)
|
||||
assert panel is not None
|
||||
|
||||
def test_snapshot_initial(self):
|
||||
panel = ActivityPanel(activer_ui=False)
|
||||
snap = panel.snapshot()
|
||||
assert snap.etat == EtatLea.INACTIVE
|
||||
assert snap.nom_workflow == ""
|
||||
assert snap.etape == 0
|
||||
|
||||
def test_definir_workflow(self):
|
||||
panel = ActivityPanel(activer_ui=False)
|
||||
panel.definir_workflow("Test", nb_etapes=10)
|
||||
snap = panel.snapshot()
|
||||
assert snap.nom_workflow == "Test"
|
||||
assert snap.nb_etapes == 10
|
||||
assert snap.etat == EtatLea.OBSERVE
|
||||
assert snap.debut_timestamp > 0
|
||||
|
||||
def test_mettre_a_jour_etape(self):
|
||||
panel = ActivityPanel(activer_ui=False)
|
||||
panel.definir_workflow("Test", 10)
|
||||
panel.mettre_a_jour(etat=EtatLea.AGIT, action="Clic", etape=3)
|
||||
snap = panel.snapshot()
|
||||
assert snap.etat == EtatLea.AGIT
|
||||
assert snap.action_courante == "Clic"
|
||||
assert snap.etape == 3
|
||||
|
||||
def test_mettre_a_jour_partiel(self):
|
||||
panel = ActivityPanel(activer_ui=False)
|
||||
panel.definir_workflow("Test", 10)
|
||||
panel.mettre_a_jour(etape=5)
|
||||
snap = panel.snapshot()
|
||||
assert snap.etape == 5
|
||||
# L'état reste OBSERVE (non modifié)
|
||||
assert snap.etat == EtatLea.OBSERVE
|
||||
|
||||
def test_progression_texte(self):
|
||||
panel = ActivityPanel(activer_ui=False)
|
||||
panel.definir_workflow("Test", 10)
|
||||
panel.mettre_a_jour(etape=3)
|
||||
snap = panel.snapshot()
|
||||
assert snap.progression_texte() == "3/10"
|
||||
|
||||
def test_progression_texte_sans_nb_etapes(self):
|
||||
panel = ActivityPanel(activer_ui=False)
|
||||
panel.definir_workflow("Test", nb_etapes=0)
|
||||
snap = panel.snapshot()
|
||||
assert snap.progression_texte() == ""
|
||||
|
||||
def test_temps_ecoule(self):
|
||||
panel = ActivityPanel(activer_ui=False)
|
||||
panel.definir_workflow("Test", 10)
|
||||
time.sleep(0.05)
|
||||
snap = panel.snapshot()
|
||||
assert snap.temps_ecoule_s() >= 0.05
|
||||
|
||||
def test_temps_ecoule_texte_secondes(self):
|
||||
panel = ActivityPanel(activer_ui=False)
|
||||
panel.definir_workflow("Test", 10)
|
||||
snap = panel.snapshot()
|
||||
# Format "Xs" pour < 60s
|
||||
texte = snap.temps_ecoule_texte()
|
||||
assert texte.endswith("s")
|
||||
|
||||
def test_terminer_succes(self):
|
||||
panel = ActivityPanel(activer_ui=False)
|
||||
panel.definir_workflow("Test", 10)
|
||||
panel.terminer(succes=True)
|
||||
snap = panel.snapshot()
|
||||
assert snap.etat == EtatLea.TERMINE
|
||||
|
||||
def test_terminer_echec(self):
|
||||
panel = ActivityPanel(activer_ui=False)
|
||||
panel.definir_workflow("Test", 10)
|
||||
panel.terminer(succes=False)
|
||||
snap = panel.snapshot()
|
||||
assert snap.etat == EtatLea.BLOQUEE
|
||||
assert snap.dernier_message # Un message par défaut est mis
|
||||
|
||||
def test_reinitialiser(self):
|
||||
panel = ActivityPanel(activer_ui=False)
|
||||
panel.definir_workflow("Test", 10)
|
||||
panel.reinitialiser()
|
||||
snap = panel.snapshot()
|
||||
assert snap.etat == EtatLea.INACTIVE
|
||||
assert snap.nom_workflow == ""
|
||||
|
||||
def test_listener_appele_sur_changement(self):
|
||||
panel = ActivityPanel(activer_ui=False)
|
||||
calls = []
|
||||
panel.on_change(lambda snap: calls.append(snap.etat))
|
||||
|
||||
panel.definir_workflow("Test", 5)
|
||||
panel.mettre_a_jour(etat=EtatLea.AGIT)
|
||||
|
||||
assert EtatLea.OBSERVE in calls
|
||||
assert EtatLea.AGIT in calls
|
||||
|
||||
def test_listener_erreur_nintervient_pas(self):
|
||||
"""Un listener qui crash ne doit pas casser le panel."""
|
||||
panel = ActivityPanel(activer_ui=False)
|
||||
|
||||
def listener_casse(snap):
|
||||
raise RuntimeError("boom")
|
||||
|
||||
panel.on_change(listener_casse)
|
||||
# Ne doit pas crasher
|
||||
panel.definir_workflow("Test", 5)
|
||||
snap = panel.snapshot()
|
||||
assert snap.nom_workflow == "Test"
|
||||
|
||||
def test_to_dict_serialisation(self):
|
||||
panel = ActivityPanel(activer_ui=False)
|
||||
panel.definir_workflow("Ma tâche", 10)
|
||||
panel.mettre_a_jour(
|
||||
etat=EtatLea.AGIT,
|
||||
action="Clic sur Valider",
|
||||
etape=3,
|
||||
)
|
||||
d = panel.snapshot().to_dict()
|
||||
assert d["nom_workflow"] == "Ma tâche"
|
||||
assert d["etat"] == "agit"
|
||||
assert d["etat_libelle"] == "Agit"
|
||||
assert d["progression"] == "3/10"
|
||||
assert d["action_courante"] == "Clic sur Valider"
|
||||
|
||||
def test_masquer_sans_ui_ne_crash_pas(self):
|
||||
panel = ActivityPanel(activer_ui=False)
|
||||
# Doit être no-op sans crasher
|
||||
panel.masquer()
|
||||
panel.afficher()
|
||||
|
||||
def test_etats_ont_couleurs_et_libelles(self):
|
||||
"""Vérifier que tous les états ont bien une couleur et un libellé."""
|
||||
for etat in EtatLea:
|
||||
assert etat.libelle
|
||||
assert etat.couleur.startswith("#")
|
||||
assert etat.code
|
||||
|
||||
def test_singleton_global(self):
|
||||
p1 = activity_panel.get_activity_panel(activer_ui=False)
|
||||
p2 = activity_panel.get_activity_panel(activer_ui=False)
|
||||
assert p1 is p2
|
||||
|
||||
def test_reset_singleton(self):
|
||||
p1 = activity_panel.get_activity_panel(activer_ui=False)
|
||||
activity_panel.reset_activity_panel()
|
||||
p2 = activity_panel.get_activity_panel(activer_ui=False)
|
||||
assert p1 is not p2
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Tests : intégration executor ↔ notifier
|
||||
# ============================================================================
|
||||
|
||||
|
||||
class TestExecutorNotifierFallback:
|
||||
"""Vérifier que le Noop fallback de l'executor couvre toutes les méthodes."""
|
||||
|
||||
def test_executor_noop_supporte_toutes_methodes(self):
|
||||
"""Le fallback _Noop doit répondre à n'importe quelle méthode."""
|
||||
# Simuler le cas où NotificationManager lève une exception
|
||||
with patch(
|
||||
"agent_v0.agent_v1.ui.notifications.NotificationManager",
|
||||
side_effect=RuntimeError("UI indisponible"),
|
||||
):
|
||||
from agent_v0.agent_v1.core.executor import ActionExecutorV1
|
||||
# Ne pas vraiment instancier (dépendances mss/pynput) — on teste
|
||||
# la logique du stub en recréant la classe noop inline.
|
||||
|
||||
# Test direct du pattern noop utilisé dans executor
|
||||
class _Noop:
|
||||
def __getattr__(self, name):
|
||||
return lambda *a, **kw: False
|
||||
|
||||
noop = _Noop()
|
||||
# Toutes ces méthodes doivent retourner False sans crasher
|
||||
assert noop.replay_target_not_found("x") is False
|
||||
assert noop.replay_wrong_window("x", "y") is False
|
||||
assert noop.replay_no_screen_change("click") is False
|
||||
assert noop.notify_message(None) is False
|
||||
assert noop.nimporte_quelle_methode() is False
|
||||
430
tests/unit/test_shadow_observer.py
Normal file
430
tests/unit/test_shadow_observer.py
Normal file
@@ -0,0 +1,430 @@
|
||||
"""
|
||||
Tests du ShadowObserver — observation temps réel de Léa.
|
||||
|
||||
Vérifie que :
|
||||
- L'observer démarre et s'arrête correctement
|
||||
- Les événements sont segmentés en étapes logiques
|
||||
- Les variables sont détectées pendant la frappe
|
||||
- Les notifications sont émises avec le bon niveau
|
||||
- La compréhension est accessible en temps réel
|
||||
- Le callback de notification est appelé
|
||||
- Les événements parasites (heartbeat) sont ignorés
|
||||
"""
|
||||
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
_ROOT = str(Path(__file__).resolve().parents[2])
|
||||
if _ROOT not in sys.path:
|
||||
sys.path.insert(0, _ROOT)
|
||||
|
||||
from core.workflow.shadow_observer import (
|
||||
NiveauNotification,
|
||||
NotificationShadow,
|
||||
ShadowObserver,
|
||||
UnderstoodStep,
|
||||
get_shared_observer,
|
||||
)
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Fixtures
|
||||
# =========================================================================
|
||||
|
||||
|
||||
def _evt_click(text="Rechercher", title="Explorateur", ts=100.0):
|
||||
return {
|
||||
"type": "mouse_click",
|
||||
"pos": [400, 580],
|
||||
"window": {"title": title, "app_name": title.split(" - ")[-1] if " - " in title else title},
|
||||
"timestamp": ts,
|
||||
"vision_info": {"text": text},
|
||||
}
|
||||
|
||||
|
||||
def _evt_type(text="bonjour", title="Bloc-notes", ts=101.0):
|
||||
return {
|
||||
"type": "text_input",
|
||||
"text": text,
|
||||
"window": {"title": title, "app_name": title},
|
||||
"timestamp": ts,
|
||||
}
|
||||
|
||||
|
||||
def _evt_key(keys=None, title="Bloc-notes", ts=102.0):
|
||||
return {
|
||||
"type": "key_combo",
|
||||
"keys": keys or ["enter"],
|
||||
"window": {"title": title, "app_name": title},
|
||||
"timestamp": ts,
|
||||
}
|
||||
|
||||
|
||||
def _evt_heartbeat(ts=103.0):
|
||||
return {"type": "heartbeat", "timestamp": ts}
|
||||
|
||||
|
||||
def _make_session_events():
|
||||
"""Scénario typique : ouvrir Bloc-notes, taper du texte, sauvegarder."""
|
||||
return [
|
||||
_evt_click(text="Rechercher", title="Menu Démarrer", ts=100.0),
|
||||
_evt_type(text="blocnote", title="Menu Démarrer", ts=101.0),
|
||||
_evt_key(keys=["enter"], title="Menu Démarrer", ts=102.0),
|
||||
_evt_heartbeat(ts=103.0), # Doit être ignoré
|
||||
# Changement d'application → nouveau segment attendu
|
||||
_evt_click(text="", title="Sans titre - Bloc-notes", ts=105.0),
|
||||
_evt_type(text="Bonjour le monde", title="Sans titre - Bloc-notes", ts=106.0),
|
||||
_evt_key(keys=["ctrl", "s"], title="Sans titre - Bloc-notes", ts=108.0),
|
||||
]
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Tests de base
|
||||
# =========================================================================
|
||||
|
||||
|
||||
class TestShadowObserverBase:
|
||||
|
||||
def test_start_et_stop(self):
|
||||
obs = ShadowObserver()
|
||||
obs.start("sess_test")
|
||||
assert obs.has_session("sess_test")
|
||||
obs.stop("sess_test")
|
||||
assert obs.has_session("sess_test") # stop ne supprime pas
|
||||
obs.reset("sess_test")
|
||||
assert not obs.has_session("sess_test")
|
||||
|
||||
def test_auto_start_sur_premier_event(self):
|
||||
"""observe_event() sans start() doit auto-démarrer la session."""
|
||||
obs = ShadowObserver()
|
||||
obs.observe_event("sess_auto", _evt_click())
|
||||
assert obs.has_session("sess_auto")
|
||||
steps = obs.get_understanding("sess_auto")
|
||||
assert len(steps) >= 1
|
||||
|
||||
def test_heartbeat_ignore(self):
|
||||
obs = ShadowObserver()
|
||||
obs.start("s1")
|
||||
obs.observe_event("s1", _evt_heartbeat())
|
||||
steps = obs.get_understanding("s1")
|
||||
assert len(steps) == 0 # Aucune étape créée par un heartbeat
|
||||
|
||||
def test_focus_change_ignore(self):
|
||||
obs = ShadowObserver()
|
||||
obs.start("s1")
|
||||
obs.observe_event("s1", {"type": "focus_change", "timestamp": 100})
|
||||
assert len(obs.get_understanding("s1")) == 0
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Segmentation
|
||||
# =========================================================================
|
||||
|
||||
|
||||
class TestShadowObserverSegmentation:
|
||||
|
||||
def test_segmentation_par_changement_app(self):
|
||||
"""Un changement d'application crée un nouveau segment."""
|
||||
obs = ShadowObserver()
|
||||
obs.start("s1")
|
||||
obs.observe_event("s1", _evt_click(title="Firefox"))
|
||||
obs.observe_event("s1", _evt_click(title="Firefox", ts=100.5))
|
||||
obs.observe_event("s1", _evt_click(title="Bloc-notes", ts=101.0))
|
||||
|
||||
steps = obs.get_understanding("s1")
|
||||
assert len(steps) >= 2 # Au moins 2 segments
|
||||
|
||||
def test_segmentation_par_pause_longue(self):
|
||||
"""Une pause > 4s coupe le segment."""
|
||||
obs = ShadowObserver()
|
||||
obs.start("s1")
|
||||
obs.observe_event("s1", _evt_click(title="App1", ts=100.0))
|
||||
obs.observe_event("s1", _evt_click(title="App1", ts=100.5))
|
||||
# Pause de 10 secondes
|
||||
obs.observe_event("s1", _evt_click(title="App1", ts=110.5))
|
||||
|
||||
steps = obs.get_understanding("s1")
|
||||
assert len(steps) >= 2
|
||||
|
||||
def test_segment_complet(self):
|
||||
"""Scénario complet : Bloc-notes + texte + save."""
|
||||
obs = ShadowObserver()
|
||||
obs.start("s1")
|
||||
for evt in _make_session_events():
|
||||
obs.observe_event("s1", evt)
|
||||
obs.stop("s1")
|
||||
|
||||
steps = obs.get_understanding("s1")
|
||||
assert len(steps) >= 2 # Au moins Menu Démarrer + Bloc-notes
|
||||
|
||||
# Au moins une étape doit mentionner le Bloc-notes
|
||||
intents = " ".join(s["intent"].lower() for s in steps)
|
||||
assert "bloc" in intents or "enregistr" in intents or "sauvegard" in intents or \
|
||||
"écrir" in intents or "text" in intents.lower()
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Intention et raffinement
|
||||
# =========================================================================
|
||||
|
||||
|
||||
class TestShadowObserverIntent:
|
||||
|
||||
def test_intent_recherche(self):
|
||||
"""Clic + saisie + entrée → 'Rechercher X'."""
|
||||
obs = ShadowObserver()
|
||||
obs.start("s1")
|
||||
obs.observe_event("s1", _evt_click(text="Champ recherche", title="Explorateur", ts=100.0))
|
||||
obs.observe_event("s1", _evt_type(text="calculatrice", title="Explorateur", ts=100.5))
|
||||
obs.observe_event("s1", _evt_key(keys=["enter"], title="Explorateur", ts=101.0))
|
||||
|
||||
current = obs.get_current_step("s1")
|
||||
assert current is not None
|
||||
assert "recherch" in current["intent"].lower() or "calculatrice" in current["intent"].lower()
|
||||
|
||||
def test_intent_ctrl_s(self):
|
||||
"""Ctrl+S → 'Sauvegarder'."""
|
||||
obs = ShadowObserver()
|
||||
obs.start("s1")
|
||||
obs.observe_event("s1", _evt_click(title="Bloc-notes", ts=100.0))
|
||||
obs.observe_event("s1", _evt_key(keys=["ctrl", "s"], title="Bloc-notes", ts=100.5))
|
||||
|
||||
current = obs.get_current_step("s1")
|
||||
assert current is not None
|
||||
assert "sauvegard" in current["intent"].lower()
|
||||
|
||||
def test_intent_ecrire(self):
|
||||
"""Saisie seule → 'Écrire'."""
|
||||
obs = ShadowObserver()
|
||||
obs.start("s1")
|
||||
obs.observe_event("s1", _evt_type(text="Un texte libre", title="Bloc-notes"))
|
||||
|
||||
current = obs.get_current_step("s1")
|
||||
assert current is not None
|
||||
assert "écri" in current["intent"].lower() or "text" in current["intent"].lower()
|
||||
|
||||
def test_confidence_augmente_avec_contexte(self):
|
||||
"""La confiance augmente quand le contexte devient clair."""
|
||||
obs = ShadowObserver()
|
||||
obs.start("s1")
|
||||
obs.observe_event("s1", _evt_click(title="App"))
|
||||
c1 = obs.get_current_step("s1")["confidence"]
|
||||
|
||||
obs.observe_event("s1", _evt_key(keys=["ctrl", "s"], title="App"))
|
||||
c2 = obs.get_current_step("s1")["confidence"]
|
||||
|
||||
assert c2 >= c1
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Détection de variables
|
||||
# =========================================================================
|
||||
|
||||
|
||||
class TestShadowObserverVariables:
|
||||
|
||||
def test_variable_detectee_lors_saisie(self):
|
||||
"""Une saisie texte > 3 caractères crée une variable."""
|
||||
obs = ShadowObserver()
|
||||
notifs = []
|
||||
obs._notify_callback = lambda n: notifs.append(n)
|
||||
obs.start("s1")
|
||||
|
||||
obs.observe_event("s1", _evt_type(text="Jean Dupont", title="Formulaire"))
|
||||
|
||||
var_notifs = [n for n in notifs if n.niveau == NiveauNotification.VARIABLE]
|
||||
assert len(var_notifs) == 1
|
||||
assert "Jean Dupont" in var_notifs[0].message
|
||||
assert var_notifs[0].data["variable_name"].startswith("texte_")
|
||||
|
||||
def test_variable_type_date(self):
|
||||
obs = ShadowObserver()
|
||||
obs.start("s1")
|
||||
obs.observe_event("s1", _evt_type(text="15/03/2026", title="Formulaire"))
|
||||
|
||||
current = obs.get_current_step("s1")
|
||||
assert len(current["variables_detectees"]) == 1
|
||||
|
||||
def test_variable_type_email(self):
|
||||
obs = ShadowObserver()
|
||||
notifs = []
|
||||
obs._notify_callback = lambda n: notifs.append(n)
|
||||
obs.start("s1")
|
||||
|
||||
obs.observe_event("s1", _evt_type(text="jean@example.com", title="Formulaire"))
|
||||
|
||||
var_notifs = [n for n in notifs if n.niveau == NiveauNotification.VARIABLE]
|
||||
assert len(var_notifs) == 1
|
||||
assert "e-mail" in var_notifs[0].message or "mail" in var_notifs[0].message
|
||||
|
||||
def test_texte_court_ignore(self):
|
||||
"""Un texte de moins de 3 caractères n'est pas une variable."""
|
||||
obs = ShadowObserver()
|
||||
notifs = []
|
||||
obs._notify_callback = lambda n: notifs.append(n)
|
||||
obs.start("s1")
|
||||
|
||||
obs.observe_event("s1", _evt_type(text="ab", title="App"))
|
||||
|
||||
var_notifs = [n for n in notifs if n.niveau == NiveauNotification.VARIABLE]
|
||||
assert len(var_notifs) == 0
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Notifications et callbacks
|
||||
# =========================================================================
|
||||
|
||||
|
||||
class TestShadowObserverNotifications:
|
||||
|
||||
def test_notification_au_demarrage(self):
|
||||
notifs = []
|
||||
obs = ShadowObserver(notify_callback=lambda n: notifs.append(n))
|
||||
obs.start("s1")
|
||||
|
||||
assert len(notifs) >= 1
|
||||
assert notifs[0].niveau == NiveauNotification.INFO
|
||||
assert "observe" in notifs[0].message.lower()
|
||||
|
||||
def test_notification_nouvelle_etape(self):
|
||||
"""Un changement d'application émet une notification DECOUVERTE."""
|
||||
notifs = []
|
||||
obs = ShadowObserver(notify_callback=lambda n: notifs.append(n))
|
||||
obs.start("s1")
|
||||
|
||||
obs.observe_event("s1", _evt_click(title="Firefox", ts=100.0))
|
||||
obs.observe_event("s1", _evt_click(title="Bloc-notes", ts=101.0))
|
||||
|
||||
decouverts = [n for n in notifs if n.niveau == NiveauNotification.DECOUVERTE]
|
||||
assert len(decouverts) >= 1
|
||||
|
||||
def test_notification_stop_resume(self):
|
||||
"""Au stop, on émet un résumé du nombre d'étapes."""
|
||||
notifs = []
|
||||
obs = ShadowObserver(notify_callback=lambda n: notifs.append(n))
|
||||
obs.start("s1")
|
||||
for evt in _make_session_events():
|
||||
obs.observe_event("s1", evt)
|
||||
obs.stop("s1")
|
||||
|
||||
messages = [n.message.lower() for n in notifs]
|
||||
assert any("étape" in m or "observ" in m for m in messages)
|
||||
|
||||
def test_notifications_since_ts(self):
|
||||
"""get_notifications(since_ts=...) filtre correctement."""
|
||||
obs = ShadowObserver()
|
||||
obs.start("s1")
|
||||
time.sleep(0.01)
|
||||
mid_ts = time.time()
|
||||
time.sleep(0.01)
|
||||
obs.observe_event("s1", _evt_click(title="Firefox"))
|
||||
obs.observe_event("s1", _evt_click(title="Bloc-notes"))
|
||||
|
||||
recentes = obs.get_notifications("s1", since_ts=mid_ts)
|
||||
toutes = obs.get_notifications("s1", since_ts=0)
|
||||
assert len(recentes) < len(toutes)
|
||||
|
||||
def test_callback_erreur_ne_plante_pas(self):
|
||||
"""Un callback qui lève ne doit pas faire planter l'observer."""
|
||||
def bad_callback(notif):
|
||||
raise RuntimeError("boom")
|
||||
|
||||
obs = ShadowObserver(notify_callback=bad_callback)
|
||||
obs.start("s1") # Devrait émettre une notification (qui plante en callback)
|
||||
# Si on arrive ici, c'est OK
|
||||
obs.observe_event("s1", _evt_click())
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Compréhension et API publique
|
||||
# =========================================================================
|
||||
|
||||
|
||||
class TestShadowObserverUnderstanding:
|
||||
|
||||
def test_get_understanding_format(self):
|
||||
"""La structure retournée est bien celle attendue."""
|
||||
obs = ShadowObserver()
|
||||
obs.start("s1")
|
||||
obs.observe_event("s1", _evt_click(title="App"))
|
||||
|
||||
steps = obs.get_understanding("s1")
|
||||
assert isinstance(steps, list)
|
||||
assert len(steps) >= 1
|
||||
step = steps[0]
|
||||
assert "step" in step
|
||||
assert "intent" in step
|
||||
assert "confidence" in step
|
||||
assert isinstance(step["step"], int)
|
||||
assert 0.0 <= step["confidence"] <= 1.0
|
||||
|
||||
def test_get_understanding_sans_session(self):
|
||||
obs = ShadowObserver()
|
||||
steps = obs.get_understanding("inexistante")
|
||||
assert steps == []
|
||||
|
||||
def test_get_current_step(self):
|
||||
obs = ShadowObserver()
|
||||
obs.start("s1")
|
||||
assert obs.get_current_step("s1") is None
|
||||
|
||||
obs.observe_event("s1", _evt_click(title="App"))
|
||||
current = obs.get_current_step("s1")
|
||||
assert current is not None
|
||||
assert current["step_index"] == 1
|
||||
|
||||
def test_get_steps_internal(self):
|
||||
"""get_steps_internal retourne des UnderstoodStep copiés."""
|
||||
obs = ShadowObserver()
|
||||
obs.start("s1")
|
||||
obs.observe_event("s1", _evt_click(title="App"))
|
||||
|
||||
internals = obs.get_steps_internal("s1")
|
||||
assert len(internals) >= 1
|
||||
assert isinstance(internals[0], UnderstoodStep)
|
||||
|
||||
# Mutation externe ne doit pas affecter l'observer
|
||||
internals[0].intent = "HACKED"
|
||||
again = obs.get_steps_internal("s1")
|
||||
assert again[0].intent != "HACKED"
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Singleton partagé
|
||||
# =========================================================================
|
||||
|
||||
|
||||
class TestSharedObserver:
|
||||
|
||||
def test_singleton(self):
|
||||
obs1 = get_shared_observer()
|
||||
obs2 = get_shared_observer()
|
||||
assert obs1 is obs2
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Performance (contrainte : observe_event doit être rapide)
|
||||
# =========================================================================
|
||||
|
||||
|
||||
class TestShadowObserverPerformance:
|
||||
|
||||
def test_observe_event_rapide(self):
|
||||
"""observe_event() doit traiter 1000 events en moins de 500ms."""
|
||||
obs = ShadowObserver()
|
||||
obs.start("s_perf")
|
||||
|
||||
events = []
|
||||
for i in range(1000):
|
||||
events.append(_evt_click(title="App", ts=100.0 + i * 0.01))
|
||||
|
||||
start = time.time()
|
||||
for evt in events:
|
||||
obs.observe_event("s_perf", evt)
|
||||
elapsed = time.time() - start
|
||||
|
||||
assert elapsed < 0.5, f"Trop lent : {elapsed:.2f}s pour 1000 events"
|
||||
531
tests/unit/test_shadow_validator.py
Normal file
531
tests/unit/test_shadow_validator.py
Normal file
@@ -0,0 +1,531 @@
|
||||
"""
|
||||
Tests du ShadowValidator — feedback utilisateur et reconstruction WorkflowIR.
|
||||
|
||||
Vérifie que :
|
||||
- Les feedbacks (validate/correct/undo/merge_next/split/cancel) sont appliqués
|
||||
- Le WorkflowIR final est bien reconstruit à partir des étapes corrigées
|
||||
- Les variables sont détectées dans les actions finales
|
||||
- L'historique des feedbacks est conservé
|
||||
- Les erreurs (index invalide, action inconnue) sont gérées proprement
|
||||
"""
|
||||
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
_ROOT = str(Path(__file__).resolve().parents[2])
|
||||
if _ROOT not in sys.path:
|
||||
sys.path.insert(0, _ROOT)
|
||||
|
||||
from core.workflow.shadow_observer import ShadowObserver, UnderstoodStep
|
||||
from core.workflow.shadow_validator import FeedbackResult, ShadowValidator
|
||||
from core.workflow.workflow_ir import WorkflowIR
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Fixtures
|
||||
# =========================================================================
|
||||
|
||||
|
||||
def _make_step(step_index=1, intent="Ouvrir Firefox", app="Firefox",
|
||||
events=None, **kwargs):
|
||||
return UnderstoodStep(
|
||||
step_index=step_index,
|
||||
intent=intent,
|
||||
app_name=app,
|
||||
window_title=app,
|
||||
events=events or [
|
||||
{"type": "mouse_click", "window": {"title": app},
|
||||
"vision_info": {"text": "bouton"}, "timestamp": 100.0}
|
||||
],
|
||||
started_at=100.0,
|
||||
ended_at=101.0,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
def _make_type_step(texte="Bonjour", app="Bloc-notes", step_index=1):
|
||||
return UnderstoodStep(
|
||||
step_index=step_index,
|
||||
intent=f"Écrire « {texte} »",
|
||||
app_name=app,
|
||||
window_title=app,
|
||||
events=[
|
||||
{"type": "text_input", "text": texte,
|
||||
"window": {"title": app}, "timestamp": 100.0}
|
||||
],
|
||||
started_at=100.0,
|
||||
)
|
||||
|
||||
|
||||
def _make_3_steps():
|
||||
return [
|
||||
_make_step(1, "Ouvrir le Bloc-notes", "Bloc-notes"),
|
||||
_make_type_step("Bonjour le monde", step_index=2),
|
||||
_make_step(3, "Sauvegarder", "Bloc-notes", events=[
|
||||
{"type": "key_combo", "keys": ["ctrl", "s"],
|
||||
"window": {"title": "Bloc-notes"}, "timestamp": 103.0}
|
||||
]),
|
||||
]
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Initialisation
|
||||
# =========================================================================
|
||||
|
||||
|
||||
class TestShadowValidatorBase:
|
||||
|
||||
def test_creation(self):
|
||||
v = ShadowValidator()
|
||||
assert v.steps == []
|
||||
assert v.history == []
|
||||
assert not v.is_cancelled
|
||||
|
||||
def test_set_steps(self):
|
||||
v = ShadowValidator()
|
||||
v.set_steps(_make_3_steps())
|
||||
assert len(v.steps) == 3
|
||||
assert v.steps[0].intent == "Ouvrir le Bloc-notes"
|
||||
|
||||
def test_clone_protege_mutation(self):
|
||||
"""set_steps clone les étapes pour éviter les mutations externes."""
|
||||
v = ShadowValidator()
|
||||
steps = _make_3_steps()
|
||||
v.set_steps(steps)
|
||||
steps[0].intent = "HACKED"
|
||||
assert v.steps[0].intent == "Ouvrir le Bloc-notes"
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# validate
|
||||
# =========================================================================
|
||||
|
||||
|
||||
class TestValidatorValidate:
|
||||
|
||||
def test_validate_etape(self):
|
||||
v = ShadowValidator()
|
||||
v.set_steps(_make_3_steps())
|
||||
|
||||
result = v.apply_feedback({"action": "validate", "step_index": 1})
|
||||
|
||||
assert result.ok is True
|
||||
assert v.steps[0].validated is True
|
||||
assert v.steps[0].confidence >= 0.95
|
||||
assert v.steps[0].intent_provisoire is False
|
||||
|
||||
def test_validate_index_invalide(self):
|
||||
v = ShadowValidator()
|
||||
v.set_steps(_make_3_steps())
|
||||
|
||||
result = v.apply_feedback({"action": "validate", "step_index": 99})
|
||||
assert result.ok is False
|
||||
assert "invalide" in result.message.lower()
|
||||
|
||||
def test_validate_toutes_les_etapes(self):
|
||||
v = ShadowValidator()
|
||||
v.set_steps(_make_3_steps())
|
||||
for i in range(1, 4):
|
||||
v.apply_feedback({"action": "validate", "step_index": i})
|
||||
|
||||
assert all(s.validated for s in v.steps)
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# correct
|
||||
# =========================================================================
|
||||
|
||||
|
||||
class TestValidatorCorrect:
|
||||
|
||||
def test_correct_intent(self):
|
||||
v = ShadowValidator()
|
||||
v.set_steps(_make_3_steps())
|
||||
|
||||
result = v.apply_feedback({
|
||||
"action": "correct",
|
||||
"step_index": 1,
|
||||
"new_intent": "Démarrer la rédaction d'un email",
|
||||
})
|
||||
|
||||
assert result.ok is True
|
||||
assert v.steps[0].intent == "Démarrer la rédaction d'un email"
|
||||
assert v.steps[0].corrected is True
|
||||
assert v.steps[0].validated is True # Corriger = valider implicitement
|
||||
assert "old_intent" in result.data
|
||||
|
||||
def test_correct_intent_vide(self):
|
||||
v = ShadowValidator()
|
||||
v.set_steps(_make_3_steps())
|
||||
|
||||
result = v.apply_feedback({
|
||||
"action": "correct",
|
||||
"step_index": 1,
|
||||
"new_intent": "",
|
||||
})
|
||||
|
||||
assert result.ok is False
|
||||
assert v.steps[0].corrected is False
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# undo
|
||||
# =========================================================================
|
||||
|
||||
|
||||
class TestValidatorUndo:
|
||||
|
||||
def test_undo_etape(self):
|
||||
v = ShadowValidator()
|
||||
v.set_steps(_make_3_steps())
|
||||
|
||||
result = v.apply_feedback({"action": "undo", "step_index": 2})
|
||||
assert result.ok is True
|
||||
assert v.steps[1].cancelled is True
|
||||
|
||||
def test_undo_exclut_etape_du_workflow(self):
|
||||
"""Une étape undo ne doit pas apparaître dans le WorkflowIR final."""
|
||||
v = ShadowValidator()
|
||||
v.set_steps(_make_3_steps())
|
||||
|
||||
v.apply_feedback({"action": "undo", "step_index": 2})
|
||||
ir = v.build_workflow_ir(session_id="s1", name="Test")
|
||||
|
||||
assert ir is not None
|
||||
assert len(ir.steps) == 2 # 3 - 1 = 2
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# merge_next
|
||||
# =========================================================================
|
||||
|
||||
|
||||
class TestValidatorMergeNext:
|
||||
|
||||
def test_merge_deux_etapes(self):
|
||||
v = ShadowValidator()
|
||||
v.set_steps(_make_3_steps())
|
||||
|
||||
result = v.apply_feedback({"action": "merge_next", "step_index": 1})
|
||||
assert result.ok is True
|
||||
assert len(v.steps) == 2 # 3 - 1 = 2
|
||||
|
||||
def test_merge_conserve_les_events(self):
|
||||
v = ShadowValidator()
|
||||
v.set_steps(_make_3_steps())
|
||||
|
||||
total_events_before = sum(len(s.events) for s in v.steps)
|
||||
v.apply_feedback({"action": "merge_next", "step_index": 2})
|
||||
total_events_after = sum(len(s.events) for s in v.steps)
|
||||
|
||||
assert total_events_before == total_events_after
|
||||
|
||||
def test_merge_derniere_etape_echoue(self):
|
||||
v = ShadowValidator()
|
||||
v.set_steps(_make_3_steps())
|
||||
|
||||
result = v.apply_feedback({"action": "merge_next", "step_index": 3})
|
||||
assert result.ok is False
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# split
|
||||
# =========================================================================
|
||||
|
||||
|
||||
class TestValidatorSplit:
|
||||
|
||||
def test_split_en_deux(self):
|
||||
v = ShadowValidator()
|
||||
multi_events_step = _make_step(
|
||||
1, "Étape composite",
|
||||
events=[
|
||||
{"type": "mouse_click", "window": {"title": "App"},
|
||||
"timestamp": 100.0, "vision_info": {}},
|
||||
{"type": "text_input", "text": "partie 1",
|
||||
"window": {"title": "App"}, "timestamp": 101.0},
|
||||
{"type": "text_input", "text": "partie 2",
|
||||
"window": {"title": "App"}, "timestamp": 102.0},
|
||||
],
|
||||
)
|
||||
v.set_steps([multi_events_step])
|
||||
|
||||
result = v.apply_feedback({
|
||||
"action": "split",
|
||||
"step_index": 1,
|
||||
"at_event_index": 2,
|
||||
})
|
||||
|
||||
assert result.ok is True
|
||||
assert len(v.steps) == 2
|
||||
assert len(v.steps[0].events) == 2
|
||||
assert len(v.steps[1].events) == 1
|
||||
|
||||
def test_split_index_invalide(self):
|
||||
v = ShadowValidator()
|
||||
v.set_steps(_make_3_steps())
|
||||
result = v.apply_feedback({
|
||||
"action": "split",
|
||||
"step_index": 1,
|
||||
"at_event_index": 99,
|
||||
})
|
||||
assert result.ok is False
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# cancel
|
||||
# =========================================================================
|
||||
|
||||
|
||||
class TestValidatorCancel:
|
||||
|
||||
def test_cancel_workflow(self):
|
||||
v = ShadowValidator()
|
||||
v.set_steps(_make_3_steps())
|
||||
|
||||
result = v.apply_feedback({"action": "cancel"})
|
||||
assert result.ok is True
|
||||
assert v.is_cancelled
|
||||
|
||||
def test_cancel_build_retourne_none(self):
|
||||
v = ShadowValidator()
|
||||
v.set_steps(_make_3_steps())
|
||||
v.apply_feedback({"action": "cancel"})
|
||||
|
||||
ir = v.build_workflow_ir(session_id="s1", name="Test")
|
||||
assert ir is None
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Action inconnue
|
||||
# =========================================================================
|
||||
|
||||
|
||||
class TestValidatorUnknownAction:
|
||||
|
||||
def test_action_inconnue(self):
|
||||
v = ShadowValidator()
|
||||
v.set_steps(_make_3_steps())
|
||||
|
||||
result = v.apply_feedback({"action": "do_magic"})
|
||||
assert result.ok is False
|
||||
assert "inconnue" in result.message.lower()
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Construction du WorkflowIR
|
||||
# =========================================================================
|
||||
|
||||
|
||||
class TestValidatorBuild:
|
||||
|
||||
def test_build_workflow_ir(self):
|
||||
v = ShadowValidator()
|
||||
v.set_steps(_make_3_steps())
|
||||
|
||||
ir = v.build_workflow_ir(
|
||||
session_id="sess_test",
|
||||
name="Mon workflow",
|
||||
domain="generic",
|
||||
)
|
||||
|
||||
assert ir is not None
|
||||
assert isinstance(ir, WorkflowIR)
|
||||
assert ir.name == "Mon workflow"
|
||||
assert ir.learned_from == "sess_test"
|
||||
assert len(ir.steps) == 3
|
||||
|
||||
def test_build_with_variables(self):
|
||||
"""Les textes saisis deviennent des variables dans le WorkflowIR."""
|
||||
v = ShadowValidator()
|
||||
v.set_steps([
|
||||
_make_type_step("Jean Dupont", step_index=1),
|
||||
_make_type_step("jean@example.com", app="Email", step_index=2),
|
||||
])
|
||||
|
||||
ir = v.build_workflow_ir(session_id="s1", name="Test")
|
||||
assert len(ir.variables) == 2
|
||||
|
||||
# Les actions de type type doivent référencer les variables
|
||||
for step in ir.steps:
|
||||
for action in step.actions:
|
||||
if action.type == "type":
|
||||
assert action.variable is True
|
||||
assert action.text.startswith("{")
|
||||
|
||||
def test_build_respecte_corrections(self):
|
||||
"""Les intentions corrigées se retrouvent dans le WorkflowIR."""
|
||||
v = ShadowValidator()
|
||||
v.set_steps(_make_3_steps())
|
||||
|
||||
v.apply_feedback({
|
||||
"action": "correct",
|
||||
"step_index": 1,
|
||||
"new_intent": "Lancer l'application de prise de notes",
|
||||
})
|
||||
|
||||
ir = v.build_workflow_ir(session_id="s1", name="Test")
|
||||
assert ir.steps[0].intent == "Lancer l'application de prise de notes"
|
||||
|
||||
def test_build_exclut_etapes_annulees(self):
|
||||
v = ShadowValidator()
|
||||
v.set_steps(_make_3_steps())
|
||||
v.apply_feedback({"action": "undo", "step_index": 2})
|
||||
|
||||
ir = v.build_workflow_ir(session_id="s1", name="Test")
|
||||
assert len(ir.steps) == 2
|
||||
|
||||
def test_build_require_all_validated(self):
|
||||
"""Avec require_all_validated, erreur si une étape n'est pas validée."""
|
||||
v = ShadowValidator()
|
||||
v.set_steps(_make_3_steps())
|
||||
v.apply_feedback({"action": "validate", "step_index": 1})
|
||||
# Étapes 2 et 3 pas validées
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
v.build_workflow_ir(
|
||||
session_id="s1", name="Test", require_all_validated=True
|
||||
)
|
||||
|
||||
def test_build_applications_detectees(self):
|
||||
v = ShadowValidator()
|
||||
v.set_steps([
|
||||
_make_step(1, "Ouvrir Firefox", "Firefox"),
|
||||
_make_step(2, "Écrire", "Bloc-notes"),
|
||||
])
|
||||
ir = v.build_workflow_ir(session_id="s1", name="Test")
|
||||
assert "Firefox" in ir.applications
|
||||
assert "Bloc-notes" in ir.applications
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Historique
|
||||
# =========================================================================
|
||||
|
||||
|
||||
class TestValidatorHistory:
|
||||
|
||||
def test_historique_feedbacks(self):
|
||||
v = ShadowValidator()
|
||||
v.set_steps(_make_3_steps())
|
||||
|
||||
v.apply_feedback({"action": "validate", "step_index": 1})
|
||||
v.apply_feedback({
|
||||
"action": "correct", "step_index": 2,
|
||||
"new_intent": "Écrire le texte"
|
||||
})
|
||||
v.apply_feedback({"action": "undo", "step_index": 3})
|
||||
|
||||
history = v.history
|
||||
assert len(history) == 3
|
||||
assert history[0].action == "validate"
|
||||
assert history[1].action == "correct"
|
||||
assert history[2].action == "undo"
|
||||
|
||||
def test_apply_feedbacks_batch(self):
|
||||
v = ShadowValidator()
|
||||
v.set_steps(_make_3_steps())
|
||||
|
||||
results = v.apply_feedbacks([
|
||||
{"action": "validate", "step_index": 1},
|
||||
{"action": "validate", "step_index": 2},
|
||||
{"action": "undo", "step_index": 3},
|
||||
])
|
||||
|
||||
assert len(results) == 3
|
||||
assert all(r.ok for r in results)
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Intégration ShadowObserver + ShadowValidator
|
||||
# =========================================================================
|
||||
|
||||
|
||||
class TestShadowObserverValidatorIntegration:
|
||||
|
||||
def test_observer_vers_validator(self):
|
||||
"""Flow complet : Observer → Validator → WorkflowIR."""
|
||||
obs = ShadowObserver()
|
||||
obs.start("sess_flow")
|
||||
|
||||
# Simuler des événements
|
||||
obs.observe_event("sess_flow", {
|
||||
"type": "mouse_click",
|
||||
"window": {"title": "Menu Démarrer", "app_name": "Menu Démarrer"},
|
||||
"vision_info": {"text": "Rechercher"},
|
||||
"timestamp": 100.0,
|
||||
})
|
||||
obs.observe_event("sess_flow", {
|
||||
"type": "text_input",
|
||||
"text": "blocnote",
|
||||
"window": {"title": "Menu Démarrer", "app_name": "Menu Démarrer"},
|
||||
"timestamp": 100.5,
|
||||
})
|
||||
obs.observe_event("sess_flow", {
|
||||
"type": "key_combo",
|
||||
"keys": ["enter"],
|
||||
"window": {"title": "Menu Démarrer", "app_name": "Menu Démarrer"},
|
||||
"timestamp": 101.0,
|
||||
})
|
||||
# Changement d'application
|
||||
obs.observe_event("sess_flow", {
|
||||
"type": "text_input",
|
||||
"text": "Hello world",
|
||||
"window": {"title": "Sans titre - Bloc-notes", "app_name": "Bloc-notes"},
|
||||
"timestamp": 105.0,
|
||||
})
|
||||
obs.stop("sess_flow")
|
||||
|
||||
# Récupérer les étapes
|
||||
internals = obs.get_steps_internal("sess_flow")
|
||||
assert len(internals) >= 2
|
||||
|
||||
# Passer au validator
|
||||
validator = ShadowValidator()
|
||||
validator.set_steps(internals)
|
||||
|
||||
# Valider la première étape, corriger la seconde
|
||||
validator.apply_feedback({"action": "validate", "step_index": 1})
|
||||
validator.apply_feedback({
|
||||
"action": "correct",
|
||||
"step_index": 2,
|
||||
"new_intent": "Écrire un texte de démonstration",
|
||||
})
|
||||
|
||||
# Construire le WorkflowIR
|
||||
ir = validator.build_workflow_ir(
|
||||
session_id="sess_flow",
|
||||
name="Flow de test",
|
||||
domain="generic",
|
||||
)
|
||||
|
||||
assert ir is not None
|
||||
assert len(ir.steps) >= 2
|
||||
assert ir.steps[1].intent == "Écrire un texte de démonstration"
|
||||
assert len(ir.variables) >= 1 # Au moins "blocnote" ou "Hello world"
|
||||
|
||||
def test_undo_puis_build(self):
|
||||
obs = ShadowObserver()
|
||||
obs.start("sess_undo")
|
||||
for i in range(3):
|
||||
obs.observe_event("sess_undo", {
|
||||
"type": "mouse_click",
|
||||
"window": {"title": f"App{i}"},
|
||||
"vision_info": {"text": "bouton"},
|
||||
"timestamp": 100.0 + i * 6.0, # > 4s pour créer des segments
|
||||
})
|
||||
obs.stop("sess_undo")
|
||||
|
||||
validator = ShadowValidator()
|
||||
validator.set_steps(obs.get_steps_internal("sess_undo"))
|
||||
nb_before = len(validator.steps)
|
||||
assert nb_before >= 2
|
||||
|
||||
validator.apply_feedback({"action": "undo", "step_index": 1})
|
||||
|
||||
ir = validator.build_workflow_ir(session_id="sess_undo", name="Test")
|
||||
assert len(ir.steps) == nb_before - 1
|
||||
353
tests/unit/test_surface_and_uia.py
Normal file
353
tests/unit/test_surface_and_uia.py
Normal file
@@ -0,0 +1,353 @@
|
||||
"""
|
||||
Tests du SurfaceClassifier et du UIAHelper.
|
||||
|
||||
Vérifie :
|
||||
- Détection correcte des 4 types de surfaces (citrix, windows_native, web, unknown)
|
||||
- Paramètres adaptés par surface (timeouts, seuils)
|
||||
- Fallback gracieux si helper UIA absent
|
||||
- Sérialisation des profils
|
||||
- Wrapper UIAHelper avec mocks subprocess
|
||||
"""
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
_ROOT = str(Path(__file__).resolve().parents[2])
|
||||
if _ROOT not in sys.path:
|
||||
sys.path.insert(0, _ROOT)
|
||||
|
||||
from core.workflow.surface_classifier import (
|
||||
SurfaceClassifier,
|
||||
SurfaceProfile,
|
||||
SurfaceType,
|
||||
)
|
||||
from core.workflow.uia_helper import UIAHelper, UiaElement
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# SurfaceClassifier — détection par processus
|
||||
# =========================================================================
|
||||
|
||||
|
||||
class TestSurfaceClassifier:
|
||||
"""Tests de détection des surfaces."""
|
||||
|
||||
def _classifier(self):
|
||||
"""Classifier sans helper UIA (pour que les tests soient reproductibles)."""
|
||||
return SurfaceClassifier(uia_helper_path="")
|
||||
|
||||
def test_detection_citrix_wfica(self):
|
||||
"""wfica32.exe → Citrix."""
|
||||
c = self._classifier()
|
||||
profile = c.classify(process_name="wfica32.exe", window_title="Session Citrix")
|
||||
assert profile.surface_type == SurfaceType.CITRIX
|
||||
assert profile.uia_available is False
|
||||
assert profile.ocr_threshold < 0.75 # Plus tolérant
|
||||
assert profile.max_retries >= 3
|
||||
|
||||
def test_detection_citrix_mstsc(self):
|
||||
"""mstsc.exe → Citrix (RDP)."""
|
||||
c = self._classifier()
|
||||
profile = c.classify(process_name="mstsc.exe", window_title="Remote Desktop")
|
||||
assert profile.surface_type == SurfaceType.CITRIX
|
||||
|
||||
def test_detection_citrix_par_titre(self):
|
||||
"""Titre 'Citrix' → Citrix même si process non listé."""
|
||||
c = self._classifier()
|
||||
profile = c.classify(process_name="chrome.exe", window_title="DxCare - Citrix Receiver")
|
||||
assert profile.surface_type == SurfaceType.CITRIX
|
||||
|
||||
def test_detection_windows_natif_notepad(self):
|
||||
"""notepad.exe → Windows natif."""
|
||||
c = self._classifier()
|
||||
profile = c.classify(process_name="notepad.exe", window_title="Sans titre – Bloc-notes")
|
||||
assert profile.surface_type == SurfaceType.WINDOWS_NATIVE
|
||||
assert profile.ocr_threshold == 0.75
|
||||
assert profile.timeout_click_ms == 8000
|
||||
|
||||
def test_detection_windows_natif_explorer(self):
|
||||
"""explorer.exe → Windows natif (cas spécial)."""
|
||||
c = self._classifier()
|
||||
profile = c.classify(process_name="explorer.exe", window_title="Lea")
|
||||
assert profile.surface_type == SurfaceType.WINDOWS_NATIVE
|
||||
|
||||
def test_detection_windows_natif_dxcare(self):
|
||||
"""dxcare.exe (DPI hospitalier) → Windows natif."""
|
||||
c = self._classifier()
|
||||
profile = c.classify(process_name="dxcare.exe", window_title="DxCare - Dossier 12345")
|
||||
assert profile.surface_type == SurfaceType.WINDOWS_NATIVE
|
||||
|
||||
def test_detection_web_chrome(self):
|
||||
"""chrome.exe → Web local."""
|
||||
c = self._classifier()
|
||||
profile = c.classify(process_name="chrome.exe", window_title="Google - Google Chrome")
|
||||
assert profile.surface_type == SurfaceType.WEB_LOCAL
|
||||
assert profile.ocr_threshold == 0.80 # Plus strict (texte bien rendu)
|
||||
assert profile.max_retries == 1 # Rapide
|
||||
|
||||
def test_detection_web_edge(self):
|
||||
c = self._classifier()
|
||||
profile = c.classify(process_name="msedge.exe", window_title="Edge")
|
||||
assert profile.surface_type == SurfaceType.WEB_LOCAL
|
||||
|
||||
def test_detection_web_firefox(self):
|
||||
c = self._classifier()
|
||||
profile = c.classify(process_name="firefox.exe", window_title="Firefox")
|
||||
assert profile.surface_type == SurfaceType.WEB_LOCAL
|
||||
|
||||
def test_detection_unknown_fallback(self):
|
||||
"""Process non reconnu → unknown avec paramètres sûrs."""
|
||||
c = self._classifier()
|
||||
profile = c.classify(process_name="", window_title="")
|
||||
assert profile.surface_type == SurfaceType.UNKNOWN
|
||||
assert profile.confidence < 1.0
|
||||
assert profile.ocr_available is True # OCR toujours dispo
|
||||
|
||||
def test_citrix_dans_navigateur(self):
|
||||
"""Citrix embedded dans Chrome → Citrix."""
|
||||
c = self._classifier()
|
||||
profile = c.classify(
|
||||
process_name="chrome.exe",
|
||||
window_title="Citrix Workspace - DxCare",
|
||||
)
|
||||
assert profile.surface_type == SurfaceType.CITRIX
|
||||
|
||||
def test_resolve_order_par_surface(self):
|
||||
"""Ordre de résolution cohérent avec la surface."""
|
||||
c = self._classifier()
|
||||
|
||||
citrix = c.classify("wfica32.exe", "Session")
|
||||
assert "uia" not in citrix.resolve_order()
|
||||
assert "ocr" in citrix.resolve_order()
|
||||
|
||||
windows = c.classify("notepad.exe", "Bloc-notes")
|
||||
# UIA pas dispo (helper path vide) donc absent
|
||||
assert "ocr" in windows.resolve_order()
|
||||
|
||||
web = c.classify("chrome.exe", "Google")
|
||||
assert "ocr" in web.resolve_order()
|
||||
|
||||
|
||||
class TestSurfaceProfile:
|
||||
"""Tests du dataclass SurfaceProfile."""
|
||||
|
||||
def test_to_dict_structure(self):
|
||||
p = SurfaceProfile(
|
||||
surface_type=SurfaceType.WINDOWS_NATIVE,
|
||||
process_name="notepad.exe",
|
||||
window_title="Test",
|
||||
)
|
||||
d = p.to_dict()
|
||||
assert d["surface_type"] == "windows_native"
|
||||
assert "capabilities" in d
|
||||
assert "parameters" in d
|
||||
assert d["capabilities"]["ocr"] is True
|
||||
assert d["capabilities"]["uia"] is False # Par défaut
|
||||
|
||||
def test_resolve_order_construction(self):
|
||||
"""L'ordre de résolution utilise les capacités dispo."""
|
||||
p = SurfaceProfile(
|
||||
surface_type=SurfaceType.WINDOWS_NATIVE,
|
||||
uia_available=True,
|
||||
)
|
||||
order = p.resolve_order()
|
||||
assert order[0] == "uia" # UIA en premier si dispo
|
||||
assert "ocr" in order
|
||||
assert "vlm" in order
|
||||
|
||||
def test_resolve_order_sans_uia(self):
|
||||
p = SurfaceProfile(
|
||||
surface_type=SurfaceType.CITRIX,
|
||||
uia_available=False,
|
||||
)
|
||||
order = p.resolve_order()
|
||||
assert "uia" not in order
|
||||
assert order[0] == "ocr" # OCR en premier
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# UIAHelper — wrapper Python
|
||||
# =========================================================================
|
||||
|
||||
|
||||
class TestUIAHelper:
|
||||
"""Tests du wrapper UIAHelper."""
|
||||
|
||||
def test_initialization_sans_helper(self):
|
||||
"""Sans helper trouvé, available = False."""
|
||||
helper = UIAHelper(helper_path="/chemin/inexistant.exe")
|
||||
assert helper.available is False
|
||||
|
||||
def test_query_retourne_none_si_indispo(self):
|
||||
"""Si le helper n'est pas dispo, query retourne None."""
|
||||
helper = UIAHelper(helper_path="/chemin/inexistant.exe")
|
||||
result = helper.query_at(100, 200)
|
||||
assert result is None
|
||||
|
||||
def test_find_retourne_none_si_indispo(self):
|
||||
helper = UIAHelper(helper_path="/chemin/inexistant.exe")
|
||||
result = helper.find_by_name("Enregistrer")
|
||||
assert result is None
|
||||
|
||||
def test_health_retourne_false_si_indispo(self):
|
||||
helper = UIAHelper(helper_path="/chemin/inexistant.exe")
|
||||
assert helper.health() is False
|
||||
|
||||
@patch("core.workflow.uia_helper.os.path.isfile", return_value=True)
|
||||
@patch("core.workflow.uia_helper.platform.system", return_value="Windows")
|
||||
@patch("core.workflow.uia_helper.subprocess.run")
|
||||
def test_query_success_mock(self, mock_run, mock_platform, mock_isfile):
|
||||
"""Query avec mock subprocess retourne un UiaElement."""
|
||||
mock_result = MagicMock()
|
||||
mock_result.returncode = 0
|
||||
mock_result.stdout = json.dumps({
|
||||
"status": "ok",
|
||||
"element": {
|
||||
"name": "Enregistrer",
|
||||
"control_type": "bouton",
|
||||
"class_name": "Button",
|
||||
"automation_id": "btnSave",
|
||||
"bounding_rect": [100, 200, 200, 250],
|
||||
"is_enabled": True,
|
||||
"is_offscreen": False,
|
||||
"parent_path": [
|
||||
{"name": "Bloc-notes", "control_type": "fenêtre"}
|
||||
],
|
||||
},
|
||||
"elapsed_ms": 15,
|
||||
})
|
||||
mock_run.return_value = mock_result
|
||||
|
||||
helper = UIAHelper(helper_path="fake_lea_uia.exe")
|
||||
element = helper.query_at(150, 225)
|
||||
|
||||
assert element is not None
|
||||
assert element.name == "Enregistrer"
|
||||
assert element.control_type == "bouton"
|
||||
assert element.bounding_rect == (100, 200, 200, 250)
|
||||
assert element.center() == (150, 225)
|
||||
assert element.is_clickable() is True
|
||||
assert len(element.parent_path) == 1
|
||||
|
||||
@patch("core.workflow.uia_helper.os.path.isfile", return_value=True)
|
||||
@patch("core.workflow.uia_helper.platform.system", return_value="Windows")
|
||||
@patch("core.workflow.uia_helper.subprocess.run")
|
||||
def test_find_success_mock(self, mock_run, mock_platform, mock_isfile):
|
||||
mock_result = MagicMock()
|
||||
mock_result.returncode = 0
|
||||
mock_result.stdout = json.dumps({
|
||||
"status": "ok",
|
||||
"element": {
|
||||
"name": "Fichier",
|
||||
"control_type": "menu",
|
||||
"bounding_rect": [0, 20, 50, 40],
|
||||
"is_enabled": True,
|
||||
"is_offscreen": False,
|
||||
},
|
||||
})
|
||||
mock_run.return_value = mock_result
|
||||
|
||||
helper = UIAHelper(helper_path="fake.exe")
|
||||
element = helper.find_by_name("Fichier", control_type="menu")
|
||||
assert element is not None
|
||||
assert element.name == "Fichier"
|
||||
|
||||
@patch("core.workflow.uia_helper.os.path.isfile", return_value=True)
|
||||
@patch("core.workflow.uia_helper.platform.system", return_value="Windows")
|
||||
@patch("core.workflow.uia_helper.subprocess.run")
|
||||
def test_not_found(self, mock_run, mock_platform, mock_isfile):
|
||||
mock_result = MagicMock()
|
||||
mock_result.returncode = 0
|
||||
mock_result.stdout = json.dumps({
|
||||
"status": "not_found",
|
||||
"reason": "Pas d'élément",
|
||||
"elapsed_ms": 5,
|
||||
})
|
||||
mock_run.return_value = mock_result
|
||||
|
||||
helper = UIAHelper(helper_path="fake.exe")
|
||||
assert helper.query_at(999, 999) is None
|
||||
|
||||
@patch("core.workflow.uia_helper.os.path.isfile", return_value=True)
|
||||
@patch("core.workflow.uia_helper.platform.system", return_value="Windows")
|
||||
@patch("core.workflow.uia_helper.subprocess.run")
|
||||
def test_timeout(self, mock_run, mock_platform, mock_isfile):
|
||||
"""Un timeout subprocess ne fait pas crash le helper."""
|
||||
import subprocess as sp
|
||||
mock_run.side_effect = sp.TimeoutExpired("lea_uia", 5)
|
||||
|
||||
helper = UIAHelper(helper_path="fake.exe")
|
||||
assert helper.query_at(100, 100) is None
|
||||
|
||||
@patch("core.workflow.uia_helper.os.path.isfile", return_value=True)
|
||||
@patch("core.workflow.uia_helper.platform.system", return_value="Windows")
|
||||
@patch("core.workflow.uia_helper.subprocess.run")
|
||||
def test_json_invalide(self, mock_run, mock_platform, mock_isfile):
|
||||
"""Une sortie non-JSON ne fait pas crash."""
|
||||
mock_result = MagicMock()
|
||||
mock_result.returncode = 0
|
||||
mock_result.stdout = "pas du JSON"
|
||||
mock_run.return_value = mock_result
|
||||
|
||||
helper = UIAHelper(helper_path="fake.exe")
|
||||
assert helper.query_at(100, 100) is None
|
||||
|
||||
|
||||
class TestUiaElement:
|
||||
"""Tests du dataclass UiaElement."""
|
||||
|
||||
def test_from_dict_minimal(self):
|
||||
e = UiaElement.from_dict({"name": "test"})
|
||||
assert e.name == "test"
|
||||
assert e.bounding_rect == (0, 0, 0, 0)
|
||||
|
||||
def test_center(self):
|
||||
e = UiaElement(bounding_rect=(100, 200, 200, 300))
|
||||
assert e.center() == (150, 250)
|
||||
|
||||
def test_is_clickable(self):
|
||||
e = UiaElement(
|
||||
bounding_rect=(100, 100, 200, 150),
|
||||
is_enabled=True,
|
||||
is_offscreen=False,
|
||||
)
|
||||
assert e.is_clickable() is True
|
||||
|
||||
e2 = UiaElement(
|
||||
bounding_rect=(100, 100, 200, 150),
|
||||
is_enabled=False,
|
||||
is_offscreen=False,
|
||||
)
|
||||
assert e2.is_clickable() is False
|
||||
|
||||
def test_path_signature(self):
|
||||
e = UiaElement(
|
||||
name="Enregistrer",
|
||||
control_type="bouton",
|
||||
parent_path=[
|
||||
{"name": "Bloc-notes", "control_type": "fenêtre"},
|
||||
{"name": "Fichier", "control_type": "menu"},
|
||||
],
|
||||
)
|
||||
sig = e.path_signature()
|
||||
assert "Bloc-notes" in sig
|
||||
assert "Enregistrer" in sig
|
||||
assert " > " in sig
|
||||
|
||||
def test_roundtrip_dict(self):
|
||||
original = UiaElement(
|
||||
name="test",
|
||||
control_type="bouton",
|
||||
bounding_rect=(10, 20, 30, 40),
|
||||
is_enabled=True,
|
||||
is_offscreen=False,
|
||||
)
|
||||
d = original.to_dict()
|
||||
copy = UiaElement.from_dict(d)
|
||||
assert copy.name == original.name
|
||||
assert copy.bounding_rect == original.bounding_rect
|
||||
assert copy.is_enabled == original.is_enabled
|
||||
305
tests/unit/test_v4_resolve_order.py
Normal file
305
tests/unit/test_v4_resolve_order.py
Normal file
@@ -0,0 +1,305 @@
|
||||
"""
|
||||
Tests du mécanisme V4 : résolution pilotée par l'ordre pré-compilé.
|
||||
|
||||
Vérifie que :
|
||||
- Le resolve_order est bien propagé du plan vers le target_spec
|
||||
- Le resolve_engine honore l'ordre au lieu de sa cascade par défaut
|
||||
- Les méthodes sont essayées dans l'ordre spécifié
|
||||
- Si toutes échouent, fallback sur la cascade legacy
|
||||
"""
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
_ROOT = str(Path(__file__).resolve().parents[2])
|
||||
if _ROOT not in sys.path:
|
||||
sys.path.insert(0, _ROOT)
|
||||
|
||||
from core.workflow.workflow_ir import WorkflowIR
|
||||
from core.workflow.execution_plan import ExecutionNode, ResolutionStrategy, ExecutionPlan
|
||||
from core.workflow.execution_compiler import ExecutionCompiler
|
||||
from agent_v0.server_v1.execution_plan_runner import (
|
||||
execution_node_to_action,
|
||||
execution_plan_to_actions,
|
||||
_strategy_to_target_spec,
|
||||
)
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Test 1 : le resolve_order est propagé du plan au target_spec
|
||||
# =========================================================================
|
||||
|
||||
|
||||
class TestResolveOrderPropagation:
|
||||
"""Le resolve_order doit être présent dans le target_spec."""
|
||||
|
||||
def test_ocr_primary_produit_resolve_order(self):
|
||||
"""OCR primaire → resolve_order commence par 'ocr'."""
|
||||
primary = ResolutionStrategy(method="ocr", target_text="Enregistrer")
|
||||
fallbacks = [
|
||||
ResolutionStrategy(method="template", anchor_b64="abc123"),
|
||||
ResolutionStrategy(method="vlm", vlm_description="bouton Enregistrer"),
|
||||
]
|
||||
spec = _strategy_to_target_spec(primary, fallbacks)
|
||||
|
||||
assert "resolve_order" in spec
|
||||
assert spec["resolve_order"] == ["ocr", "template", "vlm"]
|
||||
|
||||
def test_template_primary_produit_resolve_order(self):
|
||||
"""Template primaire → resolve_order commence par 'template'."""
|
||||
primary = ResolutionStrategy(method="template", anchor_b64="abc")
|
||||
fallbacks = [ResolutionStrategy(method="vlm", vlm_description="icône")]
|
||||
spec = _strategy_to_target_spec(primary, fallbacks)
|
||||
|
||||
assert spec["resolve_order"][0] == "template"
|
||||
|
||||
def test_vlm_only(self):
|
||||
"""Juste VLM → resolve_order = ['vlm']."""
|
||||
primary = ResolutionStrategy(method="vlm", vlm_description="popup")
|
||||
spec = _strategy_to_target_spec(primary, [])
|
||||
|
||||
assert spec["resolve_order"] == ["vlm"]
|
||||
|
||||
def test_pas_de_doublons(self):
|
||||
"""Chaque méthode apparaît une seule fois dans l'ordre."""
|
||||
primary = ResolutionStrategy(method="ocr", target_text="test")
|
||||
fallbacks = [
|
||||
ResolutionStrategy(method="template", anchor_b64="abc"),
|
||||
ResolutionStrategy(method="ocr", target_text="autre"), # Doublon
|
||||
]
|
||||
spec = _strategy_to_target_spec(primary, fallbacks)
|
||||
|
||||
assert spec["resolve_order"].count("ocr") == 1
|
||||
assert spec["resolve_order"].count("template") == 1
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Test 2 : execution_node_to_action propage bien le resolve_order
|
||||
# =========================================================================
|
||||
|
||||
|
||||
class TestExecutionNodeConversion:
|
||||
"""Les actions générées contiennent le resolve_order."""
|
||||
|
||||
def test_click_node_a_resolve_order(self):
|
||||
"""Un ExecutionNode click produit une action avec resolve_order."""
|
||||
node = ExecutionNode(
|
||||
node_id="n1",
|
||||
action_type="click",
|
||||
intent="Cliquer sur Fichier",
|
||||
strategy_primary=ResolutionStrategy(method="ocr", target_text="Fichier"),
|
||||
strategy_fallbacks=[
|
||||
ResolutionStrategy(method="vlm", vlm_description="menu Fichier"),
|
||||
],
|
||||
)
|
||||
action = execution_node_to_action(node)
|
||||
|
||||
assert action is not None
|
||||
assert action["type"] == "click"
|
||||
assert "target_spec" in action
|
||||
assert "resolve_order" in action["target_spec"]
|
||||
assert action["target_spec"]["resolve_order"] == ["ocr", "vlm"]
|
||||
assert action["target_spec"]["by_text"] == "Fichier"
|
||||
assert action["target_spec"]["vlm_description"] == "menu Fichier"
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Test 3 : le compilateur produit des plans avec resolve_order correct
|
||||
# =========================================================================
|
||||
|
||||
|
||||
class TestCompilerProduitResolveOrder:
|
||||
"""Le ExecutionCompiler produit des plans avec resolve_order."""
|
||||
|
||||
def test_workflow_complet_avec_resolve_order(self):
|
||||
"""Un workflow compilé a des actions avec resolve_order."""
|
||||
ir = WorkflowIR.new("Test", domain="generic")
|
||||
ir.add_step(
|
||||
"Cliquer sur Enregistrer",
|
||||
actions=[{
|
||||
"type": "click",
|
||||
"target": "bouton Enregistrer",
|
||||
"anchor_hint": "Enregistrer",
|
||||
}],
|
||||
)
|
||||
|
||||
compiler = ExecutionCompiler()
|
||||
plan = compiler.compile(ir)
|
||||
|
||||
actions = execution_plan_to_actions(plan)
|
||||
assert len(actions) == 1
|
||||
assert "resolve_order" in actions[0]["target_spec"]
|
||||
# OCR doit être en premier (stratégie primaire quand texte dispo)
|
||||
assert actions[0]["target_spec"]["resolve_order"][0] == "ocr"
|
||||
|
||||
def test_fallback_vlm_toujours_present(self):
|
||||
"""Le VLM est toujours présent en fallback."""
|
||||
ir = WorkflowIR.new("Test")
|
||||
ir.add_step("Clic", actions=[{"type": "click", "target": "X", "anchor_hint": "X"}])
|
||||
|
||||
compiler = ExecutionCompiler()
|
||||
plan = compiler.compile(ir)
|
||||
|
||||
actions = execution_plan_to_actions(plan)
|
||||
assert "vlm" in actions[0]["target_spec"]["resolve_order"]
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Test 4 : _resolve_with_precompiled_order respecte l'ordre
|
||||
# =========================================================================
|
||||
|
||||
|
||||
class TestResolveWithPrecompiledOrder:
|
||||
"""Le mécanisme V4 de résolution honore l'ordre."""
|
||||
|
||||
@patch("agent_v0.server_v1.resolve_engine._resolve_by_ocr_text")
|
||||
def test_ocr_appele_en_premier(self, mock_ocr):
|
||||
"""Si resolve_order=['ocr', 'vlm'], OCR est appelé en premier."""
|
||||
from agent_v0.server_v1.resolve_engine import _resolve_with_precompiled_order
|
||||
|
||||
mock_ocr.return_value = {
|
||||
"resolved": True,
|
||||
"x_pct": 0.5,
|
||||
"y_pct": 0.3,
|
||||
"score": 0.9,
|
||||
}
|
||||
|
||||
result = _resolve_with_precompiled_order(
|
||||
screenshot_path="/fake.png",
|
||||
target_spec={
|
||||
"by_text": "Enregistrer",
|
||||
"resolve_order": ["ocr", "vlm"],
|
||||
},
|
||||
resolve_order=["ocr", "vlm"],
|
||||
screen_width=1280,
|
||||
screen_height=800,
|
||||
fallback_x_pct=0.5,
|
||||
fallback_y_pct=0.5,
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
assert result.get("resolved") is True
|
||||
assert result.get("resolve_method") == "v4_ocr"
|
||||
mock_ocr.assert_called_once()
|
||||
|
||||
@patch("agent_v0.server_v1.resolve_engine._vlm_quick_find")
|
||||
@patch("agent_v0.server_v1.resolve_engine._resolve_by_ocr_text")
|
||||
def test_cascade_ocr_vers_vlm(self, mock_ocr, mock_vlm):
|
||||
"""Si OCR échoue, VLM est essayé."""
|
||||
from agent_v0.server_v1.resolve_engine import _resolve_with_precompiled_order
|
||||
|
||||
mock_ocr.return_value = None # OCR échoue
|
||||
mock_vlm.return_value = {
|
||||
"resolved": True,
|
||||
"x_pct": 0.5,
|
||||
"y_pct": 0.3,
|
||||
}
|
||||
|
||||
result = _resolve_with_precompiled_order(
|
||||
screenshot_path="/fake.png",
|
||||
target_spec={
|
||||
"by_text": "Enregistrer",
|
||||
"vlm_description": "bouton Enregistrer",
|
||||
"resolve_order": ["ocr", "vlm"],
|
||||
},
|
||||
resolve_order=["ocr", "vlm"],
|
||||
screen_width=1280,
|
||||
screen_height=800,
|
||||
fallback_x_pct=0.5,
|
||||
fallback_y_pct=0.5,
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
assert result.get("resolve_method") == "v4_vlm"
|
||||
mock_ocr.assert_called_once()
|
||||
mock_vlm.assert_called_once()
|
||||
|
||||
@patch("agent_v0.server_v1.resolve_engine._vlm_quick_find")
|
||||
@patch("agent_v0.server_v1.resolve_engine._resolve_by_ocr_text")
|
||||
def test_toutes_methodes_echouent(self, mock_ocr, mock_vlm):
|
||||
"""Si toutes les méthodes échouent, retourne None."""
|
||||
from agent_v0.server_v1.resolve_engine import _resolve_with_precompiled_order
|
||||
|
||||
mock_ocr.return_value = None
|
||||
mock_vlm.return_value = None
|
||||
|
||||
result = _resolve_with_precompiled_order(
|
||||
screenshot_path="/fake.png",
|
||||
target_spec={
|
||||
"by_text": "Inexistant",
|
||||
"vlm_description": "truc inexistant",
|
||||
"resolve_order": ["ocr", "vlm"],
|
||||
},
|
||||
resolve_order=["ocr", "vlm"],
|
||||
screen_width=1280,
|
||||
screen_height=800,
|
||||
fallback_x_pct=0.5,
|
||||
fallback_y_pct=0.5,
|
||||
)
|
||||
|
||||
assert result is None
|
||||
|
||||
def test_resolve_order_vide(self):
|
||||
"""Un resolve_order vide ne plante pas."""
|
||||
from agent_v0.server_v1.resolve_engine import _resolve_with_precompiled_order
|
||||
|
||||
result = _resolve_with_precompiled_order(
|
||||
screenshot_path="/fake.png",
|
||||
target_spec={"by_text": "test"},
|
||||
resolve_order=[],
|
||||
screen_width=1280,
|
||||
screen_height=800,
|
||||
fallback_x_pct=0.5,
|
||||
fallback_y_pct=0.5,
|
||||
)
|
||||
|
||||
assert result is None
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Test 5 : pipeline complet — IR → Plan → action avec resolve_order
|
||||
# =========================================================================
|
||||
|
||||
|
||||
class TestPipelineCompletV4:
|
||||
"""Test du pipeline V4 complet de bout en bout (sans runtime réel)."""
|
||||
|
||||
def test_ir_vers_action_avec_resolve_order(self):
|
||||
"""Un WorkflowIR produit des actions avec resolve_order correctement."""
|
||||
ir = WorkflowIR.new("Workflow complet", domain="tim_codage")
|
||||
ir.add_step(
|
||||
"Ouvrir le fichier",
|
||||
actions=[{
|
||||
"type": "click",
|
||||
"target": "bouton Ouvrir",
|
||||
"anchor_hint": "Ouvrir",
|
||||
}],
|
||||
)
|
||||
ir.add_step(
|
||||
"Saisir le nom",
|
||||
actions=[
|
||||
{"type": "type", "text": "rapport.pdf"},
|
||||
{"type": "key_combo", "keys": ["enter"]},
|
||||
],
|
||||
)
|
||||
|
||||
compiler = ExecutionCompiler()
|
||||
plan = compiler.compile(ir)
|
||||
actions = execution_plan_to_actions(plan)
|
||||
|
||||
# On doit avoir 3 actions : click, type, key_combo
|
||||
assert len(actions) == 3
|
||||
|
||||
click_action = actions[0]
|
||||
assert click_action["type"] == "click"
|
||||
assert "resolve_order" in click_action["target_spec"]
|
||||
assert click_action["target_spec"]["resolve_order"][0] == "ocr"
|
||||
assert click_action["target_spec"]["by_text"] == "Ouvrir"
|
||||
|
||||
# type et key_combo n'ont pas de target_spec
|
||||
assert actions[1]["type"] == "type"
|
||||
assert "target_spec" not in actions[1]
|
||||
assert actions[2]["type"] == "key_combo"
|
||||
349
tests/unit/test_v4_wiring.py
Normal file
349
tests/unit/test_v4_wiring.py
Normal file
@@ -0,0 +1,349 @@
|
||||
"""
|
||||
Tests de câblage complet V4 :
|
||||
- SurfaceClassifier + ExecutionCompiler : paramètres adaptés par surface
|
||||
- IRBuilder lit uia_snapshot depuis les événements
|
||||
- ExecutionCompiler crée une stratégie UIA quand dispo
|
||||
- execution_plan_runner propage uia_target dans target_spec
|
||||
- Pipeline E2E : RawTrace (avec UIA) → WorkflowIR → Plan → action runtime
|
||||
"""
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
_ROOT = str(Path(__file__).resolve().parents[2])
|
||||
if _ROOT not in sys.path:
|
||||
sys.path.insert(0, _ROOT)
|
||||
|
||||
from core.workflow.workflow_ir import WorkflowIR, Step, Action
|
||||
from core.workflow.execution_plan import ExecutionPlan, ExecutionNode, ResolutionStrategy
|
||||
from core.workflow.execution_compiler import ExecutionCompiler
|
||||
from core.workflow.surface_classifier import SurfaceClassifier, SurfaceProfile, SurfaceType
|
||||
from core.workflow.ir_builder import IRBuilder
|
||||
from agent_v0.server_v1.execution_plan_runner import (
|
||||
execution_node_to_action,
|
||||
execution_plan_to_actions,
|
||||
_strategy_to_target_spec,
|
||||
)
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# ExecutionCompiler avec SurfaceProfile
|
||||
# =========================================================================
|
||||
|
||||
|
||||
class TestCompilerWithSurfaceProfile:
|
||||
|
||||
def test_profil_citrix_impose_timeouts_longs(self):
|
||||
"""Profil Citrix → timeouts longs, retries 3x."""
|
||||
ir = WorkflowIR.new("Test")
|
||||
ir.add_step("Clic", actions=[{"type": "click", "target": "Bouton", "anchor_hint": "OK"}])
|
||||
|
||||
profile = SurfaceProfile(
|
||||
surface_type=SurfaceType.CITRIX,
|
||||
timeout_click_ms=15000,
|
||||
max_retries=3,
|
||||
ocr_threshold=0.65,
|
||||
)
|
||||
|
||||
compiler = ExecutionCompiler()
|
||||
plan = compiler.compile(ir, surface_profile=profile)
|
||||
|
||||
click_node = [n for n in plan.nodes if n.action_type == "click"][0]
|
||||
assert click_node.timeout_ms == 15000
|
||||
assert click_node.max_retries == 3
|
||||
|
||||
def test_profil_web_impose_timeouts_courts(self):
|
||||
"""Profil web → timeouts courts, 1 retry."""
|
||||
ir = WorkflowIR.new("Test")
|
||||
ir.add_step("Clic", actions=[{"type": "click", "target": "X", "anchor_hint": "Login"}])
|
||||
|
||||
profile = SurfaceProfile(
|
||||
surface_type=SurfaceType.WEB_LOCAL,
|
||||
timeout_click_ms=5000,
|
||||
max_retries=1,
|
||||
)
|
||||
|
||||
compiler = ExecutionCompiler()
|
||||
plan = compiler.compile(ir, surface_profile=profile)
|
||||
|
||||
click_node = [n for n in plan.nodes if n.action_type == "click"][0]
|
||||
assert click_node.timeout_ms == 5000
|
||||
assert click_node.max_retries == 1
|
||||
|
||||
def test_sans_profil_utilise_defauts(self):
|
||||
"""Sans surface_profile, comportement par défaut."""
|
||||
ir = WorkflowIR.new("Test")
|
||||
ir.add_step("Clic", actions=[{"type": "click", "target": "X", "anchor_hint": "Y"}])
|
||||
|
||||
compiler = ExecutionCompiler()
|
||||
plan = compiler.compile(ir)
|
||||
|
||||
click_node = [n for n in plan.nodes if n.action_type == "click"][0]
|
||||
assert click_node.timeout_ms == 10000 # Défaut
|
||||
assert click_node.max_retries == 2 # Défaut
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Stratégie UIA dans la compilation
|
||||
# =========================================================================
|
||||
|
||||
|
||||
class TestUiaStrategyCompilation:
|
||||
|
||||
def _make_ir_with_uia(self):
|
||||
"""Créer un WorkflowIR avec une action portant un uia_snapshot."""
|
||||
ir = WorkflowIR.new("Test UIA")
|
||||
action = Action(
|
||||
type="click",
|
||||
target="Bloc-notes",
|
||||
anchor_hint="Enregistrer",
|
||||
)
|
||||
# Simuler l'enrichissement avec UIA
|
||||
action._enrichment = {
|
||||
"by_text": "Enregistrer",
|
||||
"anchor_image_base64": "fake_crop_data",
|
||||
"vlm_description": "Le bouton Enregistrer du menu Fichier",
|
||||
"uia_snapshot": {
|
||||
"name": "Enregistrer",
|
||||
"control_type": "bouton",
|
||||
"automation_id": "btnSave",
|
||||
"parent_path": [
|
||||
{"name": "Bloc-notes", "control_type": "fenêtre"},
|
||||
{"name": "Fichier", "control_type": "menu"},
|
||||
],
|
||||
},
|
||||
}
|
||||
step = Step(step_id="s1", intent="Sauvegarder", actions=[action])
|
||||
ir.steps.append(step)
|
||||
return ir
|
||||
|
||||
def test_uia_strategie_creee_si_surface_windows(self):
|
||||
"""Sur Windows natif avec UIA dispo, la stratégie UIA est primaire."""
|
||||
ir = self._make_ir_with_uia()
|
||||
profile = SurfaceProfile(
|
||||
surface_type=SurfaceType.WINDOWS_NATIVE,
|
||||
uia_available=True,
|
||||
)
|
||||
|
||||
compiler = ExecutionCompiler()
|
||||
plan = compiler.compile(ir, surface_profile=profile)
|
||||
|
||||
click = [n for n in plan.nodes if n.action_type == "click"][0]
|
||||
assert click.strategy_primary is not None
|
||||
assert click.strategy_primary.method == "uia"
|
||||
assert click.strategy_primary.uia_name == "Enregistrer"
|
||||
assert click.strategy_primary.uia_control_type == "bouton"
|
||||
|
||||
def test_uia_desactive_sur_citrix(self):
|
||||
"""Sur Citrix, UIA est ignoré même si snapshot présent."""
|
||||
ir = self._make_ir_with_uia()
|
||||
profile = SurfaceProfile(
|
||||
surface_type=SurfaceType.CITRIX,
|
||||
uia_available=False,
|
||||
)
|
||||
|
||||
compiler = ExecutionCompiler()
|
||||
plan = compiler.compile(ir, surface_profile=profile)
|
||||
|
||||
click = [n for n in plan.nodes if n.action_type == "click"][0]
|
||||
assert click.strategy_primary.method != "uia"
|
||||
# OCR est la primaire (texte dispo)
|
||||
assert click.strategy_primary.method == "ocr"
|
||||
|
||||
def test_uia_fallback_sur_ocr_si_uia_manquant(self):
|
||||
"""Sans uia_snapshot, OCR primaire."""
|
||||
ir = WorkflowIR.new("Test")
|
||||
action = Action(
|
||||
type="click",
|
||||
target="Fichier",
|
||||
anchor_hint="Fichier",
|
||||
)
|
||||
action._enrichment = {
|
||||
"by_text": "Fichier",
|
||||
"vlm_description": "Menu Fichier",
|
||||
}
|
||||
step = Step(step_id="s1", intent="Ouvrir menu", actions=[action])
|
||||
ir.steps.append(step)
|
||||
|
||||
profile = SurfaceProfile(
|
||||
surface_type=SurfaceType.WINDOWS_NATIVE,
|
||||
uia_available=True,
|
||||
)
|
||||
|
||||
compiler = ExecutionCompiler()
|
||||
plan = compiler.compile(ir, surface_profile=profile)
|
||||
|
||||
click = [n for n in plan.nodes if n.action_type == "click"][0]
|
||||
assert click.strategy_primary.method == "ocr"
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# IRBuilder lit uia_snapshot depuis les événements
|
||||
# =========================================================================
|
||||
|
||||
|
||||
class TestIRBuilderLitUiaSnapshot:
|
||||
|
||||
def test_ir_builder_propage_uia_snapshot(self):
|
||||
"""Un event avec uia_snapshot → Action._enrichment contient uia_snapshot."""
|
||||
events = [
|
||||
{
|
||||
"event": {
|
||||
"type": "mouse_click",
|
||||
"pos": [500, 300],
|
||||
"window": {"title": "Bloc-notes"},
|
||||
"timestamp": 100.0,
|
||||
"uia_snapshot": {
|
||||
"name": "Enregistrer",
|
||||
"control_type": "bouton",
|
||||
"automation_id": "btnSave",
|
||||
"parent_path": [{"name": "Fichier", "control_type": "menu"}],
|
||||
},
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
builder = IRBuilder(gemma4_port="99999")
|
||||
ir = builder.build(events, name="Test")
|
||||
|
||||
# Parcourir les steps pour trouver le clic
|
||||
found_action = None
|
||||
for step in ir.steps:
|
||||
for action in step.actions:
|
||||
if action.type == "click":
|
||||
found_action = action
|
||||
break
|
||||
|
||||
assert found_action is not None
|
||||
enrichment = getattr(found_action, "_enrichment", None) or {}
|
||||
assert "uia_snapshot" in enrichment
|
||||
assert enrichment["uia_snapshot"]["name"] == "Enregistrer"
|
||||
assert enrichment["uia_snapshot"]["control_type"] == "bouton"
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# execution_plan_runner propage uia_target dans target_spec
|
||||
# =========================================================================
|
||||
|
||||
|
||||
class TestUiaTargetPropagation:
|
||||
|
||||
def test_strategy_uia_produit_uia_target(self):
|
||||
"""Une stratégie UIA primaire → target_spec contient uia_target."""
|
||||
primary = ResolutionStrategy(
|
||||
method="uia",
|
||||
uia_name="Enregistrer",
|
||||
uia_control_type="bouton",
|
||||
uia_automation_id="btnSave",
|
||||
uia_parent_path=[{"name": "Fichier", "control_type": "menu"}],
|
||||
)
|
||||
fallbacks = [
|
||||
ResolutionStrategy(method="ocr", target_text="Enregistrer"),
|
||||
ResolutionStrategy(method="vlm", vlm_description="bouton Enregistrer"),
|
||||
]
|
||||
|
||||
spec = _strategy_to_target_spec(primary, fallbacks)
|
||||
|
||||
assert "uia_target" in spec
|
||||
assert spec["uia_target"]["name"] == "Enregistrer"
|
||||
assert spec["uia_target"]["control_type"] == "bouton"
|
||||
assert spec["uia_target"]["automation_id"] == "btnSave"
|
||||
assert spec["resolve_order"][0] == "uia"
|
||||
assert "ocr" in spec["resolve_order"]
|
||||
assert "vlm" in spec["resolve_order"]
|
||||
|
||||
def test_pas_de_uia_target_si_pas_de_stratégie(self):
|
||||
"""Sans stratégie UIA → pas de uia_target."""
|
||||
primary = ResolutionStrategy(method="ocr", target_text="test")
|
||||
spec = _strategy_to_target_spec(primary, [])
|
||||
|
||||
assert "uia_target" not in spec
|
||||
assert "uia" not in spec.get("resolve_order", [])
|
||||
|
||||
def test_execution_node_to_action_avec_uia(self):
|
||||
"""Un ExecutionNode avec stratégie UIA produit une action complète."""
|
||||
node = ExecutionNode(
|
||||
node_id="n1",
|
||||
action_type="click",
|
||||
intent="Cliquer Enregistrer",
|
||||
strategy_primary=ResolutionStrategy(
|
||||
method="uia",
|
||||
uia_name="Enregistrer",
|
||||
uia_control_type="bouton",
|
||||
),
|
||||
strategy_fallbacks=[
|
||||
ResolutionStrategy(method="ocr", target_text="Enregistrer"),
|
||||
],
|
||||
)
|
||||
|
||||
action = execution_node_to_action(node)
|
||||
assert action is not None
|
||||
assert action["type"] == "click"
|
||||
assert "uia_target" in action["target_spec"]
|
||||
assert action["target_spec"]["uia_target"]["name"] == "Enregistrer"
|
||||
assert action["target_spec"]["resolve_order"] == ["uia", "ocr"]
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Pipeline E2E : événement avec UIA → action runtime avec uia_target
|
||||
# =========================================================================
|
||||
|
||||
|
||||
class TestPipelineE2EUia:
|
||||
|
||||
def test_pipeline_complet_uia(self):
|
||||
"""RawTrace (avec uia_snapshot) → WorkflowIR → Plan → action runtime."""
|
||||
# Événements simulés d'un enregistrement sur Windows natif
|
||||
events = [
|
||||
{
|
||||
"event": {
|
||||
"type": "mouse_click",
|
||||
"pos": [500, 300],
|
||||
"window": {"title": "Bloc-notes"},
|
||||
"timestamp": 100.0,
|
||||
"uia_snapshot": {
|
||||
"name": "Enregistrer",
|
||||
"control_type": "bouton",
|
||||
"automation_id": "btnSave",
|
||||
"parent_path": [
|
||||
{"name": "Bloc-notes", "control_type": "fenêtre"},
|
||||
],
|
||||
},
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
# Pipeline complet
|
||||
builder = IRBuilder(gemma4_port="99999")
|
||||
ir = builder.build(events, name="Test E2E UIA")
|
||||
|
||||
profile = SurfaceProfile(
|
||||
surface_type=SurfaceType.WINDOWS_NATIVE,
|
||||
uia_available=True,
|
||||
timeout_click_ms=8000,
|
||||
max_retries=2,
|
||||
)
|
||||
|
||||
compiler = ExecutionCompiler()
|
||||
plan = compiler.compile(ir, surface_profile=profile)
|
||||
|
||||
actions = execution_plan_to_actions(plan)
|
||||
|
||||
# Vérifier que l'action finale a toutes les données UIA
|
||||
click_actions = [a for a in actions if a["type"] == "click"]
|
||||
assert len(click_actions) == 1
|
||||
|
||||
action = click_actions[0]
|
||||
assert "target_spec" in action
|
||||
spec = action["target_spec"]
|
||||
|
||||
assert "resolve_order" in spec
|
||||
assert spec["resolve_order"][0] == "uia"
|
||||
assert "uia_target" in spec
|
||||
assert spec["uia_target"]["name"] == "Enregistrer"
|
||||
assert spec["uia_target"]["control_type"] == "bouton"
|
||||
assert action.get("timeout_ms") == 8000
|
||||
assert action.get("max_retries") == 2
|
||||
261
tests/unit/test_workflow_ir.py
Normal file
261
tests/unit/test_workflow_ir.py
Normal file
@@ -0,0 +1,261 @@
|
||||
"""
|
||||
Tests du WorkflowIR et de l'IRBuilder.
|
||||
|
||||
Vérifie que :
|
||||
- Le format WorkflowIR est correct (sérialisation, désérialisation, versioning)
|
||||
- L'IRBuilder segmente et comprend les traces brutes
|
||||
- Les variables sont détectées et substituables
|
||||
- Le tout fonctionne sans gemma4 (fallback gracieux)
|
||||
"""
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
_ROOT = str(Path(__file__).resolve().parents[2])
|
||||
if _ROOT not in sys.path:
|
||||
sys.path.insert(0, _ROOT)
|
||||
|
||||
from core.workflow.workflow_ir import WorkflowIR, Step, Action, Variable
|
||||
from core.workflow.ir_builder import IRBuilder
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# WorkflowIR — format et sérialisation
|
||||
# =========================================================================
|
||||
|
||||
|
||||
class TestWorkflowIR:
|
||||
|
||||
def test_creation_vide(self):
|
||||
ir = WorkflowIR.new("Test workflow")
|
||||
assert ir.workflow_id.startswith("wf_")
|
||||
assert ir.version == 1
|
||||
assert ir.name == "Test workflow"
|
||||
assert ir.steps == []
|
||||
assert ir.variables == []
|
||||
|
||||
def test_ajout_etapes(self):
|
||||
ir = WorkflowIR.new("Test")
|
||||
ir.add_step("Ouvrir l'application", actions=[
|
||||
{"type": "click", "target": "icône app"},
|
||||
{"type": "wait", "duration_ms": 2000},
|
||||
])
|
||||
ir.add_step("Saisir les données", actions=[
|
||||
{"type": "type", "text": "bonjour"},
|
||||
])
|
||||
assert len(ir.steps) == 2
|
||||
assert ir.steps[0].intent == "Ouvrir l'application"
|
||||
assert len(ir.steps[0].actions) == 2
|
||||
assert ir.steps[0].actions[0].type == "click"
|
||||
|
||||
def test_ajout_variables(self):
|
||||
ir = WorkflowIR.new("Test")
|
||||
ir.add_variable("patient", description="Nom du patient", source="screen")
|
||||
ir.add_variable("code", description="Code à saisir", default="A00.0")
|
||||
assert len(ir.variables) == 2
|
||||
assert ir.variables[0].name == "patient"
|
||||
assert ir.variables[1].default == "A00.0"
|
||||
|
||||
def test_serialisation_json(self):
|
||||
ir = WorkflowIR.new("Mon workflow", domain="tim_codage")
|
||||
ir.add_step("Étape 1")
|
||||
ir.add_variable("var1", description="Une variable")
|
||||
|
||||
json_str = ir.to_json()
|
||||
data = json.loads(json_str)
|
||||
|
||||
assert data["name"] == "Mon workflow"
|
||||
assert data["domain"] == "tim_codage"
|
||||
assert len(data["steps"]) == 1
|
||||
assert len(data["variables"]) == 1
|
||||
|
||||
def test_deserialisation_json(self):
|
||||
ir = WorkflowIR.new("Test roundtrip")
|
||||
ir.add_step("Ouvrir", actions=[{"type": "click", "target": "bouton"}])
|
||||
ir.add_variable("v1", description="test")
|
||||
|
||||
json_str = ir.to_json()
|
||||
ir2 = WorkflowIR.from_json(json_str)
|
||||
|
||||
assert ir2.name == "Test roundtrip"
|
||||
assert len(ir2.steps) == 1
|
||||
assert ir2.steps[0].intent == "Ouvrir"
|
||||
assert ir2.steps[0].actions[0].type == "click"
|
||||
assert len(ir2.variables) == 1
|
||||
|
||||
def test_save_et_load(self):
|
||||
tmpdir = tempfile.mkdtemp()
|
||||
try:
|
||||
ir = WorkflowIR.new("Save test")
|
||||
ir.add_step("Étape 1")
|
||||
path = ir.save(tmpdir)
|
||||
|
||||
assert path.is_file()
|
||||
|
||||
ir2 = WorkflowIR.load(str(path))
|
||||
assert ir2.name == "Save test"
|
||||
assert len(ir2.steps) == 1
|
||||
finally:
|
||||
shutil.rmtree(tmpdir)
|
||||
|
||||
def test_increment_version(self):
|
||||
ir = WorkflowIR.new("Versionning")
|
||||
assert ir.version == 1
|
||||
|
||||
ir2 = ir.increment_version()
|
||||
assert ir2.version == 2
|
||||
assert ir.version == 1 # Original inchangé
|
||||
assert ir2.name == "Versionning"
|
||||
|
||||
def test_domaine_generique(self):
|
||||
"""Le WorkflowIR est générique — pas lié à un métier."""
|
||||
for domain in ["tim_codage", "comptabilite", "rh_paie", "stocks", "generic"]:
|
||||
ir = WorkflowIR.new("Test", domain=domain)
|
||||
assert ir.domain == domain
|
||||
|
||||
def test_etape_optionnelle(self):
|
||||
ir = WorkflowIR.new("Test")
|
||||
ir.add_step("Vérification facultative", is_optional=True)
|
||||
assert ir.steps[0].is_optional is True
|
||||
|
||||
def test_etape_boucle(self):
|
||||
ir = WorkflowIR.new("Test")
|
||||
ir.add_step("Traiter chaque dossier", is_loop=True, loop_variable="dossier")
|
||||
assert ir.steps[0].is_loop is True
|
||||
assert ir.steps[0].loop_variable == "dossier"
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# IRBuilder — construction depuis RawTrace
|
||||
# =========================================================================
|
||||
|
||||
|
||||
class TestIRBuilder:
|
||||
|
||||
def _make_events(self):
|
||||
"""Créer des événements bruts simulés (comme live_events.jsonl)."""
|
||||
return [
|
||||
{"event": {"type": "mouse_click", "pos": [400, 580], "window": {"title": "Lea : Explorateur"}, "timestamp": 100.0, "vision_info": {"text": "Rechercher"}}},
|
||||
{"event": {"type": "text_input", "text": "blocnote", "window": {"title": "Rechercher"}, "timestamp": 102.0}},
|
||||
{"event": {"type": "key_combo", "keys": ["enter"], "window": {"title": "Rechercher"}, "timestamp": 103.0}},
|
||||
{"event": {"type": "heartbeat", "timestamp": 104.0}}, # Parasite — doit être filtré
|
||||
{"event": {"type": "mouse_click", "pos": [300, 200], "window": {"title": "Rechercher"}, "timestamp": 105.0, "vision_info": {"text": "Bloc-notes"}}},
|
||||
{"event": {"type": "mouse_click", "pos": [500, 300], "window": {"title": "Sans titre – Bloc-notes"}, "timestamp": 112.0, "vision_info": {"text": ""}}},
|
||||
{"event": {"type": "text_input", "text": "Bonjour le monde", "window": {"title": "*Sans titre – Bloc-notes"}, "timestamp": 113.0}},
|
||||
{"event": {"type": "key_combo", "keys": ["ctrl", "s"], "window": {"title": "*Sans titre – Bloc-notes"}, "timestamp": 115.0}},
|
||||
]
|
||||
|
||||
def test_builder_sans_gemma4(self):
|
||||
"""Le builder fonctionne même sans gemma4 (fallback gracieux)."""
|
||||
builder = IRBuilder(gemma4_port="99999") # Port invalide
|
||||
events = self._make_events()
|
||||
|
||||
ir = builder.build(events, session_id="test_sess", domain="generic", name="Test")
|
||||
|
||||
assert ir.name == "Test"
|
||||
assert ir.learned_from == "test_sess"
|
||||
assert len(ir.steps) >= 1
|
||||
assert len(ir.applications) >= 1
|
||||
|
||||
def test_filtre_heartbeat(self):
|
||||
"""Les heartbeat sont filtrés."""
|
||||
builder = IRBuilder(gemma4_port="99999")
|
||||
events = self._make_events()
|
||||
|
||||
ir = builder.build(events, name="Test")
|
||||
|
||||
# Vérifier qu'aucune action n'est de type heartbeat
|
||||
for step in ir.steps:
|
||||
for action in step.actions:
|
||||
assert action.type != "heartbeat"
|
||||
|
||||
def test_detection_applications(self):
|
||||
"""Les applications utilisées sont détectées."""
|
||||
builder = IRBuilder(gemma4_port="99999")
|
||||
events = self._make_events()
|
||||
|
||||
ir = builder.build(events, name="Test")
|
||||
|
||||
assert "Bloc-notes" in ir.applications or "Explorateur" in ir.applications
|
||||
|
||||
def test_detection_variables(self):
|
||||
"""Le texte saisi est détecté comme variable."""
|
||||
builder = IRBuilder(gemma4_port="99999")
|
||||
events = self._make_events()
|
||||
|
||||
ir = builder.build(events, name="Test")
|
||||
|
||||
# Le texte "blocnote" et "Bonjour le monde" doivent être des variables
|
||||
assert len(ir.variables) >= 1
|
||||
var_defaults = [v.default for v in ir.variables]
|
||||
assert any("blocnote" in d or "Bonjour" in d for d in var_defaults)
|
||||
|
||||
def test_segmentation_par_application(self):
|
||||
"""Les événements sont segmentés par changement d'application."""
|
||||
builder = IRBuilder(gemma4_port="99999")
|
||||
events = self._make_events()
|
||||
|
||||
ir = builder.build(events, name="Test")
|
||||
|
||||
# Au moins 2 étapes (Explorateur → Bloc-notes)
|
||||
assert len(ir.steps) >= 2
|
||||
|
||||
def test_actions_dans_les_etapes(self):
|
||||
"""Chaque étape contient les bonnes actions."""
|
||||
builder = IRBuilder(gemma4_port="99999")
|
||||
events = self._make_events()
|
||||
|
||||
ir = builder.build(events, name="Test")
|
||||
|
||||
all_actions = []
|
||||
for step in ir.steps:
|
||||
all_actions.extend(step.actions)
|
||||
|
||||
types = [a.type for a in all_actions]
|
||||
assert "click" in types
|
||||
assert "type" in types
|
||||
assert "key_combo" in types
|
||||
|
||||
def test_workflow_ir_complet_roundtrip(self):
|
||||
"""Build → JSON → reload → même contenu."""
|
||||
builder = IRBuilder(gemma4_port="99999")
|
||||
events = self._make_events()
|
||||
|
||||
ir = builder.build(events, name="Roundtrip test", domain="compta")
|
||||
json_str = ir.to_json()
|
||||
ir2 = WorkflowIR.from_json(json_str)
|
||||
|
||||
assert ir2.name == "Roundtrip test"
|
||||
assert ir2.domain == "compta"
|
||||
assert len(ir2.steps) == len(ir.steps)
|
||||
assert len(ir2.variables) == len(ir.variables)
|
||||
|
||||
@patch("requests.post")
|
||||
def test_builder_avec_gemma4_mock(self, mock_post):
|
||||
"""Avec gemma4, le builder enrichit les intentions."""
|
||||
mock_resp = MagicMock()
|
||||
mock_resp.ok = True
|
||||
mock_resp.json.return_value = {
|
||||
"message": {"content": (
|
||||
"INTENTION: Rechercher et ouvrir le Bloc-notes\n"
|
||||
"AVANT: L'explorateur de fichiers est ouvert\n"
|
||||
"APRÈS: Le Bloc-notes est ouvert et actif"
|
||||
)}
|
||||
}
|
||||
mock_post.return_value = mock_resp
|
||||
|
||||
builder = IRBuilder()
|
||||
events = self._make_events()
|
||||
|
||||
ir = builder.build(events, name="Test gemma4")
|
||||
|
||||
# Au moins une étape doit avoir une intention enrichie
|
||||
intents = [s.intent for s in ir.steps]
|
||||
has_enriched = any("Bloc-notes" in i or "Rechercher" in i for i in intents)
|
||||
assert has_enriched or len(ir.steps) >= 1 # Fallback acceptable
|
||||
@@ -1876,7 +1876,7 @@ def load_system_config():
|
||||
"version": "1.0.0",
|
||||
"services": {},
|
||||
"llm": {"provider": "ollama", "base_url": "http://localhost:11434", "model": "qwen2.5:7b"},
|
||||
"vlm": {"provider": "ollama", "base_url": "http://localhost:11434", "model": "qwen2.5vl:7b"},
|
||||
"vlm": {"provider": "ollama", "base_url": "http://localhost:11434", "model": "gemma4:e4b"},
|
||||
"detection": {"owl_model": "google/owlv2-base-patch16-ensemble", "confidence_threshold": 0.3},
|
||||
"database": {"type": "sqlite", "path": "data/training/workflows.db"},
|
||||
"security": {"enable_encryption": True, "require_authentication": False}
|
||||
@@ -2371,6 +2371,93 @@ def proxy_streaming(endpoint):
|
||||
return jsonify({'error': str(e)}), 500
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Chat conversationnel — Léa
|
||||
# =============================================================================
|
||||
|
||||
CHAT_BASE_URL = 'http://localhost:5005/api/v1/chat'
|
||||
|
||||
|
||||
@app.route('/chat')
|
||||
def chat_page():
|
||||
"""Page de chat conversationnel avec Léa."""
|
||||
return render_template('chat.html')
|
||||
|
||||
|
||||
@app.route('/api/chat/session', methods=['POST'])
|
||||
def proxy_chat_session():
|
||||
"""Proxy : créer une session de chat côté serveur streaming."""
|
||||
return _proxy_chat(
|
||||
method='POST',
|
||||
path='/session',
|
||||
payload=request.get_json(silent=True) or {},
|
||||
)
|
||||
|
||||
|
||||
@app.route('/api/chat/<session_id>/message', methods=['POST'])
|
||||
def proxy_chat_message(session_id):
|
||||
"""Proxy : envoyer un message dans une session."""
|
||||
return _proxy_chat(
|
||||
method='POST',
|
||||
path=f'/{session_id}/message',
|
||||
payload=request.get_json(silent=True) or {},
|
||||
)
|
||||
|
||||
|
||||
@app.route('/api/chat/<session_id>/history', methods=['GET'])
|
||||
def proxy_chat_history(session_id):
|
||||
"""Proxy : récupérer l'historique."""
|
||||
return _proxy_chat(method='GET', path=f'/{session_id}/history')
|
||||
|
||||
|
||||
@app.route('/api/chat/<session_id>/confirm', methods=['POST'])
|
||||
def proxy_chat_confirm(session_id):
|
||||
"""Proxy : confirmer l'exécution d'un plan."""
|
||||
return _proxy_chat(
|
||||
method='POST',
|
||||
path=f'/{session_id}/confirm',
|
||||
payload=request.get_json(silent=True) or {},
|
||||
)
|
||||
|
||||
|
||||
def _proxy_chat(method, path, payload=None):
|
||||
"""Helper pour proxyfier les requêtes vers le serveur streaming (:5005)."""
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
|
||||
url = f'{CHAT_BASE_URL}{path}'
|
||||
headers = {
|
||||
'Accept': 'application/json',
|
||||
'Content-Type': 'application/json',
|
||||
}
|
||||
# Token Bearer (lu depuis l'env — même token que le serveur streaming)
|
||||
token = os.environ.get('RPA_API_TOKEN', '')
|
||||
if token:
|
||||
headers['Authorization'] = f'Bearer {token}'
|
||||
|
||||
try:
|
||||
data_bytes = None
|
||||
if payload is not None and method != 'GET':
|
||||
data_bytes = json.dumps(payload).encode('utf-8')
|
||||
req = urllib.request.Request(url, data=data_bytes, headers=headers, method=method)
|
||||
with urllib.request.urlopen(req, timeout=15) as response:
|
||||
body = response.read().decode('utf-8')
|
||||
try:
|
||||
return jsonify(json.loads(body))
|
||||
except json.JSONDecodeError:
|
||||
return body, response.status, {'Content-Type': 'application/json'}
|
||||
except urllib.error.HTTPError as e:
|
||||
try:
|
||||
detail = json.loads(e.read().decode('utf-8'))
|
||||
except Exception:
|
||||
detail = {'error': str(e)}
|
||||
return jsonify(detail), e.code
|
||||
except urllib.error.URLError as e:
|
||||
return jsonify({'error': f'Serveur chat inaccessible : {e}'}), 502
|
||||
except Exception as e:
|
||||
return jsonify({'error': str(e)}), 500
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Main
|
||||
# =============================================================================
|
||||
|
||||
240
web_dashboard/static/js/chat.js
Normal file
240
web_dashboard/static/js/chat.js
Normal file
@@ -0,0 +1,240 @@
|
||||
// chat.js — Client Léa conversationnelle
|
||||
// Logique minimaliste : pas de framework, fetch + polling.
|
||||
|
||||
const API_BASE = "/api/chat"; // Proxyfié par le dashboard Flask vers :5005
|
||||
|
||||
let sessionId = null;
|
||||
let pollTimer = null;
|
||||
let lastMessageCount = 0;
|
||||
let currentState = "idle";
|
||||
|
||||
const STATE_LABELS = {
|
||||
idle: "En attente",
|
||||
planning: "Léa réfléchit…",
|
||||
awaiting_confirmation: "En attente de confirmation",
|
||||
executing: "Léa exécute le workflow…",
|
||||
done: "Terminé",
|
||||
error: "Erreur",
|
||||
};
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
// Initialisation
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
async function initChat() {
|
||||
try {
|
||||
const resp = await fetch(`${API_BASE}/session`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ machine_id: "default" }),
|
||||
});
|
||||
if (!resp.ok) throw new Error(`HTTP ${resp.status}`);
|
||||
const data = await resp.json();
|
||||
sessionId = data.session_id;
|
||||
currentState = data.state || "idle";
|
||||
updateStatus(currentState);
|
||||
renderMessages(data.history || []);
|
||||
document.getElementById("sessionInfo").textContent = `Session ${sessionId}`;
|
||||
startPolling();
|
||||
} catch (err) {
|
||||
console.error("Impossible de créer la session chat :", err);
|
||||
showSystemMessage(`Impossible de créer la session chat : ${err.message}. Vérifiez que le serveur streaming (5005) est démarré.`);
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
// Envoi de messages
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
async function sendMessage() {
|
||||
const input = document.getElementById("composerInput");
|
||||
const text = (input.value || "").trim();
|
||||
if (!text || !sessionId) return;
|
||||
|
||||
const sendBtn = document.getElementById("sendBtn");
|
||||
sendBtn.disabled = true;
|
||||
input.value = "";
|
||||
autosizeTextarea();
|
||||
|
||||
// Affichage optimiste
|
||||
appendMessage({
|
||||
role: "user",
|
||||
content: text,
|
||||
timestamp: Date.now() / 1000,
|
||||
});
|
||||
|
||||
try {
|
||||
updateStatus("planning");
|
||||
const resp = await fetch(`${API_BASE}/${sessionId}/message`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ message: text }),
|
||||
});
|
||||
if (!resp.ok) throw new Error(`HTTP ${resp.status}`);
|
||||
const data = await resp.json();
|
||||
currentState = data.state || "idle";
|
||||
updateStatus(currentState);
|
||||
renderMessages(data.history || []);
|
||||
} catch (err) {
|
||||
console.error("Erreur envoi message :", err);
|
||||
showSystemMessage(`Erreur : ${err.message}`);
|
||||
updateStatus("error");
|
||||
} finally {
|
||||
sendBtn.disabled = false;
|
||||
input.focus();
|
||||
}
|
||||
}
|
||||
|
||||
async function confirmPlan(confirmed) {
|
||||
if (!sessionId) return;
|
||||
const confirmBar = document.getElementById("confirmBar");
|
||||
confirmBar.classList.remove("visible");
|
||||
|
||||
try {
|
||||
const resp = await fetch(`${API_BASE}/${sessionId}/confirm`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ confirmed }),
|
||||
});
|
||||
if (!resp.ok) throw new Error(`HTTP ${resp.status}`);
|
||||
const data = await resp.json();
|
||||
currentState = data.state || "idle";
|
||||
updateStatus(currentState);
|
||||
renderMessages(data.history || []);
|
||||
} catch (err) {
|
||||
console.error("Erreur confirmation :", err);
|
||||
showSystemMessage(`Erreur confirmation : ${err.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
// Polling
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
function startPolling() {
|
||||
if (pollTimer) clearInterval(pollTimer);
|
||||
pollTimer = setInterval(pollHistory, 2000);
|
||||
}
|
||||
|
||||
async function pollHistory() {
|
||||
if (!sessionId) return;
|
||||
try {
|
||||
const resp = await fetch(`${API_BASE}/${sessionId}/history`);
|
||||
if (!resp.ok) return;
|
||||
const data = await resp.json();
|
||||
const snap = data.snapshot || {};
|
||||
currentState = snap.state || "idle";
|
||||
updateStatus(currentState, snap.progress || {});
|
||||
const messages = snap.messages || [];
|
||||
if (messages.length !== lastMessageCount) {
|
||||
renderMessages(messages);
|
||||
}
|
||||
} catch (err) {
|
||||
// Silencieux — on réessayera au prochain tick
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
// Rendu
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
function renderMessages(messages) {
|
||||
const container = document.getElementById("messages");
|
||||
container.innerHTML = "";
|
||||
messages.forEach(msg => appendMessage(msg, false));
|
||||
lastMessageCount = messages.length;
|
||||
container.scrollTop = container.scrollHeight;
|
||||
|
||||
// Afficher/masquer la barre de confirmation
|
||||
const confirmBar = document.getElementById("confirmBar");
|
||||
if (currentState === "awaiting_confirmation") {
|
||||
confirmBar.classList.add("visible");
|
||||
} else {
|
||||
confirmBar.classList.remove("visible");
|
||||
}
|
||||
}
|
||||
|
||||
function appendMessage(msg, autoscroll = true) {
|
||||
const container = document.getElementById("messages");
|
||||
const div = document.createElement("div");
|
||||
div.className = `message ${msg.role}`;
|
||||
|
||||
const avatar = document.createElement("div");
|
||||
avatar.className = "avatar";
|
||||
if (msg.role === "user") avatar.textContent = "Vous";
|
||||
else if (msg.role === "lea") avatar.textContent = "L";
|
||||
else avatar.textContent = "i";
|
||||
|
||||
const bubbleWrap = document.createElement("div");
|
||||
const bubble = document.createElement("div");
|
||||
bubble.className = "bubble";
|
||||
bubble.textContent = msg.content || "";
|
||||
bubbleWrap.appendChild(bubble);
|
||||
|
||||
const ts = document.createElement("div");
|
||||
ts.className = "timestamp";
|
||||
try {
|
||||
const d = new Date((msg.timestamp || 0) * 1000);
|
||||
ts.textContent = d.toLocaleTimeString("fr-FR");
|
||||
} catch (e) { ts.textContent = ""; }
|
||||
bubbleWrap.appendChild(ts);
|
||||
|
||||
div.appendChild(avatar);
|
||||
div.appendChild(bubbleWrap);
|
||||
container.appendChild(div);
|
||||
|
||||
if (autoscroll) container.scrollTop = container.scrollHeight;
|
||||
}
|
||||
|
||||
function showSystemMessage(text) {
|
||||
appendMessage({
|
||||
role: "system",
|
||||
content: text,
|
||||
timestamp: Date.now() / 1000,
|
||||
});
|
||||
}
|
||||
|
||||
function updateStatus(state, progress = {}) {
|
||||
const dot = document.getElementById("statusDot");
|
||||
const txt = document.getElementById("statusText");
|
||||
dot.className = `status-dot ${state}`;
|
||||
let label = STATE_LABELS[state] || state;
|
||||
|
||||
if (state === "executing" && progress && progress.total_actions) {
|
||||
const done = progress.completed_actions || 0;
|
||||
const total = progress.total_actions || 0;
|
||||
label = `Léa exécute… ${done}/${total}`;
|
||||
}
|
||||
|
||||
txt.textContent = label;
|
||||
|
||||
// Bloquer la saisie pendant planning/executing
|
||||
const input = document.getElementById("composerInput");
|
||||
const sendBtn = document.getElementById("sendBtn");
|
||||
const blocked = (state === "planning" || state === "executing");
|
||||
input.disabled = blocked;
|
||||
sendBtn.disabled = blocked;
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
// UX composer
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
function handleKeydown(event) {
|
||||
if (event.key === "Enter" && !event.shiftKey) {
|
||||
event.preventDefault();
|
||||
sendMessage();
|
||||
}
|
||||
}
|
||||
|
||||
function autosizeTextarea() {
|
||||
const input = document.getElementById("composerInput");
|
||||
input.style.height = "auto";
|
||||
input.style.height = Math.min(input.scrollHeight, 120) + "px";
|
||||
}
|
||||
|
||||
document.addEventListener("DOMContentLoaded", () => {
|
||||
const input = document.getElementById("composerInput");
|
||||
input.addEventListener("input", autosizeTextarea);
|
||||
initChat();
|
||||
});
|
||||
309
web_dashboard/templates/chat.html
Normal file
309
web_dashboard/templates/chat.html
Normal file
@@ -0,0 +1,309 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="fr">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>Léa — Chat RPA Vision V3</title>
|
||||
<style>
|
||||
* { margin: 0; padding: 0; box-sizing: border-box; }
|
||||
body {
|
||||
font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
|
||||
background: #0f172a;
|
||||
color: #e2e8f0;
|
||||
min-height: 100vh;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
}
|
||||
.header {
|
||||
background: linear-gradient(135deg, #3b82f6 0%, #8b5cf6 100%);
|
||||
color: white;
|
||||
padding: 16px 24px;
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
box-shadow: 0 2px 8px rgba(0,0,0,0.3);
|
||||
}
|
||||
.header h1 {
|
||||
font-size: 20px;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 10px;
|
||||
}
|
||||
.header .home-link {
|
||||
color: rgba(255,255,255,0.85);
|
||||
text-decoration: none;
|
||||
font-size: 13px;
|
||||
padding: 6px 14px;
|
||||
border-radius: 6px;
|
||||
background: rgba(255,255,255,0.1);
|
||||
transition: background 0.2s;
|
||||
}
|
||||
.header .home-link:hover { background: rgba(255,255,255,0.2); }
|
||||
|
||||
.chat-wrapper {
|
||||
flex: 1;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
max-width: 900px;
|
||||
width: 100%;
|
||||
margin: 0 auto;
|
||||
padding: 20px;
|
||||
gap: 16px;
|
||||
}
|
||||
|
||||
.status-bar {
|
||||
background: #1e293b;
|
||||
border: 1px solid #334155;
|
||||
border-radius: 12px;
|
||||
padding: 12px 18px;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: 12px;
|
||||
font-size: 14px;
|
||||
}
|
||||
.status-bar .status-label {
|
||||
color: #94a3b8;
|
||||
}
|
||||
.status-bar .status-value {
|
||||
font-weight: 600;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
}
|
||||
.status-dot {
|
||||
width: 10px;
|
||||
height: 10px;
|
||||
border-radius: 50%;
|
||||
background: #64748b;
|
||||
}
|
||||
.status-dot.idle { background: #64748b; }
|
||||
.status-dot.planning { background: #f59e0b; animation: pulse 1.2s infinite; }
|
||||
.status-dot.awaiting_confirmation { background: #3b82f6; animation: pulse 1.8s infinite; }
|
||||
.status-dot.executing { background: #22c55e; animation: pulse 1s infinite; }
|
||||
.status-dot.done { background: #22c55e; }
|
||||
.status-dot.error { background: #ef4444; }
|
||||
@keyframes pulse {
|
||||
0%, 100% { opacity: 1; transform: scale(1); }
|
||||
50% { opacity: 0.5; transform: scale(1.15); }
|
||||
}
|
||||
|
||||
.messages {
|
||||
flex: 1;
|
||||
background: #1e293b;
|
||||
border: 1px solid #334155;
|
||||
border-radius: 12px;
|
||||
padding: 20px;
|
||||
overflow-y: auto;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 14px;
|
||||
min-height: 400px;
|
||||
max-height: calc(100vh - 320px);
|
||||
}
|
||||
.messages::-webkit-scrollbar { width: 8px; }
|
||||
.messages::-webkit-scrollbar-thumb { background: #334155; border-radius: 4px; }
|
||||
.messages::-webkit-scrollbar-track { background: transparent; }
|
||||
|
||||
.message {
|
||||
display: flex;
|
||||
gap: 12px;
|
||||
max-width: 85%;
|
||||
animation: fadeIn 0.25s ease-out;
|
||||
}
|
||||
@keyframes fadeIn {
|
||||
from { opacity: 0; transform: translateY(6px); }
|
||||
to { opacity: 1; transform: translateY(0); }
|
||||
}
|
||||
.message.user {
|
||||
align-self: flex-end;
|
||||
flex-direction: row-reverse;
|
||||
}
|
||||
.message .avatar {
|
||||
width: 34px;
|
||||
height: 34px;
|
||||
border-radius: 50%;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
font-size: 16px;
|
||||
flex-shrink: 0;
|
||||
font-weight: 600;
|
||||
}
|
||||
.message.user .avatar {
|
||||
background: #3b82f6;
|
||||
color: white;
|
||||
}
|
||||
.message.lea .avatar {
|
||||
background: linear-gradient(135deg, #8b5cf6, #ec4899);
|
||||
color: white;
|
||||
}
|
||||
.message.system .avatar {
|
||||
background: #475569;
|
||||
color: #cbd5e1;
|
||||
}
|
||||
.message .bubble {
|
||||
background: #334155;
|
||||
padding: 12px 16px;
|
||||
border-radius: 14px;
|
||||
line-height: 1.5;
|
||||
font-size: 14px;
|
||||
white-space: pre-wrap;
|
||||
word-wrap: break-word;
|
||||
box-shadow: 0 1px 3px rgba(0,0,0,0.2);
|
||||
}
|
||||
.message.user .bubble {
|
||||
background: #1d4ed8;
|
||||
color: white;
|
||||
}
|
||||
.message.lea .bubble {
|
||||
background: #334155;
|
||||
}
|
||||
.message.system .bubble {
|
||||
background: transparent;
|
||||
border: 1px dashed #475569;
|
||||
color: #94a3b8;
|
||||
font-style: italic;
|
||||
}
|
||||
.message .timestamp {
|
||||
font-size: 11px;
|
||||
color: #64748b;
|
||||
margin-top: 4px;
|
||||
}
|
||||
|
||||
.confirm-bar {
|
||||
background: #1e293b;
|
||||
border: 1px solid #3b82f6;
|
||||
border-radius: 12px;
|
||||
padding: 14px 18px;
|
||||
display: none;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: 12px;
|
||||
}
|
||||
.confirm-bar.visible { display: flex; }
|
||||
.confirm-bar .label {
|
||||
font-size: 14px;
|
||||
color: #93c5fd;
|
||||
font-weight: 500;
|
||||
}
|
||||
.confirm-bar .actions { display: flex; gap: 10px; }
|
||||
.btn {
|
||||
padding: 9px 20px;
|
||||
border-radius: 8px;
|
||||
border: none;
|
||||
font-size: 14px;
|
||||
font-weight: 600;
|
||||
cursor: pointer;
|
||||
transition: all 0.15s;
|
||||
}
|
||||
.btn-confirm {
|
||||
background: #22c55e;
|
||||
color: white;
|
||||
}
|
||||
.btn-confirm:hover { background: #16a34a; }
|
||||
.btn-cancel {
|
||||
background: #475569;
|
||||
color: #e2e8f0;
|
||||
}
|
||||
.btn-cancel:hover { background: #64748b; }
|
||||
.btn:disabled {
|
||||
opacity: 0.5;
|
||||
cursor: not-allowed;
|
||||
}
|
||||
|
||||
.composer {
|
||||
background: #1e293b;
|
||||
border: 1px solid #334155;
|
||||
border-radius: 12px;
|
||||
padding: 14px;
|
||||
display: flex;
|
||||
gap: 10px;
|
||||
align-items: flex-end;
|
||||
}
|
||||
.composer textarea {
|
||||
flex: 1;
|
||||
background: #0f172a;
|
||||
border: 1px solid #334155;
|
||||
border-radius: 8px;
|
||||
padding: 10px 14px;
|
||||
color: #e2e8f0;
|
||||
font-family: inherit;
|
||||
font-size: 14px;
|
||||
resize: none;
|
||||
min-height: 42px;
|
||||
max-height: 120px;
|
||||
line-height: 1.5;
|
||||
}
|
||||
.composer textarea:focus {
|
||||
outline: none;
|
||||
border-color: #3b82f6;
|
||||
}
|
||||
.composer textarea:disabled {
|
||||
opacity: 0.6;
|
||||
cursor: not-allowed;
|
||||
}
|
||||
.composer .btn-send {
|
||||
background: #3b82f6;
|
||||
color: white;
|
||||
padding: 10px 22px;
|
||||
}
|
||||
.composer .btn-send:hover { background: #2563eb; }
|
||||
|
||||
.progress-bar {
|
||||
margin-top: 8px;
|
||||
height: 6px;
|
||||
background: #0f172a;
|
||||
border-radius: 3px;
|
||||
overflow: hidden;
|
||||
}
|
||||
.progress-bar .fill {
|
||||
height: 100%;
|
||||
background: linear-gradient(90deg, #3b82f6, #22c55e);
|
||||
width: 0%;
|
||||
transition: width 0.4s ease;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="header">
|
||||
<h1>Léa — Assistant RPA Vision V3</h1>
|
||||
<a href="/" class="home-link">Retour au dashboard</a>
|
||||
</div>
|
||||
|
||||
<div class="chat-wrapper">
|
||||
<div class="status-bar">
|
||||
<div>
|
||||
<span class="status-label">État :</span>
|
||||
<span class="status-value">
|
||||
<span class="status-dot idle" id="statusDot"></span>
|
||||
<span id="statusText">En attente</span>
|
||||
</span>
|
||||
</div>
|
||||
<div id="sessionInfo" style="color:#64748b;font-size:12px;">Aucune session</div>
|
||||
</div>
|
||||
|
||||
<div class="messages" id="messages"></div>
|
||||
|
||||
<div class="confirm-bar" id="confirmBar">
|
||||
<div class="label">Léa propose un plan. Confirmer l'exécution ?</div>
|
||||
<div class="actions">
|
||||
<button class="btn btn-cancel" onclick="confirmPlan(false)">Non, annuler</button>
|
||||
<button class="btn btn-confirm" onclick="confirmPlan(true)">Oui, y aller</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="composer">
|
||||
<textarea
|
||||
id="composerInput"
|
||||
placeholder="Dites à Léa ce que vous voulez faire (ex. « Ouvre le Bloc-notes et écris bonjour »)…"
|
||||
rows="1"
|
||||
onkeydown="handleKeydown(event)"
|
||||
></textarea>
|
||||
<button class="btn btn-send" id="sendBtn" onclick="sendMessage()">Envoyer</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script src="/static/js/chat.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
Reference in New Issue
Block a user