Compare commits
371 Commits
v3.0
...
f9a0531325
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f9a0531325 | ||
|
|
ab78ae390a | ||
|
|
e59489e2cd | ||
|
|
86e31ada34 | ||
|
|
94fd93ad19 | ||
|
|
50f34b5727 | ||
|
|
a1b3062991 | ||
|
|
a210e5ee32 | ||
|
|
5d235e49f1 | ||
|
|
e679804cfd | ||
|
|
e57b54a100 | ||
|
|
d34c1f2697 | ||
|
|
61664c9a36 | ||
|
|
9ab5ed4671 | ||
|
|
144a5c288a | ||
|
|
e3f61de4ad | ||
|
|
2a1b1ed80e | ||
|
|
f09b8b8cfd | ||
|
|
6a78a0059b | ||
|
|
813b33b47e | ||
|
|
a50057d499 | ||
|
|
3ed9798f06 | ||
|
|
b65710ae43 | ||
|
|
509a026cfc | ||
|
|
a62b720144 | ||
|
|
14b1bf844a | ||
|
|
c82829f2bb | ||
|
|
6075717353 | ||
|
|
13f760a3b9 | ||
|
|
9883cad012 | ||
|
|
5ed5ae2d4b | ||
|
|
7fb58195fb | ||
|
|
fccc06e4a2 | ||
|
|
6461f0a21b | ||
|
|
e84cdee393 | ||
|
|
30d8f65e9a | ||
|
|
8e4d09594c | ||
|
|
46ad5973d1 | ||
|
|
4a38000e74 | ||
|
|
2597ca9110 | ||
|
|
bbe897e614 | ||
|
|
a29b7a2f21 | ||
|
|
105ade959d | ||
|
|
29cb466595 | ||
|
|
de73cbd404 | ||
|
|
1b491326be | ||
|
|
3b592dd867 | ||
|
|
c9b7cdabb7 | ||
|
|
74df0822e2 | ||
|
|
a86c1ebb83 | ||
|
|
2cabc6cb7e | ||
|
|
d686c3ac22 | ||
|
|
e212f4141c | ||
|
|
33ddb51c3c | ||
|
|
1d6efdb1b7 | ||
|
|
cf81ce4c7b | ||
|
|
ec1fb81054 | ||
|
|
6d5ef51c60 | ||
|
|
d0c794d923 | ||
|
|
9605cc9d95 | ||
|
|
667575c3ad | ||
|
|
787dbfb0eb | ||
|
|
86b5ec18c6 | ||
|
|
b8b963059e | ||
|
|
2b1743c206 | ||
|
|
48879fb849 | ||
|
|
c12fd8e1c1 | ||
|
|
cbd3d40e39 | ||
|
|
33c1e2e0d1 | ||
|
|
c0e4c382be | ||
|
|
5c5ce747b0 | ||
|
|
b20d17882e | ||
|
|
9fb2c7bfee | ||
|
|
f7f6926410 | ||
|
|
09f65cecbe | ||
|
|
0ee54157e5 | ||
|
|
6d34b3cb68 | ||
|
|
f18de016d7 | ||
|
|
549ea0631b | ||
|
|
0e215da842 | ||
|
|
d00fe7b00b | ||
|
|
5b2afa3629 | ||
|
|
0f122a512f | ||
|
|
806cc04b82 | ||
|
|
4dc7d840d6 | ||
|
|
4e7c2a7628 | ||
|
|
3697e3ba0e | ||
|
|
5289f3de48 | ||
|
|
4b3d5ce0d7 | ||
|
|
9b8bdfdbbe | ||
|
|
f2e9aac6b7 | ||
|
|
18ed6cb751 | ||
|
|
d38f0b0f2f | ||
|
|
86b3c8f7e7 | ||
|
|
7a1a5cb6fd | ||
|
|
2dd306724c | ||
|
|
335d576830 | ||
|
|
1a58a0d1f1 | ||
|
|
eb2df539f1 | ||
|
|
c9f848273b | ||
|
|
45ec5fe969 | ||
|
|
8b6c397531 | ||
|
|
6a300a4298 | ||
|
|
0587036c17 | ||
|
|
f2a9e40502 | ||
|
|
34527b5cc5 | ||
|
|
bd3aaf7d64 | ||
|
|
05a30f2d1d | ||
|
|
47377226f2 | ||
|
|
d515b22d1b | ||
|
|
aba849324a | ||
|
|
7ad260d02f | ||
|
|
794a248dae | ||
|
|
8332b2cd37 | ||
|
|
9a45e61e2a | ||
|
|
e66bc6d452 | ||
|
|
7b1f30af1a | ||
|
|
488d14240a | ||
|
|
45b6da5e3f | ||
|
|
02211fddf2 | ||
|
|
ed36bc2b37 | ||
|
|
9677738f32 | ||
|
|
d422aa119c | ||
|
|
7b943926db | ||
|
|
99f89317cb | ||
|
|
6b8114eb97 | ||
|
|
7ef98d8089 | ||
|
|
8ea4ed0ad2 | ||
|
|
a49f59b4d6 | ||
|
|
762e75a077 | ||
|
|
c1a144c673 | ||
|
|
e8a0fb0e42 | ||
|
|
4ba426c205 | ||
|
|
7bb8d543ab | ||
|
|
debd7b423c | ||
|
|
6544ebe3f0 | ||
|
|
10136f0ee0 | ||
|
|
054279feb4 | ||
|
|
ea1f57afb1 | ||
|
|
345762330b | ||
|
|
b1b32187ba | ||
|
|
ad24d16d83 | ||
|
|
a76f3db682 | ||
|
|
9a029a221d | ||
|
|
5ed1810ef3 | ||
|
|
c9878f0a76 | ||
|
|
08701761e6 | ||
|
|
a13d6d0052 | ||
|
|
84d2d4a667 | ||
|
|
1b4e64960b | ||
|
|
bd100bc538 | ||
|
|
1647e42d32 | ||
|
|
7df51d2c79 | ||
|
|
5ea4960e65 | ||
|
|
f2212e77e3 | ||
|
|
9872f4510c | ||
|
|
2eeaa806bb | ||
|
|
df5ad59330 | ||
|
|
bfbf0f9c3e | ||
|
|
ecc5a233a7 | ||
|
|
293e54b4e6 | ||
|
|
0d7bcd18ac | ||
|
|
4df1ba5779 | ||
|
|
e9702b4df9 | ||
|
|
e0b47e4518 | ||
|
|
5dc20cc85b | ||
|
|
88ed103de5 | ||
|
|
194853cebb | ||
|
|
626823d327 | ||
|
|
2e76b44ff3 | ||
|
|
731b5bcae2 | ||
|
|
8648e375fe | ||
|
|
56e869c467 | ||
|
|
f8dc3c3af4 | ||
|
|
ca81850a20 | ||
|
|
35fd6cf4c5 | ||
|
|
7847a0e829 | ||
|
|
40440f1ca0 | ||
|
|
7233df2bb9 | ||
|
|
f62fda575f | ||
|
|
22c0a2ba61 | ||
|
|
6fdedbfe9d | ||
|
|
c969f93a23 | ||
|
|
1cbec2806e | ||
|
|
864530c851 | ||
|
|
d1ebf62217 | ||
|
|
87dbe8c5ff | ||
|
|
0a02a6ec9c | ||
|
|
83be93e121 | ||
|
|
f5c33477f0 | ||
|
|
b1a3aa16f1 | ||
|
|
0bcfddbbc4 | ||
|
|
aa47172f0f | ||
|
|
65da557310 | ||
|
|
af13cd80ff | ||
|
|
7c6945171e | ||
|
|
ca0b436a61 | ||
|
|
fc01afa59c | ||
|
|
2a51a844b9 | ||
|
|
2d71e2a249 | ||
|
|
fae95c5366 | ||
|
|
6582a69d31 | ||
|
|
5543e25f9d | ||
|
|
2a07d8084b | ||
|
|
35b27ae492 | ||
|
|
b584bbabc3 | ||
|
|
8817f527e7 | ||
|
|
964856ab30 | ||
|
|
a67d896104 | ||
|
|
90c1d8036f | ||
|
|
6261002039 | ||
|
|
0e6e61f2b1 | ||
|
|
41c1250c99 | ||
|
|
2af3bc3b93 | ||
|
|
6154423a91 | ||
|
|
41eba898c0 | ||
|
|
9452e86fd1 | ||
|
|
5e31cdf666 | ||
|
|
487bcb8618 | ||
|
|
3d6868f029 | ||
|
|
f73a2a59a9 | ||
|
|
77faa03ec9 | ||
|
|
343d6fbe95 | ||
|
|
cc64439738 | ||
|
|
90007cc7c1 | ||
|
|
73cea2385e | ||
|
|
e2046837cf | ||
|
|
b30d4b6656 | ||
|
|
e4a48e78bf | ||
|
|
ea36bba5cc | ||
|
|
9da589c8c2 | ||
|
|
16ff396dbf | ||
|
|
e44fd7b328 | ||
|
|
66815b7a1a | ||
|
|
c6b695eca8 | ||
|
|
99d2083dea | ||
|
|
a718086140 | ||
|
|
c82979e72b | ||
|
|
2185c41cc1 | ||
|
|
26804eb123 | ||
|
|
d71d5df4a8 | ||
|
|
6829ad8e79 | ||
|
|
8903f35433 | ||
|
|
4ab2c15e5c | ||
|
|
eba6fea779 | ||
|
|
f04398d5a7 | ||
|
|
4ce9c47f45 | ||
|
|
9dfcdb5fb0 | ||
|
|
3efe15d2c7 | ||
|
|
9d87ed64c5 | ||
|
|
00134963e5 | ||
|
|
0ec5e2a25b | ||
|
|
0c5fffe951 | ||
|
|
5027ed9a23 | ||
|
|
6caab2c600 | ||
|
|
552e66dbf6 | ||
|
|
de1026ee2e | ||
|
|
7b50725bf8 | ||
|
|
7feef3b6a9 | ||
|
|
0b06db222d | ||
|
|
74ee0dadee | ||
|
|
0b452f975a | ||
|
|
6ab385d671 | ||
|
|
b3eab83a0f | ||
|
|
27490849a8 | ||
|
|
cebbf0809a | ||
|
|
3e227d28ad | ||
|
|
8ce63fcba2 | ||
|
|
4202431421 | ||
|
|
4923623dd4 | ||
|
|
84181cc982 | ||
|
|
7355d315a3 | ||
|
|
c50adab3a1 | ||
|
|
2fbb305f65 | ||
|
|
ff581be397 | ||
|
|
203e5cc6c1 | ||
|
|
d1b556b6cd | ||
|
|
729cd67743 | ||
|
|
73ddcdb29d | ||
|
|
14a9442343 | ||
|
|
5da4581e76 | ||
|
|
cbe8dc95d2 | ||
|
|
04a14a56b2 | ||
|
|
2290f1846b | ||
|
|
c57b40ae1d | ||
|
|
bc21b27da7 | ||
|
|
6a2248ddcd | ||
|
|
82d7b38cff | ||
|
|
6c7f88c05d | ||
|
|
447fbb2c6e | ||
|
|
623be15bfe | ||
|
|
55d5aebbd2 | ||
|
|
73b731fef8 | ||
|
|
ffd97ae9a5 | ||
|
|
d168833609 | ||
|
|
23a06a744c | ||
|
|
af4eae28b9 | ||
|
|
c198c930a1 | ||
|
|
e3efef2fe7 | ||
|
|
95fddeebb3 | ||
|
|
71523cebd3 | ||
|
|
3aa806a630 | ||
|
|
588c8f22c1 | ||
|
|
3d243d731d | ||
|
|
2431a6c9e9 | ||
|
|
969236da03 | ||
|
|
f30461b88c | ||
|
|
f34eca20f9 | ||
|
|
309dfd5287 | ||
|
|
f5a672d7b9 | ||
|
|
1acea85fa6 | ||
|
|
4f61741420 | ||
|
|
2fa864b5c7 | ||
|
|
10739c33fa | ||
|
|
39bea1b042 | ||
|
|
26b4e6d8ce | ||
|
|
4fb84b1090 | ||
|
|
7f2bc6fe97 | ||
|
|
eded968c70 | ||
|
|
53d29d9b24 | ||
|
|
690053bd57 | ||
|
|
c7b0649716 | ||
|
|
2bfcfa4535 | ||
|
|
b808e48b1f | ||
|
|
78ee962918 | ||
|
|
c8a3618e27 | ||
|
|
9ca277a63f | ||
|
|
8c7b6e5696 | ||
|
|
af4ffa189a | ||
|
|
42f571d496 | ||
|
|
36737cfe9d | ||
|
|
93ef93e563 | ||
|
|
376e4a88b3 | ||
|
|
bb4ed2a75d | ||
|
|
f7b8cddd2b | ||
|
|
a9a99953dd | ||
|
|
aee64f54b1 | ||
|
|
c77844fa9a | ||
|
|
013fe071a2 | ||
|
|
203dc00d53 | ||
|
|
e9a028134a | ||
|
|
01bba7bc6c | ||
|
|
d5285de99c | ||
|
|
33c198b827 | ||
|
|
816b37af98 | ||
|
|
d82aad984f | ||
|
|
057c37131f | ||
|
|
9bcce3fc68 | ||
|
|
f96f6322ec | ||
|
|
02ee2d7b5b | ||
|
|
47993e2ee9 | ||
|
|
7cc03f6f10 | ||
|
|
a21f1ea9fa | ||
|
|
9188bd7df1 | ||
|
|
f82753debe | ||
|
|
b92cb9db03 | ||
|
|
e66629ce1a | ||
|
|
cecdf417b7 | ||
|
|
56e3cc052a | ||
|
|
332366b58c | ||
|
|
ac9c207474 | ||
|
|
f85d56ac05 | ||
|
|
172167f6c0 | ||
|
|
42d49dd8bd | ||
|
|
f541bb8ce4 | ||
|
|
a6eb4c168f | ||
|
|
f6ad5ff2b2 | ||
|
|
2ac781343a | ||
|
|
bffcfb2db3 | ||
|
|
cc673755f7 | ||
|
|
4509038bf0 |
12
.env.example
12
.env.example
@@ -30,7 +30,9 @@ DASHBOARD_PORT=5001
|
||||
CLIP_MODEL=ViT-B-32
|
||||
CLIP_PRETRAINED=openai
|
||||
CLIP_DEVICE=cpu # cpu or cuda
|
||||
VLM_MODEL=qwen3-vl:8b
|
||||
RPA_VLM_MODEL=gemma4:latest # gemma4:latest (défaut), qwen3-vl:8b, ui-tars (fallback)
|
||||
VLM_MODEL=gemma4:latest # alias de compatibilité
|
||||
# VLM_ALLOW_CLOUD=false # true pour activer les APIs cloud en fallback (OpenAI, Gemini, Anthropic)
|
||||
VLM_ENDPOINT=http://localhost:11434
|
||||
OWL_MODEL=google/owlv2-base-patch16-ensemble
|
||||
OWL_CONFIDENCE_THRESHOLD=0.1
|
||||
@@ -44,6 +46,14 @@ LOGS_PATH=logs
|
||||
UPLOADS_PATH=data/training/uploads
|
||||
SESSIONS_PATH=data/training/sessions
|
||||
|
||||
# ============================================================================
|
||||
# Feedback Bus (Léa parle pendant exécution)
|
||||
# ============================================================================
|
||||
# Bus SocketIO unifié 'lea:*' (action_started, action_done, need_confirm, paused).
|
||||
# Désactivé par défaut. Mettre à 1 pour activer les bulles temps réel dans ChatWindow.
|
||||
# Si la connexion bus échoue, l'exécution continue normalement (fail-safe).
|
||||
LEA_FEEDBACK_BUS=0
|
||||
|
||||
# ============================================================================
|
||||
# FAISS
|
||||
# ============================================================================
|
||||
|
||||
207
.gitea/workflows/security-audit.yml
Normal file
207
.gitea/workflows/security-audit.yml
Normal file
@@ -0,0 +1,207 @@
|
||||
# ------------------------------------------------------------------
|
||||
# Audit sécurité — bandit + pip-audit + scan secrets
|
||||
# ------------------------------------------------------------------
|
||||
# Jamais bloquant : on reporte les warnings, on ne casse pas la CI.
|
||||
# Utile pour détecter les dérives progressives (nouveaux CVE, secrets
|
||||
# oubliés dans un commit, patterns risqués).
|
||||
#
|
||||
# Fréquence : à chaque push sur main + hebdo (cron).
|
||||
# ------------------------------------------------------------------
|
||||
name: security-audit
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
schedule:
|
||||
# Tous les lundis à 6h UTC (8h Paris hiver, 7h Paris été).
|
||||
- cron: "0 6 * * 1"
|
||||
workflow_dispatch: {}
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
# ----------------------------------------------------------------
|
||||
# Job 1 — bandit (bonnes pratiques sécu Python)
|
||||
# ----------------------------------------------------------------
|
||||
bandit:
|
||||
name: Bandit (scan statique)
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
continue-on-error: true
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Python 3.12
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.12"
|
||||
cache: "pip"
|
||||
|
||||
- name: Installation bandit
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install "bandit[toml]==1.7.10"
|
||||
|
||||
- name: Scan bandit sur core/
|
||||
run: |
|
||||
# -ll : niveau LOW minimum (remonte tout)
|
||||
# -ii : confiance LOW minimum
|
||||
# --skip B101 : on ignore les asserts (usuels en tests/validation)
|
||||
bandit -r core/ \
|
||||
--skip B101,B404,B603 \
|
||||
--format txt \
|
||||
--exit-zero \
|
||||
--output bandit-report.txt
|
||||
echo "=== RAPPORT BANDIT ==="
|
||||
cat bandit-report.txt
|
||||
|
||||
- name: Upload rapport bandit
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: bandit-report
|
||||
path: bandit-report.txt
|
||||
retention-days: 30
|
||||
if-no-files-found: ignore
|
||||
|
||||
# ----------------------------------------------------------------
|
||||
# Job 2 — pip-audit (CVE sur requirements)
|
||||
# ----------------------------------------------------------------
|
||||
pip-audit:
|
||||
name: pip-audit (CVE dépendances)
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
continue-on-error: true
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Python 3.12
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.12"
|
||||
cache: "pip"
|
||||
|
||||
- name: Installation pip-audit
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install "pip-audit==2.7.3"
|
||||
|
||||
- name: Audit CVE sur requirements-ci.txt
|
||||
run: |
|
||||
if [ -f requirements-ci.txt ]; then
|
||||
pip-audit -r requirements-ci.txt \
|
||||
--format json \
|
||||
--output pip-audit-ci.json \
|
||||
--progress-spinner off \
|
||||
--disable-pip || echo "::warning::CVE détectées dans requirements-ci.txt"
|
||||
echo "=== RAPPORT pip-audit (CI) ==="
|
||||
cat pip-audit-ci.json || true
|
||||
else
|
||||
echo "::notice::requirements-ci.txt absent — skip"
|
||||
fi
|
||||
|
||||
- name: Audit CVE sur requirements.txt (best-effort)
|
||||
run: |
|
||||
# Timeout généreux car requirements.txt est massif (torch, CUDA).
|
||||
timeout 120 pip-audit -r requirements.txt \
|
||||
--format json \
|
||||
--output pip-audit-full.json \
|
||||
--progress-spinner off \
|
||||
--disable-pip 2>&1 | head -200 || \
|
||||
echo "::warning::pip-audit sur requirements.txt a timeout ou échoué (non bloquant)"
|
||||
|
||||
- name: Upload rapports pip-audit
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: pip-audit-reports
|
||||
path: |
|
||||
pip-audit-ci.json
|
||||
pip-audit-full.json
|
||||
retention-days: 30
|
||||
if-no-files-found: ignore
|
||||
|
||||
# ----------------------------------------------------------------
|
||||
# Job 3 — Scan secrets en clair (grep simple)
|
||||
# ----------------------------------------------------------------
|
||||
# Patterns recherchés : clés API Anthropic (sk-ant-), OpenAI (sk-),
|
||||
# Google (AIzaSy), AWS (AKIA), tokens Hugging Face (hf_).
|
||||
# Ne cherche QUE dans les fichiers trackés (pas .env, pas .venv).
|
||||
# ----------------------------------------------------------------
|
||||
secrets-scan:
|
||||
name: Scan secrets (grep)
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 3
|
||||
continue-on-error: true
|
||||
|
||||
steps:
|
||||
- name: Checkout (historique complet)
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Scan patterns de secrets
|
||||
run: |
|
||||
# Chemins exclus : venvs, caches, data, htmlcov, models.
|
||||
EXCLUDES='--exclude-dir=.venv --exclude-dir=venv_v3 --exclude-dir=.git \
|
||||
--exclude-dir=node_modules --exclude-dir=htmlcov --exclude-dir=models \
|
||||
--exclude-dir=data --exclude-dir=__pycache__ --exclude-dir=.pytest_cache \
|
||||
--exclude=*.lock --exclude=*.log --exclude=*.md'
|
||||
|
||||
echo "=== Recherche de secrets potentiels ==="
|
||||
FOUND=0
|
||||
|
||||
# Anthropic
|
||||
if grep -rnI $EXCLUDES -E 'sk-ant-[a-zA-Z0-9_-]{20,}' . 2>/dev/null; then
|
||||
echo "::warning::Clé Anthropic potentielle détectée"
|
||||
FOUND=1
|
||||
fi
|
||||
|
||||
# OpenAI
|
||||
if grep -rnI $EXCLUDES -E 'sk-proj-[a-zA-Z0-9_-]{20,}|sk-[a-zA-Z0-9]{40,}' . 2>/dev/null; then
|
||||
echo "::warning::Clé OpenAI potentielle détectée"
|
||||
FOUND=1
|
||||
fi
|
||||
|
||||
# Google Cloud / API Keys
|
||||
if grep -rnI $EXCLUDES -E 'AIzaSy[a-zA-Z0-9_-]{33}' . 2>/dev/null; then
|
||||
echo "::warning::Clé Google API potentielle détectée"
|
||||
FOUND=1
|
||||
fi
|
||||
|
||||
# AWS
|
||||
if grep -rnI $EXCLUDES -E 'AKIA[0-9A-Z]{16}' . 2>/dev/null; then
|
||||
echo "::warning::Clé AWS potentielle détectée"
|
||||
FOUND=1
|
||||
fi
|
||||
|
||||
# Hugging Face
|
||||
if grep -rnI $EXCLUDES -E 'hf_[a-zA-Z0-9]{30,}' . 2>/dev/null; then
|
||||
echo "::warning::Token Hugging Face potentiel détecté"
|
||||
FOUND=1
|
||||
fi
|
||||
|
||||
# Mots-clés suspects à côté d'assignations
|
||||
if grep -rnI $EXCLUDES -E '(password|passwd|secret|api_key|apikey|token)\s*=\s*["\x27][a-zA-Z0-9_\-!@#\$%]{12,}["\x27]' . 2>/dev/null \
|
||||
| grep -viE '(example|dummy|placeholder|test|fake|xxx|changeme|\$\{)' 2>/dev/null; then
|
||||
echo "::warning::Assignation suspecte d'un secret détectée"
|
||||
FOUND=1
|
||||
fi
|
||||
|
||||
if [ "$FOUND" -eq 0 ]; then
|
||||
echo "Aucun secret détecté par les patterns de base."
|
||||
else
|
||||
echo ""
|
||||
echo "::notice::Vérifier manuellement les occurrences ci-dessus."
|
||||
echo "::notice::Si faux positif : ajouter le fichier aux exclusions ou reformater."
|
||||
fi
|
||||
|
||||
# Toujours succès (job non bloquant).
|
||||
exit 0
|
||||
214
.gitea/workflows/tests.yml
Normal file
214
.gitea/workflows/tests.yml
Normal file
@@ -0,0 +1,214 @@
|
||||
# ------------------------------------------------------------------
|
||||
# CI principale — Tests unitaires + lint léger
|
||||
# ------------------------------------------------------------------
|
||||
# Déclenchement : push / pull_request sur n'importe quelle branche.
|
||||
# Objectif : feedback rapide (< 3 min) sans GPU ni Ollama.
|
||||
# Runner : self-hosted (label "ubuntu-latest" ou équivalent).
|
||||
#
|
||||
# Les tests marqués `slow`, `gpu`, `integration`, `performance`,
|
||||
# `visual` et `smoke` sont exclus volontairement — ils nécessitent
|
||||
# CUDA, Ollama, ou des captures d'écran réelles.
|
||||
# ------------------------------------------------------------------
|
||||
name: tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- "**"
|
||||
pull_request:
|
||||
branches:
|
||||
- "**"
|
||||
|
||||
# Permet à une nouvelle exécution d'annuler les précédentes
|
||||
# sur la même branche (évite l'engorgement du runner local).
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
# Empêche l'import accidentel de torch/CUDA pendant la CI.
|
||||
PYTHONDONTWRITEBYTECODE: "1"
|
||||
PIP_DISABLE_PIP_VERSION_CHECK: "1"
|
||||
PIP_NO_PYTHON_VERSION_WARNING: "1"
|
||||
# Les modules d'exécution lisent parfois ces vars ; valeurs neutres en CI.
|
||||
RPA_VISION_CI: "1"
|
||||
RPA_AUTH_VAULT_PATH: "/tmp/ci_vault.enc"
|
||||
# api_stream.py a un fail-closed P0-C : si RPA_API_TOKEN absent, sys.exit(1)
|
||||
# au module load. On fournit un token bidon pour que les imports passent en CI.
|
||||
# (Le token n'est jamais utilisé réellement — les tests mockent les requêtes.)
|
||||
RPA_API_TOKEN: "ci_test_token_not_used_for_real_auth_just_to_pass_import_check_0123456789"
|
||||
|
||||
jobs:
|
||||
# ----------------------------------------------------------------
|
||||
# Job 1 — Lint (ruff + black --check)
|
||||
# ----------------------------------------------------------------
|
||||
# Non-bloquant : si ruff/black ne sont pas installables, on log
|
||||
# un warning et on continue. L'objectif ici est d'alerter, pas de
|
||||
# casser la CI pour des espaces en trop.
|
||||
# ----------------------------------------------------------------
|
||||
lint:
|
||||
name: Lint (ruff + black)
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
continue-on-error: true
|
||||
|
||||
steps:
|
||||
- name: Checkout du code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Python 3.12
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.12"
|
||||
cache: "pip"
|
||||
|
||||
- name: Installation des linters
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install "ruff==0.6.9" "black==23.12.1" || {
|
||||
echo "::warning::Impossible d'installer ruff/black — job ignoré"
|
||||
exit 0
|
||||
}
|
||||
|
||||
- name: Ruff (lint rapide)
|
||||
run: |
|
||||
if command -v ruff >/dev/null 2>&1; then
|
||||
# Ruff : erreurs critiques uniquement (E9 syntax, F63 invalid print,
|
||||
# F7 syntax, F82 undefined in __all__).
|
||||
# F821 (undefined name) volontairement exclu le temps de nettoyer
|
||||
# la dette technique préexistante (voir docs/STATUS.md).
|
||||
# Dossiers legacy exclus :
|
||||
# - agent_v0/deploy/windows_client/ : clone obsolète (marqué OBSOLÈTE)
|
||||
# - tests/property/ : tests cassés connus (cf. MEMORY.md)
|
||||
ruff check --select=E9,F63,F7,F82 --output-format=github \
|
||||
--exclude "agent_v0/deploy/windows_client" \
|
||||
--exclude "tests/property" \
|
||||
--exclude "tests/integration/test_visual_rpa_checkpoint.py" \
|
||||
core/ agent_v0/ tests/ || {
|
||||
echo "::warning::Ruff a trouvé des erreurs critiques"
|
||||
exit 1
|
||||
}
|
||||
else
|
||||
echo "::warning::ruff indisponible — skip"
|
||||
fi
|
||||
|
||||
- name: Black (format check)
|
||||
run: |
|
||||
if command -v black >/dev/null 2>&1; then
|
||||
# --check : ne modifie pas, signale juste.
|
||||
# Dossiers legacy exclus (cohérent avec ruff).
|
||||
black --check --diff \
|
||||
--exclude "agent_v0/deploy/windows_client|tests/property" \
|
||||
core/ agent_v0/ tests/ || {
|
||||
echo "::warning::Black suggère un reformatage — non bloquant"
|
||||
exit 0
|
||||
}
|
||||
else
|
||||
echo "::warning::black indisponible — skip"
|
||||
fi
|
||||
|
||||
# ----------------------------------------------------------------
|
||||
# Job 2 — Tests unitaires
|
||||
# ----------------------------------------------------------------
|
||||
# Exclut tous les marqueurs lourds. Utilise requirements-ci.txt
|
||||
# pour éviter torch/CUDA (économie ~3 Go + ~2 min).
|
||||
# ----------------------------------------------------------------
|
||||
unit-tests:
|
||||
name: Tests unitaires (sans GPU)
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
|
||||
steps:
|
||||
- name: Checkout du code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Python 3.12
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.12"
|
||||
cache: "pip"
|
||||
cache-dependency-path: |
|
||||
requirements-ci.txt
|
||||
requirements.txt
|
||||
|
||||
- name: Installation des dépendances CI
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
if [ -f requirements-ci.txt ]; then
|
||||
echo "Utilisation de requirements-ci.txt (léger, sans torch)"
|
||||
pip install -r requirements-ci.txt
|
||||
else
|
||||
echo "::warning::requirements-ci.txt absent — fallback requirements.txt (lourd)"
|
||||
pip install -r requirements.txt
|
||||
fi
|
||||
|
||||
- name: Vérification imports critiques
|
||||
run: |
|
||||
python -c "import pytest; print(f'pytest {pytest.__version__}')"
|
||||
python -c "import sys; sys.path.insert(0, '.'); import core; print('core OK')" || {
|
||||
echo "::error::Impossible d'importer core.*"
|
||||
exit 1
|
||||
}
|
||||
|
||||
- name: Tests unitaires (hors slow/gpu/integration)
|
||||
run: |
|
||||
python -m pytest tests/unit/ \
|
||||
-m "not slow and not gpu and not integration and not performance and not visual" \
|
||||
--tb=short \
|
||||
--strict-markers \
|
||||
-q \
|
||||
--maxfail=10 \
|
||||
-o cache_dir=/tmp/.pytest_cache_ci
|
||||
|
||||
- name: Upload logs si échec
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: pytest-logs
|
||||
path: |
|
||||
/tmp/.pytest_cache_ci
|
||||
logs/
|
||||
retention-days: 3
|
||||
if-no-files-found: ignore
|
||||
|
||||
# ----------------------------------------------------------------
|
||||
# Job 3 — Tests sécurité (bloquant)
|
||||
# ----------------------------------------------------------------
|
||||
# Les tests `test_security_*` valident des invariants critiques
|
||||
# (évaluation sûre, sérialisation signée). Aucune régression tolérée.
|
||||
# ----------------------------------------------------------------
|
||||
security-tests:
|
||||
name: Tests sécurité (critique)
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
needs: [unit-tests]
|
||||
|
||||
steps:
|
||||
- name: Checkout du code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Python 3.12
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.12"
|
||||
cache: "pip"
|
||||
cache-dependency-path: |
|
||||
requirements-ci.txt
|
||||
requirements.txt
|
||||
|
||||
- name: Installation des dépendances CI
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
if [ -f requirements-ci.txt ]; then
|
||||
pip install -r requirements-ci.txt
|
||||
else
|
||||
pip install -r requirements.txt
|
||||
fi
|
||||
|
||||
- name: Tests sécurité (test_security_*)
|
||||
run: |
|
||||
python -m pytest tests/unit/test_security_*.py \
|
||||
--tb=long \
|
||||
--strict-markers \
|
||||
-v \
|
||||
-o cache_dir=/tmp/.pytest_cache_ci_sec
|
||||
59
.gitignore
vendored
59
.gitignore
vendored
@@ -74,4 +74,63 @@ htmlcov/
|
||||
|
||||
# === Backups ===
|
||||
*_backup_*
|
||||
*.db.backup_*
|
||||
backups/
|
||||
*.bak
|
||||
*.bak_*
|
||||
*.orig
|
||||
*.old
|
||||
|
||||
# === Legacy / Triage ===
|
||||
_a_trier/
|
||||
archives/
|
||||
|
||||
# === Claude Code — worktrees et données locales ===
|
||||
# Worktrees générés par la CLI Claude Code lors d'exécutions d'agents
|
||||
# parallèles. Peuvent atteindre plusieurs centaines de Mo chacun.
|
||||
# Ne jamais committer — gérer via `git worktree list` / `git worktree remove`.
|
||||
.claude/
|
||||
.kiro/
|
||||
.antigravitycli/
|
||||
.playwright-cli/
|
||||
.qwen/
|
||||
.mcp.json
|
||||
.snapshots/
|
||||
|
||||
# === Données runtime (sessions, learning, buffer, config local) ===
|
||||
data/
|
||||
**/capture_library.json
|
||||
.hypothesis/
|
||||
.deps_installed
|
||||
# Buffers SQLite locaux (streamer, cache)
|
||||
**/buffer/
|
||||
**/pending_events.db
|
||||
# Databases applicatives (instance Flask)
|
||||
**/instance/*.db
|
||||
**/instance/*.sqlite
|
||||
**/instance/*.sqlite3
|
||||
# Caches et index locaux
|
||||
*.sqlite
|
||||
*.sqlite3
|
||||
*.db-journal
|
||||
*.db-wal
|
||||
*.db-shm
|
||||
web_dashboard/static/analytics/*.bpmn
|
||||
results_vlm_bench.json
|
||||
|
||||
# Scripts locaux one-shot d'intervention/bench, non réutilisables tels quels.
|
||||
tools/bench_qwen35_evidence.py
|
||||
tools/codex_windows_correction_rapport.py
|
||||
|
||||
# Verbatims clients (sensibles, à valider avant push)
|
||||
docs/clients/
|
||||
|
||||
.qw-baseline.log
|
||||
docs/coordination/.loop_state/
|
||||
|
||||
# Runtime Python embedded pour l'installateur Inno Setup (local, ~11M, non versionné)
|
||||
deploy/installer/python-3.12-embed/
|
||||
deploy/installer/python-3.12.8-embed-amd64.zip
|
||||
# Artefacts de build installateur (EXE compilés + staging) — non versionnés
|
||||
deploy/releases/*.exe
|
||||
deploy/build/
|
||||
|
||||
8
.vite/deps/_metadata.json
Normal file
8
.vite/deps/_metadata.json
Normal file
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"hash": "cccc2566",
|
||||
"configHash": "0c083961",
|
||||
"lockfileHash": "e3b0c442",
|
||||
"browserHash": "764a8433",
|
||||
"optimized": {},
|
||||
"chunks": {}
|
||||
}
|
||||
3
.vite/deps/package.json
Normal file
3
.vite/deps/package.json
Normal file
@@ -0,0 +1,3 @@
|
||||
{
|
||||
"type": "module"
|
||||
}
|
||||
12
AGENTS.md
Normal file
12
AGENTS.md
Normal file
@@ -0,0 +1,12 @@
|
||||
## graphify
|
||||
|
||||
This project has a knowledge graph at graphify-out/ with god nodes, community structure, and cross-file relationships.
|
||||
|
||||
When the user types `/graphify`, invoke the `skill` tool with `skill: "graphify"` before doing anything else.
|
||||
|
||||
Rules:
|
||||
- For codebase questions, first run `graphify query "<question>"` when graphify-out/graph.json exists. Use `graphify path "<A>" "<B>"` for relationships and `graphify explain "<concept>"` for focused concepts. These return a scoped subgraph, usually much smaller than GRAPH_REPORT.md or raw grep output.
|
||||
- Dirty graphify-out/ files are expected after hooks or incremental updates; dirty graph files are not a reason to skip graphify. Only skip graphify if the task is about stale or incorrect graph output, or the user explicitly says not to use it.
|
||||
- If graphify-out/wiki/index.md exists, use it for broad navigation instead of raw source browsing.
|
||||
- Read graphify-out/GRAPH_REPORT.md only for broad architecture review or when query/path/explain do not surface enough context.
|
||||
- After modifying code, run `graphify update .` to keep the graph current (AST-only, no API cost).
|
||||
106
CLAUDE.md
Normal file
106
CLAUDE.md
Normal file
@@ -0,0 +1,106 @@
|
||||
# CLAUDE.md — rpa_vision_v3
|
||||
|
||||
Ce fichier prime sur le CLAUDE.md racine (`~/ai/CLAUDE.md`) pour tout travail dans ce projet.
|
||||
|
||||
## Rôle de Claude Code sur ce projet
|
||||
|
||||
Exécutant supervisé, pas architecte. Mission : garantir la **cohérence** de chaque modification avec la vision globale du projet et le **contrat "100% vision"** (résolution UI par la vue, pas par les sélecteurs DOM/API). Quand tu touches un fichier, vérifie que tu ne casses rien ailleurs.
|
||||
|
||||
Tu n'es pas en autonomie. Dom valide avant chaque étape. Tu proposes, il décide.
|
||||
|
||||
## Priorité absolue
|
||||
|
||||
**La démo Urgence_aiva_demo doit fonctionner.** Workflow 22+ steps sur Easily Assure, patiente MOREL Catherine, audience mixte DG/DSI/médecins/DIM/TIM. Tout arbitrage technique se tranche par : "est-ce que ça rapproche ou éloigne de la démo qui tourne ?"
|
||||
|
||||
## Méthode obligatoire — non négociable
|
||||
|
||||
- **Chirurgie itérative supervisée** : une modification, un test (≤ 2 min), validation explicite de Dom avant la suivante.
|
||||
- **Pas de batch** : jamais plusieurs changements groupés sans validation intermédiaire.
|
||||
- **Rustine interdite** : tu corriges la cause, pas le symptôme. Si tu ne comprends pas la cause, tu le dis et tu arrêtes.
|
||||
- **Lire la doc avant d'agir** : code existant, `docs/`, specs. Pas de proposition basée sur des suppositions.
|
||||
- **Un commit = une intention** : message explicite, daté.
|
||||
- **Diff review systématique** sur tout code de production avant commit.
|
||||
|
||||
## Anti-patterns à proscrire
|
||||
|
||||
- Réponses longues. Si Dom dit "trop long" ou "déjà vu", tu raccourcis sans débattre.
|
||||
- Propositions structurelles avant d'avoir compris l'intention de Dom.
|
||||
- Re-proposer ce qui est déjà en place dans le code.
|
||||
- Raisonner sur un composant trouvé via grep **sans vérifier qu'il est effectivement appelé au runtime**. Le projet contient beaucoup de code écrit mais non wired.
|
||||
- Présenter la première solution qui marche. Toujours explorer 2-3 approches, présenter la meilleure avec justification.
|
||||
|
||||
## Architecture runtime réelle (à valider/raffiner avec Dom)
|
||||
|
||||
```
|
||||
[VWB frontend React :3002]
|
||||
↓ (HTTP)
|
||||
[VWB backend Flask + SQLite]
|
||||
↓ (envoi step par step)
|
||||
[agent_v1 — Linux]
|
||||
↓ (SSH vers Windows)
|
||||
[Léa — chatbot exécutant — PC Windows]
|
||||
↓
|
||||
[Easily Assure — interface cible]
|
||||
```
|
||||
|
||||
**Ollama** : sert le ou les modèles utilisés pour la résolution VLM, l'extraction texte, et la décision t2a. Sert aussi de **proxy vers cloud** pour certains appels.
|
||||
|
||||
**Cascade de résolution UI** (à confirmer composant par composant au runtime) :
|
||||
1. OCR (docTR ou EasyOCR selon module)
|
||||
2. cv2 template matching
|
||||
3. YOLO v4 grounding
|
||||
4. VLM grounding
|
||||
|
||||
**UI-DETR-1** : utilisé par VWB **au recording** pour overlays numérotés (équivalent OmniParser). `crop_hash` volontairement non persisté.
|
||||
|
||||
**Asymétrie connue, sujet ouvert post-démo** : VWB direct utilise UI-DETR-1 au runtime, le replay sur Léa ne l'utilise pas (cascade OCR/template/VLM seulement). Ne pas tenter de "fixer" cette asymétrie maintenant.
|
||||
|
||||
## ⚠️ Champs de mines — code orphelin
|
||||
|
||||
`core/` contient ~40 sous-modules. **Beaucoup ne sont pas wired au runtime actif.** Avant de raisonner sur un composant trouvé dans `core/` (coaching, healing, federation, learning, cognition, etc.) :
|
||||
|
||||
1. Vérifier qu'il est importé par un point d'entrée actif.
|
||||
2. Vérifier qu'il est effectivement appelé en runtime (traces, logs).
|
||||
3. Si doute, demander à Dom.
|
||||
|
||||
**Cas spécifique agent_v1** : suspicion de code orphelin à rebrancher. Si tu trouves un appel codé mais non exécuté en runtime (ex. appel Ollama de commentaire d'action présent dans le code mais jamais déclenché), c'est prioritaire à signaler.
|
||||
|
||||
## Debug — où regarder en premier
|
||||
|
||||
- `logs/` (racine projet) — logs runtime généraux
|
||||
- `logs/audit/` — traces d'exécution
|
||||
- `logs/healing/` — si concerne le healing
|
||||
- `data/runner_captures/` — captures d'exécution
|
||||
- `visual_workflow_builder/logs/` — logs VWB
|
||||
- `server/logs/` — logs serveur
|
||||
|
||||
**Vérifier qu'un appel Ollama se déclenche vraiment au runtime** : ne pas se fier à la présence de l'appel dans le code. Tracer effectivement (log d'entrée de fonction, requête vue côté Ollama `:11434`).
|
||||
|
||||
## Inspirations externes
|
||||
|
||||
Voir `docs/INSPIRATION_FRAMEWORKS_2026-05-10.md` pour les patterns convergents (OpenAdapt, Skyvern, OmniParser : Policy/Grounding, Safety Gate, Abstraction Ladder, Planner-Actor-Validator). Le projet est techniquement plus mature que sa documentation ne le suggère — s'inspirer des bons patterns sans complexe.
|
||||
|
||||
## Recherche d'information
|
||||
|
||||
Ta connaissance interne est datée. Pour tout sujet technique évoluant vite (modèles VLM, frameworks RPA visuels, librairies de grounding, versions d'outils), **chercher sur internet d'abord**. Privilégier les sources de moins de 6 mois.
|
||||
|
||||
## Stack
|
||||
|
||||
- Python 3.10-3.12, venv `venv_v3/`
|
||||
- Backend VWB : Flask + SQLite
|
||||
- Frontend VWB : React (port 3002), dashboard :5001, API :8000
|
||||
- LLM local : Ollama `:11434`
|
||||
- GUI legacy : PyQt5
|
||||
- Tests : pytest avec marqueurs (unit/integration/slow/smoke)
|
||||
- Langue : français (code, commentaires, logs, GUI)
|
||||
|
||||
## Commandes utiles
|
||||
|
||||
```bash
|
||||
cd ~/ai/rpa_vision_v3 && source venv_v3/bin/activate
|
||||
./run.sh --full # Écosystème complet
|
||||
./run.sh --gui # GUI PyQt5 seule
|
||||
./run.sh --test # Tests complets
|
||||
make test-fast # Tests rapides
|
||||
make check # Validation imports + tests rapides
|
||||
```
|
||||
@@ -21,7 +21,12 @@ ollama serve
|
||||
### 3. Télécharger le modèle VLM
|
||||
|
||||
```bash
|
||||
ollama pull qwen3-vl:8b
|
||||
# Modèle par défaut du projet (voir .env.example)
|
||||
ollama pull gemma4:latest
|
||||
|
||||
# Alternatives supportées
|
||||
# ollama pull qwen3-vl:8b
|
||||
# ollama pull 0000/ui-tars-1.5-7b-q8_0:7b # grounder visuel
|
||||
```
|
||||
|
||||
## Utilisation
|
||||
|
||||
339
README.md
339
README.md
@@ -1,207 +1,204 @@
|
||||
# RPA Vision V3 - 100% Vision-Based Workflow Automation
|
||||
# RPA Vision V3 — Automatisation basée sur la compréhension visuelle des interfaces
|
||||
|
||||
## 📊 Status
|
||||
> ⚠️ **Projet en phase POC** — voir [`docs/STATUS.md`](docs/STATUS.md) pour l'état
|
||||
> réel par module. Certaines briques sont opérationnelles bout en bout,
|
||||
> d'autres sont en cours de stabilisation. Ce dépôt n'est pas production-ready.
|
||||
|
||||
🚀 **PRODUCTION-READY** - Phase 12 Complete (77% System Completion) ✅
|
||||
*Dernière mise à jour : 14 avril 2026*
|
||||
|
||||
**Latest Update**: 14 Décembre 2024
|
||||
- ✅ **10/13 Phases Complétées** - Système mature et fonctionnel
|
||||
- ✅ **Performance Exceptionnelle** - 500-6250x plus rapide que requis
|
||||
- ✅ **Architecture Entreprise** - 148k+ lignes, 19 modules, 6 specs complètes
|
||||
- ✅ **Innovations Techniques** - Self-healing, Multi-modal, GPU management
|
||||
- 📊 **Audit Complet** - [Rapport détaillé](AUDIT_COMPLET_SYSTEME_RPA_VISION_V3.md)
|
||||
## Intention
|
||||
|
||||
**Quick Test**: `bash test_clip.sh`
|
||||
Automatiser des workflows métier par **compréhension sémantique de l'écran**
|
||||
plutôt que par coordonnées de clic fixes. Le système observe l'utilisateur,
|
||||
reconstruit un graphe d'états de l'interface, et cherche à rejouer la
|
||||
procédure en reconnaissant visuellement les éléments cibles — y compris
|
||||
quand l'UI change légèrement.
|
||||
|
||||
## 🎯 Vision
|
||||
Terrain cible principal : postes hospitaliers (Citrix, applications métier
|
||||
web et desktop). Contrainte forte : **100 % local**, pas d'appel à un LLM
|
||||
cloud dans le pipeline par défaut.
|
||||
|
||||
RPA basé sur la **compréhension sémantique** des interfaces, pas sur des coordonnées de clics.
|
||||
|
||||
Le système apprend des workflows en observant l'utilisateur et les automatise de manière robuste grâce à une architecture en 5 couches.
|
||||
|
||||
## 🏗️ Architecture en 5 Couches
|
||||
## Architecture en couches
|
||||
|
||||
```
|
||||
RawSession (Couche 0)
|
||||
↓
|
||||
ScreenState (Couche 1) - 4 niveaux d'abstraction
|
||||
↓
|
||||
UIElement Detection (Couche 2) - Types + Rôles sémantiques
|
||||
↓
|
||||
State Embedding (Couche 3) - Fusion multi-modale
|
||||
↓
|
||||
Workflow Graph (Couche 4) - Nodes + Edges + Learning States
|
||||
RawSession (couche 0) — capture événements + screenshots
|
||||
↓
|
||||
ScreenState (couche 1) — états d'écran à plusieurs niveaux d'abstraction
|
||||
↓
|
||||
UIElement (couche 2) — détection sémantique (cascade OCR + templates + VLM)
|
||||
↓
|
||||
State Embedding (couche 3) — fusion multi-modale + index FAISS
|
||||
↓
|
||||
Workflow Graph (couche 4) — nœuds, transitions, résolution de cibles
|
||||
```
|
||||
|
||||
## 📁 Structure
|
||||
## État des fonctionnalités (synthèse)
|
||||
|
||||
```
|
||||
rpa_vision_v3/
|
||||
├── core/
|
||||
│ ├── models/ # Couches 0-4 : Structures de données
|
||||
│ ├── capture/ # Couche 0 : Capture événements + screenshots
|
||||
│ ├── detection/ # Couche 2 : Détection UI sémantique
|
||||
│ ├── embedding/ # Couche 3 : Fusion multi-modale + FAISS
|
||||
│ ├── graph/ # Couche 4 : Construction + Matching + Exécution
|
||||
│ └── persistence/ # Sauvegarde/Chargement
|
||||
├── data/
|
||||
│ ├── sessions/ # RawSessions
|
||||
│ ├── screen_states/ # ScreenStates
|
||||
│ ├── embeddings/ # Vecteurs .npy
|
||||
│ ├── faiss_index/ # Index FAISS
|
||||
│ └── workflows/ # Workflow Graphs
|
||||
└── tests/ # Tests unitaires + intégration
|
||||
```
|
||||
Le détail par module est dans [`docs/STATUS.md`](docs/STATUS.md).
|
||||
|
||||
## 🚀 Démarrage Rapide
|
||||
**Opérationnel**
|
||||
- Capture Windows (Agent V1) + streaming vers serveur Linux
|
||||
- Stockage des sessions brutes (screenshots + événements)
|
||||
- Streaming server FastAPI, sessions en mémoire
|
||||
- Build du package Windows (`deploy/build_package.sh`)
|
||||
|
||||
**Alpha (fonctionnel sur un cas de référence, encore peu généralisé)**
|
||||
- Détection UI par cascade VLM + OCR + templates
|
||||
- Construction de workflow graph depuis une session
|
||||
- Replay E2E supervisé — premier succès sur Notepad le 13 avril 2026
|
||||
- Mode apprentissage : pause et demande d'aide humaine quand la résolution échoue
|
||||
- Embeddings CLIP + index FAISS
|
||||
- Module auth (Fernet + TOTP), federation (LearningPack)
|
||||
- Web Dashboard, Agent Chat
|
||||
|
||||
**En cours**
|
||||
- Visual Workflow Builder (VWB) — bugs DB runtime connus
|
||||
- Self-healing / recovery global
|
||||
- Analytics / reporting
|
||||
- Worker de compilation sessions → ExecutionPlan
|
||||
- Tests E2E multi-applications
|
||||
|
||||
## Limitations connues
|
||||
|
||||
- Le pipeline de replay est validé sur un nombre très restreint d'applications.
|
||||
- `TargetMemoryStore` (apprentissage Phase 1) est câblé mais sa base reste
|
||||
vide tant qu'un replay complet n'a pas été cristallisé.
|
||||
- Certaines asymétries entre chemins stricts et legacy dans le serveur de
|
||||
streaming peuvent provoquer des arrêts au lieu de pauses d'apprentissage.
|
||||
- VWB n'est pas encore stable en écriture ; un outil dédié plus simple est
|
||||
envisagé.
|
||||
|
||||
## Démarrage
|
||||
|
||||
### Prérequis
|
||||
|
||||
- Python 3.10 à 3.12
|
||||
- [Ollama](https://ollama.ai) installé et démarré localement
|
||||
- Recommandé : GPU NVIDIA pour l'inférence VLM
|
||||
- Windows 10/11 uniquement pour le client Agent V1
|
||||
|
||||
### Installation
|
||||
|
||||
```bash
|
||||
# 1. Installer Ollama
|
||||
curl -fsSL https://ollama.ai/install.sh | sh # Linux
|
||||
# ou
|
||||
brew install ollama # macOS
|
||||
|
||||
# 2. Démarrer Ollama
|
||||
ollama serve
|
||||
|
||||
# 3. Télécharger le modèle VLM
|
||||
ollama pull qwen3-vl:8b
|
||||
|
||||
# 4. Installer dépendances Python
|
||||
# 1) Cloner puis créer le venv
|
||||
python3 -m venv .venv
|
||||
source .venv/bin/activate
|
||||
pip install -r requirements.txt
|
||||
|
||||
# 2) Démarrer Ollama et récupérer le modèle VLM par défaut
|
||||
ollama serve &
|
||||
ollama pull gemma4:latest # défaut du projet
|
||||
# Alternatives supportées :
|
||||
# ollama pull qwen3-vl:8b
|
||||
# ollama pull 0000/ui-tars-1.5-7b-q8_0:7b # grounder visuel
|
||||
|
||||
# 3) Copier et ajuster la configuration
|
||||
cp .env.example .env
|
||||
# éditer .env pour vérifier RPA_VLM_MODEL, VLM_ENDPOINT, ports, etc.
|
||||
```
|
||||
|
||||
### Test Rapide
|
||||
### Lancer les services
|
||||
|
||||
Tous les services sont pilotés par `svc.sh` (source de vérité des ports :
|
||||
`services.conf`).
|
||||
|
||||
```bash
|
||||
# Diagnostic système
|
||||
python3 rpa_vision_v3/examples/diagnostic_vlm.py
|
||||
|
||||
# Test de détection
|
||||
./rpa_vision_v3/test_quick.sh
|
||||
./svc.sh status # État de tous les services
|
||||
./svc.sh start # Tout démarrer
|
||||
./svc.sh start streaming # Streaming server uniquement (port 5005)
|
||||
./svc.sh restart api # Redémarrer l'API (port 8000)
|
||||
./svc.sh stop # Tout arrêter
|
||||
```
|
||||
|
||||
### Utilisation - Détection UI
|
||||
| Port | Service |
|
||||
|---|---|
|
||||
| 8000 | API Server (upload / traitement core) |
|
||||
| 5001 | Web Dashboard |
|
||||
| 5002 | VWB Backend (Flask) |
|
||||
| 5003 | Monitoring |
|
||||
| 5004 | Agent Chat |
|
||||
| 5005 | Streaming Server (Agent V1 → pipeline core) |
|
||||
| 5006 | Session Cleaner |
|
||||
| 5099 | Worker de compilation (optionnel) |
|
||||
| 3002 | VWB Frontend (Vite/React) |
|
||||
|
||||
```python
|
||||
from rpa_vision_v3.core.detection import create_detector
|
||||
### Client Windows (Agent V1)
|
||||
|
||||
# Créer le détecteur
|
||||
detector = create_detector()
|
||||
|
||||
# Détecter les éléments UI
|
||||
elements = detector.detect("screenshot.png")
|
||||
|
||||
# Utiliser les résultats
|
||||
for elem in elements:
|
||||
print(f"{elem.type:15s} | {elem.role:20s} | {elem.label}")
|
||||
```
|
||||
|
||||
### Utilisation - Workflow (Phase 4 - À venir)
|
||||
|
||||
```python
|
||||
from rpa_vision_v3.core.models import RawSession, ScreenState, Workflow
|
||||
from rpa_vision_v3.core.graph import GraphBuilder, NodeMatcher
|
||||
|
||||
# 1. Capturer une session
|
||||
session = RawSession(...)
|
||||
# ... capturer événements et screenshots
|
||||
|
||||
# 2. Construire workflow automatiquement
|
||||
builder = GraphBuilder(...)
|
||||
workflow = builder.build_from_session(session)
|
||||
|
||||
# 3. Matcher état actuel
|
||||
matcher = NodeMatcher(...)
|
||||
current_state = ScreenState(...)
|
||||
match = matcher.match(current_state, workflow)
|
||||
|
||||
# 4. Exécuter action
|
||||
if match:
|
||||
edge = workflow.get_outgoing_edges(match.node.node_id)[0]
|
||||
executor.execute_edge(edge, current_state)
|
||||
```
|
||||
|
||||
## 📚 Documentation
|
||||
|
||||
### Guides Principaux
|
||||
- **Quick Start** : `QUICK_START.md` - Démarrage rapide
|
||||
- **Prochaines Étapes** : `NEXT_STEPS.md` - Roadmap et Phase 4
|
||||
- **Phase 3 Complète** : `PHASE3_COMPLETE.md` - Résumé Phase 3
|
||||
|
||||
### Documentation Technique
|
||||
- **Spec complète** : `.kiro/specs/workflow-graph-implementation/`
|
||||
- **Architecture** : `docs/reference/ARCHITECTURE_VISION_COMPLETE.md`
|
||||
- **Détection Hybride** : `HYBRID_DETECTION_SUMMARY.md`
|
||||
- **Intégration Ollama** : `docs/OLLAMA_INTEGRATION.md`
|
||||
|
||||
## 🎓 Concepts Clés
|
||||
|
||||
### RPA 100% Vision
|
||||
|
||||
- ❌ Pas de coordonnées (x, y) fixes
|
||||
- ✅ Rôles sémantiques (primary_action, form_input, etc.)
|
||||
- ✅ Matching par similarité visuelle et textuelle
|
||||
- ✅ Robuste aux changements d'UI
|
||||
|
||||
### Apprentissage Progressif
|
||||
|
||||
```
|
||||
OBSERVATION (5+ exécutions)
|
||||
↓
|
||||
COACHING (10+ assistances, succès >90%)
|
||||
↓
|
||||
AUTO_CANDIDATE (20+ exécutions, succès >95%)
|
||||
↓
|
||||
AUTO_CONFIRMÉ (validation utilisateur)
|
||||
```
|
||||
|
||||
### State Embedding
|
||||
|
||||
Fusion multi-modale :
|
||||
- 50% Image (screenshot complet)
|
||||
- 30% Texte (texte détecté)
|
||||
- 10% Titre (fenêtre)
|
||||
- 10% UI (éléments détectés)
|
||||
|
||||
## 🧪 Tests
|
||||
Le client capture souris, clavier et écran sur le poste Windows et envoie
|
||||
les données au streaming server Linux.
|
||||
|
||||
```bash
|
||||
# Tests unitaires
|
||||
pytest tests/unit/
|
||||
|
||||
# Tests d'intégration
|
||||
pytest tests/integration/
|
||||
|
||||
# Tests de performance
|
||||
pytest tests/performance/ --benchmark-only
|
||||
# Build du package Windows depuis le repo Linux
|
||||
./deploy/build_package.sh
|
||||
# produit deploy/Lea_v<version>.zip
|
||||
```
|
||||
|
||||
## 📈 Roadmap - 77% Complété (10/13 Phases)
|
||||
Voir [`docs/DEV_SETUP.md`](docs/DEV_SETUP.md) pour la maintenance du dépôt
|
||||
(worktrees, build, services).
|
||||
|
||||
### ✅ **Phases Complétées**
|
||||
- [x] **Phase 1-2** : Fondations + Embeddings FAISS ✅
|
||||
- [x] **Phase 4-6** : Détection UI + Workflow Graphs + Action Execution ✅
|
||||
- [x] **Phase 7-8** : Learning System + Training System ✅
|
||||
- [x] **Phase 10-12** : GPU Management + Performance + Monitoring ✅
|
||||
## Arborescence du dépôt
|
||||
|
||||
### 🎯 **Phases Restantes**
|
||||
- [ ] **Phase 3** : Checkpoint Final (tests storage)
|
||||
- [ ] **Phase 9** : Visual Workflow Builder (90% → 100%)
|
||||
- [ ] **Phase 13** : Tests End-to-End + Documentation finale
|
||||
```
|
||||
rpa_vision_v3/
|
||||
├── agent_v0/ # Agent V1 (client Windows) + serveur de streaming
|
||||
│ ├── agent_v1/ # Source de l'agent (capture, UI tray, exécution)
|
||||
│ └── server_v1/ # FastAPI streaming + processeurs
|
||||
├── core/ # Pipeline core
|
||||
│ ├── detection/ # Cascade VLM + OCR + templates
|
||||
│ ├── embedding/ # CLIP + FAISS
|
||||
│ ├── graph/ # Construction / matching de workflow graphs
|
||||
│ ├── execution/ # Résolution de cibles, actions LLM
|
||||
│ ├── learning/ # TargetMemoryStore (apprentissage)
|
||||
│ ├── auth/ # Vault Fernet + TOTP
|
||||
│ └── federation/ # Export/import de LearningPacks
|
||||
├── visual_workflow_builder/ # VWB (backend Flask + frontend React Vite)
|
||||
├── web_dashboard/ # Dashboard Flask + SocketIO
|
||||
├── agent_chat/ # Interface conversationnelle + planner
|
||||
├── deploy/ # Scripts de build et unités systemd
|
||||
├── data/ # Sessions, embeddings, index FAISS, apprentissage
|
||||
├── docs/ # Documentation technique
|
||||
├── tests/ # pytest (unit, integration, e2e)
|
||||
├── services.conf # Source de vérité des ports
|
||||
├── svc.sh # Orchestrateur des services
|
||||
└── run.sh # Démarrage tout-en-un (legacy, préférer svc.sh)
|
||||
```
|
||||
|
||||
### 🚀 **Composants Production-Ready**
|
||||
- **Agent V0** : Capture cross-platform + Encryption ✅
|
||||
- **Server API** : Processing pipeline + Web dashboard ✅
|
||||
- **Analytics System** : Monitoring + Insights + Reporting ✅
|
||||
- **Self-Healing** : Automatic adaptation + Recovery ✅
|
||||
## Tests
|
||||
|
||||
## 🤝 Contribution
|
||||
```bash
|
||||
source .venv/bin/activate
|
||||
|
||||
Voir `.kiro/specs/workflow-graph-implementation/tasks.md` pour les tâches en cours.
|
||||
# Tests rapides (hors marqueur slow)
|
||||
pytest -m "not slow" -q
|
||||
|
||||
## 📄 Licence
|
||||
# Tests d'intégration (streaming, pipeline)
|
||||
pytest tests/integration/ -q
|
||||
|
||||
Propriétaire - Tous droits réservés
|
||||
# Tests E2E
|
||||
pytest tests/test_pipeline_e2e.py -q
|
||||
```
|
||||
|
||||
Quelques tests legacy sont connus comme cassés — voir la mémoire projet et
|
||||
`docs/` pour la liste.
|
||||
|
||||
## Documentation
|
||||
|
||||
- [`docs/STATUS.md`](docs/STATUS.md) — état réel par module
|
||||
- [`docs/DEV_SETUP.md`](docs/DEV_SETUP.md) — tâches d'administration (worktrees, build)
|
||||
- [`docs/EXECUTION_LOOP_FLAGS.md`](docs/EXECUTION_LOOP_FLAGS.md) — flags C1 vision-aware (`enable_ui_detection`, `enable_ocr`, `analyze_timeout_ms`, `window_info_provider`)
|
||||
- [`docs/VISION_RPA_INTELLIGENT.md`](docs/VISION_RPA_INTELLIGENT.md) — cahier des charges
|
||||
- [`docs/PLAN_ACTEUR_V1.md`](docs/PLAN_ACTEUR_V1.md) — architecture 3 niveaux (Macro / Méso / Micro)
|
||||
- [`docs/CONFORMITE_AI_ACT.md`](docs/CONFORMITE_AI_ACT.md) — journalisation, floutage, rétention
|
||||
|
||||
## Concepts clés
|
||||
|
||||
- **RPA 100 % vision** : pas de coordonnées fixes ; l'agent localise un
|
||||
élément par ce qu'il voit (label + contexte visuel), pas par `x,y`.
|
||||
- **Apprentissage progressif** : mode shadow → assisté → autonome, validé
|
||||
par supervision humaine sur les échecs.
|
||||
- **LLM 100 % local** : Ollama sur la machine. Aucun appel cloud dans le
|
||||
pipeline par défaut (cf. feedback projet `feedback_local_only.md`).
|
||||
|
||||
## Licence
|
||||
|
||||
Propriétaire — tous droits réservés.
|
||||
|
||||
@@ -125,25 +125,47 @@ class WorkflowPipelineEnhanced:
|
||||
current_node_id = match_result["node_id"]
|
||||
logger.info(f"Matched current state to node: {current_node_id} (confidence: {match_result['confidence']:.3f})")
|
||||
|
||||
# 2. Obtenir la prochaine action
|
||||
# 2. Obtenir la prochaine action (contrat dict avec status explicite)
|
||||
action_info = self.get_next_action(workflow_id, current_node_id)
|
||||
|
||||
if not action_info:
|
||||
# Workflow terminé
|
||||
action_status = action_info.get("status")
|
||||
|
||||
if action_status == "terminal":
|
||||
# Workflow terminé (aucun outgoing_edge = fin légitime)
|
||||
performance_metrics.total_execution_time_ms = (datetime.now() - start_time).total_seconds() * 1000
|
||||
|
||||
|
||||
result = WorkflowExecutionResult.workflow_complete(
|
||||
execution_id=execution_id,
|
||||
workflow_id=workflow_id,
|
||||
current_node=current_node_id,
|
||||
performance_metrics=performance_metrics
|
||||
performance_metrics=performance_metrics,
|
||||
)
|
||||
result.correlation_id = correlation_id
|
||||
result.match_result = match_result
|
||||
|
||||
|
||||
logger.info(f"Workflow {workflow_id} completed at node {current_node_id}")
|
||||
return result
|
||||
|
||||
|
||||
if action_status == "blocked":
|
||||
# Des edges existent mais aucun ne passe les filtres :
|
||||
# c'est un blocage, pas une fin de workflow.
|
||||
performance_metrics.total_execution_time_ms = (datetime.now() - start_time).total_seconds() * 1000
|
||||
|
||||
result = WorkflowExecutionResult.error(
|
||||
execution_id=execution_id,
|
||||
workflow_id=workflow_id,
|
||||
error_message=f"No valid edge: {action_info.get('reason', 'unknown')}",
|
||||
step_type="action_selection",
|
||||
current_node=current_node_id,
|
||||
performance_metrics=performance_metrics,
|
||||
)
|
||||
result.correlation_id = correlation_id
|
||||
|
||||
logger.warning(
|
||||
f"Workflow {workflow_id} blocked at node {current_node_id}: "
|
||||
f"{action_info.get('reason')}"
|
||||
)
|
||||
return result
|
||||
|
||||
logger.info(f"Next action: {action_info['action']['type']} -> {action_info['target_node']}")
|
||||
|
||||
# 3. Charger le workflow pour obtenir l'edge complet
|
||||
@@ -14,8 +14,9 @@ import asyncio
|
||||
import logging
|
||||
import json
|
||||
import base64
|
||||
import pickle
|
||||
import gzip
|
||||
import pickle # noqa: S403 - usage legacy restreint au fallback de migration
|
||||
import io
|
||||
from typing import Dict, List, Optional, Any, Tuple
|
||||
from dataclasses import dataclass, asdict
|
||||
from datetime import datetime
|
||||
@@ -24,6 +25,12 @@ import numpy as np
|
||||
|
||||
from core.visual.visual_target_manager import VisualTarget, VisualTargetManager
|
||||
from core.visual.screenshot_validation_manager import ScreenshotValidationManager, ValidationResult
|
||||
from core.security.signed_serializer import (
|
||||
SignatureVerificationError,
|
||||
UnsupportedFormatError,
|
||||
dumps_signed,
|
||||
loads_signed,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -435,19 +442,19 @@ class VisualPersistenceManager:
|
||||
return None
|
||||
|
||||
async def _serialize_workflow_data(self, workflow_data: VisualWorkflowData) -> bytes:
|
||||
"""Sérialise les données d'un workflow"""
|
||||
"""Sérialise les données d'un workflow en JSON signé HMAC."""
|
||||
# Convertir en dictionnaire
|
||||
data_dict = asdict(workflow_data)
|
||||
|
||||
|
||||
# Traiter les types spéciaux
|
||||
data_dict['created_at'] = workflow_data.created_at.isoformat()
|
||||
|
||||
|
||||
# Sérialiser les cibles visuelles
|
||||
serialized_targets = {}
|
||||
for signature, target in workflow_data.visual_targets.items():
|
||||
serialized_targets[signature] = await self._serialize_visual_target(target)
|
||||
data_dict['visual_targets'] = serialized_targets
|
||||
|
||||
|
||||
# Sérialiser l'historique de validation
|
||||
serialized_history = {}
|
||||
for signature, history in workflow_data.validation_history.items():
|
||||
@@ -455,15 +462,30 @@ class VisualPersistenceManager:
|
||||
self._serialize_validation_result(result) for result in history
|
||||
]
|
||||
data_dict['validation_history'] = serialized_history
|
||||
|
||||
# Convertir en bytes
|
||||
return pickle.dumps(data_dict)
|
||||
|
||||
|
||||
# JSON signé HMAC (cf. core.security.signed_serializer)
|
||||
return dumps_signed(data_dict)
|
||||
|
||||
async def _deserialize_workflow_data(self, data: bytes) -> VisualWorkflowData:
|
||||
"""Désérialise les données d'un workflow"""
|
||||
# Désérialiser le dictionnaire
|
||||
data_dict = pickle.loads(data)
|
||||
|
||||
"""Désérialise les données d'un workflow (JSON signé HMAC ;
|
||||
fallback pickle legacy avec WARNING pour migrer les anciens fichiers)."""
|
||||
try:
|
||||
data_dict = loads_signed(data)
|
||||
except SignatureVerificationError:
|
||||
# Fichier altéré ou clé différente : on refuse sans fallback.
|
||||
logger.error("Workflow visuel : signature HMAC invalide — refus.")
|
||||
raise
|
||||
except UnsupportedFormatError:
|
||||
# Ancien format pickle : fallback explicite et bruyant.
|
||||
import os
|
||||
if os.getenv("RPA_ALLOW_PICKLE_FALLBACK", "1") == "0":
|
||||
raise
|
||||
logger.warning(
|
||||
"Workflow visuel au format pickle legacy — lecture de compat, "
|
||||
"ré-écrire en JSON signé dès que possible."
|
||||
)
|
||||
data_dict = pickle.loads(data) # noqa: S301 - fallback legacy
|
||||
|
||||
# Reconstruire les objets
|
||||
workflow_data = VisualWorkflowData(
|
||||
workflow_id=data_dict['workflow_id'],
|
||||
@@ -38,6 +38,7 @@ from werkzeug.utils import secure_filename
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from core.workflow import SemanticMatcher, VariableManager
|
||||
from core.detection.vlm_config import get_reasoning_model
|
||||
|
||||
# Import des composants conversationnels
|
||||
from .intent_parser import IntentParser, IntentType, get_intent_parser
|
||||
@@ -83,9 +84,24 @@ app.config['MAX_CONTENT_LENGTH'] = 50 * 1024 * 1024 # 50 MB max upload (sécuri
|
||||
_ALLOWED_ORIGINS = [
|
||||
"http://localhost:3002",
|
||||
"http://localhost:5002",
|
||||
"http://localhost:5004",
|
||||
"https://vwb.labs.laurinebazin.design",
|
||||
"https://lea.labs.laurinebazin.design",
|
||||
# LAN local : serveur Linux (192.168.1.40) + Léa Windows (192.168.1.11).
|
||||
# Sans ces origines, engineio rejette la ChatWindow tkinter Windows et
|
||||
# même les requêtes self-loopback (cf. journal 2026-05-24 11:00:47).
|
||||
"http://192.168.1.40:5004",
|
||||
"http://192.168.1.40:5005",
|
||||
"http://192.168.1.11:5004",
|
||||
"http://192.168.1.11:5005",
|
||||
]
|
||||
# Override possible via LEA_CORS_ALLOWED_ORIGINS=comma,separated,list pour
|
||||
# environnements non-LAN. Vide ou absent → garde la liste par défaut ci-dessus.
|
||||
_extra_origins = os.environ.get("LEA_CORS_ALLOWED_ORIGINS", "").strip()
|
||||
if _extra_origins:
|
||||
_ALLOWED_ORIGINS.extend(
|
||||
o.strip() for o in _extra_origins.split(",") if o.strip()
|
||||
)
|
||||
socketio = SocketIO(app, cors_allowed_origins=_ALLOWED_ORIGINS)
|
||||
|
||||
|
||||
@@ -133,6 +149,28 @@ def _streaming_headers() -> dict:
|
||||
headers["Authorization"] = f"Bearer {_STREAMING_API_TOKEN}"
|
||||
return headers
|
||||
|
||||
|
||||
# ============================================================
|
||||
# Feedback Bus — events 'lea:*' temps réel vers ChatWindow
|
||||
# ============================================================
|
||||
LEA_FEEDBACK_BUS = os.environ.get("LEA_FEEDBACK_BUS", "0").lower() in ("1", "true", "yes", "on")
|
||||
|
||||
|
||||
def _emit_lea(event: str, payload: Dict[str, Any]) -> None:
|
||||
"""Émet 'lea:{event}' sur le bus SocketIO. No-op silencieux si flag off ou erreur."""
|
||||
if not LEA_FEEDBACK_BUS:
|
||||
return
|
||||
try:
|
||||
socketio.emit(f"lea:{event}", payload)
|
||||
except Exception:
|
||||
logger.debug("_emit_lea silenced", exc_info=True)
|
||||
|
||||
|
||||
def _emit_dual(legacy_event: str, lea_event: str, payload: Dict[str, Any], **kwargs) -> None:
|
||||
"""Émet l'event legacy (compat dashboard) ET l'alias lea:* (ChatWindow tkinter)."""
|
||||
socketio.emit(legacy_event, payload, **kwargs)
|
||||
_emit_lea(lea_event, payload)
|
||||
|
||||
execution_status = {
|
||||
"running": False,
|
||||
"workflow": None,
|
||||
@@ -177,6 +215,9 @@ _pending_imports: Dict[str, Dict[str, Any]] = {}
|
||||
# Copilot state — suivi du mode pas-à-pas
|
||||
_copilot_sessions: Dict[str, Dict[str, Any]] = {}
|
||||
|
||||
# LearnActionOrchestrator — P1-LEA SHADOW (apprentissage Léa-first)
|
||||
learn_action_orchestrator = None # injecté par init_system()
|
||||
|
||||
_COPILOT_KEYWORDS = [
|
||||
"copilot", "co-pilot",
|
||||
"pas à pas", "pas-à-pas", "pas a pas",
|
||||
@@ -197,6 +238,7 @@ def init_system():
|
||||
global matcher, gpu_manager
|
||||
global intent_parser, confirmation_loop, response_generator, conversation_manager
|
||||
global autonomous_planner
|
||||
reasoning_model = get_reasoning_model()
|
||||
|
||||
# 1. SemanticMatcher — multi-répertoires (P0-6) + matching LLM (P0-7)
|
||||
# Scan data/workflows/ + data/training/workflows/ + data/training/live_sessions/workflows/
|
||||
@@ -204,7 +246,7 @@ def init_system():
|
||||
matcher = SemanticMatcher(
|
||||
workflows_dir=None, # None = scan tous les répertoires par défaut
|
||||
use_llm=True, # Matching sémantique via Ollama (P0-7)
|
||||
llm_model="qwen2.5:7b",
|
||||
llm_model=reasoning_model,
|
||||
)
|
||||
dirs_info = matcher.get_directories()
|
||||
dirs_summary = ", ".join(
|
||||
@@ -229,7 +271,10 @@ def init_system():
|
||||
|
||||
# 3. Composants conversationnels
|
||||
try:
|
||||
intent_parser = get_intent_parser(use_llm=True) # LLM activé (Ollama)
|
||||
intent_parser = get_intent_parser(
|
||||
use_llm=True,
|
||||
llm_model=reasoning_model,
|
||||
) # LLM activé (Ollama)
|
||||
confirmation_loop = get_confirmation_loop()
|
||||
response_generator = get_response_generator()
|
||||
conversation_manager = get_conversation_manager()
|
||||
@@ -256,8 +301,24 @@ def init_system():
|
||||
if EXECUTION_AVAILABLE:
|
||||
try:
|
||||
# Pipeline de workflow (matching + actions)
|
||||
workflow_pipeline = WorkflowPipeline()
|
||||
logger.info("✓ WorkflowPipeline initialisé")
|
||||
# Depuis C1c 2026-05-25 : désactiver UI detection (OWL/VLM côté
|
||||
# UIDetector via DetectionConfig) par défaut pour économiser
|
||||
# ~900 MiB VRAM au boot du chat service. Le chemin SocketIO 5004
|
||||
# / narration ChatWindow / ExecutionLoop n'utilise pas
|
||||
# workflow_pipeline.ui_detector (grep confirmé). Activation
|
||||
# explicite : AGENT_CHAT_ENABLE_UI_DETECTION=1.
|
||||
_ui_detection_enabled = os.environ.get(
|
||||
"AGENT_CHAT_ENABLE_UI_DETECTION", "0"
|
||||
).strip() in ("1", "true", "yes")
|
||||
workflow_pipeline = WorkflowPipeline(
|
||||
enable_ui_detection=_ui_detection_enabled,
|
||||
enable_vlm=_ui_detection_enabled,
|
||||
)
|
||||
logger.info(
|
||||
f"✓ WorkflowPipeline initialisé "
|
||||
f"(ui_detection={_ui_detection_enabled}, "
|
||||
f"économie ~900 MiB VRAM si False)"
|
||||
)
|
||||
|
||||
# Capture d'écran
|
||||
screen_capturer = ScreenCapturer()
|
||||
@@ -294,7 +355,7 @@ def init_system():
|
||||
|
||||
# 5. Autonomous Planner (Agent Libre)
|
||||
try:
|
||||
autonomous_planner = get_autonomous_planner(llm_model="qwen2.5:7b")
|
||||
autonomous_planner = get_autonomous_planner(llm_model=reasoning_model)
|
||||
|
||||
# Configurer les callbacks pour l'exécution
|
||||
if screen_capturer:
|
||||
@@ -334,6 +395,26 @@ def init_system():
|
||||
else:
|
||||
logger.info("ℹ Import Excel non disponible (openpyxl manquant ?)")
|
||||
|
||||
# 8. LearnActionOrchestrator (P1-LEA SHADOW) — apprentissage Léa-first
|
||||
global learn_action_orchestrator
|
||||
try:
|
||||
from .handlers.learn_action import get_learn_action_orchestrator
|
||||
|
||||
def _learn_emit(event: str, payload: Dict[str, Any]) -> None:
|
||||
try:
|
||||
socketio.emit(event, payload)
|
||||
except Exception:
|
||||
logger.debug("learn emit silenced", exc_info=True)
|
||||
|
||||
learn_action_orchestrator = get_learn_action_orchestrator(emit=_learn_emit)
|
||||
resumed = learn_action_orchestrator.resume_sessions()
|
||||
logger.info(
|
||||
f"✓ LearnActionOrchestrator initialisé (sessions reprises: {len(resumed)})"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"⚠ LearnActionOrchestrator: {e}")
|
||||
learn_action_orchestrator = None
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Routes Web
|
||||
@@ -623,7 +704,7 @@ def api_execute():
|
||||
}
|
||||
|
||||
# Notifier via WebSocket
|
||||
socketio.emit('execution_started', {
|
||||
_emit_dual('execution_started', 'action_started', {
|
||||
"workflow": match.workflow_name,
|
||||
"params": all_params
|
||||
})
|
||||
@@ -650,7 +731,7 @@ def api_history():
|
||||
# =============================================================================
|
||||
|
||||
# Modèle texte pour les réponses conversationnelles (pas besoin de vision)
|
||||
_LEA_LLM_MODEL = os.environ.get("LEA_LLM_MODEL", "qwen3:8b")
|
||||
_LEA_LLM_MODEL = os.environ.get("LEA_LLM_MODEL") or get_reasoning_model()
|
||||
|
||||
_LEA_SYSTEM_PROMPT = """Tu es Léa, une assistante professionnelle chaleureuse et bienveillante.
|
||||
|
||||
@@ -746,6 +827,24 @@ def api_chat():
|
||||
if not message:
|
||||
return jsonify({"error": "Message vide"}), 400
|
||||
|
||||
# 0. Routage P1-LEA : si une session d'apprentissage est active pour ce
|
||||
# session_id, l'orchestrateur traite le message ; sinon on tombe sur le
|
||||
# flux normal (intent_parser / matcher / confirmation).
|
||||
if learn_action_orchestrator is not None and session_id:
|
||||
try:
|
||||
learn_reply = learn_action_orchestrator.handle_chat_message(
|
||||
session_id, message
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("learn_action_orchestrator error")
|
||||
learn_reply = None
|
||||
if learn_reply is not None:
|
||||
return jsonify({
|
||||
"session_id": session_id,
|
||||
"response": learn_reply,
|
||||
"handler": "learn_action",
|
||||
})
|
||||
|
||||
# 1. Obtenir ou créer la session
|
||||
session = conversation_manager.get_or_create_session(session_id=session_id)
|
||||
|
||||
@@ -1181,28 +1280,28 @@ def _execute_gesture(gesture):
|
||||
)
|
||||
|
||||
if resp.status_code == 200:
|
||||
socketio.emit('execution_completed', {
|
||||
_emit_dual('execution_completed', 'done', {
|
||||
"workflow": gesture.name,
|
||||
"success": True,
|
||||
"message": f"Geste '{gesture.name}' ({'+'.join(gesture.keys)}) envoyé",
|
||||
})
|
||||
else:
|
||||
error = resp.text[:200]
|
||||
socketio.emit('execution_completed', {
|
||||
_emit_dual('execution_completed', 'done', {
|
||||
"workflow": gesture.name,
|
||||
"success": False,
|
||||
"message": f"Erreur: {error}",
|
||||
})
|
||||
|
||||
except http_requests.ConnectionError:
|
||||
socketio.emit('execution_completed', {
|
||||
_emit_dual('execution_completed', 'done', {
|
||||
"workflow": gesture.name,
|
||||
"success": False,
|
||||
"message": "Serveur de streaming non disponible (port 5005).",
|
||||
})
|
||||
except Exception as e:
|
||||
logger.error(f"Gesture execution error: {e}")
|
||||
socketio.emit('execution_completed', {
|
||||
_emit_dual('execution_completed', 'done', {
|
||||
"workflow": gesture.name,
|
||||
"success": False,
|
||||
"message": f"Erreur: {str(e)}",
|
||||
@@ -1661,6 +1760,52 @@ def handle_copilot_abort():
|
||||
})
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Bulle paused_need_help — handlers SocketIO depuis ChatWindow (J3.5)
|
||||
# =============================================================================
|
||||
|
||||
@socketio.on('lea:replay_resume')
|
||||
def handle_lea_replay_resume(data):
|
||||
"""Bouton Continuer : relayer le resume vers le streaming server."""
|
||||
replay_id = (data or {}).get("replay_id")
|
||||
if not replay_id:
|
||||
_emit_lea("resume_acked", {"status": "error", "detail": "replay_id manquant"})
|
||||
return
|
||||
try:
|
||||
resp = http_requests.post(
|
||||
f"{STREAMING_SERVER_URL}/api/v1/traces/stream/replay/{replay_id}/resume",
|
||||
headers=_streaming_headers(),
|
||||
timeout=5,
|
||||
)
|
||||
if resp.ok:
|
||||
logger.info(f"Replay {replay_id} resume relayé OK")
|
||||
_emit_lea("resume_acked", {"replay_id": replay_id, "status": "ok"})
|
||||
else:
|
||||
detail = resp.text[:200]
|
||||
logger.warning(f"Resume échoué (HTTP {resp.status_code}): {detail}")
|
||||
_emit_lea("resume_acked", {
|
||||
"replay_id": replay_id, "status": "error",
|
||||
"http_status": resp.status_code, "detail": detail,
|
||||
})
|
||||
except Exception as e:
|
||||
logger.warning(f"Resume relay error: {e}")
|
||||
_emit_lea("resume_acked", {
|
||||
"replay_id": replay_id, "status": "error", "detail": str(e),
|
||||
})
|
||||
|
||||
|
||||
@socketio.on('lea:replay_abort')
|
||||
def handle_lea_replay_abort(data):
|
||||
"""Bouton Annuler : arrêter le polling local. Le replay côté streaming sera
|
||||
cleaned up naturellement au prochain replay (cf api_stream._replay_states stale)."""
|
||||
global execution_status
|
||||
replay_id = (data or {}).get("replay_id")
|
||||
execution_status["running"] = False
|
||||
execution_status["message"] = "Annulé par l'utilisateur"
|
||||
logger.info(f"Replay {replay_id or '?'} abort par l'utilisateur (paused bubble)")
|
||||
_emit_lea("abort_acked", {"replay_id": replay_id, "status": "ok"})
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Exécution de workflow
|
||||
# =============================================================================
|
||||
@@ -1730,14 +1875,20 @@ def _poll_replay_progress(replay_id: str, workflow_name: str, total_actions: int
|
||||
"""Suivre la progression d'un replay distant via polling."""
|
||||
import time
|
||||
|
||||
max_wait = 120 # 2 minutes max
|
||||
max_wait_running = 120 # 2 min en exécution active
|
||||
max_wait_paused = 600 # 10 min en pause supervisée (humain peut prendre son temps)
|
||||
poll_interval = 2.0
|
||||
elapsed = 0
|
||||
was_paused = False
|
||||
|
||||
while elapsed < max_wait and execution_status.get("running"):
|
||||
while execution_status.get("running"):
|
||||
time.sleep(poll_interval)
|
||||
elapsed += poll_interval
|
||||
|
||||
cap = max_wait_paused if was_paused else max_wait_running
|
||||
if elapsed >= cap:
|
||||
break
|
||||
|
||||
try:
|
||||
resp = http_requests.get(
|
||||
f"{STREAMING_SERVER_URL}/api/v1/traces/stream/replay/{replay_id}",
|
||||
@@ -1753,7 +1904,32 @@ def _poll_replay_progress(replay_id: str, workflow_name: str, total_actions: int
|
||||
failed = data.get("failed_actions", 0)
|
||||
progress = int(10 + (completed / max(total_actions, 1)) * 80)
|
||||
|
||||
socketio.emit('execution_progress', {
|
||||
if status == "paused_need_help" and not was_paused:
|
||||
_emit_lea("paused", {
|
||||
"workflow": workflow_name,
|
||||
"replay_id": replay_id,
|
||||
"completed": completed,
|
||||
"total": total_actions,
|
||||
"failed_action": data.get("failed_action"),
|
||||
"reason": (
|
||||
data.get("pause_message")
|
||||
or data.get("message")
|
||||
or data.get("error")
|
||||
or "Action incertaine"
|
||||
),
|
||||
"safety_checks": data.get("safety_checks") or [],
|
||||
})
|
||||
was_paused = True
|
||||
elapsed = 0
|
||||
elif was_paused and status != "paused_need_help":
|
||||
_emit_lea("resumed", {
|
||||
"workflow": workflow_name,
|
||||
"replay_id": replay_id,
|
||||
"status_after": status,
|
||||
})
|
||||
was_paused = False
|
||||
|
||||
_emit_dual('execution_progress', 'action_progress', {
|
||||
"progress": progress,
|
||||
"step": f"Action {completed}/{total_actions} exécutée",
|
||||
"current": completed,
|
||||
@@ -1922,7 +2098,7 @@ def execute_workflow_copilot(match, params: Dict[str, Any]):
|
||||
|
||||
actions = _build_actions_from_workflow(match, params)
|
||||
if not actions:
|
||||
socketio.emit('copilot_complete', {
|
||||
_emit_dual('copilot_complete', 'done', {
|
||||
"workflow": workflow_name,
|
||||
"status": "error",
|
||||
"message": "Aucune action exécutable dans ce workflow.",
|
||||
@@ -1959,7 +2135,7 @@ def execute_workflow_copilot(match, params: Dict[str, Any]):
|
||||
break
|
||||
|
||||
copilot_state["status"] = "waiting_approval"
|
||||
socketio.emit('copilot_step', {
|
||||
_emit_dual('copilot_step', 'need_confirm', {
|
||||
"workflow": workflow_name,
|
||||
"step_index": idx,
|
||||
"total": total,
|
||||
@@ -1982,7 +2158,7 @@ def execute_workflow_copilot(match, params: Dict[str, Any]):
|
||||
|
||||
if waited >= max_wait:
|
||||
copilot_state["status"] = "aborted"
|
||||
socketio.emit('copilot_complete', {
|
||||
_emit_dual('copilot_complete', 'done', {
|
||||
"workflow": workflow_name,
|
||||
"status": "timeout",
|
||||
"message": f"Timeout : pas de réponse après {max_wait}s.",
|
||||
@@ -1999,7 +2175,7 @@ def execute_workflow_copilot(match, params: Dict[str, Any]):
|
||||
elif decision == "skipped":
|
||||
copilot_state["skipped"] += 1
|
||||
logger.info(f"Copilot skip étape {idx + 1}/{total}")
|
||||
socketio.emit('copilot_step_result', {
|
||||
_emit_dual('copilot_step_result', 'step_result', {
|
||||
"step_index": idx,
|
||||
"total": total,
|
||||
"status": "skipped",
|
||||
@@ -2034,7 +2210,7 @@ def execute_workflow_copilot(match, params: Dict[str, Any]):
|
||||
|
||||
if action_success:
|
||||
copilot_state["completed"] += 1
|
||||
socketio.emit('copilot_step_result', {
|
||||
_emit_dual('copilot_step_result', 'step_result', {
|
||||
"step_index": idx,
|
||||
"total": total,
|
||||
"status": "completed",
|
||||
@@ -2042,7 +2218,7 @@ def execute_workflow_copilot(match, params: Dict[str, Any]):
|
||||
})
|
||||
else:
|
||||
copilot_state["failed"] += 1
|
||||
socketio.emit('copilot_step_result', {
|
||||
_emit_dual('copilot_step_result', 'step_result', {
|
||||
"step_index": idx,
|
||||
"total": total,
|
||||
"status": "failed",
|
||||
@@ -2051,7 +2227,7 @@ def execute_workflow_copilot(match, params: Dict[str, Any]):
|
||||
else:
|
||||
error = resp.text[:200]
|
||||
copilot_state["failed"] += 1
|
||||
socketio.emit('copilot_step_result', {
|
||||
_emit_dual('copilot_step_result', 'step_result', {
|
||||
"step_index": idx,
|
||||
"total": total,
|
||||
"status": "failed",
|
||||
@@ -2060,7 +2236,7 @@ def execute_workflow_copilot(match, params: Dict[str, Any]):
|
||||
|
||||
except http_requests.ConnectionError:
|
||||
copilot_state["failed"] += 1
|
||||
socketio.emit('copilot_step_result', {
|
||||
_emit_dual('copilot_step_result', 'step_result', {
|
||||
"step_index": idx,
|
||||
"total": total,
|
||||
"status": "failed",
|
||||
@@ -2070,7 +2246,7 @@ def execute_workflow_copilot(match, params: Dict[str, Any]):
|
||||
except Exception as e:
|
||||
copilot_state["failed"] += 1
|
||||
logger.error(f"Copilot action error: {e}")
|
||||
socketio.emit('copilot_step_result', {
|
||||
_emit_dual('copilot_step_result', 'step_result', {
|
||||
"step_index": idx,
|
||||
"total": total,
|
||||
"status": "failed",
|
||||
@@ -2098,7 +2274,7 @@ def execute_workflow_copilot(match, params: Dict[str, Any]):
|
||||
f"Copilot terminé : {completed} réussies, "
|
||||
f"{skipped} passées, {failed} échouées sur {total} étapes."
|
||||
)
|
||||
socketio.emit('copilot_complete', {
|
||||
_emit_dual('copilot_complete', 'done', {
|
||||
"workflow": workflow_name,
|
||||
"status": "completed" if success else "partial",
|
||||
"message": message,
|
||||
@@ -2175,7 +2351,7 @@ def execute_workflow(match, params):
|
||||
execution_status["progress"] = 10
|
||||
execution_status["message"] = f"Envoyé à l'Agent V1 ({target_session})"
|
||||
|
||||
socketio.emit('execution_progress', {
|
||||
_emit_dual('execution_progress', 'action_progress', {
|
||||
"progress": 10,
|
||||
"step": f"Replay envoyé à l'Agent V1 — {total_actions} actions en attente",
|
||||
"current": 0,
|
||||
@@ -2523,7 +2699,7 @@ def update_progress(progress: int, message: str, current: int, total: int):
|
||||
execution_status["progress"] = progress
|
||||
execution_status["message"] = message
|
||||
|
||||
socketio.emit('execution_progress', {
|
||||
_emit_dual('execution_progress', 'action_progress', {
|
||||
"progress": progress,
|
||||
"step": message,
|
||||
"current": current,
|
||||
@@ -2543,13 +2719,149 @@ def finish_execution(workflow_name: str, success: bool, message: str):
|
||||
if command_history:
|
||||
command_history[-1]["status"] = "completed" if success else "failed"
|
||||
|
||||
socketio.emit('execution_completed', {
|
||||
_emit_dual('execution_completed', 'done', {
|
||||
"workflow": workflow_name,
|
||||
"success": success,
|
||||
"message": message
|
||||
})
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Orchestration démo GHT Sud 95 — "traite N dossiers"
|
||||
# =============================================================================
|
||||
# Délégué à agent_chat.urgences_orchestrator (gemma3:1b NLP + thread orchestrateur).
|
||||
# Routes :
|
||||
# POST /api/urgences/parse — test parsing intent (debug)
|
||||
# POST /api/urgences/start — démarrer une orchestration
|
||||
# GET /api/urgences/status/<id>— état d'une orchestration
|
||||
# GET /api/urgences/list — toutes les orchestrations en mémoire
|
||||
|
||||
try:
|
||||
from agent_chat.urgences_orchestrator import (
|
||||
parse_lea_command,
|
||||
start_orchestration,
|
||||
get_orchestration,
|
||||
list_orchestrations,
|
||||
)
|
||||
_URGENCES_AVAILABLE = True
|
||||
except Exception as _e_urg:
|
||||
logger.warning("Module urgences_orchestrator indisponible : %s", _e_urg)
|
||||
_URGENCES_AVAILABLE = False
|
||||
|
||||
|
||||
@app.route('/api/urgences/parse', methods=['POST'])
|
||||
def urgences_parse():
|
||||
if not _URGENCES_AVAILABLE:
|
||||
return jsonify({"error": "module urgences_orchestrator indisponible"}), 503
|
||||
payload = request.get_json(silent=True) or {}
|
||||
text = (payload.get("text") or "").strip()
|
||||
if not text:
|
||||
return jsonify({"error": "champ 'text' manquant"}), 400
|
||||
intent = parse_lea_command(text)
|
||||
return jsonify(intent)
|
||||
|
||||
|
||||
@app.route('/api/urgences/start', methods=['POST'])
|
||||
def urgences_start():
|
||||
if not _URGENCES_AVAILABLE:
|
||||
return jsonify({"error": "module urgences_orchestrator indisponible"}), 503
|
||||
payload = request.get_json(silent=True) or {}
|
||||
text = (payload.get("text") or "").strip()
|
||||
session_id = payload.get("session_id") or ""
|
||||
machine_id = payload.get("machine_id") or None
|
||||
if not text:
|
||||
return jsonify({"error": "champ 'text' manquant"}), 400
|
||||
intent = parse_lea_command(text)
|
||||
if intent.get("action") != "process_patients":
|
||||
return jsonify({"intent": intent, "started": False,
|
||||
"reply": "Je n'ai pas compris la commande. Exemples : 'traite-moi 3 dossiers', 'code les 5 premiers'."})
|
||||
state = start_orchestration(intent, session_id=session_id, machine_id=machine_id)
|
||||
return jsonify({"intent": intent, "started": True, "orchestration": state.to_dict()})
|
||||
|
||||
|
||||
@app.route('/api/urgences/status/<orch_id>')
|
||||
def urgences_status(orch_id):
|
||||
if not _URGENCES_AVAILABLE:
|
||||
return jsonify({"error": "module urgences_orchestrator indisponible"}), 503
|
||||
state = get_orchestration(orch_id)
|
||||
if not state:
|
||||
return jsonify({"error": f"orchestration {orch_id} introuvable"}), 404
|
||||
return jsonify(state.to_dict())
|
||||
|
||||
|
||||
@app.route('/api/urgences/list')
|
||||
def urgences_list():
|
||||
if not _URGENCES_AVAILABLE:
|
||||
return jsonify({"error": "module urgences_orchestrator indisponible"}), 503
|
||||
return jsonify({"orchestrations": list_orchestrations()})
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# P1-LEA SHADOW — déclenchement d'apprentissage depuis l'extérieur
|
||||
# =============================================================================
|
||||
|
||||
@app.route('/api/learn/start', methods=['POST'])
|
||||
def api_learn_start():
|
||||
"""Déclenche une session d'apprentissage Léa-first.
|
||||
|
||||
Endpoint utilisé par le bouton Windows (ChatWindow tkinter) ou tout autre
|
||||
client externe pour démarrer le cycle Shadow → Persist côté agent-chat.
|
||||
|
||||
Payload JSON :
|
||||
- machine_id (str, obligatoire) : identifiant de la machine où
|
||||
l'apprentissage est en cours (sera repris pour le persist).
|
||||
- session_name (str | None, optionnel) : nom d'affichage de la
|
||||
session (ignoré pour l'instant — réservé futur).
|
||||
- user_id (str | None, optionnel) : défaut "default".
|
||||
- trigger_source (str, optionnel) : défaut "windows_button".
|
||||
Utilisé pour distinguer du "magic_phrase" ou "proactive".
|
||||
|
||||
Retours :
|
||||
- 200 : {"session_id": str, "state": str, "message": str}
|
||||
- 400 : machine_id absent ou vide
|
||||
- 503 : orchestrateur non initialisé (init_system pas appelé)
|
||||
- 500 : exception interne (shadow_start, état illégal, etc.)
|
||||
|
||||
Auth/CORS : suit le pattern des autres routes API du module (pas d'auth
|
||||
Flask explicite — l'API est en LAN derrière le reverse proxy /
|
||||
SocketIO cors_allowed_origins).
|
||||
"""
|
||||
if learn_action_orchestrator is None:
|
||||
return jsonify({
|
||||
"error": "LearnActionOrchestrator non initialisé",
|
||||
}), 503
|
||||
|
||||
data = request.get_json(silent=True) or {}
|
||||
machine_id = (data.get("machine_id") or "").strip()
|
||||
if not machine_id:
|
||||
return jsonify({
|
||||
"error": "machine_id requis (str non vide)",
|
||||
}), 400
|
||||
|
||||
user_id = (data.get("user_id") or "default").strip() or "default"
|
||||
trigger_source = (data.get("trigger_source") or "windows_button").strip() or "windows_button"
|
||||
# session_name reçu mais non utilisé pour l'instant (réservé futur)
|
||||
_session_name = data.get("session_name")
|
||||
|
||||
try:
|
||||
st, reply = learn_action_orchestrator.start_session(
|
||||
user_id=user_id,
|
||||
trigger_source=trigger_source,
|
||||
machine_id=machine_id,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.exception("api_learn_start failed")
|
||||
return jsonify({
|
||||
"error": f"démarrage apprentissage impossible: {exc}",
|
||||
}), 500
|
||||
|
||||
return jsonify({
|
||||
"session_id": st.session_id,
|
||||
"state": st.state.value if hasattr(st.state, "value") else str(st.state),
|
||||
"message": reply,
|
||||
})
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Main
|
||||
# =============================================================================
|
||||
|
||||
@@ -27,6 +27,8 @@ import requests
|
||||
# Ajouter le chemin du projet pour les imports core
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
|
||||
|
||||
from core.detection.vlm_config import get_reasoning_model
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Essayer d'importer les composants de détection visuelle
|
||||
@@ -49,7 +51,10 @@ try:
|
||||
from PIL import Image as PILImage
|
||||
import pyautogui
|
||||
PYAUTOGUI_AVAILABLE = True
|
||||
except ImportError:
|
||||
except Exception:
|
||||
# pyautogui peut lever Xlib.error.DisplayConnectionError (pas un ImportError)
|
||||
# quand X n'est pas accessible — typique d'un service systemd headless côté
|
||||
# serveur. Le serveur n'a pas besoin de pyautogui (utilisé côté client agent).
|
||||
PYAUTOGUI_AVAILABLE = False
|
||||
PILImage = None
|
||||
pyautogui = None
|
||||
@@ -110,11 +115,11 @@ class AutonomousPlanner:
|
||||
def __init__(
|
||||
self,
|
||||
llm_endpoint: str = "http://localhost:11434/api/generate",
|
||||
llm_model: str = "qwen2.5:7b",
|
||||
llm_model: Optional[str] = None,
|
||||
timeout: int = 60
|
||||
):
|
||||
self.llm_endpoint = llm_endpoint
|
||||
self.llm_model = llm_model
|
||||
self.llm_model = llm_model or get_reasoning_model()
|
||||
self.timeout = timeout
|
||||
self.llm_available = self._check_llm()
|
||||
|
||||
@@ -134,11 +139,31 @@ class AutonomousPlanner:
|
||||
logger.info(f"AutonomousPlanner initialized (LLM: {self.llm_model}, available: {self.llm_available}, visual: {self._owl_detector is not None}, vlm: {self._vlm_client is not None})")
|
||||
|
||||
def _init_visual_detection(self):
|
||||
"""Initialise le détecteur visuel OWL-v2."""
|
||||
"""Initialise le détecteur visuel OWL-v2.
|
||||
|
||||
Désactivé par défaut depuis 2026-05-25 (C1b) : OWL-v2 chargeait sur
|
||||
CUDA au boot et retenait ~600 MiB VRAM même en cas d'OOM silencieux,
|
||||
fausssant les benchs perf et contribuant à l'offload Ollama VLM.
|
||||
Comme `autonomous_planner` est largement non-wired au runtime actif
|
||||
(cf. mémoire projet : HTTP 410 dépréciés), le défaut est skip.
|
||||
|
||||
Activation : `AGENT_CHAT_ENABLE_OWL=1` (env var).
|
||||
Device : `AGENT_CHAT_OWL_DEVICE=cuda|cpu` (override l'auto-détect).
|
||||
"""
|
||||
if os.environ.get("AGENT_CHAT_ENABLE_OWL", "0").strip() not in ("1", "true", "yes"):
|
||||
logger.info(
|
||||
"OWL-v2 visual detector skipped at boot "
|
||||
"(AGENT_CHAT_ENABLE_OWL!=1, économie ~600 MiB VRAM)"
|
||||
)
|
||||
return
|
||||
if VISUAL_DETECTION_AVAILABLE and OwlDetector:
|
||||
try:
|
||||
self._owl_detector = OwlDetector(confidence_threshold=0.1)
|
||||
logger.info("OWL-v2 visual detector initialized")
|
||||
device = os.environ.get("AGENT_CHAT_OWL_DEVICE", "").strip() or None
|
||||
self._owl_detector = OwlDetector(
|
||||
confidence_threshold=0.1,
|
||||
device=device,
|
||||
)
|
||||
logger.info(f"OWL-v2 visual detector initialized (device={device or 'auto'})")
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not initialize OWL detector: {e}")
|
||||
self._owl_detector = None
|
||||
@@ -147,8 +172,10 @@ class AutonomousPlanner:
|
||||
"""Initialise le client VLM pour analyse intelligente."""
|
||||
if VLM_AVAILABLE and OllamaClient:
|
||||
try:
|
||||
self._vlm_client = OllamaClient(model="qwen2.5vl:7b")
|
||||
logger.info("VLM client initialized (qwen2.5vl:7b)")
|
||||
from core.detection.vlm_config import get_vlm_model
|
||||
_planner_vlm = get_vlm_model()
|
||||
self._vlm_client = OllamaClient(model=_planner_vlm)
|
||||
logger.info("VLM client initialized (%s)", _planner_vlm)
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not initialize VLM client: {e}")
|
||||
self._vlm_client = None
|
||||
@@ -1003,12 +1030,12 @@ _planner_instance: Optional[AutonomousPlanner] = None
|
||||
|
||||
|
||||
def get_autonomous_planner(
|
||||
llm_model: str = "qwen2.5:7b"
|
||||
llm_model: Optional[str] = None
|
||||
) -> AutonomousPlanner:
|
||||
"""Retourne l'instance singleton du planner."""
|
||||
global _planner_instance
|
||||
|
||||
if _planner_instance is None:
|
||||
_planner_instance = AutonomousPlanner(llm_model=llm_model)
|
||||
_planner_instance = AutonomousPlanner(llm_model=llm_model or get_reasoning_model())
|
||||
|
||||
return _planner_instance
|
||||
|
||||
@@ -16,6 +16,7 @@ Auteur: Dom — Mars 2026
|
||||
|
||||
import logging
|
||||
import re
|
||||
import unicodedata
|
||||
import uuid
|
||||
from dataclasses import dataclass, field
|
||||
from difflib import SequenceMatcher
|
||||
@@ -24,6 +25,11 @@ from typing import Dict, List, Optional, Tuple
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
SAVE_COMMAND_LABELS = {"enregistrer", "save", "sauvegarder"}
|
||||
SAVE_AS_LABELS = {"enregistrer sous", "save as", "sauvegarder sous"}
|
||||
FILE_MENU_LABELS = {"fichier", "file", "menu fichier", "file menu"}
|
||||
|
||||
|
||||
@dataclass
|
||||
class Gesture:
|
||||
"""Un geste primitif universel."""
|
||||
@@ -564,6 +570,7 @@ class GestureCatalog:
|
||||
Patterns :
|
||||
- Clic en haut à droite de la fenêtre (x > 95%, y < 5%) → fermer
|
||||
- target_text contenant ✕, ×, X, □, ─, etc.
|
||||
- Commande applicative "Enregistrer" sûre → Ctrl+S
|
||||
"""
|
||||
# Vérifier le target_text
|
||||
target_text = (
|
||||
@@ -583,6 +590,9 @@ class GestureCatalog:
|
||||
if target_lower in ("─", "—", "_", "minimize", "réduire"):
|
||||
return self._by_id.get("win_minimize")
|
||||
|
||||
if self._is_save_command_action(action):
|
||||
return self._by_id.get("edit_save")
|
||||
|
||||
# Vérifier la position relative (coin haut-droite = fermer)
|
||||
x_pct = action.get("x_pct", 0)
|
||||
y_pct = action.get("y_pct", 0)
|
||||
@@ -596,6 +606,128 @@ class GestureCatalog:
|
||||
|
||||
return None
|
||||
|
||||
def _normalize_ui_text(self, value: str) -> str:
|
||||
"""Normaliser un libellé UI pour comparer accents, casse et raccourcis."""
|
||||
text = str(value or "").strip().lower()
|
||||
text = unicodedata.normalize("NFKD", text)
|
||||
text = "".join(ch for ch in text if not unicodedata.combining(ch))
|
||||
text = text.replace("’", "'")
|
||||
text = re.sub(r"\s+", " ", text)
|
||||
text = re.sub(r"\s*\([^)]*ctrl\s*\+?\s*s[^)]*\)\s*$", "", text)
|
||||
text = re.sub(r"\s+ctrl\s*\+?\s*s\s*$", "", text)
|
||||
return text.strip()
|
||||
|
||||
def _action_text_candidates(self, action: Dict) -> List[str]:
|
||||
"""Retourner les libellés utiles d'une action et de son target_spec."""
|
||||
target_spec = action.get("target_spec") or {}
|
||||
candidates = [
|
||||
action.get("target_text", ""),
|
||||
action.get("target_description", ""),
|
||||
action.get("description", ""),
|
||||
target_spec.get("by_text", ""),
|
||||
target_spec.get("target_text", ""),
|
||||
target_spec.get("vlm_description", ""),
|
||||
]
|
||||
return [str(c) for c in candidates if c]
|
||||
|
||||
def _action_role_text(self, action: Dict) -> str:
|
||||
target_spec = action.get("target_spec") or {}
|
||||
uia = action.get("uia_snapshot") or {}
|
||||
role_parts = [
|
||||
action.get("role", ""),
|
||||
action.get("control_type", ""),
|
||||
target_spec.get("by_role", ""),
|
||||
target_spec.get("role", ""),
|
||||
target_spec.get("control_type", ""),
|
||||
uia.get("control_type", ""),
|
||||
uia.get("class_name", ""),
|
||||
]
|
||||
return " ".join(self._normalize_ui_text(part) for part in role_parts if part)
|
||||
|
||||
def _action_context_text(self, action: Dict) -> str:
|
||||
target_spec = action.get("target_spec") or {}
|
||||
hints = target_spec.get("context_hints") or {}
|
||||
context_parts = [
|
||||
action.get("window_title", ""),
|
||||
target_spec.get("window_title", ""),
|
||||
target_spec.get("vlm_description", ""),
|
||||
hints.get("window_title", ""),
|
||||
hints.get("interaction", ""),
|
||||
hints.get("source", ""),
|
||||
hints.get("menu_path", ""),
|
||||
]
|
||||
return " ".join(self._normalize_ui_text(part) for part in context_parts if part)
|
||||
|
||||
def _is_file_menu_action(self, action: Dict) -> bool:
|
||||
labels = {self._normalize_ui_text(text) for text in self._action_text_candidates(action)}
|
||||
return bool(labels & FILE_MENU_LABELS)
|
||||
|
||||
def _is_save_command_label(self, action: Dict) -> bool:
|
||||
for text in self._action_text_candidates(action):
|
||||
label = self._normalize_ui_text(text)
|
||||
if not label:
|
||||
continue
|
||||
if any(save_as in label for save_as in SAVE_AS_LABELS):
|
||||
return False
|
||||
if label in SAVE_COMMAND_LABELS:
|
||||
return True
|
||||
return False
|
||||
|
||||
def _is_save_dialog_action(self, action: Dict) -> bool:
|
||||
context = self._action_context_text(action)
|
||||
if any(save_as in context for save_as in SAVE_AS_LABELS):
|
||||
return True
|
||||
dialog_markers = (
|
||||
"save dialog",
|
||||
"save_dialog",
|
||||
"dialog",
|
||||
"boite de dialogue",
|
||||
"fenetre enregistrer sous",
|
||||
"confirmer l'enregistrement",
|
||||
"save changes",
|
||||
)
|
||||
return any(marker in context for marker in dialog_markers)
|
||||
|
||||
def _is_save_command_action(self, action: Dict) -> bool:
|
||||
if not self._is_save_command_label(action):
|
||||
return False
|
||||
if self._is_save_dialog_action(action):
|
||||
return False
|
||||
|
||||
role = self._action_role_text(action)
|
||||
context = self._action_context_text(action)
|
||||
command_markers = (
|
||||
"menu",
|
||||
"menuitem",
|
||||
"item de menu",
|
||||
"toolbar",
|
||||
"barre d'outils",
|
||||
"tool bar",
|
||||
"ruban",
|
||||
"ribbon",
|
||||
"commande",
|
||||
"command",
|
||||
)
|
||||
return any(marker in role or marker in context for marker in command_markers)
|
||||
|
||||
def _substitute_action(
|
||||
self,
|
||||
action: Dict,
|
||||
gesture: Gesture,
|
||||
*,
|
||||
original_type: str,
|
||||
source_action_ids: Optional[List[str]] = None,
|
||||
reason: str = "",
|
||||
) -> Dict:
|
||||
new_action = gesture.to_replay_action()
|
||||
new_action["action_id"] = action.get("action_id", new_action["action_id"])
|
||||
new_action["original_type"] = original_type
|
||||
if source_action_ids:
|
||||
new_action["substitution_source_action_ids"] = source_action_ids
|
||||
if reason:
|
||||
new_action["substitution_reason"] = reason
|
||||
return new_action
|
||||
|
||||
def optimize_replay_actions(self, actions: List[Dict]) -> List[Dict]:
|
||||
"""
|
||||
Optimiser une liste d'actions de replay en substituant les gestes connus.
|
||||
@@ -610,13 +742,45 @@ class GestureCatalog:
|
||||
substitutions = 0
|
||||
|
||||
for action in actions:
|
||||
if (
|
||||
action.get("type") == "click"
|
||||
and optimized
|
||||
and optimized[-1].get("type") == "click"
|
||||
and self._is_file_menu_action(optimized[-1])
|
||||
and self._is_save_command_label(action)
|
||||
and not self._is_save_dialog_action(action)
|
||||
):
|
||||
gesture = self._by_id.get("edit_save")
|
||||
previous = optimized.pop()
|
||||
source_ids = [
|
||||
source_id for source_id in (
|
||||
previous.get("action_id"),
|
||||
action.get("action_id"),
|
||||
)
|
||||
if source_id
|
||||
]
|
||||
optimized.append(
|
||||
self._substitute_action(
|
||||
action,
|
||||
gesture,
|
||||
original_type="click_sequence",
|
||||
source_action_ids=source_ids,
|
||||
reason="file_menu_save_to_ctrl_s",
|
||||
)
|
||||
)
|
||||
substitutions += 1
|
||||
logger.debug("Séquence Fichier > Enregistrer substituée par Ctrl+S")
|
||||
continue
|
||||
|
||||
gesture = self.match_action(action)
|
||||
if gesture and action.get("type") != "key_combo":
|
||||
# Substituer par le raccourci clavier
|
||||
new_action = gesture.to_replay_action()
|
||||
# Conserver l'action_id original pour le tracking
|
||||
new_action["action_id"] = action.get("action_id", new_action["action_id"])
|
||||
new_action["original_type"] = action.get("type")
|
||||
new_action = self._substitute_action(
|
||||
action,
|
||||
gesture,
|
||||
original_type=action.get("type", ""),
|
||||
reason=f"{gesture.id}_gesture_substitution",
|
||||
)
|
||||
optimized.append(new_action)
|
||||
substitutions += 1
|
||||
logger.debug(
|
||||
|
||||
29
agent_chat/handlers/__init__.py
Normal file
29
agent_chat/handlers/__init__.py
Normal file
@@ -0,0 +1,29 @@
|
||||
"""Agent-chat handlers package.
|
||||
|
||||
Contient les orchestrateurs spécialisés (apprentissage Léa, etc.) appelés
|
||||
par `agent_chat.app` quand le routage normal d'intent ne suffit pas.
|
||||
"""
|
||||
|
||||
from .learn_action import (
|
||||
LearnActionOrchestrator,
|
||||
LearnState,
|
||||
LearnIntent,
|
||||
LearnIntentParser,
|
||||
OptionCFormatter,
|
||||
StreamingClient,
|
||||
StateStore,
|
||||
PersistPayloadBuilder,
|
||||
get_learn_action_orchestrator,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"LearnActionOrchestrator",
|
||||
"LearnState",
|
||||
"LearnIntent",
|
||||
"LearnIntentParser",
|
||||
"OptionCFormatter",
|
||||
"StreamingClient",
|
||||
"StateStore",
|
||||
"PersistPayloadBuilder",
|
||||
"get_learn_action_orchestrator",
|
||||
]
|
||||
1192
agent_chat/handlers/learn_action.py
Normal file
1192
agent_chat/handlers/learn_action.py
Normal file
File diff suppressed because it is too large
Load Diff
@@ -19,6 +19,8 @@ from enum import Enum
|
||||
from typing import Dict, Any, List, Optional, Tuple
|
||||
from pathlib import Path
|
||||
|
||||
from core.detection.vlm_config import get_reasoning_model
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -280,7 +282,7 @@ class IntentParser:
|
||||
self,
|
||||
use_llm: bool = False,
|
||||
llm_endpoint: str = "http://localhost:11434",
|
||||
llm_model: str = "qwen2.5:7b"
|
||||
llm_model: Optional[str] = None
|
||||
):
|
||||
"""
|
||||
Initialiser le parseur d'intentions.
|
||||
@@ -292,7 +294,7 @@ class IntentParser:
|
||||
"""
|
||||
self.use_llm = use_llm
|
||||
self.llm_endpoint = llm_endpoint
|
||||
self.llm_model = llm_model
|
||||
self.llm_model = llm_model or get_reasoning_model()
|
||||
self.llm_available = False
|
||||
self._workflows_cache: List[Dict[str, Any]] = []
|
||||
|
||||
@@ -687,7 +689,7 @@ _intent_parser: Optional[IntentParser] = None
|
||||
|
||||
def get_intent_parser(
|
||||
use_llm: bool = False,
|
||||
llm_model: str = "qwen2.5:7b",
|
||||
llm_model: Optional[str] = None,
|
||||
llm_endpoint: str = "http://localhost:11434"
|
||||
) -> IntentParser:
|
||||
"""
|
||||
@@ -695,20 +697,21 @@ def get_intent_parser(
|
||||
|
||||
Args:
|
||||
use_llm: Activer le LLM (Ollama)
|
||||
llm_model: Modèle à utiliser (qwen2.5:7b par défaut)
|
||||
llm_model: Modèle à utiliser (défaut: modèle reasoning central)
|
||||
llm_endpoint: URL de l'endpoint Ollama
|
||||
"""
|
||||
global _intent_parser
|
||||
resolved_model = llm_model or get_reasoning_model()
|
||||
if _intent_parser is None:
|
||||
_intent_parser = IntentParser(
|
||||
use_llm=use_llm,
|
||||
llm_endpoint=llm_endpoint,
|
||||
llm_model=llm_model
|
||||
llm_model=resolved_model
|
||||
)
|
||||
elif use_llm and not _intent_parser.use_llm:
|
||||
# Réactiver le LLM si demandé
|
||||
_intent_parser.use_llm = True
|
||||
_intent_parser.llm_model = llm_model
|
||||
_intent_parser.llm_model = resolved_model
|
||||
_intent_parser._check_llm_availability()
|
||||
return _intent_parser
|
||||
|
||||
|
||||
518
agent_chat/urgences_orchestrator.py
Normal file
518
agent_chat/urgences_orchestrator.py
Normal file
@@ -0,0 +1,518 @@
|
||||
"""Orchestrateur démo GHT Sud 95 — pilotage du scénario "traite N dossiers".
|
||||
|
||||
Reçoit une commande naturelle de Léa (chat) et orchestre :
|
||||
1. Parsing intent via gemma3:1b (mini-LLM local, ~400 ms)
|
||||
2. Setup Chrome (Win+R → URL maquette → Enter) via /replay/raw
|
||||
3. extract_table sur la liste des patients (regex IPP, limit=N)
|
||||
4. Boucle : pour chaque IPP, lance le workflow "Urgence_unit" via /replay
|
||||
avec `variables={"patient_id": ipp}` pour la résolution `{{patient_id}}`
|
||||
5. Synthèse finale postée dans le chat
|
||||
|
||||
L'orchestration tourne dans un thread daemon. L'état est stocké en mémoire,
|
||||
poll-able via /api/urgences/status/<orch_id>.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Chargement explicite de .env.local du repo (le service systemd peut ne pas
|
||||
# voir cet env file). Cherche dans le parent de agent_chat/.
|
||||
def _load_env_local() -> None:
|
||||
env_path = Path(__file__).resolve().parent.parent / ".env.local"
|
||||
if not env_path.is_file():
|
||||
return
|
||||
try:
|
||||
for line in env_path.read_text().splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
k, v = line.split("=", 1)
|
||||
k = k.strip()
|
||||
v = v.strip().strip('"').strip("'")
|
||||
os.environ.setdefault(k, v)
|
||||
except Exception as e:
|
||||
logger.warning("Erreur chargement .env.local: %s", e)
|
||||
|
||||
|
||||
_load_env_local()
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────
|
||||
# Config
|
||||
# ─────────────────────────────────────────────────────────────────────
|
||||
STREAM_BASE = os.environ.get("RPA_STREAM_BASE", "http://localhost:5005")
|
||||
OLLAMA_URL = os.environ.get("OLLAMA_URL", "http://localhost:11434/api/generate")
|
||||
NLP_MODEL = os.environ.get("LEA_NLP_MODEL", "gemma3:1b")
|
||||
RPA_API_TOKEN = os.environ.get("RPA_API_TOKEN", "")
|
||||
|
||||
URGENCE_WORKFLOW_ID = os.environ.get("LEA_URGENCE_WORKFLOW_ID", "wf_urgence_unit")
|
||||
# URL LAN locale (sans Basic Auth ni HTTPS) pour éviter le prompt Windows Hello
|
||||
# de Chrome (lecteur d'empreintes digitales) qui bloque le replay automatique.
|
||||
# L'URL publique HTTPS reste disponible (https://urgence.labs.laurinebazin.design)
|
||||
# pour usage humain, mais n'est PAS utilisée par Léa pendant la démo.
|
||||
MAQUETTE_URL = os.environ.get("LEA_MAQUETTE_URL", "http://192.168.1.40:8765/index.html")
|
||||
|
||||
|
||||
|
||||
# Session de replay stable de l'agent V1. L'agent polle /replay/next sur
|
||||
# `agent_<user_id>` indépendamment des sessions d'enregistrement (sess_*).
|
||||
# user_id default côté agent V1 = "demo_user" (cf. agent_v1/main.py:62).
|
||||
AGENT_SESSION_ID = os.environ.get("LEA_AGENT_SESSION_ID", "agent_demo_user")
|
||||
|
||||
# machine_id de l'agent V1 cible. DOIT matcher self.machine_id côté agent V1
|
||||
# (sinon /replay/next ne distribue pas la queue à cette machine — le serveur
|
||||
# isole les machines pour éviter le vol cross-machine d'actions).
|
||||
# Valeur par défaut = hostname du PC Windows de démo GHT.
|
||||
AGENT_MACHINE_ID = os.environ.get("LEA_AGENT_MACHINE_ID", "DESKTOP-58D5CAC_windows")
|
||||
|
||||
# Pattern IPP : 8 chiffres, premier groupe "25" (cohort 2025), reste libre
|
||||
IPP_PATTERN = r"^25\d{6}$"
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────
|
||||
# NLP : parsing de commande naturelle via gemma3:1b
|
||||
# ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
NLP_PROMPT = """Tu es un parseur d'intentions pour Léa, assistant RPA médical.
|
||||
Réponds UNIQUEMENT en JSON valide, sans texte avant/après, selon ce schéma :
|
||||
{"action": "process_patients" | "stop" | "unknown", "count": <int|null>, "order": "first" | "last" | "all" | "specific" | null, "ipp": "<string>" | null}
|
||||
|
||||
Règles :
|
||||
- "traite N dossiers" / "code N dossiers" / "fais les N premiers" → action=process_patients, count=N, order="first"
|
||||
- "traite tous les dossiers" → action=process_patients, count=null, order="all"
|
||||
- "traite le dossier 25003364" → action=process_patients, count=1, order="specific", ipp="25003364"
|
||||
- "stop" / "arrête" / "annule" → action=stop
|
||||
- Question ("comment", "pourquoi") → action=unknown
|
||||
- Si tu ne comprends pas → action=unknown"""
|
||||
|
||||
|
||||
def parse_lea_command(text: str, model: str = NLP_MODEL, timeout: int = 8) -> Dict[str, Any]:
|
||||
"""Parse une commande naturelle en intent structuré via gemma3:1b.
|
||||
|
||||
Fallback regex si Ollama est indisponible — pour ne pas bloquer la démo.
|
||||
Returns : dict {action, count, order, ipp} ou {action: "unknown"}.
|
||||
"""
|
||||
payload = {
|
||||
"model": model,
|
||||
"prompt": NLP_PROMPT + "\n\nUtilisateur : " + text + "\n\nJSON :",
|
||||
"stream": False,
|
||||
"format": "json",
|
||||
"options": {"temperature": 0.0, "num_predict": 120, "num_ctx": 1024},
|
||||
}
|
||||
data = json.dumps(payload).encode("utf-8")
|
||||
req = urllib.request.Request(OLLAMA_URL, data=data, headers={"Content-Type": "application/json"})
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
body = json.loads(resp.read().decode("utf-8"))
|
||||
raw = (body.get("response") or "").strip()
|
||||
if raw.startswith("```"):
|
||||
raw = raw.split("\n", 1)[-1].rsplit("```", 1)[0].strip()
|
||||
intent = json.loads(raw)
|
||||
except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as e:
|
||||
logger.warning("parse_lea_command: gemma3:1b indisponible (%s), fallback regex", e)
|
||||
return _parse_fallback_regex(text)
|
||||
|
||||
# Post-processing : gemma3:1b a tendance à remplir tous les champs même
|
||||
# quand non pertinent. On nettoie :
|
||||
# - ipp ne doit être conservé que si présent LITTÉRALEMENT dans le texte source
|
||||
# (sinon le LLM hallucine un IPP plausible)
|
||||
if intent.get("ipp") and str(intent["ipp"]) not in text:
|
||||
intent["ipp"] = None
|
||||
# Si le LLM a forcé order=specific sans vrai IPP, on bascule en first
|
||||
if intent.get("order") == "specific":
|
||||
intent["order"] = "first"
|
||||
# - ipp ne doit être conservé que si order="specific" ET format IPP valide
|
||||
if intent.get("ipp") and intent.get("order") != "specific":
|
||||
intent["ipp"] = None
|
||||
if intent.get("ipp") and not re.match(r"^\d{8,10}$", str(intent["ipp"])):
|
||||
intent["ipp"] = None
|
||||
# - si count est défini ET order="all", l'humain demande "N dossiers" et
|
||||
# non "tous les dossiers" : on bascule en "first" (cohérence sémantique)
|
||||
if intent.get("count") and intent.get("order") == "all":
|
||||
intent["order"] = "first"
|
||||
return intent
|
||||
|
||||
|
||||
def _parse_fallback_regex(text: str) -> Dict[str, Any]:
|
||||
"""Fallback regex robuste si LLM HS — couvre les phrasings classiques."""
|
||||
t = text.lower()
|
||||
if any(w in t for w in ("stop", "arrête", "annule", "annuler")):
|
||||
return {"action": "stop", "count": None, "order": None, "ipp": None}
|
||||
# IPP spécifique : "traite le dossier 25003364"
|
||||
m = re.search(r"\b(25\d{6})\b", text)
|
||||
if m and any(w in t for w in ("traite", "code", "analyse")):
|
||||
return {"action": "process_patients", "count": 1, "order": "specific", "ipp": m.group(1)}
|
||||
if any(w in t for w in ("tous", "toutes")) and any(w in t for w in ("traite", "code")):
|
||||
return {"action": "process_patients", "count": None, "order": "all", "ipp": None}
|
||||
# Quantifié : "traite 3 dossiers"
|
||||
m = re.search(r"(\d+)\s*(?:premiers?\s*)?(?:dossiers?|cas|patients?)", t)
|
||||
if m and any(w in t for w in ("traite", "code", "fais", "analyse")):
|
||||
return {"action": "process_patients", "count": int(m.group(1)), "order": "first", "ipp": None}
|
||||
return {"action": "unknown", "count": None, "order": None, "ipp": None}
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────
|
||||
# Helpers HTTP vers le streaming server (port 5005)
|
||||
# ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
def _stream_headers() -> Dict[str, str]:
|
||||
h = {"Content-Type": "application/json"}
|
||||
if RPA_API_TOKEN:
|
||||
h["Authorization"] = f"Bearer {RPA_API_TOKEN}"
|
||||
return h
|
||||
|
||||
|
||||
def _post(path: str, body: dict, timeout: int = 30) -> dict:
|
||||
req = urllib.request.Request(
|
||||
STREAM_BASE + path,
|
||||
data=json.dumps(body).encode("utf-8"),
|
||||
headers=_stream_headers(),
|
||||
method="POST",
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
return json.loads(resp.read().decode("utf-8"))
|
||||
|
||||
|
||||
def _get(path: str, timeout: int = 10) -> dict:
|
||||
req = urllib.request.Request(
|
||||
STREAM_BASE + path,
|
||||
headers=_stream_headers(),
|
||||
method="GET",
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
return json.loads(resp.read().decode("utf-8"))
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────
|
||||
# Orchestration : état + thread d'exécution
|
||||
# ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
@dataclass
|
||||
class DossierResult:
|
||||
ipp: str
|
||||
decision: Optional[str] = None # "REQUALIFICATION_HOSPITALISATION" | "FORFAIT_URGENCE"
|
||||
decision_court: Optional[str] = None # "UHCD" | "Forfait Urgences"
|
||||
confiance: Optional[str] = None
|
||||
duree_passage_heures: Optional[float] = None
|
||||
concordance: Optional[bool] = None
|
||||
error: Optional[str] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class OrchestrationState:
|
||||
orch_id: str
|
||||
status: str = "starting" # starting | running | done | error | cancelled
|
||||
progress: int = 0 # 0 → count
|
||||
count: int = 0
|
||||
current_step: str = "" # "setup_chrome" | "extract_table" | "process_dossier_X" | "synthese"
|
||||
intent: Dict[str, Any] = field(default_factory=dict)
|
||||
patients: List[str] = field(default_factory=list)
|
||||
results: List[DossierResult] = field(default_factory=list)
|
||||
synthese: Optional[str] = None
|
||||
error: Optional[str] = None
|
||||
started_at: float = field(default_factory=time.time)
|
||||
finished_at: Optional[float] = None
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"orch_id": self.orch_id,
|
||||
"status": self.status,
|
||||
"progress": self.progress,
|
||||
"count": self.count,
|
||||
"current_step": self.current_step,
|
||||
"intent": self.intent,
|
||||
"patients": self.patients,
|
||||
"results": [r.__dict__ for r in self.results],
|
||||
"synthese": self.synthese,
|
||||
"error": self.error,
|
||||
"elapsed_s": round((self.finished_at or time.time()) - self.started_at, 1),
|
||||
}
|
||||
|
||||
|
||||
# Registry global des orchestrations en cours (thread-safe via lock)
|
||||
_ORCH_REGISTRY: Dict[str, OrchestrationState] = {}
|
||||
_ORCH_LOCK = threading.Lock()
|
||||
|
||||
|
||||
def get_orchestration(orch_id: str) -> Optional[OrchestrationState]:
|
||||
with _ORCH_LOCK:
|
||||
return _ORCH_REGISTRY.get(orch_id)
|
||||
|
||||
|
||||
def list_orchestrations() -> List[Dict[str, Any]]:
|
||||
with _ORCH_LOCK:
|
||||
return [s.to_dict() for s in _ORCH_REGISTRY.values()]
|
||||
|
||||
|
||||
def start_orchestration(
|
||||
intent: Dict[str, Any],
|
||||
session_id: str = "",
|
||||
machine_id: Optional[str] = None,
|
||||
) -> OrchestrationState:
|
||||
"""Lance une orchestration en thread daemon. Retourne l'état initial.
|
||||
|
||||
Args:
|
||||
intent: dict {action, count, order, ipp} (sortie de parse_lea_command)
|
||||
session_id: session de replay (default: agent_demo_user, le canal stable
|
||||
sur lequel l'agent V1 polle /replay/next)
|
||||
machine_id: machine cible (optionnel, pour multi-machines futurs)
|
||||
"""
|
||||
if not session_id:
|
||||
session_id = AGENT_SESSION_ID
|
||||
if not machine_id:
|
||||
machine_id = AGENT_MACHINE_ID
|
||||
orch_id = "orch_" + uuid.uuid4().hex[:10]
|
||||
count = intent.get("count") or 3 # default 3 si "tous" ou "first" sans nombre
|
||||
state = OrchestrationState(
|
||||
orch_id=orch_id,
|
||||
status="starting",
|
||||
count=count,
|
||||
intent=intent,
|
||||
)
|
||||
with _ORCH_LOCK:
|
||||
_ORCH_REGISTRY[orch_id] = state
|
||||
|
||||
th = threading.Thread(
|
||||
target=_run_orchestration,
|
||||
args=(state, session_id, machine_id),
|
||||
daemon=True,
|
||||
name=f"orch-{orch_id}",
|
||||
)
|
||||
th.start()
|
||||
return state
|
||||
|
||||
|
||||
def _run_orchestration(state: OrchestrationState, session_id: str, machine_id: Optional[str]) -> None:
|
||||
"""Boucle d'orchestration exécutée dans un thread.
|
||||
|
||||
Phases :
|
||||
1. Setup Chrome (raw actions Win+R)
|
||||
2. extract_table sur liste patients
|
||||
3. Boucle workflow Urgence_unit
|
||||
4. Synthèse
|
||||
"""
|
||||
try:
|
||||
state.status = "running"
|
||||
intent = state.intent
|
||||
|
||||
# Cas "specific" : court-circuiter, juste 1 IPP
|
||||
if intent.get("order") == "specific" and intent.get("ipp"):
|
||||
state.patients = [intent["ipp"]]
|
||||
state.count = 1
|
||||
state.current_step = "process_dossier"
|
||||
_process_dossiers(state, session_id, machine_id)
|
||||
else:
|
||||
# 1. Setup Chrome → URL maquette
|
||||
state.current_step = "setup_chrome"
|
||||
_setup_chrome(session_id, machine_id)
|
||||
|
||||
# 2. Lire la liste des IPP via extract_table
|
||||
state.current_step = "extract_table"
|
||||
patients = _extract_patient_list(session_id, machine_id, limit=state.count)
|
||||
state.patients = patients
|
||||
if not patients:
|
||||
raise RuntimeError("extract_table n'a trouvé aucun IPP — vérifier que Chrome est sur index.html")
|
||||
|
||||
# 3. Pour chaque IPP : lancer workflow Urgence_unit
|
||||
_process_dossiers(state, session_id, machine_id)
|
||||
|
||||
# 4. Synthèse
|
||||
state.current_step = "synthese"
|
||||
state.synthese = _build_synthese(state)
|
||||
state.status = "done"
|
||||
except Exception as e:
|
||||
logger.exception("Orchestration %s : erreur fatale", state.orch_id)
|
||||
state.status = "error"
|
||||
state.error = str(e)
|
||||
finally:
|
||||
state.finished_at = time.time()
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────
|
||||
# Phases de l'orchestration
|
||||
# ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
def _setup_chrome(session_id: str, machine_id: Optional[str]) -> None:
|
||||
"""Composer "ouvrir Chrome sur l'URL maquette" via le catalogue de réflexes.
|
||||
|
||||
Léa ne fait PAS un workflow appris pour cette étape : c'est une composition
|
||||
de primitives natives (réflexes du catalogue) + une saisie texte.
|
||||
|
||||
Séquence :
|
||||
1. réflexe `sys_run` (Win+R) ← gesture_catalog
|
||||
2. type "chrome.exe <URL>" ← saisie atomique
|
||||
3. réflexe `nav_enter` (Entrée) ← gesture_catalog
|
||||
"""
|
||||
from agent_chat.gesture_catalog import get_gesture_catalog
|
||||
|
||||
catalog = get_gesture_catalog()
|
||||
show_desktop = catalog.get_by_id("win_minimize_all") # Win+D — minimise tout (Léa incl.)
|
||||
sys_run = catalog.get_by_id("sys_run")
|
||||
nav_enter = catalog.get_by_id("nav_enter")
|
||||
if sys_run is None or nav_enter is None or show_desktop is None:
|
||||
raise RuntimeError("Réflexes catalogue manquants : win_minimize_all / sys_run / nav_enter")
|
||||
|
||||
actions = [
|
||||
show_desktop.to_replay_action(), # réflexe Win+D — Léa se réduit complètement
|
||||
{
|
||||
"action_id": f"setup_wait_desktop_{uuid.uuid4().hex[:6]}",
|
||||
"type": "wait",
|
||||
"duration_ms": 400,
|
||||
"intention": "Attendre que le bureau soit affiché",
|
||||
},
|
||||
sys_run.to_replay_action(), # réflexe Win+R
|
||||
{
|
||||
"action_id": f"setup_wait_{uuid.uuid4().hex[:6]}",
|
||||
"type": "wait",
|
||||
"duration_ms": 800,
|
||||
"intention": "Attendre que la boîte Exécuter soit prête",
|
||||
},
|
||||
{
|
||||
"action_id": f"setup_typeurl_{uuid.uuid4().hex[:6]}",
|
||||
"type": "type",
|
||||
"text": f"chrome.exe {MAQUETTE_URL}",
|
||||
"intention": "Taper la commande Chrome + URL maquette",
|
||||
},
|
||||
nav_enter.to_replay_action(), # réflexe Entrée
|
||||
{
|
||||
"action_id": f"setup_wait_load_{uuid.uuid4().hex[:6]}",
|
||||
"type": "wait",
|
||||
"duration_ms": 3500,
|
||||
"intention": "Attendre le chargement de la maquette",
|
||||
},
|
||||
]
|
||||
payload = {
|
||||
"actions": actions,
|
||||
"session_id": session_id,
|
||||
"task_description": "Setup démo GHT — composition réflexes (sys_run + type + nav_enter)",
|
||||
}
|
||||
if machine_id:
|
||||
payload["machine_id"] = machine_id
|
||||
resp = _post("/api/v1/traces/stream/replay/raw", payload, timeout=20)
|
||||
replay_id = resp.get("replay_id")
|
||||
if not replay_id:
|
||||
raise RuntimeError(f"setup_chrome : pas de replay_id ({resp})")
|
||||
# Setup Chrome ≈ 13s observé (Win+D + Win+R + type URL + Enter + wait 3500ms),
|
||||
# mais le PC peut être chargé → 60s donne de la marge.
|
||||
_wait_replay_done(replay_id, timeout_s=60)
|
||||
|
||||
|
||||
def _extract_patient_list(session_id: str, machine_id: Optional[str], limit: int) -> List[str]:
|
||||
"""Lance une action extract_table seule pour lire la liste des IPP."""
|
||||
actions = [
|
||||
{
|
||||
"action_id": f"extract_table_{uuid.uuid4().hex[:6]}",
|
||||
"type": "extract_table",
|
||||
"parameters": {
|
||||
"output_var": "patients_list",
|
||||
"pattern": IPP_PATTERN,
|
||||
"limit": limit,
|
||||
},
|
||||
"intention": "Lire la liste des IPP visible à l'écran",
|
||||
},
|
||||
]
|
||||
payload = {
|
||||
"actions": actions,
|
||||
"session_id": session_id,
|
||||
"task_description": "Extraction liste patients GHT",
|
||||
}
|
||||
if machine_id:
|
||||
payload["machine_id"] = machine_id
|
||||
resp = _post("/api/v1/traces/stream/replay/raw", payload, timeout=15)
|
||||
replay_id = resp.get("replay_id")
|
||||
if not replay_id:
|
||||
raise RuntimeError(f"extract_table : pas de replay_id ({resp})")
|
||||
final = _wait_replay_done(replay_id, timeout_s=20)
|
||||
return list(final.get("variables", {}).get("patients_list") or [])
|
||||
|
||||
|
||||
def _process_dossiers(state: OrchestrationState, session_id: str, machine_id: Optional[str]) -> None:
|
||||
"""Boucle : pour chaque IPP, lance le workflow Urgence_unit."""
|
||||
for i, ipp in enumerate(state.patients):
|
||||
state.current_step = f"process_dossier_{i+1}_of_{len(state.patients)}"
|
||||
result = DossierResult(ipp=ipp)
|
||||
try:
|
||||
payload = {
|
||||
"workflow_id": URGENCE_WORKFLOW_ID,
|
||||
"session_id": session_id,
|
||||
"variables": {"patient_id": ipp},
|
||||
}
|
||||
if machine_id:
|
||||
payload["machine_id"] = machine_id
|
||||
resp = _post("/api/v1/traces/stream/replay", payload, timeout=20)
|
||||
replay_id = resp.get("replay_id")
|
||||
if not replay_id:
|
||||
raise RuntimeError(f"replay_id manquant ({resp})")
|
||||
final = _wait_replay_done(replay_id, timeout_s=180)
|
||||
t2a = final.get("variables", {}).get("t2a_result") or {}
|
||||
result.decision = t2a.get("decision")
|
||||
result.decision_court = t2a.get("decision_court")
|
||||
result.confiance = t2a.get("confiance")
|
||||
result.duree_passage_heures = t2a.get("duree_passage_heures")
|
||||
result.concordance = t2a.get("concordance")
|
||||
except Exception as e:
|
||||
result.error = str(e)
|
||||
logger.warning("Dossier %s : erreur %s", ipp, e)
|
||||
state.results.append(result)
|
||||
state.progress = i + 1
|
||||
|
||||
|
||||
def _wait_replay_done(replay_id: str, timeout_s: int = 60, poll_s: float = 1.0) -> Dict[str, Any]:
|
||||
"""Poll /replay/<id> jusqu'à status terminal."""
|
||||
deadline = time.time() + timeout_s
|
||||
last = {}
|
||||
while time.time() < deadline:
|
||||
try:
|
||||
last = _get(f"/api/v1/traces/stream/replay/{replay_id}", timeout=5)
|
||||
except Exception as e:
|
||||
logger.warning("poll replay %s : %s", replay_id, e)
|
||||
status = last.get("status", "")
|
||||
if status in ("done", "completed", "finished", "error", "cancelled", "paused_need_help"):
|
||||
return last
|
||||
time.sleep(poll_s)
|
||||
raise TimeoutError(f"replay {replay_id} non terminé après {timeout_s}s (status={last.get('status')})")
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────
|
||||
# Synthèse finale
|
||||
# ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
def _build_synthese(state: OrchestrationState) -> str:
|
||||
"""Construit le message de synthèse posté dans le chat à la fin."""
|
||||
n = len(state.results)
|
||||
if n == 0:
|
||||
return "Aucun dossier traité."
|
||||
n_uhcd = sum(1 for r in state.results if r.decision == "REQUALIFICATION_HOSPITALISATION")
|
||||
n_forfait = sum(1 for r in state.results if r.decision == "FORFAIT_URGENCE")
|
||||
n_concord = sum(1 for r in state.results if r.concordance is True)
|
||||
lines = [f"✅ Terminé. {n} dossier(s) traité(s) : {n_forfait} forfait(s) urgences, {n_uhcd} UHCD."]
|
||||
if any(r.concordance is not None for r in state.results):
|
||||
lines.append(f"Concordance vérité-terrain : {n_concord}/{n}.")
|
||||
lines.append("")
|
||||
for r in state.results:
|
||||
if r.error:
|
||||
lines.append(f" • {r.ipp} : ❌ erreur — {r.error}")
|
||||
continue
|
||||
decision_label = r.decision_court or r.decision or "—"
|
||||
conf = f"confiance {r.confiance}" if r.confiance else ""
|
||||
duree = f"{r.duree_passage_heures:.1f}h" if r.duree_passage_heures else ""
|
||||
concord_mark = ""
|
||||
if r.concordance is True:
|
||||
concord_mark = " ✓"
|
||||
elif r.concordance is False:
|
||||
concord_mark = " ⚠ écart vérité-terrain"
|
||||
details = ", ".join(x for x in (conf, duree) if x)
|
||||
lines.append(f" • {r.ipp} : {decision_label}{concord_mark}" + (f" ({details})" if details else ""))
|
||||
return "\n".join(lines)
|
||||
3
agent_rust/lea_uia/.gitignore
vendored
Normal file
3
agent_rust/lea_uia/.gitignore
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
target/
|
||||
**/target/
|
||||
|
||||
384
agent_rust/lea_uia/Cargo.lock
generated
Normal file
384
agent_rust/lea_uia/Cargo.lock
generated
Normal file
@@ -0,0 +1,384 @@
|
||||
# This file is automatically @generated by Cargo.
|
||||
# It is not intended for manual editing.
|
||||
version = 4
|
||||
|
||||
[[package]]
|
||||
name = "anstream"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d"
|
||||
dependencies = [
|
||||
"anstyle",
|
||||
"anstyle-parse",
|
||||
"anstyle-query",
|
||||
"anstyle-wincon",
|
||||
"colorchoice",
|
||||
"is_terminal_polyfill",
|
||||
"utf8parse",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstyle"
|
||||
version = "1.0.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000"
|
||||
|
||||
[[package]]
|
||||
name = "anstyle-parse"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e"
|
||||
dependencies = [
|
||||
"utf8parse",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstyle-query"
|
||||
version = "1.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
|
||||
dependencies = [
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstyle-wincon"
|
||||
version = "3.0.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
|
||||
dependencies = [
|
||||
"anstyle",
|
||||
"once_cell_polyfill",
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap"
|
||||
version = "4.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b193af5b67834b676abd72466a96c1024e6a6ad978a1f484bd90b85c94041351"
|
||||
dependencies = [
|
||||
"clap_builder",
|
||||
"clap_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap_builder"
|
||||
version = "4.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f"
|
||||
dependencies = [
|
||||
"anstream",
|
||||
"anstyle",
|
||||
"clap_lex",
|
||||
"strsim",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap_derive"
|
||||
version = "4.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1110bd8a634a1ab8cb04345d8d878267d57c3cf1b38d91b71af6686408bbca6a"
|
||||
dependencies = [
|
||||
"heck",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap_lex"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9"
|
||||
|
||||
[[package]]
|
||||
name = "colorchoice"
|
||||
version = "1.0.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570"
|
||||
|
||||
[[package]]
|
||||
name = "heck"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
|
||||
|
||||
[[package]]
|
||||
name = "is_terminal_polyfill"
|
||||
version = "1.70.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695"
|
||||
|
||||
[[package]]
|
||||
name = "itoa"
|
||||
version = "1.0.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
|
||||
|
||||
[[package]]
|
||||
name = "lea_uia"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"clap",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"windows",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"
|
||||
|
||||
[[package]]
|
||||
name = "once_cell_polyfill"
|
||||
version = "1.70.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.106"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934"
|
||||
dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.45"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.228"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"
|
||||
dependencies = [
|
||||
"serde_core",
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_core"
|
||||
version = "1.0.228"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"
|
||||
dependencies = [
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_derive"
|
||||
version = "1.0.228"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_json"
|
||||
version = "1.0.149"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86"
|
||||
dependencies = [
|
||||
"itoa",
|
||||
"memchr",
|
||||
"serde",
|
||||
"serde_core",
|
||||
"zmij",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "strsim"
|
||||
version = "0.11.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "2.0.117"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "unicode-ident"
|
||||
version = "1.0.24"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
|
||||
|
||||
[[package]]
|
||||
name = "utf8parse"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
|
||||
|
||||
[[package]]
|
||||
name = "windows"
|
||||
version = "0.59.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7f919aee0a93304be7f62e8e5027811bbba96bcb1de84d6618be56e43f8a32a1"
|
||||
dependencies = [
|
||||
"windows-core",
|
||||
"windows-targets",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-core"
|
||||
version = "0.59.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "810ce18ed2112484b0d4e15d022e5f598113e220c53e373fb31e67e21670c1ce"
|
||||
dependencies = [
|
||||
"windows-implement",
|
||||
"windows-interface",
|
||||
"windows-result",
|
||||
"windows-strings",
|
||||
"windows-targets",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-implement"
|
||||
version = "0.59.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "83577b051e2f49a058c308f17f273b570a6a758386fc291b5f6a934dd84e48c1"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-interface"
|
||||
version = "0.59.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-link"
|
||||
version = "0.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a"
|
||||
|
||||
[[package]]
|
||||
name = "windows-link"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
|
||||
|
||||
[[package]]
|
||||
name = "windows-result"
|
||||
version = "0.3.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6"
|
||||
dependencies = [
|
||||
"windows-link 0.1.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-strings"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "87fa48cc5d406560701792be122a10132491cff9d0aeb23583cc2dcafc847319"
|
||||
dependencies = [
|
||||
"windows-link 0.1.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-sys"
|
||||
version = "0.61.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc"
|
||||
dependencies = [
|
||||
"windows-link 0.2.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-targets"
|
||||
version = "0.53.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3"
|
||||
dependencies = [
|
||||
"windows-link 0.2.1",
|
||||
"windows_aarch64_gnullvm",
|
||||
"windows_aarch64_msvc",
|
||||
"windows_i686_gnu",
|
||||
"windows_i686_gnullvm",
|
||||
"windows_i686_msvc",
|
||||
"windows_x86_64_gnu",
|
||||
"windows_x86_64_gnullvm",
|
||||
"windows_x86_64_msvc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows_aarch64_gnullvm"
|
||||
version = "0.53.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53"
|
||||
|
||||
[[package]]
|
||||
name = "windows_aarch64_msvc"
|
||||
version = "0.53.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_gnu"
|
||||
version = "0.53.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_gnullvm"
|
||||
version = "0.53.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_msvc"
|
||||
version = "0.53.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_gnu"
|
||||
version = "0.53.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_gnullvm"
|
||||
version = "0.53.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_msvc"
|
||||
version = "0.53.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650"
|
||||
|
||||
[[package]]
|
||||
name = "zmij"
|
||||
version = "1.0.21"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa"
|
||||
34
agent_rust/lea_uia/Cargo.toml
Normal file
34
agent_rust/lea_uia/Cargo.toml
Normal file
@@ -0,0 +1,34 @@
|
||||
[package]
|
||||
name = "lea_uia"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
authors = ["Dom <dom@rpa-vision-v3>"]
|
||||
description = "Helper Windows UI Automation pour Léa (agent RPA V3)"
|
||||
license = "Proprietary"
|
||||
|
||||
[[bin]]
|
||||
name = "lea_uia"
|
||||
path = "src/main.rs"
|
||||
|
||||
[dependencies]
|
||||
clap = { version = "4.5", features = ["derive"] }
|
||||
serde = { version = "1.0", features = ["derive"] }
|
||||
serde_json = "1.0"
|
||||
|
||||
[target.'cfg(windows)'.dependencies]
|
||||
windows = { version = "0.59", features = [
|
||||
"Win32_Foundation",
|
||||
"Win32_System_Com",
|
||||
"Win32_System_Ole",
|
||||
"Win32_System_Variant",
|
||||
"Win32_UI_Accessibility",
|
||||
"Win32_UI_WindowsAndMessaging",
|
||||
"Win32_Graphics_Gdi",
|
||||
] }
|
||||
|
||||
[profile.release]
|
||||
opt-level = "z" # Taille minimale
|
||||
lto = true # Link-time optimization
|
||||
codegen-units = 1 # Meilleure optimisation
|
||||
strip = true # Retirer les symboles
|
||||
panic = "abort" # Pas d'unwinding → binaire plus petit
|
||||
564
agent_rust/lea_uia/src/main.rs
Normal file
564
agent_rust/lea_uia/src/main.rs
Normal file
@@ -0,0 +1,564 @@
|
||||
// lea_uia — Helper Windows UI Automation pour Léa
|
||||
//
|
||||
// Binaire standalone qui expose 3 commandes UIA :
|
||||
// query → retourne l'élément UIA à une position (x, y)
|
||||
// find → retrouve un élément par son chemin logique
|
||||
// capture → liste les éléments visibles (debug)
|
||||
//
|
||||
// Communication avec l'agent Python via stdin/stdout JSON.
|
||||
// Tous les appels sont non-bloquants et retournent du JSON structuré.
|
||||
//
|
||||
// Sur Linux (développement) : retourne des stubs d'erreur.
|
||||
// Sur Windows : utilise UIAutomationCore via `windows-rs`.
|
||||
|
||||
use clap::{Parser, Subcommand};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Parser)]
|
||||
#[command(name = "lea_uia")]
|
||||
#[command(about = "Helper UI Automation pour Léa", long_about = None)]
|
||||
#[command(version)]
|
||||
struct Cli {
|
||||
#[command(subcommand)]
|
||||
command: Commands,
|
||||
}
|
||||
|
||||
#[derive(Subcommand)]
|
||||
enum Commands {
|
||||
/// Retourner l'élément UIA à une position donnée (x, y en pixels écran)
|
||||
Query {
|
||||
/// Coordonnée X (pixels)
|
||||
#[arg(long)]
|
||||
x: i32,
|
||||
/// Coordonnée Y (pixels)
|
||||
#[arg(long)]
|
||||
y: i32,
|
||||
/// Inclure la hiérarchie des parents (peut être lent)
|
||||
#[arg(long, default_value_t = true)]
|
||||
with_parents: bool,
|
||||
},
|
||||
/// Rechercher un élément par son chemin logique ou son nom
|
||||
Find {
|
||||
/// Nom de l'élément (Name property)
|
||||
#[arg(long)]
|
||||
name: Option<String>,
|
||||
/// Type de contrôle (Button, Edit, MenuItem, etc.)
|
||||
#[arg(long)]
|
||||
control_type: Option<String>,
|
||||
/// AutomationId
|
||||
#[arg(long)]
|
||||
automation_id: Option<String>,
|
||||
/// Limite la recherche à cette fenêtre (titre exact)
|
||||
#[arg(long)]
|
||||
window: Option<String>,
|
||||
/// Timeout en millisecondes
|
||||
#[arg(long, default_value_t = 2000)]
|
||||
timeout_ms: u32,
|
||||
},
|
||||
/// Lister tous les éléments visibles de la fenêtre active (debug)
|
||||
Capture {
|
||||
/// Profondeur maximale de l'arbre
|
||||
#[arg(long, default_value_t = 3)]
|
||||
max_depth: u32,
|
||||
},
|
||||
/// Vérifier que UIA est disponible et fonctionnel
|
||||
Health,
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// Modèles de sortie JSON
|
||||
// =========================================================================
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone)]
|
||||
struct UiaElement {
|
||||
/// Nom visible de l'élément
|
||||
name: String,
|
||||
/// Type de contrôle (Button, Edit, MenuItem, Window, ...)
|
||||
control_type: String,
|
||||
/// Classe Windows (Edit, Static, #32770, ...)
|
||||
class_name: String,
|
||||
/// AutomationId (ID interne, parfois vide)
|
||||
automation_id: String,
|
||||
/// Rectangle absolu [x1, y1, x2, y2] en pixels écran
|
||||
bounding_rect: [i32; 4],
|
||||
/// Est-ce que l'élément est activable
|
||||
is_enabled: bool,
|
||||
/// Est-ce que l'élément est visible
|
||||
is_offscreen: bool,
|
||||
/// Hiérarchie des parents (chemin logique)
|
||||
#[serde(skip_serializing_if = "Vec::is_empty")]
|
||||
parent_path: Vec<ParentHint>,
|
||||
/// Process owning this element
|
||||
#[serde(skip_serializing_if = "String::is_empty")]
|
||||
process_name: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone)]
|
||||
struct ParentHint {
|
||||
name: String,
|
||||
control_type: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug)]
|
||||
#[serde(tag = "status")]
|
||||
enum UiaResponse {
|
||||
#[serde(rename = "ok")]
|
||||
Ok {
|
||||
element: Option<UiaElement>,
|
||||
#[serde(skip_serializing_if = "Vec::is_empty")]
|
||||
elements: Vec<UiaElement>,
|
||||
elapsed_ms: u64,
|
||||
},
|
||||
#[serde(rename = "not_found")]
|
||||
NotFound {
|
||||
reason: String,
|
||||
elapsed_ms: u64,
|
||||
},
|
||||
#[serde(rename = "error")]
|
||||
Error {
|
||||
message: String,
|
||||
code: String,
|
||||
},
|
||||
#[serde(rename = "unavailable")]
|
||||
Unavailable {
|
||||
reason: String,
|
||||
},
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// Implémentation Windows
|
||||
// =========================================================================
|
||||
|
||||
#[cfg(windows)]
|
||||
mod uia_impl {
|
||||
use super::*;
|
||||
use std::time::Instant;
|
||||
use windows::Win32::Foundation::POINT;
|
||||
use windows::Win32::System::Com::{
|
||||
CoCreateInstance, CoInitializeEx, CoUninitialize, CLSCTX_INPROC_SERVER,
|
||||
COINIT_APARTMENTTHREADED,
|
||||
};
|
||||
use windows::Win32::UI::Accessibility::{
|
||||
CUIAutomation, IUIAutomation, IUIAutomationElement, IUIAutomationTreeWalker,
|
||||
};
|
||||
|
||||
struct ComGuard;
|
||||
impl ComGuard {
|
||||
fn new() -> windows::core::Result<Self> {
|
||||
unsafe {
|
||||
let hr = CoInitializeEx(None, COINIT_APARTMENTTHREADED);
|
||||
if hr.is_err() {
|
||||
// RPC_E_CHANGED_MODE : le thread est déjà initialisé → OK
|
||||
let code = hr.0 as u32;
|
||||
if code != 0x80010106 {
|
||||
return Err(windows::core::Error::from(hr));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(Self)
|
||||
}
|
||||
}
|
||||
impl Drop for ComGuard {
|
||||
fn drop(&mut self) {
|
||||
unsafe { CoUninitialize() };
|
||||
}
|
||||
}
|
||||
|
||||
fn get_automation() -> windows::core::Result<IUIAutomation> {
|
||||
unsafe { CoCreateInstance(&CUIAutomation, None, CLSCTX_INPROC_SERVER) }
|
||||
}
|
||||
|
||||
fn element_to_struct(
|
||||
element: &IUIAutomationElement,
|
||||
with_parents: bool,
|
||||
) -> windows::core::Result<UiaElement> {
|
||||
let mut result = UiaElement {
|
||||
name: String::new(),
|
||||
control_type: String::new(),
|
||||
class_name: String::new(),
|
||||
automation_id: String::new(),
|
||||
bounding_rect: [0, 0, 0, 0],
|
||||
is_enabled: false,
|
||||
is_offscreen: true,
|
||||
parent_path: Vec::new(),
|
||||
process_name: String::new(),
|
||||
};
|
||||
|
||||
unsafe {
|
||||
if let Ok(name) = element.CurrentName() {
|
||||
result.name = name.to_string();
|
||||
}
|
||||
if let Ok(ct) = element.CurrentLocalizedControlType() {
|
||||
result.control_type = ct.to_string();
|
||||
}
|
||||
if let Ok(cn) = element.CurrentClassName() {
|
||||
result.class_name = cn.to_string();
|
||||
}
|
||||
if let Ok(aid) = element.CurrentAutomationId() {
|
||||
result.automation_id = aid.to_string();
|
||||
}
|
||||
if let Ok(rect) = element.CurrentBoundingRectangle() {
|
||||
result.bounding_rect = [rect.left, rect.top, rect.right, rect.bottom];
|
||||
}
|
||||
if let Ok(enabled) = element.CurrentIsEnabled() {
|
||||
result.is_enabled = enabled.as_bool();
|
||||
}
|
||||
if let Ok(offscreen) = element.CurrentIsOffscreen() {
|
||||
result.is_offscreen = offscreen.as_bool();
|
||||
}
|
||||
if with_parents {
|
||||
// Remonter la hiérarchie jusqu'à la Window root
|
||||
if let Ok(automation) = get_automation() {
|
||||
let walker = automation.ControlViewWalker();
|
||||
if let Ok(walker) = walker {
|
||||
let mut current = element.clone();
|
||||
for _ in 0..10 {
|
||||
match walker.GetParentElement(¤t) {
|
||||
Ok(parent) => {
|
||||
let name = parent
|
||||
.CurrentName()
|
||||
.map(|n| n.to_string())
|
||||
.unwrap_or_default();
|
||||
let ct = parent
|
||||
.CurrentLocalizedControlType()
|
||||
.map(|c| c.to_string())
|
||||
.unwrap_or_default();
|
||||
if name.is_empty() && ct.is_empty() {
|
||||
break;
|
||||
}
|
||||
result.parent_path.insert(
|
||||
0,
|
||||
ParentHint {
|
||||
name,
|
||||
control_type: ct,
|
||||
},
|
||||
);
|
||||
current = parent;
|
||||
}
|
||||
Err(_) => break,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
pub fn query_at_point(x: i32, y: i32, with_parents: bool) -> UiaResponse {
|
||||
let start = Instant::now();
|
||||
let _com = match ComGuard::new() {
|
||||
Ok(g) => g,
|
||||
Err(e) => {
|
||||
return UiaResponse::Error {
|
||||
message: format!("CoInitializeEx: {}", e),
|
||||
code: "com_init_failed".into(),
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let automation = match get_automation() {
|
||||
Ok(a) => a,
|
||||
Err(e) => {
|
||||
return UiaResponse::Error {
|
||||
message: format!("CUIAutomation: {}", e),
|
||||
code: "automation_failed".into(),
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let point = POINT { x, y };
|
||||
let element = unsafe { automation.ElementFromPoint(point) };
|
||||
match element {
|
||||
Ok(el) => match element_to_struct(&el, with_parents) {
|
||||
Ok(e) => UiaResponse::Ok {
|
||||
element: Some(e),
|
||||
elements: Vec::new(),
|
||||
elapsed_ms: start.elapsed().as_millis() as u64,
|
||||
},
|
||||
Err(e) => UiaResponse::Error {
|
||||
message: format!("element_to_struct: {}", e),
|
||||
code: "extract_failed".into(),
|
||||
},
|
||||
},
|
||||
Err(_) => UiaResponse::NotFound {
|
||||
reason: format!("Aucun élément UIA à ({}, {})", x, y),
|
||||
elapsed_ms: start.elapsed().as_millis() as u64,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub fn find_element(
|
||||
name: Option<String>,
|
||||
_control_type: Option<String>,
|
||||
_automation_id: Option<String>,
|
||||
_window: Option<String>,
|
||||
_timeout_ms: u32,
|
||||
) -> UiaResponse {
|
||||
let start = Instant::now();
|
||||
let _com = match ComGuard::new() {
|
||||
Ok(g) => g,
|
||||
Err(e) => {
|
||||
return UiaResponse::Error {
|
||||
message: format!("CoInitializeEx: {}", e),
|
||||
code: "com_init_failed".into(),
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let automation = match get_automation() {
|
||||
Ok(a) => a,
|
||||
Err(e) => {
|
||||
return UiaResponse::Error {
|
||||
message: format!("CUIAutomation: {}", e),
|
||||
code: "automation_failed".into(),
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let root = match unsafe { automation.GetRootElement() } {
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
return UiaResponse::Error {
|
||||
message: format!("GetRootElement: {}", e),
|
||||
code: "root_failed".into(),
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// Recherche simple par parcours d'arbre (MVP)
|
||||
// L'arbre UIA peut être énorme → on limite la profondeur
|
||||
if let Some(target_name) = name {
|
||||
let walker = unsafe { automation.ControlViewWalker() };
|
||||
if let Ok(walker) = walker {
|
||||
if let Some(found) =
|
||||
walk_and_find(&walker, &root, &target_name, 0, 6, &_control_type, &_automation_id)
|
||||
{
|
||||
match element_to_struct(&found, true) {
|
||||
Ok(e) => {
|
||||
return UiaResponse::Ok {
|
||||
element: Some(e),
|
||||
elements: Vec::new(),
|
||||
elapsed_ms: start.elapsed().as_millis() as u64,
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
return UiaResponse::Error {
|
||||
message: format!("element_to_struct: {}", e),
|
||||
code: "extract_failed".into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
UiaResponse::NotFound {
|
||||
reason: "Aucun élément trouvé".into(),
|
||||
elapsed_ms: start.elapsed().as_millis() as u64,
|
||||
}
|
||||
}
|
||||
|
||||
/// Parcours récursif de l'arbre UIA pour trouver un élément par nom
|
||||
fn walk_and_find(
|
||||
walker: &IUIAutomationTreeWalker,
|
||||
element: &IUIAutomationElement,
|
||||
target_name: &str,
|
||||
depth: u32,
|
||||
max_depth: u32,
|
||||
target_control_type: &Option<String>,
|
||||
target_automation_id: &Option<String>,
|
||||
) -> Option<IUIAutomationElement> {
|
||||
if depth > max_depth {
|
||||
return None;
|
||||
}
|
||||
|
||||
// Tester l'élément courant
|
||||
unsafe {
|
||||
if let Ok(name) = element.CurrentName() {
|
||||
if name.to_string() == target_name {
|
||||
// Vérifier les filtres additionnels
|
||||
let mut matches = true;
|
||||
if let Some(ct) = target_control_type {
|
||||
if let Ok(local_ct) = element.CurrentLocalizedControlType() {
|
||||
if !local_ct.to_string().to_lowercase().contains(&ct.to_lowercase()) {
|
||||
matches = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
if matches {
|
||||
if let Some(aid) = target_automation_id {
|
||||
if let Ok(local_aid) = element.CurrentAutomationId() {
|
||||
if local_aid.to_string() != *aid {
|
||||
matches = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if matches {
|
||||
return Some(element.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Parcourir les enfants
|
||||
if let Ok(first_child) = walker.GetFirstChildElement(element) {
|
||||
let mut current = first_child;
|
||||
loop {
|
||||
if let Some(found) = walk_and_find(
|
||||
walker,
|
||||
¤t,
|
||||
target_name,
|
||||
depth + 1,
|
||||
max_depth,
|
||||
target_control_type,
|
||||
target_automation_id,
|
||||
) {
|
||||
return Some(found);
|
||||
}
|
||||
match walker.GetNextSiblingElement(¤t) {
|
||||
Ok(next) => current = next,
|
||||
Err(_) => break,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
pub fn capture_tree(_max_depth: u32) -> UiaResponse {
|
||||
let start = Instant::now();
|
||||
let _com = match ComGuard::new() {
|
||||
Ok(g) => g,
|
||||
Err(e) => {
|
||||
return UiaResponse::Error {
|
||||
message: format!("CoInitializeEx: {}", e),
|
||||
code: "com_init_failed".into(),
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let automation = match get_automation() {
|
||||
Ok(a) => a,
|
||||
Err(e) => {
|
||||
return UiaResponse::Error {
|
||||
message: format!("CUIAutomation: {}", e),
|
||||
code: "automation_failed".into(),
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let focused = unsafe { automation.GetFocusedElement() };
|
||||
match focused {
|
||||
Ok(el) => match element_to_struct(&el, true) {
|
||||
Ok(e) => UiaResponse::Ok {
|
||||
element: Some(e),
|
||||
elements: Vec::new(),
|
||||
elapsed_ms: start.elapsed().as_millis() as u64,
|
||||
},
|
||||
Err(e) => UiaResponse::Error {
|
||||
message: format!("element_to_struct: {}", e),
|
||||
code: "extract_failed".into(),
|
||||
},
|
||||
},
|
||||
Err(e) => UiaResponse::Error {
|
||||
message: format!("GetFocusedElement: {}", e),
|
||||
code: "focused_failed".into(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub fn health_check() -> UiaResponse {
|
||||
let _com = match ComGuard::new() {
|
||||
Ok(g) => g,
|
||||
Err(e) => {
|
||||
return UiaResponse::Unavailable {
|
||||
reason: format!("COM init failed: {}", e),
|
||||
}
|
||||
}
|
||||
};
|
||||
match get_automation() {
|
||||
Ok(_) => UiaResponse::Ok {
|
||||
element: None,
|
||||
elements: Vec::new(),
|
||||
elapsed_ms: 0,
|
||||
},
|
||||
Err(e) => UiaResponse::Unavailable {
|
||||
reason: format!("UIA not available: {}", e),
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// Stub Linux (pour développement et tests)
|
||||
// =========================================================================
|
||||
|
||||
#[cfg(not(windows))]
|
||||
mod uia_impl {
|
||||
use super::*;
|
||||
|
||||
pub fn query_at_point(_x: i32, _y: i32, _with_parents: bool) -> UiaResponse {
|
||||
UiaResponse::Unavailable {
|
||||
reason: "UIA n'est disponible que sur Windows".into(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn find_element(
|
||||
_name: Option<String>,
|
||||
_control_type: Option<String>,
|
||||
_automation_id: Option<String>,
|
||||
_window: Option<String>,
|
||||
_timeout_ms: u32,
|
||||
) -> UiaResponse {
|
||||
UiaResponse::Unavailable {
|
||||
reason: "UIA n'est disponible que sur Windows".into(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn capture_tree(_max_depth: u32) -> UiaResponse {
|
||||
UiaResponse::Unavailable {
|
||||
reason: "UIA n'est disponible que sur Windows".into(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn health_check() -> UiaResponse {
|
||||
UiaResponse::Unavailable {
|
||||
reason: "UIA n'est disponible que sur Windows".into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// Main
|
||||
// =========================================================================
|
||||
|
||||
fn main() {
|
||||
let cli = Cli::parse();
|
||||
|
||||
let response = match cli.command {
|
||||
Commands::Query {
|
||||
x,
|
||||
y,
|
||||
with_parents,
|
||||
} => uia_impl::query_at_point(x, y, with_parents),
|
||||
Commands::Find {
|
||||
name,
|
||||
control_type,
|
||||
automation_id,
|
||||
window,
|
||||
timeout_ms,
|
||||
} => uia_impl::find_element(name, control_type, automation_id, window, timeout_ms),
|
||||
Commands::Capture { max_depth } => uia_impl::capture_tree(max_depth),
|
||||
Commands::Health => uia_impl::health_check(),
|
||||
};
|
||||
|
||||
// Sortie JSON sur stdout
|
||||
match serde_json::to_string(&response) {
|
||||
Ok(json) => println!("{}", json),
|
||||
Err(e) => {
|
||||
eprintln!("{{\"status\":\"error\",\"message\":\"JSON serialization: {}\"}}", e);
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -27,7 +27,7 @@ if platform.system() == "Windows":
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
AGENT_VERSION = "1.0.0"
|
||||
AGENT_VERSION = os.environ.get("RPA_AGENT_VERSION", "1.0.2")
|
||||
|
||||
# Identifiant unique de la machine (utilisé pour le multi-machine)
|
||||
# Configurable via variable d'environnement, sinon auto-généré depuis hostname + OS
|
||||
@@ -40,14 +40,29 @@ MACHINE_ID = os.environ.get(
|
||||
BASE_DIR = Path(__file__).resolve().parent
|
||||
|
||||
# Endpoint du serveur Streaming (port 5005)
|
||||
# SERVER_URL contient TOUJOURS /api/v1 à la fin (convention unifiée).
|
||||
SERVER_URL = os.getenv("RPA_SERVER_URL", "http://localhost:5005/api/v1")
|
||||
# Base sans /api/v1 — pour les routes à la racine (/health)
|
||||
SERVER_BASE = SERVER_URL.rsplit("/api/v1", 1)[0]
|
||||
UPLOAD_ENDPOINT = f"{SERVER_URL}/traces/upload"
|
||||
STREAMING_ENDPOINT = f"{SERVER_URL}/traces/stream"
|
||||
|
||||
# Host Ollama — SÉPARÉ du serveur RPA.
|
||||
# Ollama tourne en local sur la machine serveur, jamais exposé via le reverse proxy.
|
||||
# Défaut : localhost (exécution locale ou accès LAN direct).
|
||||
OLLAMA_HOST = os.getenv("RPA_OLLAMA_HOST", "localhost")
|
||||
|
||||
# Token d'authentification API (doit correspondre au token du serveur)
|
||||
# Configurable via variable d'environnement RPA_API_TOKEN
|
||||
API_TOKEN = os.environ.get("RPA_API_TOKEN", "")
|
||||
|
||||
# --- Orchestrateur Léa-first (agent-chat Linux) ---
|
||||
# Endpoint racine du service agent-chat qui héberge POST /api/learn/start
|
||||
# (P1-LEA-SHADOW). Configurable via RPA_AGENT_CHAT_URL.
|
||||
# Défaut : localhost:5004 (même machine en dev). En POC clinique, doit
|
||||
# pointer vers le DGX Spark (ex. http://agent-chat.dgx-local:5004).
|
||||
AGENT_CHAT_URL = os.environ.get("RPA_AGENT_CHAT_URL", "http://localhost:5004")
|
||||
|
||||
# Paramètres de session
|
||||
MAX_SESSION_DURATION_S = 60 * 60 # 1 heure
|
||||
SESSIONS_ROOT = BASE_DIR / "sessions"
|
||||
@@ -67,6 +82,38 @@ BLUR_SENSITIVE = os.environ.get("RPA_BLUR_SENSITIVE", "true").lower() in ("true"
|
||||
# Configurable via variable d'environnement pour permettre l'ajustement
|
||||
LOG_RETENTION_DAYS = int(os.environ.get("RPA_LOG_RETENTION_DAYS", "180"))
|
||||
|
||||
# Remontée automatique des logs vers le serveur (push-log-DGX).
|
||||
# Diagnostic des postes clinique SANS AnyDesk : les logs (déjà écrits sur disque)
|
||||
# sont poussés au serveur, rangés par machine_id, consultables au dashboard.
|
||||
# Défaut PRUDENT = désactivé : on l'active poste par poste via config.txt /
|
||||
# variable d'environnement, sans rebuild de l'installateur.
|
||||
LOG_SHIP_ENABLED = os.environ.get("RPA_LOG_SHIP_ENABLED", "false").lower() in (
|
||||
"true", "1", "yes",
|
||||
)
|
||||
# Intervalle de flush du buffer de logs (secondes).
|
||||
LOG_SHIP_INTERVAL_S = float(os.environ.get("RPA_LOG_SHIP_INTERVAL_S", "30"))
|
||||
|
||||
# Mise à jour silencieuse du client Léa (DETTE-022 v2).
|
||||
# Le client interroge le serveur (GET /api/v1/agents/update/check), télécharge
|
||||
# le ZIP en staging et vérifie le SHA256. Le SWAP réel des fichiers / l'édition
|
||||
# de Lea.bat / le redémarrage restent RÉSERVÉS RÉVISION HUMAINE (voir
|
||||
# network/updater.py : stubs apply_update / write_boot_ok_marker).
|
||||
# Défaut PRUDENT = désactivé : activé poste par poste via config.txt / variable
|
||||
# d'environnement, sans rebuild de l'installateur (même esprit que LOG_SHIP).
|
||||
AUTO_UPDATE_ENABLED = os.environ.get("RPA_AUTO_UPDATE_ENABLED", "false").lower() in (
|
||||
"true", "1", "yes", "on",
|
||||
)
|
||||
# Intervalle entre deux interrogations serveur pour une MAJ (secondes).
|
||||
# Défaut 1 h : une MAJ n'est jamais urgente ; on interroge peu pour ne pas
|
||||
# charger le réseau clinique. Le check ne fait de toute façon aucun swap.
|
||||
AUTO_UPDATE_INTERVAL_S = float(os.environ.get("RPA_AUTO_UPDATE_INTERVAL_S", "3600"))
|
||||
# Dossier de STAGING des ZIP d'update (jamais les fichiers vivants). Équivalent
|
||||
# de `Lea_next\\`. Sous LOCALAPPDATA en prod Windows, sinon à côté de l'agent.
|
||||
AUTO_UPDATE_STAGING_DIR = os.environ.get(
|
||||
"RPA_AUTO_UPDATE_STAGING_DIR",
|
||||
str(BASE_DIR / "_update_staging"),
|
||||
)
|
||||
|
||||
# Monitoring
|
||||
PERF_MONITOR_INTERVAL_S = 30
|
||||
LOGS_DIR = BASE_DIR / "logs"
|
||||
|
||||
82
agent_v0/agent_v1/core/anchor_catalog.py
Normal file
82
agent_v0/agent_v1/core/anchor_catalog.py
Normal file
@@ -0,0 +1,82 @@
|
||||
"""Catalog d'ancres visuelles — Phase 1 standalone.
|
||||
|
||||
Ce module fournit un catalog Python (pas YAML) listant les trios
|
||||
(window_title, anchor_label, target_label) connus pour lesquels la
|
||||
résolution par triangulation visuelle est applicable.
|
||||
|
||||
Phase 1 : non branché au runtime, prouvé sur fixtures par
|
||||
`tests/unit/test_anchor_relative.py`.
|
||||
|
||||
Edition simple : ajouter une entrée à `ANCHOR_ENTRIES`.
|
||||
Validation : `find_entry_for_title(title)` retourne la première entrée
|
||||
dont un `title_patterns` matche (case-insensitive, substring).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
|
||||
# Catalog des entrées d'ancres visuelles connues.
|
||||
#
|
||||
# Format d'une entrée :
|
||||
# id (str) : identifiant stable pour audit
|
||||
# title_patterns (tuple) : sous-chaines case-insensitive du titre fenêtre
|
||||
# anchor_label (list) : labels d'ancres a essayer dans l'ordre (FR puis EN)
|
||||
# target_label (str) : libelle cible (ex. "Enregistrer")
|
||||
# geometry_hint (dict) :
|
||||
# region (str) : indicatif ("bottom-right", "bottom-center", ...)
|
||||
# min_x_norm/min_y_norm/max_x_norm/max_y_norm (float) : zone valide
|
||||
# (normalisée 0..1 sur la fenêtre/écran)
|
||||
# offset_from_anchor (dict) : {"x_px": int, "y_px": int} delta ancre→cible
|
||||
ANCHOR_ENTRIES: List[Dict[str, Any]] = [
|
||||
{
|
||||
"id": "notepad_save_as_enregistrer",
|
||||
"title_patterns": ("enregistrer sous", "save as"),
|
||||
"anchor_label": ["Annuler", "Cancel"],
|
||||
"target_label": "Enregistrer",
|
||||
"geometry_hint": {
|
||||
"region": "bottom-right",
|
||||
"min_x_norm": 0.55,
|
||||
"min_y_norm": 0.75,
|
||||
"max_x_norm": 1.0,
|
||||
"max_y_norm": 1.0,
|
||||
"offset_from_anchor": {"x_px": -100, "y_px": 0},
|
||||
},
|
||||
},
|
||||
{
|
||||
"id": "notepad_unsaved_changes_enregistrer",
|
||||
"title_patterns": ("bloc-notes", "notepad"),
|
||||
"anchor_label": ["Ne pas enregistrer", "Don't Save"],
|
||||
"target_label": "Enregistrer",
|
||||
"geometry_hint": {
|
||||
"region": "bottom-center",
|
||||
"min_x_norm": 0.30,
|
||||
"min_y_norm": 0.50,
|
||||
"max_x_norm": 0.85,
|
||||
"max_y_norm": 1.0,
|
||||
"offset_from_anchor": {"x_px": -120, "y_px": 0},
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def find_entry_for_title(title: str) -> Optional[Dict[str, Any]]:
|
||||
"""Retourne la première entrée dont un title_pattern matche (substring CI).
|
||||
|
||||
Args:
|
||||
title: titre de fenêtre courant (ex. "Enregistrer sous").
|
||||
|
||||
Returns:
|
||||
L'entrée catalog matchante, ou None si aucun match.
|
||||
Aucun raise — l'absence de match est un cas normal.
|
||||
"""
|
||||
if not title:
|
||||
return None
|
||||
title_lower = title.lower()
|
||||
for entry in ANCHOR_ENTRIES:
|
||||
patterns = entry.get("title_patterns") or ()
|
||||
for pat in patterns:
|
||||
if pat and pat.lower() in title_lower:
|
||||
return entry
|
||||
return None
|
||||
292
agent_v0/agent_v1/core/anchor_relative.py
Normal file
292
agent_v0/agent_v1/core/anchor_relative.py
Normal file
@@ -0,0 +1,292 @@
|
||||
"""Localisation par triangulation depuis une ancre visuelle.
|
||||
|
||||
Module standalone Phase 1 — non branché au runtime.
|
||||
|
||||
Principe : étant donnée une ancre texte fiable (ex. "Annuler"),
|
||||
localiser une cible voisine ("Enregistrer") par offset géométrique.
|
||||
Validation optionnelle par cross-check du label cible.
|
||||
|
||||
Détecteur injectable (`detector=`) pour faciliter les tests offline ;
|
||||
au runtime (Phase 2), on injectera `ActionExecutorV1._find_text_on_screen`.
|
||||
|
||||
Pas de dépendance nouvelle. Pas de VLM, pas d'UIA, pas de persistance.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Callable, Dict, Optional, Tuple
|
||||
|
||||
# Type alias : un détecteur prend (screenshot_b64, label) et retourne
|
||||
# (x_px, y_px) ou None.
|
||||
DetectorFn = Callable[[str, str], Optional[Tuple[int, int]]]
|
||||
|
||||
|
||||
@dataclass
|
||||
class AnchorMatch:
|
||||
"""Résultat d'une recherche par ancre relative.
|
||||
|
||||
Tous les champs sont remplis même si `found=False` (zéros pour les
|
||||
coordonnées, reason explicite, evidence pour audit).
|
||||
"""
|
||||
|
||||
found: bool
|
||||
target_x_pct: float
|
||||
target_y_pct: float
|
||||
anchor_x_pct: float
|
||||
anchor_y_pct: float
|
||||
confidence: float
|
||||
reason: str
|
||||
evidence: Dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
def _default_detector(screenshot_b64: str, label: str) -> Optional[Tuple[int, int]]:
|
||||
"""Détecteur OCR par défaut : rendu TTF + cv2.matchTemplate.
|
||||
|
||||
Reprend la logique de `ActionExecutorV1._find_text_on_screen`
|
||||
(executor.py:3277) sans dépendre de l'instance ActionExecutorV1
|
||||
(qui amène mss/pynput inutiles ici).
|
||||
"""
|
||||
try:
|
||||
from PIL import Image, ImageDraw, ImageFont
|
||||
import cv2
|
||||
import numpy as np
|
||||
except ImportError:
|
||||
return None
|
||||
|
||||
if not label or not screenshot_b64:
|
||||
return None
|
||||
|
||||
try:
|
||||
img_bytes = base64.b64decode(screenshot_b64)
|
||||
img_array = np.frombuffer(img_bytes, dtype=np.uint8)
|
||||
screenshot_bgr = cv2.imdecode(img_array, cv2.IMREAD_COLOR)
|
||||
if screenshot_bgr is None:
|
||||
return None
|
||||
gray = cv2.cvtColor(screenshot_bgr, cv2.COLOR_BGR2GRAY)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
font_paths = [
|
||||
"C:/Windows/Fonts/arial.ttf",
|
||||
"C:/Windows/Fonts/segoeui.ttf",
|
||||
"C:/Windows/Fonts/tahoma.ttf",
|
||||
"/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf",
|
||||
"/usr/share/fonts/truetype/liberation/LiberationSans-Regular.ttf",
|
||||
]
|
||||
|
||||
def _get_font(size: int):
|
||||
for fp in font_paths:
|
||||
try:
|
||||
return ImageFont.truetype(fp, size)
|
||||
except (OSError, IOError):
|
||||
continue
|
||||
return ImageFont.load_default()
|
||||
|
||||
best_match: Optional[Tuple[int, int]] = None
|
||||
best_val = 0.0
|
||||
threshold = 0.75
|
||||
|
||||
for font_size in (14, 16, 18, 20, 22, 24, 12, 26, 28, 10):
|
||||
font = _get_font(font_size)
|
||||
tmp = Image.new("L", (1, 1), 255)
|
||||
tmp_draw = ImageDraw.Draw(tmp)
|
||||
bbox = tmp_draw.textbbox((0, 0), label, font=font)
|
||||
text_w = bbox[2] - bbox[0] + 6
|
||||
text_h = bbox[3] - bbox[1] + 6
|
||||
if text_w <= 0 or text_h <= 0:
|
||||
continue
|
||||
if text_w >= gray.shape[1] or text_h >= gray.shape[0]:
|
||||
continue
|
||||
text_img = Image.new("L", (text_w, text_h), 255)
|
||||
draw = ImageDraw.Draw(text_img)
|
||||
draw.text((3, 3), label, fill=0, font=font)
|
||||
template = np.array(text_img)
|
||||
result = cv2.matchTemplate(gray, template, cv2.TM_CCOEFF_NORMED)
|
||||
_, max_val, _, max_loc = cv2.minMaxLoc(result)
|
||||
if max_val > best_val:
|
||||
best_val = max_val
|
||||
best_match = (
|
||||
max_loc[0] + template.shape[1] // 2,
|
||||
max_loc[1] + template.shape[0] // 2,
|
||||
)
|
||||
if max_val > 0.75:
|
||||
break
|
||||
|
||||
if best_match and best_val >= threshold:
|
||||
return best_match
|
||||
return None
|
||||
|
||||
|
||||
def _try_detect(
|
||||
detector: DetectorFn,
|
||||
screenshot_b64: str,
|
||||
labels: Any,
|
||||
) -> Tuple[Optional[Tuple[int, int]], str]:
|
||||
"""Essaye chaque label de la liste (ou string unique) jusqu'à un hit.
|
||||
|
||||
Retourne (position_px, label_qui_a_matche) ou (None, "").
|
||||
"""
|
||||
if isinstance(labels, str):
|
||||
labels_list = [labels]
|
||||
else:
|
||||
labels_list = list(labels or [])
|
||||
for label in labels_list:
|
||||
pos = detector(screenshot_b64, label)
|
||||
if pos:
|
||||
return pos, label
|
||||
return None, ""
|
||||
|
||||
|
||||
def _is_in_zone(
|
||||
x_norm: float,
|
||||
y_norm: float,
|
||||
geometry_hint: Dict[str, Any],
|
||||
) -> bool:
|
||||
"""Vérifie que (x_norm, y_norm) tombe dans la zone du geometry_hint."""
|
||||
min_x = float(geometry_hint.get("min_x_norm", 0.0))
|
||||
max_x = float(geometry_hint.get("max_x_norm", 1.0))
|
||||
min_y = float(geometry_hint.get("min_y_norm", 0.0))
|
||||
max_y = float(geometry_hint.get("max_y_norm", 1.0))
|
||||
return (min_x <= x_norm <= max_x) and (min_y <= y_norm <= max_y)
|
||||
|
||||
|
||||
def find_target_via_anchor(
|
||||
anchor_label: Any,
|
||||
target_label: str,
|
||||
geometry_hint: Dict[str, Any],
|
||||
screenshot_b64: str,
|
||||
screen_width: int,
|
||||
screen_height: int,
|
||||
detector: Optional[DetectorFn] = None,
|
||||
cross_check_target: bool = True,
|
||||
) -> AnchorMatch:
|
||||
"""Localise `target_label` par triangulation depuis `anchor_label`.
|
||||
|
||||
Args:
|
||||
anchor_label: label (str) ou liste de labels essayés dans l'ordre
|
||||
(ex. ["Annuler", "Cancel"] pour fallback FR→EN).
|
||||
target_label: libellé cible (ex. "Enregistrer"). Utilisé pour le
|
||||
cross-check uniquement.
|
||||
geometry_hint: dict décrivant la zone valide pour l'ancre et
|
||||
l'offset ancre→cible. Voir `anchor_catalog.ANCHOR_ENTRIES`
|
||||
pour le format exact.
|
||||
screenshot_b64: capture encodée base64 (JPEG/PNG).
|
||||
screen_width: largeur de référence en pixels (écran ou fenêtre).
|
||||
screen_height: hauteur de référence en pixels.
|
||||
detector: callable (b64, label) → (x_px, y_px) | None. Si None,
|
||||
utilise un détecteur OCR par défaut (rendu TTF + cv2).
|
||||
Pour les tests, injecter un mock.
|
||||
cross_check_target: si True (défaut), tente de détecter aussi
|
||||
`target_label` près de la position candidate et ajuste la
|
||||
confidence en conséquence.
|
||||
|
||||
Returns:
|
||||
AnchorMatch toujours retourné (jamais None). `found=False` si
|
||||
l'ancre n'est pas trouvée ou hors zone ; `reason` explique.
|
||||
"""
|
||||
det = detector or _default_detector
|
||||
ev: Dict[str, Any] = {
|
||||
"anchor_candidates_tried": (
|
||||
list(anchor_label) if not isinstance(anchor_label, str) else [anchor_label]
|
||||
),
|
||||
"target_label": target_label,
|
||||
"geometry_hint": geometry_hint,
|
||||
}
|
||||
|
||||
# 1. Détection ancre (FR puis EN)
|
||||
anchor_px, matched_anchor_label = _try_detect(det, screenshot_b64, anchor_label)
|
||||
if not anchor_px:
|
||||
return AnchorMatch(
|
||||
found=False,
|
||||
target_x_pct=0.0,
|
||||
target_y_pct=0.0,
|
||||
anchor_x_pct=0.0,
|
||||
anchor_y_pct=0.0,
|
||||
confidence=0.0,
|
||||
reason="anchor_not_found",
|
||||
evidence=ev,
|
||||
)
|
||||
|
||||
ax, ay = anchor_px
|
||||
anchor_x_pct = ax / float(screen_width) if screen_width else 0.0
|
||||
anchor_y_pct = ay / float(screen_height) if screen_height else 0.0
|
||||
ev["anchor_matched_label"] = matched_anchor_label
|
||||
ev["anchor_px"] = [ax, ay]
|
||||
ev["anchor_norm"] = [anchor_x_pct, anchor_y_pct]
|
||||
|
||||
# 2. Garde géométrique : ancre dans la zone autorisée
|
||||
if not _is_in_zone(anchor_x_pct, anchor_y_pct, geometry_hint):
|
||||
return AnchorMatch(
|
||||
found=False,
|
||||
target_x_pct=0.0,
|
||||
target_y_pct=0.0,
|
||||
anchor_x_pct=anchor_x_pct,
|
||||
anchor_y_pct=anchor_y_pct,
|
||||
confidence=0.0,
|
||||
reason="anchor_out_of_zone",
|
||||
evidence=ev,
|
||||
)
|
||||
|
||||
# 3. Déduction position cible par offset
|
||||
offset = geometry_hint.get("offset_from_anchor", {}) or {}
|
||||
dx = int(offset.get("x_px", 0))
|
||||
dy = int(offset.get("y_px", 0))
|
||||
target_x_px = ax + dx
|
||||
target_y_px = ay + dy
|
||||
target_x_pct = target_x_px / float(screen_width) if screen_width else 0.0
|
||||
target_y_pct = target_y_px / float(screen_height) if screen_height else 0.0
|
||||
ev["target_px_from_offset"] = [target_x_px, target_y_px]
|
||||
|
||||
if not (0.0 <= target_x_pct <= 1.0 and 0.0 <= target_y_pct <= 1.0):
|
||||
return AnchorMatch(
|
||||
found=False,
|
||||
target_x_pct=target_x_pct,
|
||||
target_y_pct=target_y_pct,
|
||||
anchor_x_pct=anchor_x_pct,
|
||||
anchor_y_pct=anchor_y_pct,
|
||||
confidence=0.0,
|
||||
reason="target_out_of_bounds",
|
||||
evidence=ev,
|
||||
)
|
||||
|
||||
# 4. Cross-check : tenter de détecter target_label
|
||||
confidence = 0.5 # ancre seule
|
||||
reason = "anchor_only"
|
||||
if cross_check_target and target_label:
|
||||
target_pos = det(screenshot_b64, target_label)
|
||||
if target_pos:
|
||||
tx, ty = target_pos
|
||||
dist_px = ((tx - target_x_px) ** 2 + (ty - target_y_px) ** 2) ** 0.5
|
||||
ev["target_detected_px"] = [tx, ty]
|
||||
ev["target_cross_check_dist_px"] = round(dist_px, 1)
|
||||
# Tolerance proche de l'offset (cf. design 2200 §3.2)
|
||||
if dist_px <= 50:
|
||||
# Cross-check OK : on raffine sur la position détectée
|
||||
target_x_px, target_y_px = tx, ty
|
||||
target_x_pct = tx / float(screen_width) if screen_width else 0.0
|
||||
target_y_pct = ty / float(screen_height) if screen_height else 0.0
|
||||
confidence = 0.85
|
||||
reason = "anchor_plus_target_cross_check"
|
||||
else:
|
||||
# target_label détecté mais loin de l'offset attendu : suspect.
|
||||
# On garde la position offset mais on dégrade confidence.
|
||||
confidence = 0.4
|
||||
reason = "anchor_ok_target_drift_high"
|
||||
else:
|
||||
# Cross-check absent : comportement documenté (cf. test 7).
|
||||
# On garde la position offset mais confidence reste à 0.5.
|
||||
ev["target_cross_check_dist_px"] = None
|
||||
reason = "anchor_only_target_not_visible"
|
||||
|
||||
return AnchorMatch(
|
||||
found=True,
|
||||
target_x_pct=target_x_pct,
|
||||
target_y_pct=target_y_pct,
|
||||
anchor_x_pct=anchor_x_pct,
|
||||
anchor_y_pct=anchor_y_pct,
|
||||
confidence=confidence,
|
||||
reason=reason,
|
||||
evidence=ev,
|
||||
)
|
||||
@@ -32,6 +32,7 @@ from pynput.keyboard import Key, KeyCode
|
||||
# Importation relative pour rester dans le module v1
|
||||
from ..vision.capturer import VisionCapturer
|
||||
from ..vision.system_info import get_screen_metadata
|
||||
from .log_safe import _sanitize_metadata
|
||||
# from ..monitoring.system import SystemMonitor
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -56,6 +57,8 @@ class EventCaptorV1:
|
||||
|
||||
# État des touches modificatrices
|
||||
self.modifiers = set()
|
||||
self._pending_standalone_win = False
|
||||
self._suppress_release_only_win_combo = False
|
||||
|
||||
# Tracking du focus fenêtre
|
||||
self.last_window = None
|
||||
@@ -178,8 +181,41 @@ class EventCaptorV1:
|
||||
"timestamp": now,
|
||||
}
|
||||
self._inject_screen_metadata(event)
|
||||
# Capturer le snapshot UIA à la position du clic (si helper dispo)
|
||||
# Non-bloquant : si UIA échoue, l'event est enrichi uniquement
|
||||
# des données vision comme aujourd'hui.
|
||||
self._inject_uia_snapshot(event, x, y)
|
||||
self.on_event(event)
|
||||
|
||||
def _inject_uia_snapshot(self, event: dict, x: int, y: int) -> None:
|
||||
"""Ajouter un uia_snapshot à l'événement si le helper UIA est dispo.
|
||||
|
||||
Appelle lea_uia.exe query --x N --y N en ~10-20ms.
|
||||
Fallback silencieux si le helper n'est pas dispo ou échoue.
|
||||
"""
|
||||
try:
|
||||
from .uia_helper import get_shared_helper
|
||||
helper = get_shared_helper()
|
||||
if not helper.available:
|
||||
return
|
||||
element = helper.query_at(int(x), int(y), with_parents=True)
|
||||
if element is None:
|
||||
return
|
||||
event["uia_snapshot"] = {
|
||||
"name": element.name,
|
||||
"control_type": element.control_type,
|
||||
"class_name": element.class_name,
|
||||
"automation_id": element.automation_id,
|
||||
"bounding_rect": list(element.bounding_rect),
|
||||
"is_enabled": element.is_enabled,
|
||||
"is_offscreen": element.is_offscreen,
|
||||
"parent_path": element.parent_path,
|
||||
}
|
||||
except Exception as e:
|
||||
# Non bloquant — on continue sans UIA
|
||||
import logging
|
||||
logging.getLogger(__name__).debug(f"UIA snapshot skip: {e}")
|
||||
|
||||
def _on_scroll(self, x, y, dx, dy):
|
||||
event = {
|
||||
"type": "mouse_scroll",
|
||||
@@ -294,6 +330,56 @@ class EventCaptorV1:
|
||||
return {"kind": "key", "name": key.name}
|
||||
return {"kind": "unknown", "str": str(key)}
|
||||
|
||||
@staticmethod
|
||||
def _raw_key_name(raw_key: Dict[str, Any]) -> Optional[str]:
|
||||
"""Nom lisible depuis un raw_key sérialisé."""
|
||||
if raw_key.get("kind") == "vk":
|
||||
char = raw_key.get("char")
|
||||
if char and len(str(char)) == 1:
|
||||
return str(char).lower()
|
||||
if raw_key.get("kind") == "key":
|
||||
name = raw_key.get("name")
|
||||
return str(name).lower() if name else None
|
||||
return None
|
||||
|
||||
def _emit_release_only_windows_combo(self) -> bool:
|
||||
"""Infère Win+<touche> si Windows/NoMachine n'a livré que les releases.
|
||||
|
||||
Certaines sessions ne remontent pas les press de Win+S via pynput,
|
||||
mais livrent ensuite release('s') puis release('cmd'). Sans cette
|
||||
inférence ciblée, le geste système est perdu et les releases polluent
|
||||
le prochain text_input.
|
||||
"""
|
||||
with self._text_lock:
|
||||
raw_keys = list(self._raw_key_buffer)
|
||||
if len(raw_keys) < 2:
|
||||
return False
|
||||
cmd_names = {"cmd", "cmd_l", "cmd_r"}
|
||||
last = raw_keys[-1]
|
||||
if last.get("action") != "release" or self._raw_key_name(last) not in cmd_names:
|
||||
return False
|
||||
combo_key = None
|
||||
for raw in reversed(raw_keys[:-1]):
|
||||
if raw.get("action") != "release":
|
||||
continue
|
||||
name = self._raw_key_name(raw)
|
||||
if name and name not in self._MODIFIER_KEY_NAMES:
|
||||
combo_key = name
|
||||
break
|
||||
if not combo_key:
|
||||
return False
|
||||
self._raw_key_buffer.clear()
|
||||
|
||||
event = {
|
||||
"type": "key_combo",
|
||||
"keys": ["win", combo_key],
|
||||
"raw_keys": raw_keys,
|
||||
"timestamp": time.time(),
|
||||
}
|
||||
self._inject_screen_metadata(event)
|
||||
self.on_event(event)
|
||||
return True
|
||||
|
||||
def _on_press(self, key):
|
||||
# TOUJOURS enregistrer le press brut dans le buffer raw_keys
|
||||
with self._text_lock:
|
||||
@@ -311,6 +397,7 @@ class EventCaptorV1:
|
||||
self.modifiers.add("shift")
|
||||
elif key in (Key.cmd, Key.cmd_l, Key.cmd_r):
|
||||
self.modifiers.add("win")
|
||||
self._pending_standalone_win = True
|
||||
|
||||
# --- Combos avec modificateur (sauf Shift seul) ---
|
||||
# Shift seul n'est pas un « vrai » modificateur pour les combos :
|
||||
@@ -336,6 +423,9 @@ class EventCaptorV1:
|
||||
# Ne PAS émettre de combo si c'est un modificateur seul
|
||||
# (ex: appui sur Ctrl sans autre touche = pas de combo)
|
||||
if key_name and key_name not in self._MODIFIER_KEY_NAMES:
|
||||
self._pending_standalone_win = False
|
||||
if "win" in self.modifiers:
|
||||
self._suppress_release_only_win_combo = True
|
||||
# Un combo interrompt la saisie texte en cours
|
||||
self._flush_text_buffer()
|
||||
# Attacher les raw_keys accumulés (press des modificateurs + press de la touche)
|
||||
@@ -367,6 +457,7 @@ class EventCaptorV1:
|
||||
- Enter / Tab : flush immédiat + émission de l'événement
|
||||
- Escape : vide le buffer sans émettre
|
||||
"""
|
||||
escape_raw_keys = None
|
||||
with self._text_lock:
|
||||
# --- Touches spéciales ---
|
||||
if key == Key.backspace:
|
||||
@@ -378,12 +469,14 @@ class EventCaptorV1:
|
||||
if key == Key.esc:
|
||||
# Annuler la saisie en cours
|
||||
self._text_buffer.clear()
|
||||
self._raw_key_buffer.clear()
|
||||
self._text_start_pos = None
|
||||
self._cancel_flush_timer()
|
||||
return
|
||||
escape_raw_keys = list(self._raw_key_buffer)
|
||||
self._raw_key_buffer.clear()
|
||||
# Émettre hors lock après le bloc critique.
|
||||
pass
|
||||
|
||||
if key in (Key.enter, Key.tab):
|
||||
elif key in (Key.enter, Key.tab):
|
||||
# Flush immédiat — on relâche le lock avant d'appeler
|
||||
# _flush_text_buffer (qui prend aussi le lock)
|
||||
pass # on sort du with et on flush après
|
||||
@@ -421,6 +514,18 @@ class EventCaptorV1:
|
||||
# Touche spéciale non gérée (F1, Insert, etc.) — on ignore
|
||||
return
|
||||
|
||||
if escape_raw_keys is not None:
|
||||
event = {
|
||||
"type": "key_combo",
|
||||
"keys": ["escape"],
|
||||
"timestamp": time.time(),
|
||||
}
|
||||
if escape_raw_keys:
|
||||
event["raw_keys"] = escape_raw_keys
|
||||
self._inject_screen_metadata(event)
|
||||
self.on_event(event)
|
||||
return
|
||||
|
||||
# Si on arrive ici, c'est Enter ou Tab → flush le buffer en cours
|
||||
# puis émettre le caractère spécial comme text_input séparé
|
||||
self._flush_text_buffer()
|
||||
@@ -518,6 +623,35 @@ class EventCaptorV1:
|
||||
**self._encode_key(key),
|
||||
})
|
||||
|
||||
if key in (Key.cmd, Key.cmd_l, Key.cmd_r) and self._suppress_release_only_win_combo:
|
||||
with self._text_lock:
|
||||
self._raw_key_buffer.clear()
|
||||
self._pending_standalone_win = False
|
||||
self._suppress_release_only_win_combo = False
|
||||
self.modifiers.discard("win")
|
||||
return
|
||||
|
||||
if key in (Key.cmd, Key.cmd_l, Key.cmd_r) and self._emit_release_only_windows_combo():
|
||||
self._pending_standalone_win = False
|
||||
self._suppress_release_only_win_combo = False
|
||||
self.modifiers.discard("win")
|
||||
return
|
||||
|
||||
if key in (Key.cmd, Key.cmd_l, Key.cmd_r) and self._pending_standalone_win:
|
||||
with self._text_lock:
|
||||
raw_keys = list(self._raw_key_buffer)
|
||||
self._raw_key_buffer.clear()
|
||||
event = {
|
||||
"type": "key_combo",
|
||||
"keys": ["win"],
|
||||
"raw_keys": raw_keys,
|
||||
"timestamp": time.time(),
|
||||
}
|
||||
self._inject_screen_metadata(event)
|
||||
self.on_event(event)
|
||||
self._pending_standalone_win = False
|
||||
self._suppress_release_only_win_combo = False
|
||||
|
||||
if key in (Key.ctrl, Key.ctrl_l, Key.ctrl_r):
|
||||
self.modifiers.discard("ctrl")
|
||||
elif key in (Key.alt, Key.alt_l, Key.alt_r):
|
||||
@@ -526,6 +660,8 @@ class EventCaptorV1:
|
||||
self.modifiers.discard("shift")
|
||||
elif key in (Key.cmd, Key.cmd_l, Key.cmd_r):
|
||||
self.modifiers.discard("win")
|
||||
self._pending_standalone_win = False
|
||||
self._suppress_release_only_win_combo = False
|
||||
|
||||
# ----------------------------------------------------------------
|
||||
# Métadonnées système
|
||||
@@ -541,7 +677,7 @@ class EventCaptorV1:
|
||||
metadata = get_screen_metadata()
|
||||
with self._screen_metadata_lock:
|
||||
self._screen_metadata = metadata
|
||||
logger.debug(f"Métadonnées système rafraîchies : {metadata}")
|
||||
logger.debug(f"Métadonnées système rafraîchies : {_sanitize_metadata(metadata)}")
|
||||
except Exception as e:
|
||||
logger.error(f"Erreur refresh métadonnées système : {e}")
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -15,6 +15,7 @@ Ref: docs/PLAN_ACTEUR_V1.md — Architecture MICRO (grounding + exécution)
|
||||
"""
|
||||
|
||||
import base64
|
||||
import io
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
@@ -73,6 +74,171 @@ class GroundingEngine:
|
||||
"""
|
||||
self._executor = executor
|
||||
|
||||
@staticmethod
|
||||
def _should_scope_to_active_window(target_spec: Dict[str, Any]) -> bool:
|
||||
"""Déterminer si le grounding doit être limité à la fenêtre active."""
|
||||
if str(target_spec.get("screen_scope", "")).strip().lower() == "full_screen":
|
||||
return False
|
||||
|
||||
by_role = str(target_spec.get("by_role", "")).strip().lower()
|
||||
if by_role in {"start_button"}:
|
||||
return False
|
||||
|
||||
has_anchor = bool(target_spec.get("anchor_image_base64"))
|
||||
context_hints = target_spec.get("context_hints") or {}
|
||||
has_window_or_text_hint = any(
|
||||
str(target_spec.get(key, "") or "").strip()
|
||||
for key in ("window_title", "by_text", "vlm_description")
|
||||
) or bool(str(context_hints.get("window_title", "") or "").strip())
|
||||
if has_anchor and not has_window_or_text_hint and not by_role:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _targets_lea_window(target_spec: Dict[str, Any]) -> bool:
|
||||
"""Déterminer si la cible pointe explicitement vers l'UI de Léa."""
|
||||
try:
|
||||
from ..ui.messages import est_fenetre_lea
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
context_hints = target_spec.get("context_hints") or {}
|
||||
hints = [
|
||||
target_spec.get("window_title", ""),
|
||||
context_hints.get("window_title", ""),
|
||||
target_spec.get("vlm_description", ""),
|
||||
target_spec.get("by_text", ""),
|
||||
]
|
||||
return any(est_fenetre_lea(str(hint)) for hint in hints if hint)
|
||||
|
||||
@staticmethod
|
||||
def _is_plausible_window_rect(
|
||||
rect: Optional[List[int]],
|
||||
title: str,
|
||||
screen_width: int,
|
||||
screen_height: int,
|
||||
) -> bool:
|
||||
"""Valider qu'un rect actif ressemble à une vraie fenêtre utilisable.
|
||||
|
||||
Rejette explicitement les zones système "bar-like" (taskbar, systray)
|
||||
et les titres inconnus/bruités. Le grounding ne doit jamais se
|
||||
contraindre à une zone non validée.
|
||||
"""
|
||||
if not rect or len(rect) != 4:
|
||||
return False
|
||||
|
||||
try:
|
||||
from ..ui.messages import est_fenetre_bruit
|
||||
except Exception:
|
||||
def est_fenetre_bruit(_title: str) -> bool:
|
||||
return not _title or _title.strip().lower() == "unknown_window"
|
||||
|
||||
w = rect[2] - rect[0]
|
||||
h = rect[3] - rect[1]
|
||||
title_clean = str(title or "").strip()
|
||||
if w <= 50 or h <= 50:
|
||||
return False
|
||||
title_lower = title_clean.lower()
|
||||
is_unknown_title = not title_clean or title_lower == "unknown_window"
|
||||
if not is_unknown_title and est_fenetre_bruit(title_clean):
|
||||
return False
|
||||
|
||||
# Une zone très plate, surtout en bas d'écran et très large, est
|
||||
# typiquement une barre des tâches / systray, pas une vraie fenêtre.
|
||||
# On réduit le seuil de hauteur à 120px pour ne pas rejeter les petits modaux.
|
||||
is_bar_like = (
|
||||
h < 120
|
||||
or (w > 0.9 * screen_width and h < 0.15 * screen_height)
|
||||
)
|
||||
|
||||
# Exception : si le titre contient un mot-clé de dialogue connu,
|
||||
# on considère que c'est plausible même si c'est petit.
|
||||
keywords = ["enregistrer sous", "save as", "voulez-vous", "confirm", "attention", "error", "erreur"]
|
||||
if any(k in title_lower for k in keywords):
|
||||
return h >= 80 # Un dialogue fait au moins 80px (titre + bouton)
|
||||
|
||||
return not is_bar_like
|
||||
|
||||
@staticmethod
|
||||
def _visual_scope_hints(target_spec: Dict[str, Any]) -> List[str]:
|
||||
"""Construire des indices textuels à chercher dans le crop fenêtre."""
|
||||
hints: List[str] = []
|
||||
raw_hints = [
|
||||
target_spec.get("window_title", ""),
|
||||
(target_spec.get("context_hints") or {}).get("window_title", ""),
|
||||
target_spec.get("by_text", ""),
|
||||
]
|
||||
for raw in raw_hints:
|
||||
text = str(raw or "").strip()
|
||||
if not text:
|
||||
continue
|
||||
text = text.lstrip("*").strip()
|
||||
variants = [text]
|
||||
for sep in (" – ", " - ", " — "):
|
||||
if sep in text:
|
||||
variants.extend(part.strip().lstrip("*") for part in text.split(sep))
|
||||
for variant in variants:
|
||||
if variant and len(variant) >= 3 and variant not in hints:
|
||||
hints.append(variant)
|
||||
return hints
|
||||
|
||||
@staticmethod
|
||||
def _server_rejects_text_fallback(raw: Optional[Dict[str, Any]]) -> bool:
|
||||
"""Dire si un rejet serveur doit bloquer le fallback texte local.
|
||||
|
||||
Un rejet explicite n'est pas un simple "non trouvé": le serveur a vu
|
||||
un candidat et l'a refusé pour une raison de qualité/zone. Refaire une
|
||||
recherche OCR large côté client contournerait ce garde-fou.
|
||||
"""
|
||||
if not raw or raw.get("resolved"):
|
||||
return False
|
||||
|
||||
reason = str(raw.get("reason") or "")
|
||||
method = str(raw.get("method") or "")
|
||||
return (
|
||||
method.startswith("rejected_")
|
||||
or reason.startswith("close_tab_")
|
||||
or reason.startswith("drift_")
|
||||
or "below_threshold" in reason
|
||||
)
|
||||
|
||||
def _window_crop_matches_target_visually(
|
||||
self,
|
||||
screenshot_b64: str,
|
||||
target_spec: Dict[str, Any],
|
||||
) -> bool:
|
||||
"""Vérifier visuellement qu'un crop contraint contient la bonne cible.
|
||||
|
||||
Principe: ne jamais faire confiance au rect système seul. Si aucun
|
||||
indice textuel n'est disponible, on laisse passer le crop plausible
|
||||
pour ne pas sur-bloquer les cibles purement iconiques.
|
||||
"""
|
||||
hints = self._visual_scope_hints(target_spec)
|
||||
if not hints:
|
||||
return True
|
||||
|
||||
finder = getattr(self._executor, "_find_text_on_screen", None)
|
||||
if not callable(finder):
|
||||
return True
|
||||
|
||||
for hint in hints:
|
||||
try:
|
||||
if finder(screenshot_b64, hint):
|
||||
logger.info(
|
||||
"Grounding fenêtre validé visuellement via '%s'",
|
||||
hint,
|
||||
)
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.debug("Validation visuelle du crop échouée pour '%s': %s", hint, e)
|
||||
logger.info(
|
||||
"Grounding plein écran : crop fenêtre rejeté par validation visuelle "
|
||||
"(hints=%s)",
|
||||
hints,
|
||||
)
|
||||
return False
|
||||
|
||||
def locate(
|
||||
self,
|
||||
server_url: str,
|
||||
@@ -126,28 +292,164 @@ class GroundingEngine:
|
||||
)
|
||||
|
||||
t_start = time.time()
|
||||
screenshot_b64 = self._executor._capture_screenshot_b64(max_width=0, quality=75)
|
||||
|
||||
window_rect = None
|
||||
active_title = ""
|
||||
if self._should_scope_to_active_window(target_spec):
|
||||
# ── Capture contrainte à la fenêtre active ──
|
||||
# Le grounding ne voit QUE la fenêtre attendue — pas la taskbar,
|
||||
# pas le systray, pas les autres apps. Comme un humain qui regarde
|
||||
# l'application sur laquelle il travaille.
|
||||
try:
|
||||
from ..window_info_crossplatform import get_active_window_rect
|
||||
from ..ui.messages import est_fenetre_lea
|
||||
|
||||
win_info = get_active_window_rect()
|
||||
if win_info and win_info.get("rect"):
|
||||
active_title = str(win_info.get("title", "") or "")
|
||||
if est_fenetre_lea(active_title) and not self._targets_lea_window(target_spec):
|
||||
logger.info(
|
||||
"Grounding plein écran : fenêtre active Léa ignorée pour "
|
||||
"cible externe (%s)",
|
||||
target_spec.get("by_text", "") or target_spec.get("by_role", ""),
|
||||
)
|
||||
win_info = None
|
||||
if win_info and win_info.get("rect"):
|
||||
r = win_info["rect"] # [left, top, right, bottom]
|
||||
if self._is_plausible_window_rect(r, active_title, screen_width, screen_height):
|
||||
w = r[2] - r[0]
|
||||
h = r[3] - r[1]
|
||||
window_rect = {
|
||||
"left": max(0, r[0]),
|
||||
"top": max(0, r[1]),
|
||||
"width": min(w, screen_width),
|
||||
"height": min(h, screen_height),
|
||||
}
|
||||
logger.info(
|
||||
f"Grounding contraint à la fenêtre : "
|
||||
f"{window_rect['width']}x{window_rect['height']} "
|
||||
f"à ({window_rect['left']}, {window_rect['top']})"
|
||||
)
|
||||
else:
|
||||
logger.info(
|
||||
"Grounding plein écran : rect actif rejeté "
|
||||
"(title='%s', rect=%s)",
|
||||
active_title,
|
||||
r,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug(f"Pas de window rect disponible : {e}")
|
||||
else:
|
||||
logger.info(
|
||||
"Grounding plein écran pour by_role='%s'",
|
||||
target_spec.get("by_role", ""),
|
||||
)
|
||||
|
||||
screenshot_b64 = self._capture_window_or_screen(window_rect)
|
||||
if window_rect and screenshot_b64:
|
||||
if not self._window_crop_matches_target_visually(screenshot_b64, target_spec):
|
||||
window_rect = None
|
||||
screenshot_b64 = self._capture_window_or_screen(None)
|
||||
if not screenshot_b64:
|
||||
return GroundingResult(
|
||||
found=False, detail="Capture screenshot échouée",
|
||||
elapsed_ms=(time.time() - t_start) * 1000,
|
||||
)
|
||||
|
||||
# Dimensions de la zone capturée (fenêtre ou écran entier)
|
||||
cap_w = window_rect["width"] if window_rect else screen_width
|
||||
cap_h = window_rect["height"] if window_rect else screen_height
|
||||
|
||||
skip_text_fallback_after_server_reject = False
|
||||
for strategy in strategies:
|
||||
if (
|
||||
strategy == "vlm_local"
|
||||
and skip_text_fallback_after_server_reject
|
||||
and target_spec.get("by_text")
|
||||
):
|
||||
by_text = target_spec.get("by_text", "")
|
||||
logger.info(
|
||||
"[GROUNDING] Rejet serveur explicite pour '%s' — "
|
||||
"skip fallback local hybrid_text_direct",
|
||||
by_text,
|
||||
)
|
||||
print(
|
||||
f" [GROUNDING] Rejet serveur explicite pour '{by_text}' "
|
||||
"→ pas de fallback texte local"
|
||||
)
|
||||
continue
|
||||
|
||||
result = self._try_strategy(
|
||||
strategy, server_url, screenshot_b64, target_spec,
|
||||
fallback_x, fallback_y, screen_width, screen_height,
|
||||
fallback_x, fallback_y, cap_w, cap_h,
|
||||
)
|
||||
if strategy == "server" and self._server_rejects_text_fallback(result.raw):
|
||||
skip_text_fallback_after_server_reject = True
|
||||
if result.found:
|
||||
# ── Conversion coords fenêtre → coords écran ──
|
||||
if window_rect:
|
||||
# Le grounding a retourné des coords relatives à la fenêtre
|
||||
# On les convertit en coords relatives à l'écran entier
|
||||
abs_x = window_rect["left"] + result.x_pct * cap_w
|
||||
abs_y = window_rect["top"] + result.y_pct * cap_h
|
||||
result.x_pct = abs_x / screen_width
|
||||
result.y_pct = abs_y / screen_height
|
||||
result.detail = f"{result.detail} [fenêtre {cap_w}x{cap_h}]"
|
||||
|
||||
result.elapsed_ms = (time.time() - t_start) * 1000
|
||||
return result
|
||||
|
||||
if target_spec.get("allow_position_fallback"):
|
||||
if 0.0 <= fallback_x <= 1.0 and 0.0 <= fallback_y <= 1.0:
|
||||
return GroundingResult(
|
||||
found=True,
|
||||
x_pct=fallback_x,
|
||||
y_pct=fallback_y,
|
||||
method="position_fallback",
|
||||
score=0.2,
|
||||
detail="fallback positionnel explicite",
|
||||
elapsed_ms=(time.time() - t_start) * 1000,
|
||||
)
|
||||
|
||||
return GroundingResult(
|
||||
found=False,
|
||||
detail=f"Toutes les stratégies ont échoué ({', '.join(strategies)})",
|
||||
elapsed_ms=(time.time() - t_start) * 1000,
|
||||
)
|
||||
|
||||
def _capture_window_or_screen(self, window_rect: Optional[Dict]) -> str:
|
||||
"""Capturer soit la fenêtre active (croppée), soit l'écran entier.
|
||||
|
||||
Si window_rect est fourni, capture uniquement cette zone.
|
||||
Sinon, capture l'écran entier (fallback).
|
||||
"""
|
||||
try:
|
||||
from PIL import Image
|
||||
import mss as mss_lib
|
||||
|
||||
with mss_lib.mss() as local_sct:
|
||||
if window_rect:
|
||||
# Capture de la zone fenêtre uniquement
|
||||
region = {
|
||||
"left": window_rect["left"],
|
||||
"top": window_rect["top"],
|
||||
"width": window_rect["width"],
|
||||
"height": window_rect["height"],
|
||||
}
|
||||
raw = local_sct.grab(region)
|
||||
else:
|
||||
# Fallback écran entier
|
||||
raw = local_sct.grab(local_sct.monitors[1])
|
||||
|
||||
img = Image.frombytes("RGB", raw.size, raw.bgra, "raw", "BGRX")
|
||||
buffer = io.BytesIO()
|
||||
img.save(buffer, format="JPEG", quality=75)
|
||||
return base64.b64encode(buffer.getvalue()).decode("utf-8")
|
||||
except Exception as e:
|
||||
logger.warning(f"Capture échouée : {e}")
|
||||
# Fallback sur la méthode existante de l'executor
|
||||
return self._executor._capture_screenshot_b64(max_width=0, quality=75)
|
||||
|
||||
def _try_strategy(
|
||||
self,
|
||||
strategy: str,
|
||||
@@ -176,12 +478,25 @@ class GroundingEngine:
|
||||
detail=raw.get("matched_element", {}).get("label", ""),
|
||||
raw=raw,
|
||||
)
|
||||
if raw:
|
||||
return GroundingResult(
|
||||
found=False,
|
||||
method=raw.get("method", "server"),
|
||||
score=raw.get("score", 0.0),
|
||||
detail=raw.get("reason", "server: pas trouvé"),
|
||||
raw=raw,
|
||||
)
|
||||
|
||||
elif strategy == "template":
|
||||
anchor_b64 = target_spec.get("anchor_image_base64", "")
|
||||
if anchor_b64:
|
||||
raw = self._executor._template_match_anchor(
|
||||
screenshot_b64, anchor_b64, screen_width, screen_height,
|
||||
screenshot_b64,
|
||||
anchor_b64,
|
||||
screen_width,
|
||||
screen_height,
|
||||
fallback_x_pct=fallback_x,
|
||||
fallback_y_pct=fallback_y,
|
||||
)
|
||||
if raw and raw.get("resolved"):
|
||||
return GroundingResult(
|
||||
|
||||
48
agent_v0/agent_v1/core/log_safe.py
Normal file
48
agent_v0/agent_v1/core/log_safe.py
Normal file
@@ -0,0 +1,48 @@
|
||||
"""Helpers de logging PII-safe pour le client Léa (agent_v1).
|
||||
|
||||
Convention : ne jamais logger le contenu brut d'une variable utilisateur
|
||||
(texte tapé, titre de fenêtre, nom de workflow, réponse VLM, chemin fichier).
|
||||
Le remplacer par :
|
||||
- une longueur ou un hash court (corrélation de diagnostic sans révéler) ;
|
||||
- un dict de métadonnées filtré (sans titre / fenêtre active).
|
||||
|
||||
À importer dans tout module d'agent_v1 qui logge une donnée potentiellement
|
||||
sensible. Branche feat/push-log-dgx — DETTE-020 (assainissement à la source).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
|
||||
|
||||
def _title_hash(title: str) -> str:
|
||||
"""Hash SHA1 tronqué (8 hex) d'un titre.
|
||||
|
||||
Corrélation stable (même titre → même hash → « même popup re-détectée »)
|
||||
sans exposer le contenu. `errors="replace"` pour ne jamais lever sur un
|
||||
encodage exotique (titres Windows multi-langues).
|
||||
"""
|
||||
return hashlib.sha1((title or "").encode("utf-8", errors="replace")).hexdigest()[:8]
|
||||
|
||||
|
||||
# Clés de métadonnées susceptibles de contenir du contenu utilisateur (PII).
|
||||
_PII_METADATA_KEYS = ("title", "active_window", "window_title")
|
||||
|
||||
|
||||
def _sanitize_metadata(metadata: dict) -> dict:
|
||||
"""Copie d'un dict de métadonnées sans les clés porteuses de PII.
|
||||
|
||||
Garde les champs techniques (resolution, dpi, theme, langue…), retire
|
||||
titre / fenêtre active. Ne mute pas le dict d'origine.
|
||||
"""
|
||||
return {k: v for k, v in metadata.items() if k not in _PII_METADATA_KEYS}
|
||||
|
||||
|
||||
def _path_ext(path: str) -> str:
|
||||
"""Extension seule d'un chemin (ex. « .png »), sans nom ni dossier.
|
||||
|
||||
Un chemin peut nommer un patient ; l'extension suffit au diagnostic.
|
||||
Chaîne vide si pas de chemin ou pas d'extension.
|
||||
"""
|
||||
return os.path.splitext(path)[1] if path else ""
|
||||
@@ -85,6 +85,10 @@ class PolicyEngine:
|
||||
2. Si retry déjà fait → demander à l'acteur gemma4
|
||||
3. Selon gemma4 : SKIP, ABORT, ou SUPERVISE
|
||||
|
||||
**SÉCURITÉ** : si, pendant l'étape 1, le handler popup détecte un
|
||||
dialogue système Windows (UAC, CredUI, SmartScreen…), on bascule
|
||||
immédiatement en SUPERVISE. Cf. system_dialog_guard.py.
|
||||
|
||||
Args:
|
||||
action: L'action qui a échoué
|
||||
target_spec: La cible non trouvée
|
||||
@@ -96,6 +100,22 @@ class PolicyEngine:
|
||||
# ── Étape 1 : Tentative de fermeture popup (premier essai) ──
|
||||
if retry_count == 0:
|
||||
popup_handled = self._try_close_popup()
|
||||
|
||||
# Si le popup handler a détecté un dialogue système, on
|
||||
# bascule immédiatement en SUPERVISE — pas de retry, pas de
|
||||
# gemma4 : on rend la main à l'humain.
|
||||
if getattr(self._executor, "_system_dialog_pause", None):
|
||||
sd = self._executor._system_dialog_pause
|
||||
return PolicyDecision(
|
||||
decision=Decision.SUPERVISE,
|
||||
reason=(
|
||||
f"Dialogue système détecté ({sd.get('category', '?')}) — "
|
||||
f"refus d'interaction automatique"
|
||||
),
|
||||
action_taken="system_dialog_blocked",
|
||||
elapsed_ms=(time.time() - t_start) * 1000,
|
||||
)
|
||||
|
||||
if popup_handled:
|
||||
return PolicyDecision(
|
||||
decision=Decision.RETRY,
|
||||
|
||||
@@ -24,6 +24,8 @@ from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from .log_safe import _title_hash
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -168,8 +170,8 @@ class RecoveryEngine:
|
||||
from ..window_info_crossplatform import get_active_window_info
|
||||
active = get_active_window_info()
|
||||
active_title = active.get("title", "")
|
||||
logger.info(f"Recovery : Alt+F4 sur '{active_title}'")
|
||||
print(f" [RECOVERY] Alt+F4 — fermeture de '{active_title}'")
|
||||
logger.info(f"Recovery : Alt+F4 sur [title_hash={_title_hash(active_title)}]")
|
||||
print(f" [RECOVERY] Alt+F4 — fermeture de [title_hash={_title_hash(active_title)}]")
|
||||
except Exception:
|
||||
logger.info("Recovery : Alt+F4 (fenêtre active inconnue)")
|
||||
print(" [RECOVERY] Alt+F4 — fermeture fenêtre indésirable")
|
||||
@@ -182,7 +184,7 @@ class RecoveryEngine:
|
||||
return RecoveryResult(
|
||||
action_taken=RecoveryAction.CLOSE_WINDOW,
|
||||
success=True,
|
||||
detail=f"Alt+F4 exécuté sur '{active_title if 'active_title' in dir() else '?'}'",
|
||||
detail=f"Alt+F4 exécuté sur [title_hash={_title_hash(active_title) if 'active_title' in dir() else '?'}]",
|
||||
)
|
||||
|
||||
elif strategy == RecoveryAction.CLICK_AWAY:
|
||||
|
||||
448
agent_v0/agent_v1/core/system_dialog_guard.py
Normal file
448
agent_v0/agent_v1/core/system_dialog_guard.py
Normal file
@@ -0,0 +1,448 @@
|
||||
# agent_v1/core/system_dialog_guard.py
|
||||
"""
|
||||
Garde-fou sécurité : détection des dialogues système Windows critiques.
|
||||
|
||||
==============================================================================
|
||||
POURQUOI ?
|
||||
==============================================================================
|
||||
|
||||
Pendant un replay, si un dialogue UAC, CredUI (mot de passe Windows),
|
||||
SmartScreen ou une notification de sécurité Windows apparaît, Léa pourrait
|
||||
demander au VLM "quel bouton cliquer" et recevoir "Oui" en réponse.
|
||||
|
||||
→ **Léa cliquerait OUI sur une élévation UAC** → vecteur d'attaque ransomware.
|
||||
|
||||
Ce module fournit la détection de ces dialogues pour que l'exécuteur
|
||||
**ne clique JAMAIS dessus automatiquement**. La décision est renvoyée à
|
||||
l'humain (pause supervisée).
|
||||
|
||||
==============================================================================
|
||||
PRINCIPE
|
||||
==============================================================================
|
||||
|
||||
- **Faux positif tolérable** : on préfère pauser pour rien plutôt que cliquer
|
||||
sur un UAC.
|
||||
- **Faux négatif catastrophique** : mieux vaut être trop prudent.
|
||||
- **Multi-signal** : titre, ClassName UIA, nom de processus, parent_path.
|
||||
Un seul signal suffit à bloquer.
|
||||
- **Compatible Citrix** : les dialogues UAC d'un client Citrix apparaissent
|
||||
aussi dans la VM distante — la détection par classe UIA fonctionne.
|
||||
|
||||
==============================================================================
|
||||
PATTERNS DE DÉTECTION (ordre de criticité décroissant)
|
||||
==============================================================================
|
||||
|
||||
1. UAC Consent (élévation de privilèges)
|
||||
- ClassName : `$$$Secure UAP Dummy Window Class$$$`
|
||||
- Process : `consent.exe`
|
||||
- Titre : "Contrôle de compte d'utilisateur", "User Account Control"
|
||||
|
||||
2. CredUI (prompt mot de passe Windows)
|
||||
- ClassName : `Credential Dialog Xaml Host`
|
||||
- Process : `credentialuibroker.exe`, `credui.exe`
|
||||
- Titre : "Sécurité Windows", "Windows Security"
|
||||
|
||||
3. SmartScreen (protection contre applications inconnues)
|
||||
- Process : `smartscreen.exe`
|
||||
- Titre : "Windows a protégé votre ordinateur", "Windows protected your PC"
|
||||
|
||||
4. Windows Defender / Security Center
|
||||
- Process : `securityhealthhost.exe`, `msmpeng.exe`
|
||||
- Titre : "Sécurité Windows", "Windows Defender"
|
||||
|
||||
5. Signatures pilotes / driver install
|
||||
- Titre : "Installer ce pilote", "Driver signature"
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Dict, Optional, Tuple
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Catégories de dialogues système (pour logging + messages)
|
||||
# =============================================================================
|
||||
|
||||
|
||||
class SystemDialogCategory:
|
||||
"""Catégories de dialogues système à bloquer absolument."""
|
||||
UAC = "uac_consent" # Élévation de privilèges
|
||||
CREDUI = "windows_credential_prompt" # Prompt de mot de passe
|
||||
SMARTSCREEN = "smartscreen" # Protection SmartScreen
|
||||
DEFENDER = "windows_defender" # Alerte Windows Defender
|
||||
DRIVER = "driver_install" # Installation pilote signé
|
||||
SECURITY_TOAST = "security_toast" # Toast de sécurité Windows
|
||||
UNKNOWN_DIALOG = "unknown_system_dialog" # Dialogue #32770 sans app connue
|
||||
|
||||
|
||||
@dataclass
|
||||
class SystemDialogDetection:
|
||||
"""Résultat d'une analyse de dialogue système."""
|
||||
is_system_dialog: bool
|
||||
category: str = "" # Valeur de SystemDialogCategory
|
||||
matched_signal: str = "" # Ex: "class_name=Consent.exe"
|
||||
matched_value: str = "" # La valeur qui a matché
|
||||
reason: str = "" # Explication lisible
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"is_system_dialog": self.is_system_dialog,
|
||||
"category": self.category,
|
||||
"matched_signal": self.matched_signal,
|
||||
"matched_value": self.matched_value,
|
||||
"reason": self.reason,
|
||||
}
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Signatures de détection
|
||||
# =============================================================================
|
||||
|
||||
|
||||
# ClassName UIA (casse préservée — Windows exposées telle quelle par UIA).
|
||||
# Utilisées telles quelles puis en minuscules pour matcher avec souplesse.
|
||||
_CLASS_NAMES_SYSTEM = {
|
||||
# UAC Consent
|
||||
"$$$Secure UAP Dummy Window Class$$$": SystemDialogCategory.UAC,
|
||||
"Credential Dialog Xaml Host": SystemDialogCategory.CREDUI,
|
||||
# Windows Credential UI ancien nom
|
||||
"CredentialDialogXamlHost": SystemDialogCategory.CREDUI,
|
||||
}
|
||||
|
||||
# Nom de processus (comparaison insensible à la casse, .exe normalisé)
|
||||
_PROCESS_NAMES_SYSTEM = {
|
||||
"consent.exe": SystemDialogCategory.UAC,
|
||||
"credentialuibroker.exe": SystemDialogCategory.CREDUI,
|
||||
"credui.exe": SystemDialogCategory.CREDUI,
|
||||
"credwiz.exe": SystemDialogCategory.CREDUI,
|
||||
"smartscreen.exe": SystemDialogCategory.SMARTSCREEN,
|
||||
"securityhealthhost.exe": SystemDialogCategory.DEFENDER,
|
||||
"securityhealthui.exe": SystemDialogCategory.DEFENDER,
|
||||
"securityhealthsystray.exe": SystemDialogCategory.DEFENDER,
|
||||
"msmpeng.exe": SystemDialogCategory.DEFENDER,
|
||||
"windowsdefender.exe": SystemDialogCategory.DEFENDER,
|
||||
"msiexec.exe": SystemDialogCategory.DRIVER, # prompts pilotes signés
|
||||
"drvinst.exe": SystemDialogCategory.DRIVER,
|
||||
}
|
||||
|
||||
# Motifs titre (insensibles à la casse, regex avec word boundaries)
|
||||
# On ne matche pas les titres génériques trop larges pour limiter les faux
|
||||
# positifs sur OSIRIS/OBSIUS/MEDSPHERE.
|
||||
_TITLE_PATTERNS_SYSTEM: Tuple[Tuple[re.Pattern, str], ...] = (
|
||||
# UAC
|
||||
(re.compile(r"contr[oô]le\s+de\s+compte\s+d'?utilisateur", re.IGNORECASE),
|
||||
SystemDialogCategory.UAC),
|
||||
(re.compile(r"\buser\s+account\s+control\b", re.IGNORECASE),
|
||||
SystemDialogCategory.UAC),
|
||||
(re.compile(r"voulez-vous\s+autoriser\s+cette\s+application", re.IGNORECASE),
|
||||
SystemDialogCategory.UAC),
|
||||
(re.compile(r"do\s+you\s+want\s+to\s+allow\s+this\s+app", re.IGNORECASE),
|
||||
SystemDialogCategory.UAC),
|
||||
|
||||
# CredUI / Sécurité Windows
|
||||
(re.compile(r"\bs[eé]curit[eé]\s+windows\b", re.IGNORECASE),
|
||||
SystemDialogCategory.CREDUI),
|
||||
(re.compile(r"\bwindows\s+security\b", re.IGNORECASE),
|
||||
SystemDialogCategory.CREDUI),
|
||||
(re.compile(r"entrer\s+les\s+informations\s+d'?identification", re.IGNORECASE),
|
||||
SystemDialogCategory.CREDUI),
|
||||
(re.compile(r"enter\s+(?:your\s+)?credentials?", re.IGNORECASE),
|
||||
SystemDialogCategory.CREDUI),
|
||||
(re.compile(r"connectez-vous\s+[aà]\s+votre\s+compte", re.IGNORECASE),
|
||||
SystemDialogCategory.CREDUI),
|
||||
(re.compile(r"\bsign\s+in\s+to\s+your\s+account\b", re.IGNORECASE),
|
||||
SystemDialogCategory.CREDUI),
|
||||
|
||||
# SmartScreen
|
||||
(re.compile(r"windows\s+a\s+prot[eé]g[eé]", re.IGNORECASE),
|
||||
SystemDialogCategory.SMARTSCREEN),
|
||||
(re.compile(r"windows\s+protected\s+your\s+pc", re.IGNORECASE),
|
||||
SystemDialogCategory.SMARTSCREEN),
|
||||
(re.compile(r"\bsmartscreen\b", re.IGNORECASE),
|
||||
SystemDialogCategory.SMARTSCREEN),
|
||||
(re.compile(r"\b[eé]diteur\s+inconnu\b", re.IGNORECASE),
|
||||
SystemDialogCategory.SMARTSCREEN),
|
||||
(re.compile(r"\bunknown\s+publisher\b", re.IGNORECASE),
|
||||
SystemDialogCategory.SMARTSCREEN),
|
||||
|
||||
# Windows Defender
|
||||
(re.compile(r"windows\s+defender", re.IGNORECASE),
|
||||
SystemDialogCategory.DEFENDER),
|
||||
(re.compile(r"menace\s+d[eé]tect[eé]e", re.IGNORECASE),
|
||||
SystemDialogCategory.DEFENDER),
|
||||
(re.compile(r"threat\s+detected", re.IGNORECASE),
|
||||
SystemDialogCategory.DEFENDER),
|
||||
|
||||
# Driver
|
||||
(re.compile(r"installer\s+ce\s+pilote", re.IGNORECASE),
|
||||
SystemDialogCategory.DRIVER),
|
||||
(re.compile(r"install\s+this\s+driver", re.IGNORECASE),
|
||||
SystemDialogCategory.DRIVER),
|
||||
(re.compile(r"signature\s+num[eé]rique\s+du\s+pilote", re.IGNORECASE),
|
||||
SystemDialogCategory.DRIVER),
|
||||
)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Fonctions de détection
|
||||
# =============================================================================
|
||||
|
||||
|
||||
def _normalize_process(name: str) -> str:
|
||||
"""Normaliser un nom de processus pour comparaison."""
|
||||
if not name:
|
||||
return ""
|
||||
name = name.strip().lower()
|
||||
# Enlever le chemin éventuel
|
||||
if "\\" in name or "/" in name:
|
||||
name = name.replace("\\", "/").split("/")[-1]
|
||||
# Assurer suffixe .exe pour matcher le dictionnaire
|
||||
if not name.endswith(".exe") and name:
|
||||
# Les process_name peuvent venir sans .exe (psutil) — on ajoute
|
||||
# pour avoir une clé uniforme
|
||||
name_with_exe = name + ".exe"
|
||||
if name_with_exe in _PROCESS_NAMES_SYSTEM:
|
||||
return name_with_exe
|
||||
return name
|
||||
|
||||
|
||||
def _check_class_name(class_name: str) -> Optional[Tuple[str, str, str]]:
|
||||
"""Vérifier si un ClassName UIA matche un dialogue système.
|
||||
|
||||
Returns:
|
||||
(category, matched_class, reason) si match, None sinon.
|
||||
"""
|
||||
if not class_name:
|
||||
return None
|
||||
|
||||
# Match exact
|
||||
if class_name in _CLASS_NAMES_SYSTEM:
|
||||
cat = _CLASS_NAMES_SYSTEM[class_name]
|
||||
return (cat, class_name, f"ClassName UIA '{class_name}' = dialogue système {cat}")
|
||||
|
||||
# Match insensible à la casse + normalisation espaces
|
||||
cn_norm = class_name.strip()
|
||||
for known, cat in _CLASS_NAMES_SYSTEM.items():
|
||||
if cn_norm.lower() == known.lower():
|
||||
return (cat, class_name, f"ClassName UIA ~= '{known}' ({cat})")
|
||||
|
||||
# Détection souple UAC (il existe quelques variantes de la classe secure)
|
||||
if "secure uap" in class_name.lower() or "uap dummy" in class_name.lower():
|
||||
return (SystemDialogCategory.UAC, class_name,
|
||||
f"ClassName '{class_name}' contient 'Secure UAP' → UAC")
|
||||
|
||||
# Credential XAML Host
|
||||
if "credential" in class_name.lower() and "xaml" in class_name.lower():
|
||||
return (SystemDialogCategory.CREDUI, class_name,
|
||||
f"ClassName '{class_name}' contient Credential+Xaml → CredUI")
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _check_process_name(process_name: str) -> Optional[Tuple[str, str, str]]:
|
||||
"""Vérifier si un nom de processus est un dialogue système.
|
||||
|
||||
Returns:
|
||||
(category, matched_process, reason) si match, None sinon.
|
||||
"""
|
||||
if not process_name:
|
||||
return None
|
||||
|
||||
norm = _normalize_process(process_name)
|
||||
if norm in _PROCESS_NAMES_SYSTEM:
|
||||
cat = _PROCESS_NAMES_SYSTEM[norm]
|
||||
return (cat, process_name, f"Processus '{norm}' = {cat}")
|
||||
return None
|
||||
|
||||
|
||||
def _check_title(title: str) -> Optional[Tuple[str, str, str]]:
|
||||
"""Vérifier si un titre de fenêtre matche un dialogue système.
|
||||
|
||||
Returns:
|
||||
(category, matched_pattern, reason) si match, None sinon.
|
||||
"""
|
||||
if not title:
|
||||
return None
|
||||
|
||||
for pattern, cat in _TITLE_PATTERNS_SYSTEM:
|
||||
m = pattern.search(title)
|
||||
if m:
|
||||
return (cat, m.group(0),
|
||||
f"Titre '{title[:60]}' matche '{pattern.pattern}' → {cat}")
|
||||
return None
|
||||
|
||||
|
||||
def is_system_dialog(
|
||||
uia_snapshot: Optional[Dict[str, Any]] = None,
|
||||
window_info: Optional[Dict[str, Any]] = None,
|
||||
) -> SystemDialogDetection:
|
||||
"""Déterminer si la fenêtre active est un dialogue système critique.
|
||||
|
||||
La détection combine plusieurs signaux — **un seul suffit à bloquer**.
|
||||
On préfère un faux positif (pause inutile) à un faux négatif (clic UAC).
|
||||
|
||||
Args:
|
||||
uia_snapshot: Dict avec champs `class_name`, `process_name`,
|
||||
`parent_path`, `name`. Peut être None si UIA indisponible.
|
||||
window_info: Dict avec champs `title`, `app_name`. Peut être None.
|
||||
|
||||
Returns:
|
||||
SystemDialogDetection avec is_system_dialog=True si un dialogue
|
||||
système est détecté.
|
||||
|
||||
Exemples::
|
||||
|
||||
det = is_system_dialog(window_info={"title": "User Account Control"})
|
||||
assert det.is_system_dialog # UAC détecté
|
||||
|
||||
det = is_system_dialog(uia_snapshot={"class_name": "$$$Secure UAP Dummy Window Class$$$"})
|
||||
assert det.is_system_dialog # UAC via ClassName
|
||||
|
||||
det = is_system_dialog(window_info={"title": "OSIRIS - Patient Dupont"})
|
||||
assert not det.is_system_dialog # Application métier → OK
|
||||
"""
|
||||
# ── Signal 1 : ClassName UIA ──
|
||||
if uia_snapshot:
|
||||
cn = uia_snapshot.get("class_name", "") or ""
|
||||
r = _check_class_name(cn)
|
||||
if r:
|
||||
cat, matched, reason = r
|
||||
return SystemDialogDetection(
|
||||
is_system_dialog=True,
|
||||
category=cat,
|
||||
matched_signal="class_name",
|
||||
matched_value=matched,
|
||||
reason=reason,
|
||||
)
|
||||
|
||||
# Explorer aussi les parents (le champ cliqué peut être un bouton
|
||||
# interne dont la ClassName est "Button", mais le root de la fenêtre
|
||||
# est le Consent.exe).
|
||||
for parent in uia_snapshot.get("parent_path", []) or []:
|
||||
p_cn = parent.get("class_name", "") or ""
|
||||
r = _check_class_name(p_cn)
|
||||
if r:
|
||||
cat, matched, reason = r
|
||||
return SystemDialogDetection(
|
||||
is_system_dialog=True,
|
||||
category=cat,
|
||||
matched_signal="parent_class_name",
|
||||
matched_value=matched,
|
||||
reason=f"Parent : {reason}",
|
||||
)
|
||||
|
||||
# ── Signal 2 : Process name ──
|
||||
if uia_snapshot:
|
||||
pn = uia_snapshot.get("process_name", "") or ""
|
||||
r = _check_process_name(pn)
|
||||
if r:
|
||||
cat, matched, reason = r
|
||||
return SystemDialogDetection(
|
||||
is_system_dialog=True,
|
||||
category=cat,
|
||||
matched_signal="process_name",
|
||||
matched_value=matched,
|
||||
reason=reason,
|
||||
)
|
||||
|
||||
if window_info:
|
||||
app = window_info.get("app_name", "") or ""
|
||||
r = _check_process_name(app)
|
||||
if r:
|
||||
cat, matched, reason = r
|
||||
return SystemDialogDetection(
|
||||
is_system_dialog=True,
|
||||
category=cat,
|
||||
matched_signal="app_name",
|
||||
matched_value=matched,
|
||||
reason=reason,
|
||||
)
|
||||
|
||||
# ── Signal 3 : Titre de fenêtre ──
|
||||
if window_info:
|
||||
title = window_info.get("title", "") or ""
|
||||
r = _check_title(title)
|
||||
if r:
|
||||
cat, matched, reason = r
|
||||
return SystemDialogDetection(
|
||||
is_system_dialog=True,
|
||||
category=cat,
|
||||
matched_signal="window_title",
|
||||
matched_value=matched,
|
||||
reason=reason,
|
||||
)
|
||||
|
||||
if uia_snapshot:
|
||||
# Certains dialogues système remontent leur titre dans uia.name
|
||||
uia_name = uia_snapshot.get("name", "") or ""
|
||||
r = _check_title(uia_name)
|
||||
if r:
|
||||
cat, matched, reason = r
|
||||
return SystemDialogDetection(
|
||||
is_system_dialog=True,
|
||||
category=cat,
|
||||
matched_signal="uia_name",
|
||||
matched_value=matched,
|
||||
reason=reason,
|
||||
)
|
||||
|
||||
return SystemDialogDetection(is_system_dialog=False)
|
||||
|
||||
|
||||
def detect_current_system_dialog() -> SystemDialogDetection:
|
||||
"""Analyser l'écran actuel et détecter un dialogue système.
|
||||
|
||||
Helper autonome qui interroge à la fois `get_active_window_info()` et
|
||||
le helper UIA (si dispo) pour obtenir la détection la plus fiable.
|
||||
|
||||
Returns:
|
||||
SystemDialogDetection. Si un signal matche, is_system_dialog=True.
|
||||
Si rien n'est disponible (Linux, UIA absent), is_system_dialog=False
|
||||
mais le caller peut encore fallback sur une analyse par titre.
|
||||
"""
|
||||
window_info: Optional[Dict[str, Any]] = None
|
||||
uia_snapshot: Optional[Dict[str, Any]] = None
|
||||
|
||||
# Fenêtre active (cross-platform)
|
||||
try:
|
||||
from ..window_info_crossplatform import get_active_window_info
|
||||
window_info = get_active_window_info()
|
||||
except Exception as e: # pragma: no cover — best-effort
|
||||
logger.debug(f"[SYS-DIALOG] window_info indisponible : {e}")
|
||||
|
||||
# UIA local (Windows uniquement, via lea_uia.exe)
|
||||
try:
|
||||
from .uia_helper import get_shared_helper
|
||||
helper = get_shared_helper()
|
||||
if helper.available:
|
||||
# On capture l'élément focalisé (root = fenêtre active)
|
||||
element = helper.capture_focused(max_depth=2)
|
||||
if element is not None:
|
||||
uia_snapshot = element.to_dict()
|
||||
except Exception as e: # pragma: no cover
|
||||
logger.debug(f"[SYS-DIALOG] UIA indisponible : {e}")
|
||||
|
||||
detection = is_system_dialog(
|
||||
uia_snapshot=uia_snapshot, window_info=window_info,
|
||||
)
|
||||
|
||||
if detection.is_system_dialog:
|
||||
logger.warning(
|
||||
f"[SYS-DIALOG] BLOCAGE — dialogue système détecté "
|
||||
f"[{detection.category}] via {detection.matched_signal}='{detection.matched_value}' "
|
||||
f"— {detection.reason}"
|
||||
)
|
||||
return detection
|
||||
|
||||
|
||||
__all__ = [
|
||||
"SystemDialogCategory",
|
||||
"SystemDialogDetection",
|
||||
"is_system_dialog",
|
||||
"detect_current_system_dialog",
|
||||
]
|
||||
294
agent_v0/agent_v1/core/uia_helper.py
Normal file
294
agent_v0/agent_v1/core/uia_helper.py
Normal file
@@ -0,0 +1,294 @@
|
||||
# core/workflow/uia_helper.py
|
||||
"""
|
||||
UIAHelper — Wrapper Python pour lea_uia.exe (helper Rust UI Automation).
|
||||
|
||||
Expose une API Python simple pour interroger UIA via le binaire Rust.
|
||||
Communique via subprocess + stdin/stdout JSON.
|
||||
|
||||
Pourquoi un helper Rust ?
|
||||
- 5-10x plus rapide que pywinauto (10-20ms vs 50-200ms)
|
||||
- Binaire standalone ~500 Ko, aucune dépendance runtime
|
||||
- Pas de problèmes de threading COM en Python
|
||||
- Crash-safe (le crash du helper n'affecte pas l'agent Python)
|
||||
|
||||
Architecture :
|
||||
Python executor
|
||||
↓ subprocess.run
|
||||
lea_uia.exe query --x 812 --y 436
|
||||
↓ UIA API Windows
|
||||
JSON response
|
||||
↓ stdout
|
||||
Python executor parse JSON
|
||||
|
||||
Si lea_uia.exe n'est pas disponible (Linux, binaire absent, crash) :
|
||||
toutes les méthodes retournent None → fallback vision automatique.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import platform
|
||||
import subprocess
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Timeout par défaut pour les appels UIA (en secondes)
|
||||
_DEFAULT_TIMEOUT = 5.0
|
||||
|
||||
# Masquer la fenêtre console lors du spawn de lea_uia.exe sur Windows.
|
||||
# Sans ce flag, chaque appel (à chaque clic utilisateur pendant
|
||||
# l'enregistrement) fait apparaître une fenêtre cmd noire brièvement
|
||||
# visible à l'écran → ralentit la souris et pollue les screenshots
|
||||
# capturés (le VLM peut "voir" le chemin lea_uia.exe comme texte cliqué).
|
||||
#
|
||||
# La valeur 0x08000000 correspond à CREATE_NO_WINDOW défini dans
|
||||
# l'API Windows. Sur Linux/Mac, la valeur est 0 et `creationflags`
|
||||
# est ignoré. getattr() gère le cas où Python expose déjà la constante
|
||||
# sur Windows.
|
||||
if platform.system() == "Windows":
|
||||
_SUBPROCESS_CREATION_FLAGS = getattr(subprocess, "CREATE_NO_WINDOW", 0x08000000)
|
||||
else:
|
||||
_SUBPROCESS_CREATION_FLAGS = 0
|
||||
|
||||
|
||||
@dataclass
|
||||
class UiaElement:
|
||||
"""Représentation Python d'un élément UIA."""
|
||||
name: str = ""
|
||||
control_type: str = ""
|
||||
class_name: str = ""
|
||||
automation_id: str = ""
|
||||
bounding_rect: Tuple[int, int, int, int] = (0, 0, 0, 0)
|
||||
is_enabled: bool = False
|
||||
is_offscreen: bool = True
|
||||
parent_path: List[Dict[str, str]] = field(default_factory=list)
|
||||
process_name: str = ""
|
||||
|
||||
def center(self) -> Tuple[int, int]:
|
||||
"""Retourner le centre du rectangle (pixels)."""
|
||||
x1, y1, x2, y2 = self.bounding_rect
|
||||
return ((x1 + x2) // 2, (y1 + y2) // 2)
|
||||
|
||||
def width(self) -> int:
|
||||
return self.bounding_rect[2] - self.bounding_rect[0]
|
||||
|
||||
def height(self) -> int:
|
||||
return self.bounding_rect[3] - self.bounding_rect[1]
|
||||
|
||||
def is_clickable(self) -> bool:
|
||||
"""Peut-on cliquer dessus ?"""
|
||||
return (
|
||||
self.is_enabled
|
||||
and not self.is_offscreen
|
||||
and self.width() > 0
|
||||
and self.height() > 0
|
||||
)
|
||||
|
||||
def path_signature(self) -> str:
|
||||
"""Signature du chemin parent (pour retrouver l'élément)."""
|
||||
parts = [f"{p['control_type']}[{p['name']}]" for p in self.parent_path if p.get("name")]
|
||||
parts.append(f"{self.control_type}[{self.name}]")
|
||||
return " > ".join(parts)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"name": self.name,
|
||||
"control_type": self.control_type,
|
||||
"class_name": self.class_name,
|
||||
"automation_id": self.automation_id,
|
||||
"bounding_rect": list(self.bounding_rect),
|
||||
"is_enabled": self.is_enabled,
|
||||
"is_offscreen": self.is_offscreen,
|
||||
"parent_path": self.parent_path,
|
||||
"process_name": self.process_name,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, d: Dict[str, Any]) -> "UiaElement":
|
||||
rect = d.get("bounding_rect", [0, 0, 0, 0])
|
||||
if isinstance(rect, list) and len(rect) >= 4:
|
||||
rect = tuple(rect[:4])
|
||||
else:
|
||||
rect = (0, 0, 0, 0)
|
||||
return cls(
|
||||
name=d.get("name", ""),
|
||||
control_type=d.get("control_type", ""),
|
||||
class_name=d.get("class_name", ""),
|
||||
automation_id=d.get("automation_id", ""),
|
||||
bounding_rect=rect,
|
||||
is_enabled=d.get("is_enabled", False),
|
||||
is_offscreen=d.get("is_offscreen", True),
|
||||
parent_path=d.get("parent_path", []),
|
||||
process_name=d.get("process_name", ""),
|
||||
)
|
||||
|
||||
|
||||
class UIAHelper:
|
||||
"""Wrapper Python pour lea_uia.exe."""
|
||||
|
||||
def __init__(self, helper_path: str = "", timeout: float = _DEFAULT_TIMEOUT):
|
||||
self._helper_path = helper_path or self._find_helper()
|
||||
self._timeout = timeout
|
||||
self._available = self._check_available()
|
||||
|
||||
def _find_helper(self) -> str:
|
||||
"""Trouver lea_uia.exe dans les emplacements standards."""
|
||||
candidates = [
|
||||
r"C:\Lea\helpers\lea_uia.exe",
|
||||
os.path.join(os.path.dirname(__file__), "..", "..",
|
||||
"agent_rust", "lea_uia", "target",
|
||||
"x86_64-pc-windows-gnu", "release", "lea_uia.exe"),
|
||||
"./helpers/lea_uia.exe",
|
||||
"lea_uia.exe",
|
||||
]
|
||||
for path in candidates:
|
||||
if os.path.isfile(path):
|
||||
return os.path.abspath(path)
|
||||
return ""
|
||||
|
||||
def _check_available(self) -> bool:
|
||||
"""Vérifier que le helper est utilisable (Windows + binaire + health OK)."""
|
||||
if platform.system() != "Windows":
|
||||
logger.debug("UIAHelper: Linux/Mac — helper désactivé")
|
||||
return False
|
||||
if not self._helper_path:
|
||||
logger.debug("UIAHelper: lea_uia.exe introuvable")
|
||||
return False
|
||||
if not os.path.isfile(self._helper_path):
|
||||
logger.debug(f"UIAHelper: chemin invalide {self._helper_path}")
|
||||
return False
|
||||
return True
|
||||
|
||||
@property
|
||||
def available(self) -> bool:
|
||||
return self._available
|
||||
|
||||
@property
|
||||
def helper_path(self) -> str:
|
||||
return self._helper_path
|
||||
|
||||
def _run(self, args: List[str]) -> Optional[Dict[str, Any]]:
|
||||
"""Exécuter lea_uia.exe avec les arguments et parser le JSON."""
|
||||
if not self._available:
|
||||
return None
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[self._helper_path] + args,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=self._timeout,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
creationflags=_SUBPROCESS_CREATION_FLAGS,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
logger.debug(
|
||||
f"UIAHelper: exit code {result.returncode}, "
|
||||
f"stderr: {result.stderr[:200]}"
|
||||
)
|
||||
return None
|
||||
output = result.stdout.strip()
|
||||
if not output:
|
||||
return None
|
||||
return json.loads(output)
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.debug(f"UIAHelper: timeout ({self._timeout}s) sur {args}")
|
||||
return None
|
||||
except json.JSONDecodeError as e:
|
||||
logger.debug(f"UIAHelper: JSON invalide — {e}")
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.debug(f"UIAHelper: erreur {e}")
|
||||
return None
|
||||
|
||||
def health(self) -> bool:
|
||||
"""Vérifier que UIA répond."""
|
||||
data = self._run(["health"])
|
||||
return data is not None and data.get("status") == "ok"
|
||||
|
||||
def query_at(
|
||||
self,
|
||||
x: int,
|
||||
y: int,
|
||||
with_parents: bool = True,
|
||||
) -> Optional[UiaElement]:
|
||||
"""Récupérer l'élément UIA à une position écran.
|
||||
|
||||
Args:
|
||||
x, y: Coordonnées pixel absolues
|
||||
with_parents: Inclure la hiérarchie des parents
|
||||
|
||||
Returns:
|
||||
UiaElement si trouvé, None sinon (pas d'élément ou UIA indispo)
|
||||
"""
|
||||
args = ["query", "--x", str(x), "--y", str(y)]
|
||||
if not with_parents:
|
||||
args.append("--with-parents=false")
|
||||
|
||||
data = self._run(args)
|
||||
if not data or data.get("status") != "ok":
|
||||
return None
|
||||
|
||||
elem_data = data.get("element")
|
||||
if not elem_data:
|
||||
return None
|
||||
return UiaElement.from_dict(elem_data)
|
||||
|
||||
def find_by_name(
|
||||
self,
|
||||
name: str,
|
||||
control_type: Optional[str] = None,
|
||||
automation_id: Optional[str] = None,
|
||||
window: Optional[str] = None,
|
||||
timeout_ms: int = 2000,
|
||||
) -> Optional[UiaElement]:
|
||||
"""Rechercher un élément par son nom (+ filtres optionnels).
|
||||
|
||||
Args:
|
||||
name: Nom exact de l'élément
|
||||
control_type: Type de contrôle (Button, Edit, MenuItem...)
|
||||
automation_id: ID d'automation
|
||||
window: Restreindre à une fenêtre spécifique
|
||||
timeout_ms: Timeout de recherche en millisecondes
|
||||
"""
|
||||
args = ["find", "--name", name, "--timeout-ms", str(timeout_ms)]
|
||||
if control_type:
|
||||
args.extend(["--control-type", control_type])
|
||||
if automation_id:
|
||||
args.extend(["--automation-id", automation_id])
|
||||
if window:
|
||||
args.extend(["--window", window])
|
||||
|
||||
data = self._run(args)
|
||||
if not data or data.get("status") != "ok":
|
||||
return None
|
||||
|
||||
elem_data = data.get("element")
|
||||
if not elem_data:
|
||||
return None
|
||||
return UiaElement.from_dict(elem_data)
|
||||
|
||||
def capture_focused(self, max_depth: int = 3) -> Optional[UiaElement]:
|
||||
"""Capturer l'élément ayant le focus + son contexte."""
|
||||
data = self._run(["capture", "--max-depth", str(max_depth)])
|
||||
if not data or data.get("status") != "ok":
|
||||
return None
|
||||
|
||||
elem_data = data.get("element")
|
||||
if not elem_data:
|
||||
return None
|
||||
return UiaElement.from_dict(elem_data)
|
||||
|
||||
|
||||
# Instance globale partagée (singleton léger)
|
||||
_SHARED_HELPER: Optional[UIAHelper] = None
|
||||
|
||||
|
||||
def get_shared_helper() -> UIAHelper:
|
||||
"""Retourner une instance partagée de UIAHelper."""
|
||||
global _SHARED_HELPER
|
||||
if _SHARED_HELPER is None:
|
||||
_SHARED_HELPER = UIAHelper()
|
||||
return _SHARED_HELPER
|
||||
39
agent_v0/agent_v1/finalize_contract.py
Normal file
39
agent_v0/agent_v1/finalize_contract.py
Normal file
@@ -0,0 +1,39 @@
|
||||
"""Dispatch léger du contrat enrichi de /finalize côté agent."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any, Dict
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def dispatch_finalize_result(ui: Any, payload: Dict[str, Any], replay_name: str) -> None:
|
||||
"""Router le résultat de /finalize vers la bonne surface UI agent."""
|
||||
if not isinstance(payload, dict):
|
||||
return
|
||||
|
||||
replay_request = payload.get("replay_request") or {}
|
||||
replay_launch = payload.get("replay_launch") or {}
|
||||
|
||||
if replay_launch.get("status") == "started":
|
||||
logger.info("Replay direct déjà lancé par le serveur après finalize")
|
||||
return
|
||||
|
||||
if not payload.get("replay_ready") or not replay_request:
|
||||
return
|
||||
|
||||
if replay_launch.get("status") == "failed":
|
||||
logger.warning(
|
||||
"Auto-replay serveur échoué après finalize, proposition manuelle"
|
||||
)
|
||||
|
||||
if ui is None or not hasattr(ui, "offer_finalize_replay"):
|
||||
logger.info("UI indisponible pour proposer un test immédiat")
|
||||
return
|
||||
|
||||
ui.offer_finalize_replay(
|
||||
replay_request,
|
||||
replay_name or "la tâche que vous venez d'enregistrer",
|
||||
)
|
||||
56
agent_v0/agent_v1/logging_setup.py
Normal file
56
agent_v0/agent_v1/logging_setup.py
Normal file
@@ -0,0 +1,56 @@
|
||||
"""Journalisation client Léa — DETTE-021.
|
||||
|
||||
Branche un handler **fichier** (`TimedRotatingFileHandler`) sur le logger racine,
|
||||
en plus de la console. Sans cela, sous `pythonw.exe` (pas de console), les logs
|
||||
partent sur stderr et sont **perdus** — diagnostic terrain impossible.
|
||||
|
||||
Rotation quotidienne + rétention `retention_days` (Règlement IA Art. 12 :
|
||||
journalisation automatique + conservation minimum 180 j).
|
||||
"""
|
||||
import logging
|
||||
from logging.handlers import TimedRotatingFileHandler
|
||||
from pathlib import Path
|
||||
|
||||
_FMT = "%(asctime)s %(levelname)-7s %(name)-25s %(message)s"
|
||||
|
||||
|
||||
def setup_logging(log_file, level=logging.INFO, retention_days=180):
|
||||
"""Configure le logging racine : fichier (rotation quotidienne, `retention_days`
|
||||
fichiers conservés) + console. **Idempotent** : ne réempile pas nos handlers.
|
||||
|
||||
Args:
|
||||
log_file: chemin du fichier de log (`config.LOG_FILE` en prod).
|
||||
level: niveau racine (INFO par défaut ; DEBUG géré par l'appelant).
|
||||
retention_days: nb de fichiers quotidiens conservés (180 = Règlement IA Art. 12).
|
||||
|
||||
Returns:
|
||||
Le `TimedRotatingFileHandler` créé.
|
||||
"""
|
||||
log_file = Path(log_file)
|
||||
log_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
root = logging.getLogger()
|
||||
root.setLevel(level)
|
||||
|
||||
# Idempotence : retirer nos propres handlers posés par un appel précédent.
|
||||
for h in list(root.handlers):
|
||||
if getattr(h, "_lea_managed", False):
|
||||
h.close()
|
||||
root.removeHandler(h)
|
||||
|
||||
file_handler = TimedRotatingFileHandler(
|
||||
str(log_file), when="midnight", backupCount=retention_days, encoding="utf-8"
|
||||
)
|
||||
file_handler.setFormatter(logging.Formatter(_FMT, datefmt="%Y-%m-%d %H:%M:%S"))
|
||||
file_handler.setLevel(level)
|
||||
file_handler._lea_managed = True
|
||||
root.addHandler(file_handler)
|
||||
|
||||
# Console conservée (utile en dev / si lancé avec une console).
|
||||
console = logging.StreamHandler()
|
||||
console.setFormatter(logging.Formatter(_FMT, datefmt="%H:%M:%S"))
|
||||
console.setLevel(level)
|
||||
console._lea_managed = True
|
||||
root.addHandler(console)
|
||||
|
||||
return file_handler
|
||||
@@ -15,8 +15,10 @@ import time
|
||||
import logging
|
||||
import threading
|
||||
from .config import (
|
||||
SESSIONS_ROOT, AGENT_VERSION, SERVER_URL, MACHINE_ID, LOG_RETENTION_DAYS,
|
||||
SESSIONS_ROOT, AGENT_VERSION, SERVER_URL, MACHINE_ID, LOG_RETENTION_DAYS, LOG_FILE,
|
||||
SCREEN_RESOLUTION, DPI_SCALE, OS_THEME, API_TOKEN, MAX_SESSION_DURATION_S,
|
||||
STREAMING_ENDPOINT, LOG_SHIP_ENABLED, LOG_SHIP_INTERVAL_S,
|
||||
AUTO_UPDATE_ENABLED, AUTO_UPDATE_INTERVAL_S, AUTO_UPDATE_STAGING_DIR,
|
||||
)
|
||||
from .core.captor import EventCaptorV1
|
||||
from .core.executor import ActionExecutorV1
|
||||
@@ -27,6 +29,8 @@ from .ui.chat_window import ChatWindow
|
||||
from .ui.capture_server import CaptureServer
|
||||
from .session.storage import SessionStorage
|
||||
from .vision.capturer import VisionCapturer
|
||||
from .finalize_contract import dispatch_finalize_result
|
||||
from .core.log_safe import _title_hash
|
||||
|
||||
# Import optionnel du client serveur (pour le chat et les workflows)
|
||||
# Deux chemins : relatif (depuis agent_v0.agent_v1) ou absolu (depuis C:\rpa_vision\agent_v1)
|
||||
@@ -38,8 +42,47 @@ except (ImportError, ValueError):
|
||||
except ImportError:
|
||||
LeaServerClient = None
|
||||
|
||||
# Configuration du logging
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
|
||||
# Configuration du logging — format structuré et lisible pour un TIM
|
||||
# Niveau de détail : INFO par défaut, DEBUG si RPA_AGENT_DEBUG=1
|
||||
_log_level = logging.DEBUG if os.environ.get("RPA_AGENT_DEBUG") == "1" else logging.INFO
|
||||
# DETTE-021 : journaliser dans un FICHIER (rotation quotidienne + rétention 180 j,
|
||||
# Règlement IA Art. 12). Sous `pythonw.exe` (sans console), un basicConfig→stderr
|
||||
# serait perdu. Fallback console si le fichier est indisponible — ne JAMAIS
|
||||
# empêcher Léa de démarrer pour un problème de log.
|
||||
try:
|
||||
from .logging_setup import setup_logging
|
||||
setup_logging(LOG_FILE, level=_log_level, retention_days=LOG_RETENTION_DAYS)
|
||||
except Exception:
|
||||
logging.basicConfig(
|
||||
level=_log_level,
|
||||
format="%(asctime)s %(levelname)-7s %(name)-25s %(message)s",
|
||||
datefmt="%H:%M:%S",
|
||||
)
|
||||
|
||||
# Réduire le bruit de certaines libs
|
||||
for _noisy in ("urllib3", "requests.packages.urllib3", "PIL", "mss"):
|
||||
logging.getLogger(_noisy).setLevel(logging.WARNING)
|
||||
|
||||
# push-log-DGX : remontée automatique des logs vers le serveur (diagnostic des
|
||||
# postes SANS AnyDesk). GARDÉ derrière RPA_LOG_SHIP_ENABLED (défaut désactivé) —
|
||||
# activable poste par poste via config.txt, sans rebuild. Le handler est attaché
|
||||
# au logger racine APRÈS setup_logging (les logs partent aussi dans le fichier).
|
||||
_log_shipper = None
|
||||
if LOG_SHIP_ENABLED:
|
||||
try:
|
||||
from .network.log_shipper import LogShipper
|
||||
_log_shipper = LogShipper(
|
||||
machine_id=MACHINE_ID,
|
||||
max_batch=int(os.environ.get("RPA_AGENT_LOGS_MAX_BATCH", "1000")),
|
||||
flush_interval_s=LOG_SHIP_INTERVAL_S,
|
||||
)
|
||||
logging.getLogger().addHandler(_log_shipper.handler)
|
||||
_log_shipper.start()
|
||||
except Exception as _e:
|
||||
# Ne JAMAIS empêcher Léa de démarrer pour un problème de remontée de logs.
|
||||
logging.getLogger(__name__).warning("Log shipper non démarré : %s", _e)
|
||||
_log_shipper = None
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Intervalle de polling replay (secondes)
|
||||
@@ -68,6 +111,7 @@ class AgentV1:
|
||||
self._executor = None
|
||||
# Flag pour indiquer qu'un replay est en cours (eviter les conflits)
|
||||
self._replay_active = False
|
||||
self._last_recording_name = ""
|
||||
|
||||
# Etat partage entre systray et chat (source de verite unique)
|
||||
self._state = AgentState()
|
||||
@@ -75,22 +119,23 @@ class AgentV1:
|
||||
self._state.set_on_stop(self.stop_session)
|
||||
|
||||
# Client serveur pour le chat et les workflows
|
||||
# Plus de RPA_SERVER_HOST : le LeaServerClient derive tout de SERVER_URL
|
||||
self._server_client = None
|
||||
if LeaServerClient is not None:
|
||||
# Forcer le token API pour éviter les 401
|
||||
# (le token est set par start.bat dans l'environnement)
|
||||
from .config import API_TOKEN as _token
|
||||
server_host = os.getenv("RPA_SERVER_HOST", "localhost")
|
||||
self._server_client = LeaServerClient(server_host=server_host)
|
||||
self._server_client = LeaServerClient()
|
||||
if _token and not self._server_client._api_token:
|
||||
self._server_client._api_token = _token
|
||||
logger.info("Token API forcé dans LeaServerClient")
|
||||
|
||||
# Fenetre de chat Lea (tkinter natif)
|
||||
# Le host est derive de SERVER_URL (plus de RPA_SERVER_HOST)
|
||||
server_host = (
|
||||
self._server_client.server_host
|
||||
if self._server_client is not None
|
||||
else os.getenv("RPA_SERVER_HOST", "localhost")
|
||||
else "localhost"
|
||||
)
|
||||
self._chat_window = ChatWindow(
|
||||
server_client=self._server_client,
|
||||
@@ -103,12 +148,42 @@ class AgentV1:
|
||||
# Executeur pour le replay (doit exister avant le poll)
|
||||
self._executor = ActionExecutorV1()
|
||||
|
||||
# Wiring ChatWindow → Executor pour Plan B (pause_message → bulle interactive)
|
||||
# Permet à l'executor d'afficher une bulle paused dans la fenêtre Léa V1
|
||||
# quand le serveur signale replay_paused=True via /replay/next.
|
||||
self._wire_chat_window_to_executor()
|
||||
|
||||
# Boucles permanentes (pas besoin de session active)
|
||||
self.running = True
|
||||
self._bg_vision = VisionCapturer(str(SESSIONS_ROOT / "_background"))
|
||||
threading.Thread(target=self._replay_poll_loop, daemon=True).start()
|
||||
threading.Thread(target=self._background_heartbeat_loop, daemon=True).start()
|
||||
|
||||
# DETTE-022 v2 : MAJ silencieuse — boucle de check GATED (défaut OFF).
|
||||
# Interroge le serveur (canary-aware) et télécharge en STAGING ; le swap
|
||||
# réel reste réservé révision humaine (updater.apply_update = stub no-op).
|
||||
# Activable poste par poste via RPA_AUTO_UPDATE_ENABLED, sans rebuild.
|
||||
if AUTO_UPDATE_ENABLED:
|
||||
threading.Thread(
|
||||
target=self._auto_update_loop, daemon=True, name="lea-auto-update"
|
||||
).start()
|
||||
|
||||
# MAJ silencieuse — confirmation de boot post-swap. Si Lea.bat vient
|
||||
# d'appliquer une MAJ (marqueur PENDING_BOOT), on désarme le rollback
|
||||
# après ~90 s de tourne STABLE (liveness LOCALE, indépendante du DGX).
|
||||
# Un quit propre avant 90 s confirme aussi (cf. main()). Seul un vrai
|
||||
# crash laisse PENDING_BOOT → rollback au prochain lancement.
|
||||
if _pending_boot_marker_exists():
|
||||
def _boot_confirm():
|
||||
import os as _os
|
||||
import time as _time
|
||||
_time.sleep(float(_os.environ.get("RPA_BOOT_CONFIRM_DELAY_S", "90")))
|
||||
if self.running:
|
||||
_confirm_boot_ok()
|
||||
threading.Thread(
|
||||
target=_boot_confirm, daemon=True, name="lea-boot-confirm"
|
||||
).start()
|
||||
|
||||
# Mini-serveur HTTP pour captures a la demande (port 5006)
|
||||
self._capture_server = CaptureServer()
|
||||
self._capture_server.start()
|
||||
@@ -131,6 +206,15 @@ class AgentV1:
|
||||
shared_state=self._state,
|
||||
)
|
||||
|
||||
def _wire_chat_window_to_executor(self) -> None:
|
||||
"""Relie l'executor courant à la ChatWindow pour les pauses supervisees."""
|
||||
if self._executor is None or self._chat_window is None:
|
||||
return
|
||||
try:
|
||||
self._executor._chat_window_ref = self._chat_window
|
||||
except Exception:
|
||||
logger.debug("Wiring chat_window->executor echoue (non bloquant)", exc_info=True)
|
||||
|
||||
def _delayed_cleanup(self):
|
||||
"""Nettoyage en arrière-plan après 30s pour ne pas bloquer le démarrage."""
|
||||
time.sleep(30)
|
||||
@@ -189,16 +273,19 @@ class AgentV1:
|
||||
time.sleep(30) # Vérifier toutes les 30s
|
||||
|
||||
def start_session(self, workflow_name):
|
||||
self._last_recording_name = workflow_name
|
||||
self.session_id = f"sess_{time.strftime('%Y%m%dT%H%M%S')}_{uuid.uuid4().hex[:6]}"
|
||||
self.session_dir = self.storage.get_session_dir(self.session_id)
|
||||
|
||||
self.vision = VisionCapturer(str(self.session_dir))
|
||||
|
||||
self.streamer = TraceStreamer(self.session_id, machine_id=self.machine_id)
|
||||
self.streamer.set_on_finalize_result(self._on_finalize_result)
|
||||
self.captor = EventCaptorV1(self._on_event_bridge)
|
||||
|
||||
# Initialiser l'executeur partage
|
||||
self._executor = ActionExecutorV1()
|
||||
self._wire_chat_window_to_executor()
|
||||
|
||||
self.shot_counter = 0
|
||||
self.running = True
|
||||
@@ -221,7 +308,7 @@ class AgentV1:
|
||||
# Ne PAS en relancer une ici — deux threads poll simultanés causent
|
||||
# une race condition où les actions sont consommées mais pas exécutées.
|
||||
|
||||
logger.info(f"Session {self.session_id} ({workflow_name}) sur machine {self.machine_id} en cours...")
|
||||
logger.info(f"Session {self.session_id} [wf_hash={_title_hash(workflow_name)}] sur machine {self.machine_id} en cours...")
|
||||
|
||||
def _command_watchdog_loop(self):
|
||||
"""Surveille un fichier de commande pour executer des ordres visuels (legacy)."""
|
||||
@@ -304,6 +391,15 @@ class AgentV1:
|
||||
# pour enchainer les actions du workflow
|
||||
time.sleep(0.2)
|
||||
else:
|
||||
if getattr(self._executor, "_replay_paused", False):
|
||||
if not self._replay_active:
|
||||
self._replay_active = True
|
||||
self.ui.set_replay_active(True)
|
||||
self._state.set_replay_active(True)
|
||||
poll_delay = getattr(self._executor, '_poll_backoff', REPLAY_POLL_INTERVAL)
|
||||
time.sleep(max(poll_delay, REPLAY_POLL_INTERVAL))
|
||||
continue
|
||||
|
||||
# Pas d'action en attente — utiliser le backoff de l'executor
|
||||
# (augmente si le serveur est indisponible, reset a 1s sinon)
|
||||
if self._replay_active:
|
||||
@@ -352,11 +448,11 @@ class AgentV1:
|
||||
continue
|
||||
self._last_bg_hash = img_hash
|
||||
|
||||
# Envoyer au streaming server (avec token auth)
|
||||
# Envoyer au streaming server (via STREAMING_ENDPOINT unifié)
|
||||
headers = {"Authorization": f"Bearer {API_TOKEN}"} if API_TOKEN else {}
|
||||
with open(full_path, 'rb') as f:
|
||||
req.post(
|
||||
f"{SERVER_URL}/traces/stream/image",
|
||||
f"{STREAMING_ENDPOINT}/image",
|
||||
params={
|
||||
"session_id": bg_session,
|
||||
"shot_id": f"heartbeat_{int(time.time())}",
|
||||
@@ -365,18 +461,90 @@ class AgentV1:
|
||||
headers=headers,
|
||||
files={"file": ("screenshot.png", f, "image/png")},
|
||||
timeout=10,
|
||||
allow_redirects=False,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug(f"[HEARTBEAT] Erreur: {e}")
|
||||
time.sleep(5)
|
||||
|
||||
def stop_session(self):
|
||||
# Arrêter la capture et le streaming de la session d'enregistrement
|
||||
if self.captor: self.captor.stop()
|
||||
if self.streamer: self.streamer.stop()
|
||||
logger.info(f"Session {self.session_id} terminée.")
|
||||
def _auto_update_loop(self):
|
||||
"""DETTE-022 v2 — boucle de MAJ silencieuse GATED (défaut OFF).
|
||||
|
||||
# Reset le session_id pour que le poll replay utilise l'ID stable
|
||||
Interroge périodiquement le serveur (endpoint canary-aware), et si une
|
||||
MAJ est proposée pour CE poste, la télécharge dans le STAGING après
|
||||
vérif SHA256. Le swap réel N'EST PAS fait ici : `updater.run_update_cycle`
|
||||
s'arrête au staging (apply_update = stub réservé révision humaine + swap
|
||||
hors-process par Lea.bat au prochain démarrage).
|
||||
|
||||
SÉCURITÉ — « au bon moment » : on NE stage PAS pendant un enregistrement
|
||||
ou un replay actif (self.session_id / self._replay_active), pour ne pas
|
||||
perturber le travail utilisateur ni consommer du réseau au mauvais
|
||||
moment. Best-effort : aucune exception ne remonte (ne casse jamais Léa).
|
||||
"""
|
||||
try:
|
||||
from .network.updater import run_update_cycle
|
||||
except Exception as e:
|
||||
logger.warning("[UPDATE] Module updater indisponible : %s", e)
|
||||
return
|
||||
|
||||
logger.info(
|
||||
"[UPDATE] Boucle MAJ silencieuse démarrée (intervalle=%.0fs, "
|
||||
"version=%s) — check seul, swap réservé révision humaine",
|
||||
AUTO_UPDATE_INTERVAL_S, AGENT_VERSION,
|
||||
)
|
||||
|
||||
while self.running:
|
||||
# Découpe l'attente pour réagir vite à l'arrêt.
|
||||
waited = 0.0
|
||||
step = 1.0
|
||||
while self.running and waited < AUTO_UPDATE_INTERVAL_S:
|
||||
time.sleep(step)
|
||||
waited += step
|
||||
if not self.running:
|
||||
break
|
||||
|
||||
# « Au bon moment » : jamais en plein travail (enregistrement/replay).
|
||||
if self.session_id or getattr(self, "_replay_active", False):
|
||||
logger.debug("[UPDATE] Report du check (session/replay active)")
|
||||
continue
|
||||
|
||||
try:
|
||||
result = run_update_cycle(
|
||||
local_version=AGENT_VERSION,
|
||||
machine_id=self.machine_id,
|
||||
staging_dir=AUTO_UPDATE_STAGING_DIR,
|
||||
)
|
||||
status = result.get("status")
|
||||
if status == "staged":
|
||||
logger.info(
|
||||
"[UPDATE] MAJ %s téléchargée en staging (SHA256=%s) — "
|
||||
"swap réservé révision humaine, non appliqué",
|
||||
result.get("target_version"),
|
||||
result.get("sha256_verified"),
|
||||
)
|
||||
elif status not in ("up_to_date", "disabled"):
|
||||
logger.debug("[UPDATE] Cycle: %s", result)
|
||||
except Exception as e:
|
||||
# run_update_cycle est déjà best-effort ; double filet ici.
|
||||
logger.debug("[UPDATE] Erreur boucle MAJ : %s", e)
|
||||
|
||||
def stop_session(self):
|
||||
# Sauvegarder le session_id avant de l'annuler (pour les logs)
|
||||
ended_session_id = self.session_id
|
||||
|
||||
# Arrêter la capture d'abord (plus d'events entrants)
|
||||
if self.captor: self.captor.stop()
|
||||
|
||||
# Attendre que les events en cours de traitement dans _on_event_bridge
|
||||
# aient le temps d'être envoyés au streamer (capture duale + push)
|
||||
import time
|
||||
time.sleep(1.5)
|
||||
|
||||
# Maintenant arrêter le streamer (drain queue + finalize)
|
||||
if self.streamer: self.streamer.stop()
|
||||
logger.info(f"Session {ended_session_id} terminée.")
|
||||
|
||||
# Reset le session_id APRÈS le stop complet du streamer
|
||||
self.session_id = None
|
||||
|
||||
# Reset le backoff de l'executor pour reprendre le polling immédiatement
|
||||
@@ -397,12 +565,18 @@ class AgentV1:
|
||||
f"agent_{self.user_id}"
|
||||
)
|
||||
|
||||
def _on_finalize_result(self, payload: dict) -> None:
|
||||
"""Réagir au contrat enrichi de /finalize côté agent."""
|
||||
replay_name = self._last_recording_name or "la tâche que vous venez d'enregistrer"
|
||||
dispatch_finalize_result(self.ui, payload, replay_name)
|
||||
|
||||
_last_heartbeat_hash: str = ""
|
||||
|
||||
def _heartbeat_loop(self):
|
||||
"""Capture périodique pour donner du contexte au stagiaire.
|
||||
Déduplication : n'envoie que si l'écran a changé.
|
||||
Tourne tant que session_id est défini (= enregistrement actif).
|
||||
Enrichi avec le titre de la fenêtre active pour contextualisation.
|
||||
"""
|
||||
while self.running and self.session_id:
|
||||
try:
|
||||
@@ -413,7 +587,23 @@ class AgentV1:
|
||||
if img_hash != self._last_heartbeat_hash:
|
||||
self._last_heartbeat_hash = img_hash
|
||||
self.streamer.push_image(full_path, f"heartbeat_{int(time.time())}")
|
||||
self.streamer.push_event({"type": "heartbeat", "image": full_path, "timestamp": time.time(), "machine_id": self.machine_id})
|
||||
heartbeat_event = {
|
||||
"type": "heartbeat",
|
||||
"image": full_path,
|
||||
"timestamp": time.time(),
|
||||
"machine_id": self.machine_id,
|
||||
}
|
||||
# Ajouter le titre de la fenêtre active (léger, pas de crop)
|
||||
window_title = self.vision.get_active_window_title()
|
||||
if window_title:
|
||||
heartbeat_event["active_window_title"] = window_title
|
||||
# QW1 — enrichissement multi-écrans (additif, fallback gracieux)
|
||||
try:
|
||||
from .vision.capturer import _enrich_with_monitor_info
|
||||
_enrich_with_monitor_info(heartbeat_event)
|
||||
except Exception:
|
||||
pass
|
||||
self.streamer.push_event(heartbeat_event)
|
||||
except Exception as e:
|
||||
logger.error(f"Heartbeat error: {e}")
|
||||
time.sleep(5)
|
||||
@@ -448,20 +638,33 @@ class AgentV1:
|
||||
event["screenshot_context"] = full_path
|
||||
self.streamer.push_image(full_path, f"focus_{int(time.time())}")
|
||||
|
||||
# 🔴 Capture Interactive (Dual)
|
||||
# Capture Interactive (Dual + Fenêtre active)
|
||||
if event["type"] in ["mouse_click", "key_combo"]:
|
||||
self.shot_counter += 1
|
||||
shot_id = f"shot_{self.shot_counter:04d}"
|
||||
|
||||
|
||||
pos = event.get("pos", (0, 0))
|
||||
capture_info = self.vision.capture_dual(pos[0], pos[1], shot_id)
|
||||
|
||||
|
||||
event["screenshot_id"] = shot_id
|
||||
event["vision_info"] = capture_info
|
||||
|
||||
|
||||
# Enrichir l'event avec les métadonnées de la fenêtre active
|
||||
# (titre, rect, coordonnées clic relatives, taille fenêtre)
|
||||
window_capture = capture_info.get("window_capture")
|
||||
if window_capture:
|
||||
event["window_capture"] = {
|
||||
"title": window_capture.get("window_title", ""),
|
||||
"app_name": window_capture.get("app_name", ""),
|
||||
"rect": window_capture.get("window_rect"),
|
||||
"click_relative": window_capture.get("click_in_window"),
|
||||
"window_size": window_capture.get("window_size"),
|
||||
"click_inside_window": window_capture.get("click_inside_window", True),
|
||||
}
|
||||
|
||||
self._stream_capture_info(capture_info, shot_id)
|
||||
|
||||
# 🕒 POST-ACTION : Capture du résultat après 1s (pour voir le résultat du clic)
|
||||
|
||||
# POST-ACTION : Capture du résultat après 1s (pour voir le résultat du clic)
|
||||
threading.Timer(1.0, self._capture_result, args=(shot_id,)).start()
|
||||
|
||||
self.ui.update_stats(self.shot_counter)
|
||||
@@ -481,13 +684,113 @@ class AgentV1:
|
||||
self.streamer.push_image(capture_info["full"], f"{shot_id}_full")
|
||||
if "crop" in capture_info:
|
||||
self.streamer.push_image(capture_info["crop"], f"{shot_id}_crop")
|
||||
# Streamer l'image de la fenêtre active si disponible
|
||||
window_capture = capture_info.get("window_capture")
|
||||
if window_capture and "window_image" in window_capture:
|
||||
self.streamer.push_image(
|
||||
window_capture["window_image"], f"{shot_id}_window"
|
||||
)
|
||||
|
||||
def run(self):
|
||||
self.ui.run()
|
||||
|
||||
def _install_signal_handlers(agent, watchdog) -> None:
|
||||
"""Installe SIGTERM/SIGINT/SIGBREAK pour un arrêt propre du main thread.
|
||||
|
||||
Met ``agent.running=False`` (les daemon threads s'arrêtent) et réveille
|
||||
le watchdog (qui sort de sa boucle de surveillance). Sans session
|
||||
interactive (pystray.Icon.stop indisponible), c'est le SEUL moyen
|
||||
d'arrêter Léa proprement : ``kill -TERM <pid>`` ou Ctrl+C.
|
||||
"""
|
||||
import signal as _sig
|
||||
|
||||
def _handler(sig, frame):
|
||||
logger.info(f"[MAIN] Signal {sig} recu — arret propre")
|
||||
agent.running = False
|
||||
watchdog.stop()
|
||||
|
||||
for sig_name in ("SIGTERM", "SIGINT", "SIGBREAK"):
|
||||
sig_obj = getattr(_sig, sig_name, None)
|
||||
if sig_obj is None:
|
||||
continue
|
||||
try:
|
||||
_sig.signal(sig_obj, _handler)
|
||||
except (ValueError, OSError):
|
||||
pass
|
||||
|
||||
|
||||
def _agent_should_live(agent) -> bool:
|
||||
"""Vrai tant que Léa doit vivre : agent actif ET pas de Quitter explicite.
|
||||
|
||||
Un « Quitter » utilisateur (``ui._quit_requested``) doit stopper le
|
||||
watchdog pour de bon ; une simple déconnexion RDP ne met JAMAIS ce flag
|
||||
→ le tray revient tout seul à la reconnexion.
|
||||
"""
|
||||
if not getattr(agent, "running", False):
|
||||
return False
|
||||
ui = getattr(agent, "ui", None)
|
||||
if ui is not None and getattr(ui, "_quit_requested", False):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _pending_boot_marker_exists() -> bool:
|
||||
"""True si Lea.bat a posé PENDING_BOOT (boot post-MAJ à valider)."""
|
||||
try:
|
||||
from .network.updater import _resolve_app_dir
|
||||
return (_resolve_app_dir(None) / "PENDING_BOOT").exists()
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _confirm_boot_ok() -> None:
|
||||
"""Confirme un boot post-MAJ : écrit boot_ok + retire PENDING_BOOT.
|
||||
|
||||
Désarme le rollback de Lea.bat. No-op si pas de PENDING_BOOT (boot normal).
|
||||
Best-effort — ne doit jamais casser l'arrêt/la vie de Léa.
|
||||
"""
|
||||
try:
|
||||
if not _pending_boot_marker_exists():
|
||||
return
|
||||
from .network import updater
|
||||
updater.write_boot_ok_marker(AGENT_VERSION)
|
||||
logger.info("[MAJ] Boot confirmé (v%s) — rollback désarmé", AGENT_VERSION)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.debug("confirm_boot_ok: %s", e)
|
||||
|
||||
|
||||
def main():
|
||||
from .ui.session_watchdog import InteractiveSessionWatchdog
|
||||
|
||||
agent = AgentV1()
|
||||
agent.run()
|
||||
|
||||
# Résilience RDP/Citrix : au lieu de bloquer le main thread pour toujours
|
||||
# quand pystray sort (session interactive perdue), on surveille la
|
||||
# session et on ré-affiche le tray + le chat à chaque reconnexion.
|
||||
# agent.run() (== agent.ui.run()) est ré-entrant : les threads de fond
|
||||
# ne démarrent qu'une fois, seule l'icône est recréée. Les daemon threads
|
||||
# de capture/heartbeat/replay tournent contre agent.running et restent
|
||||
# uniques — le watchdog n'y touche pas.
|
||||
watchdog = InteractiveSessionWatchdog(
|
||||
run_ui=agent.run,
|
||||
is_running=lambda: _agent_should_live(agent),
|
||||
)
|
||||
_install_signal_handlers(agent, watchdog)
|
||||
|
||||
try:
|
||||
watchdog.run()
|
||||
# Sortie normale du watchdog = quit propre (tray / session) → le boot
|
||||
# était sain : on confirme (couvre un quit AVANT les 90 s, évite un faux
|
||||
# rollback). No-op si ce n'est pas un boot post-MAJ.
|
||||
_confirm_boot_ok()
|
||||
except KeyboardInterrupt:
|
||||
logger.info("[MAIN] Interruption clavier — arret propre")
|
||||
except Exception:
|
||||
logger.exception("[MAIN] Le watchdog de session a leve une exception")
|
||||
finally:
|
||||
agent.running = False
|
||||
logger.info("[MAIN] Sortie — agent.running=False, daemon threads vont s'arreter")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
149
agent_v0/agent_v1/network/feedback_bus.py
Normal file
149
agent_v0/agent_v1/network/feedback_bus.py
Normal file
@@ -0,0 +1,149 @@
|
||||
# agent_v1/network/feedback_bus.py
|
||||
"""Client SocketIO pour le bus feedback Léa.
|
||||
|
||||
Consomme les events 'lea:*' émis par agent_chat (port 5004) et les dispatche
|
||||
vers ChatWindow pour affichage en bulles temps réel.
|
||||
|
||||
Events écoutés :
|
||||
lea:action_started — début d'un workflow ou d'une action
|
||||
lea:action_progress — progression dans le workflow
|
||||
lea:done — fin d'un workflow ou d'un copilot
|
||||
lea:need_confirm — étape copilot en attente de validation
|
||||
lea:step_result — résultat d'une étape copilot
|
||||
lea:paused — basculement en paused_need_help (asset démo)
|
||||
lea:resumed — sortie de pause supervisée
|
||||
|
||||
Fail-safe : toute erreur de connexion ou de dispatch est silencieusement
|
||||
loggée. Le ChatWindow continue de fonctionner même si le bus est mort
|
||||
(comportement strictement identique au pré-J3).
|
||||
|
||||
Usage :
|
||||
bus = FeedbackBusClient(
|
||||
server_url="http://localhost:5004",
|
||||
token=os.environ.get("RPA_API_TOKEN", ""),
|
||||
on_event=lambda event, payload: print(event, payload),
|
||||
)
|
||||
bus.start() # connexion en arrière-plan, non-bloquant
|
||||
# ... ChatWindow tourne ...
|
||||
bus.stop()
|
||||
"""
|
||||
|
||||
import logging
|
||||
import threading
|
||||
from typing import Callable, Optional
|
||||
|
||||
import socketio
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
LEA_EVENTS = (
|
||||
'lea:action_started',
|
||||
'lea:action_progress',
|
||||
'lea:done',
|
||||
'lea:need_confirm',
|
||||
'lea:step_result',
|
||||
'lea:paused',
|
||||
'lea:resumed',
|
||||
)
|
||||
|
||||
EventCallback = Callable[[str, dict], None]
|
||||
|
||||
|
||||
class FeedbackBusClient:
|
||||
"""Client SocketIO non-bloquant pour le bus 'lea:*'."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
server_url: str,
|
||||
token: Optional[str] = None,
|
||||
on_event: Optional[EventCallback] = None,
|
||||
):
|
||||
self._url = server_url.rstrip('/')
|
||||
self._token = token or None
|
||||
self._on_event: EventCallback = on_event or (lambda e, p: None)
|
||||
self._sio = socketio.Client(
|
||||
reconnection=True,
|
||||
reconnection_attempts=0, # 0 = illimité
|
||||
reconnection_delay=2,
|
||||
reconnection_delay_max=30,
|
||||
logger=False,
|
||||
engineio_logger=False,
|
||||
)
|
||||
self._thread: Optional[threading.Thread] = None
|
||||
self._register_handlers()
|
||||
|
||||
def _register_handlers(self) -> None:
|
||||
@self._sio.event
|
||||
def connect():
|
||||
logger.info("FeedbackBus connecté à %s", self._url)
|
||||
|
||||
@self._sio.event
|
||||
def disconnect():
|
||||
logger.info("FeedbackBus déconnecté")
|
||||
|
||||
for ev in LEA_EVENTS:
|
||||
self._sio.on(ev, lambda data, e=ev: self._dispatch(e, data))
|
||||
|
||||
def _dispatch(self, event: str, payload: Optional[dict]) -> None:
|
||||
try:
|
||||
self._on_event(event, payload or {})
|
||||
except Exception:
|
||||
logger.debug("FeedbackBus dispatch silenced", exc_info=True)
|
||||
|
||||
def start(self) -> None:
|
||||
"""Démarrer la connexion en arrière-plan (idempotent, non-bloquant)."""
|
||||
if self._thread is not None and self._thread.is_alive():
|
||||
return
|
||||
self._thread = threading.Thread(
|
||||
target=self._run, daemon=True, name="LeaFeedbackBus",
|
||||
)
|
||||
self._thread.start()
|
||||
|
||||
def _run(self) -> None:
|
||||
headers = {}
|
||||
if self._token:
|
||||
headers['Authorization'] = f'Bearer {self._token}'
|
||||
try:
|
||||
self._sio.connect(self._url, headers=headers, wait=True)
|
||||
self._sio.wait()
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"FeedbackBus connect échoué (%s) — ChatWindow continue normalement", e,
|
||||
)
|
||||
|
||||
def stop(self) -> None:
|
||||
"""Arrêter proprement la connexion (idempotent, fail-safe)."""
|
||||
try:
|
||||
if self._sio.connected:
|
||||
self._sio.disconnect()
|
||||
except Exception:
|
||||
logger.debug("FeedbackBus stop silenced", exc_info=True)
|
||||
|
||||
@property
|
||||
def connected(self) -> bool:
|
||||
return bool(self._sio.connected)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Actions utilisateur depuis la bulle paused_need_help (J3.5)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def resume_replay(self, replay_id: str) -> bool:
|
||||
"""Bouton Continuer : émet 'lea:replay_resume' vers agent_chat.
|
||||
|
||||
Retourne True si l'event a pu être émis, False sinon (déconnecté/erreur).
|
||||
"""
|
||||
return self._safe_emit("lea:replay_resume", {"replay_id": replay_id})
|
||||
|
||||
def abort_replay(self, replay_id: str) -> bool:
|
||||
"""Bouton Annuler : émet 'lea:replay_abort' vers agent_chat."""
|
||||
return self._safe_emit("lea:replay_abort", {"replay_id": replay_id})
|
||||
|
||||
def _safe_emit(self, event: str, payload: dict) -> bool:
|
||||
try:
|
||||
if not self._sio.connected:
|
||||
return False
|
||||
self._sio.emit(event, payload)
|
||||
return True
|
||||
except Exception:
|
||||
logger.debug("FeedbackBus _safe_emit silenced", exc_info=True)
|
||||
return False
|
||||
147
agent_v0/agent_v1/network/lea_orchestrator_client.py
Normal file
147
agent_v0/agent_v1/network/lea_orchestrator_client.py
Normal file
@@ -0,0 +1,147 @@
|
||||
"""
|
||||
Client HTTP minimal pour l'orchestrateur Léa-first (agent-chat Linux).
|
||||
|
||||
Rebranchement P1-LEA-SHADOW : le bouton "Apprenez-moi" côté Windows déclenche
|
||||
la création d'une session d'apprentissage côté agent-chat (REST) AVANT de
|
||||
lancer la capture locale. Le pipeline streaming (capture frames/événements
|
||||
via start_recording) n'est PAS modifié — seule la prise de contact initiale
|
||||
avec Léa change.
|
||||
|
||||
Contrat :
|
||||
POST {AGENT_CHAT_URL}/api/learn/start
|
||||
Headers : Authorization: Bearer <RPA_API_TOKEN>, Content-Type: application/json
|
||||
Body : { machine_id, session_name, user_id?, trigger_source }
|
||||
Réponse : { session_id, state, message }
|
||||
|
||||
Politique :
|
||||
- Timeout 10s (connect + read)
|
||||
- Retry x2 avec backoff 0.5s puis 1.0s
|
||||
- En cas d'échec définitif : lève LeaOrchestratorError (le caller doit
|
||||
basculer en mode dégradé : start_recording local sans assistance).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# Timeout HTTP (connect + read) — 10s comme spec
|
||||
_HTTP_TIMEOUT_S = 10.0
|
||||
# Nombre de tentatives totales (1 + 2 retry)
|
||||
_MAX_ATTEMPTS = 3
|
||||
# Backoff progressif entre les tentatives
|
||||
_BACKOFF_S = (0.5, 1.0)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LearnStartResponse:
|
||||
"""Réponse normalisée de POST /api/learn/start."""
|
||||
|
||||
session_id: str
|
||||
state: str
|
||||
message: str
|
||||
|
||||
|
||||
class LeaOrchestratorError(RuntimeError):
|
||||
"""Erreur définitive de communication avec l'orchestrateur Léa."""
|
||||
|
||||
|
||||
def start_learning_session(
|
||||
base_url: str,
|
||||
*,
|
||||
machine_id: str,
|
||||
session_name: str,
|
||||
api_token: str = "",
|
||||
user_id: Optional[str] = None,
|
||||
trigger_source: str = "windows_button",
|
||||
timeout_s: float = _HTTP_TIMEOUT_S,
|
||||
max_attempts: int = _MAX_ATTEMPTS,
|
||||
backoff_s: tuple = _BACKOFF_S,
|
||||
) -> LearnStartResponse:
|
||||
"""Démarre une session d'apprentissage via l'orchestrateur agent-chat.
|
||||
|
||||
Args:
|
||||
base_url: URL racine de l'agent-chat (ex. http://localhost:5004).
|
||||
machine_id: Identifiant unique du poste Windows.
|
||||
session_name: Nom humain de la tâche (saisi par l'utilisateur).
|
||||
api_token: Bearer token (RPA_API_TOKEN). Vide => header omis.
|
||||
user_id: Identifiant utilisateur optionnel.
|
||||
trigger_source: Source du déclenchement (windows_button, tray, ...).
|
||||
timeout_s: Timeout total connect+read par tentative.
|
||||
max_attempts: Nombre total de tentatives (1 + retry).
|
||||
backoff_s: Tuple des délais en secondes entre tentatives (len = max_attempts-1).
|
||||
|
||||
Returns:
|
||||
LearnStartResponse normalisée.
|
||||
|
||||
Raises:
|
||||
LeaOrchestratorError: si toutes les tentatives échouent.
|
||||
"""
|
||||
# Import local : httpx peut ne pas être installé sur tous les postes
|
||||
# Windows historiques. On veut un message d'erreur clair plutôt qu'un
|
||||
# ImportError en chaîne au moment du clic bouton.
|
||||
try:
|
||||
import httpx
|
||||
except ImportError as exc: # pragma: no cover (dépend du venv)
|
||||
raise LeaOrchestratorError(
|
||||
"httpx non disponible — installer httpx>=0.27 sur le poste Windows."
|
||||
) from exc
|
||||
|
||||
url = base_url.rstrip("/") + "/api/learn/start"
|
||||
payload = {
|
||||
"machine_id": machine_id,
|
||||
"session_name": session_name,
|
||||
"trigger_source": trigger_source,
|
||||
}
|
||||
if user_id:
|
||||
payload["user_id"] = user_id
|
||||
|
||||
headers = {"Content-Type": "application/json"}
|
||||
if api_token:
|
||||
headers["Authorization"] = f"Bearer {api_token}"
|
||||
|
||||
last_exc: Optional[Exception] = None
|
||||
for attempt in range(max_attempts):
|
||||
try:
|
||||
logger.info(
|
||||
"POST %s (tentative %d/%d) machine_id=%s session=%s",
|
||||
url, attempt + 1, max_attempts, machine_id, session_name,
|
||||
)
|
||||
with httpx.Client(timeout=timeout_s) as client:
|
||||
resp = client.post(url, json=payload, headers=headers)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
session_id = data.get("session_id", "")
|
||||
state = data.get("state", "")
|
||||
message = data.get("message", "")
|
||||
if not session_id:
|
||||
raise LeaOrchestratorError(
|
||||
f"Réponse invalide (pas de session_id) : {data!r}"
|
||||
)
|
||||
logger.info(
|
||||
"Session Léa démarrée : session_id=%s state=%s",
|
||||
session_id, state,
|
||||
)
|
||||
return LearnStartResponse(
|
||||
session_id=str(session_id),
|
||||
state=str(state),
|
||||
message=str(message),
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 — on retry sur toute erreur réseau/HTTP
|
||||
last_exc = exc
|
||||
logger.warning(
|
||||
"Echec tentative %d/%d POST %s : %s",
|
||||
attempt + 1, max_attempts, url, exc,
|
||||
)
|
||||
if attempt < max_attempts - 1:
|
||||
delay = backoff_s[attempt] if attempt < len(backoff_s) else backoff_s[-1]
|
||||
time.sleep(delay)
|
||||
|
||||
raise LeaOrchestratorError(
|
||||
f"Echec définitif POST {url} après {max_attempts} tentatives : {last_exc}"
|
||||
)
|
||||
317
agent_v0/agent_v1/network/log_shipper.py
Normal file
317
agent_v0/agent_v1/network/log_shipper.py
Normal file
@@ -0,0 +1,317 @@
|
||||
# agent_v1/network/log_shipper.py
|
||||
"""Remontée AUTOMATIQUE des logs du client Léa vers le serveur (push-log-DGX).
|
||||
|
||||
But : diagnostiquer les postes Windows clinique SANS AnyDesk. Les logs déjà
|
||||
écrits sur disque par `logging_setup.py` (rotation quotidienne, rétention 180 j,
|
||||
Règlement IA Art. 12) sont en plus poussés au serveur, rangés par `machine_id`,
|
||||
consultables au dashboard.
|
||||
|
||||
Serveur (déjà prêt — NE PAS toucher) :
|
||||
POST /api/v1/agents/logs
|
||||
body = {machine_id: str, logs: [{ts, level, logger, message}]}
|
||||
borne RPA_AGENT_LOGS_MAX_BATCH (défaut 1000) — 413 si dépassée.
|
||||
|
||||
Conception :
|
||||
- `LogShipperHandler(logging.Handler)` : sur `emit(record)`, formate au
|
||||
schéma EXACT `{ts, level, logger, message}`, applique un assainissement
|
||||
PII au message (défense en profondeur — la discipline `log_safe` à la
|
||||
source logue déjà des hashes/longueurs, pas du contenu brut), puis
|
||||
empile dans un buffer borné.
|
||||
- `LogShipper` : flush par BATCH (≤ max_batch) via un `sender` callable
|
||||
INJECTABLE `(machine_id, logs) -> bool`. Défaut = POST réel Bearer
|
||||
(pattern `streamer.py`).
|
||||
- Résilience (ZÉRO perte) : si `sender` renvoie False ou lève, les logs
|
||||
RESTENT dans le buffer et sont rejoués au flush suivant. Le fichier de
|
||||
log local reste de toute façon la source durable (survit au crash) ; le
|
||||
buffer RAM est un best-effort de remontée, volontairement NON persisté en
|
||||
SQLite (le `PersistentBuffer` est session/event-scoped — y mêler des logs
|
||||
polluerait la DB d'events). Borne mémoire = `max_buffer` (drop des plus
|
||||
VIEUX au-delà — un log récent vaut mieux qu'un vieux pour le diagnostic).
|
||||
|
||||
Pattern d'import PII : on tente `anonymize_text` (server_v1.pii_sanitizer,
|
||||
source de vérité des tokens typés) via le même import paresseux tolérant que
|
||||
`ui/messages.py`. Sur un vrai poste (sans server_v1), on retombe sur l'identité :
|
||||
acceptable car la PII de message est déjà neutralisée à la source par la
|
||||
discipline `log_safe`. Le sanitizer reste INJECTABLE pour les tests/évolutions.
|
||||
|
||||
Branche feat/push-log-dgx.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from collections import deque
|
||||
from typing import Callable, Deque, Dict, List, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Schéma d'une entrée de log poussée au serveur.
|
||||
# ts : epoch (float) — l'heure de l'évènement
|
||||
# level : nom du niveau ("INFO", "WARNING"...)
|
||||
# logger : nom du logger (record.name)
|
||||
# message : message formaté (args interpolés) ET assaini PII
|
||||
|
||||
# Défaut aligné sur la borne serveur RPA_AGENT_LOGS_MAX_BATCH (api_stream.py).
|
||||
DEFAULT_MAX_BATCH = 1000
|
||||
|
||||
# Borne mémoire du buffer : au-delà, on droppe les plus VIEUX (diagnostic =
|
||||
# on préfère les logs récents). Quelques milliers d'entrées = quelques Mo RAM.
|
||||
DEFAULT_MAX_BUFFER = 5000
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Assainissement PII du message (défense en profondeur)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _default_message_sanitizer(text: str) -> str:
|
||||
"""Sanitizer par défaut côté client = identité.
|
||||
|
||||
Le **rempart PII des logs est le SERVEUR** : `sanitize_log_entries`
|
||||
ré-assainit chaque message à la réception (`/api/v1/agents/logs`), via le
|
||||
même `anonymize_text` que les events. Tenter un import de `server_v1` côté
|
||||
poste à CHAQUE ligne de log est inutile (absent du bundle client) et coûteux
|
||||
(exception attrapée par emit). La discipline `log_safe` neutralise déjà la
|
||||
PII à la source. Reste INJECTABLE pour tests/évolutions.
|
||||
"""
|
||||
return text
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Handler — empile les LogRecords dans un buffer partagé
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class LogShipperHandler(logging.Handler):
|
||||
"""Handler logging qui sérialise chaque record et l'empile pour envoi.
|
||||
|
||||
Ne fait AUCUN réseau : il alimente seulement le buffer du `LogShipper`.
|
||||
L'envoi est piloté par `LogShipper.flush()` (thread dédié périodique).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
buffer: Deque[Dict],
|
||||
lock: threading.Lock,
|
||||
message_sanitizer: Callable[[str], str],
|
||||
max_buffer: int = DEFAULT_MAX_BUFFER,
|
||||
level=logging.NOTSET,
|
||||
):
|
||||
super().__init__(level=level)
|
||||
self._buffer = buffer
|
||||
self._lock = lock
|
||||
self._sanitize = message_sanitizer
|
||||
self._max_buffer = max_buffer
|
||||
|
||||
def _format_record(self, record: logging.LogRecord) -> Dict:
|
||||
"""Construit l'entrée au schéma EXACT {ts, level, logger, message}.
|
||||
|
||||
`record.getMessage()` interpole les args (%s...). Le message est ensuite
|
||||
passé au sanitizer PII. Tolérant : un message non formatable ne doit pas
|
||||
faire perdre l'entrée.
|
||||
"""
|
||||
try:
|
||||
message = record.getMessage()
|
||||
except Exception:
|
||||
message = str(record.msg)
|
||||
try:
|
||||
message = self._sanitize(message)
|
||||
except Exception:
|
||||
# Le sanitizer ne doit jamais casser le logging.
|
||||
pass
|
||||
return {
|
||||
"ts": record.created,
|
||||
"level": record.levelname,
|
||||
"logger": record.name,
|
||||
"message": message,
|
||||
}
|
||||
|
||||
def emit(self, record: logging.LogRecord) -> None:
|
||||
"""Sérialise et empile le record (best-effort, ne lève jamais)."""
|
||||
try:
|
||||
entry = self._format_record(record)
|
||||
with self._lock:
|
||||
# deque(maxlen) droppe automatiquement le plus VIEUX au-delà
|
||||
# de la borne — pas de croissance mémoire non bornée.
|
||||
self._buffer.append(entry)
|
||||
except Exception:
|
||||
# handleError respecte logging.raiseExceptions (silencieux en prod).
|
||||
self.handleError(record)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Shipper — flush périodique par batch via un sender injectable
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class LogShipper:
|
||||
"""Orchestre la remontée des logs : buffer + flush par batch.
|
||||
|
||||
Args:
|
||||
machine_id : identifiant du poste (config.MACHINE_ID en prod).
|
||||
sender : callable INJECTABLE `(machine_id, logs) -> bool`. True =
|
||||
accusé de réception serveur. Défaut = POST réel Bearer.
|
||||
max_batch : taille max d'un batch (≤ borne serveur). Défaut 1000.
|
||||
max_buffer : borne mémoire du buffer (drop des plus vieux au-delà).
|
||||
message_sanitizer : assainissement PII du message. Défaut = pii_sanitizer
|
||||
si disponible, sinon identité.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
machine_id: str,
|
||||
sender: Optional[Callable[[str, List[Dict]], bool]] = None,
|
||||
max_batch: int = DEFAULT_MAX_BATCH,
|
||||
max_buffer: int = DEFAULT_MAX_BUFFER,
|
||||
message_sanitizer: Optional[Callable[[str], str]] = None,
|
||||
flush_interval_s: float = 30.0,
|
||||
):
|
||||
self.machine_id = machine_id
|
||||
self.max_batch = max(1, int(max_batch))
|
||||
self.flush_interval_s = flush_interval_s
|
||||
self._sender = sender if sender is not None else self._default_sender
|
||||
self._sanitize = message_sanitizer or _default_message_sanitizer
|
||||
self._lock = threading.Lock()
|
||||
self._buffer: Deque[Dict] = deque(maxlen=max_buffer)
|
||||
self.handler = LogShipperHandler(
|
||||
buffer=self._buffer,
|
||||
lock=self._lock,
|
||||
message_sanitizer=self._sanitize,
|
||||
max_buffer=max_buffer,
|
||||
)
|
||||
self._running = False
|
||||
self._thread: Optional[threading.Thread] = None
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Introspection (diagnostic / tests)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def peek_buffer(self) -> List[Dict]:
|
||||
"""Copie des entrées en attente (lecture seule, pour diagnostic/tests)."""
|
||||
with self._lock:
|
||||
return list(self._buffer)
|
||||
|
||||
def pending(self) -> int:
|
||||
with self._lock:
|
||||
return len(self._buffer)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Flush — envoie le buffer par batches ≤ max_batch
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def flush(self) -> int:
|
||||
"""Envoie le buffer par batches successifs. Retourne le nb de logs ACK.
|
||||
|
||||
Résilience ZÉRO perte : on retire un batch du buffer, on tente l'envoi.
|
||||
- Succès → les entrées sont définitivement consommées.
|
||||
- Échec (False ou exception) → on REMET les entrées en tête du buffer
|
||||
et on ARRÊTE la passe (serveur probablement down) ; rejeu au flush
|
||||
suivant. Les entrées non encore extraites restent en place.
|
||||
"""
|
||||
sent = 0
|
||||
while True:
|
||||
with self._lock:
|
||||
if not self._buffer:
|
||||
break
|
||||
batch: List[Dict] = []
|
||||
for _ in range(min(self.max_batch, len(self._buffer))):
|
||||
batch.append(self._buffer.popleft())
|
||||
|
||||
try:
|
||||
ok = self._sender(self.machine_id, batch)
|
||||
except Exception as e:
|
||||
ok = False
|
||||
logger.debug("Log shipper sender a levé : %s", e)
|
||||
|
||||
if ok:
|
||||
sent += len(batch)
|
||||
continue
|
||||
|
||||
# Échec : on remet le batch en tête (ordre préservé) et on arrête.
|
||||
with self._lock:
|
||||
self._buffer.extendleft(reversed(batch))
|
||||
break
|
||||
|
||||
return sent
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Sender réel — POST Bearer (pattern streamer.py)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@staticmethod
|
||||
def _auth_headers() -> dict:
|
||||
"""Headers Bearer (pattern streamer.py)."""
|
||||
try:
|
||||
from ..config import API_TOKEN
|
||||
except Exception:
|
||||
API_TOKEN = ""
|
||||
if API_TOKEN:
|
||||
return {"Authorization": f"Bearer {API_TOKEN}"}
|
||||
return {}
|
||||
|
||||
def _default_sender(self, machine_id: str, logs: List[Dict]) -> bool:
|
||||
"""POST réel vers /api/v1/agents/logs. True si HTTP 2xx.
|
||||
|
||||
Best-effort : tout échec réseau/serveur → False (logs conservés,
|
||||
rejoués). Aucune exception ne remonte au-delà du sender.
|
||||
"""
|
||||
try:
|
||||
import requests
|
||||
|
||||
from ..config import SERVER_URL
|
||||
|
||||
url = f"{SERVER_URL}/agents/logs"
|
||||
resp = requests.post(
|
||||
url,
|
||||
json={"machine_id": machine_id, "logs": logs},
|
||||
headers=self._auth_headers(),
|
||||
timeout=5,
|
||||
allow_redirects=False,
|
||||
)
|
||||
return bool(resp.ok)
|
||||
except Exception as e:
|
||||
logger.debug("Log shipper POST échoué : %s", e)
|
||||
return False
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Boucle de flush périodique (thread daemon)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def start(self) -> None:
|
||||
"""Démarre le thread de flush périodique (idempotent)."""
|
||||
if self._running:
|
||||
return
|
||||
self._running = True
|
||||
self._thread = threading.Thread(
|
||||
target=self._flush_loop, daemon=True, name="lea-log-shipper"
|
||||
)
|
||||
self._thread.start()
|
||||
logger.info(
|
||||
"Log shipper démarré (machine_id=%s, intervalle=%.0fs, batch≤%d)",
|
||||
self.machine_id, self.flush_interval_s, self.max_batch,
|
||||
)
|
||||
|
||||
def stop(self, final_flush: bool = True) -> None:
|
||||
"""Arrête la boucle et tente un dernier flush (best-effort)."""
|
||||
self._running = False
|
||||
if self._thread:
|
||||
self._thread.join(timeout=2.0)
|
||||
if final_flush:
|
||||
try:
|
||||
self.flush()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _flush_loop(self) -> None:
|
||||
while self._running:
|
||||
# Découpe l'attente pour réagir vite à stop().
|
||||
waited = 0.0
|
||||
step = 0.5
|
||||
while self._running and waited < self.flush_interval_s:
|
||||
time.sleep(step)
|
||||
waited += step
|
||||
if not self._running:
|
||||
break
|
||||
try:
|
||||
self.flush()
|
||||
except Exception as e:
|
||||
logger.debug("Log shipper flush loop : %s", e)
|
||||
380
agent_v0/agent_v1/network/persistent_buffer.py
Normal file
380
agent_v0/agent_v1/network/persistent_buffer.py
Normal file
@@ -0,0 +1,380 @@
|
||||
# agent_v1/network/persistent_buffer.py
|
||||
"""
|
||||
Buffer persistant SQLite pour les événements/images qui n'ont pas pu être envoyés.
|
||||
|
||||
Résout le bloquant AI Act Article 12 : en cas de coupure serveur ou de queue pleine,
|
||||
les événements prioritaires (click, key, action, screenshot) sont persistés sur disque
|
||||
au lieu d'être silencieusement perdus. Ils sont rejoués à la reconnexion.
|
||||
|
||||
Caractéristiques :
|
||||
- SQLite fichier unique (agent_v1/buffer/pending_events.db), thread-safe
|
||||
- Async : les écritures se font depuis un thread daemon, jamais bloquant
|
||||
- Quota : compteur d'attempts par item, abandon après MAX_ATTEMPTS
|
||||
- Robustesse : un fichier corrompu est renommé et recréé vide
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sqlite3
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Nombre max de tentatives avant abandon définitif d'un item
|
||||
MAX_ATTEMPTS = 10
|
||||
|
||||
# Taille max du buffer en items pour éviter une explosion disque
|
||||
# (typiquement : 1000 events + 1000 images = quelques Mo de SQLite)
|
||||
MAX_BUFFER_ITEMS = 2000
|
||||
|
||||
|
||||
class PersistentBuffer:
|
||||
"""Buffer SQLite pour événements/images en attente d'envoi.
|
||||
|
||||
Deux tables :
|
||||
- pending_events (id, session_id, payload_json, attempts, created_at)
|
||||
- pending_images (id, session_id, shot_id, image_path, attempts, created_at)
|
||||
|
||||
Usage :
|
||||
buf = PersistentBuffer(base_dir / "buffer")
|
||||
buf.add_event(session_id, event_dict) # persiste un event
|
||||
buf.add_image(session_id, image_path, shot_id) # persiste une image
|
||||
for row in buf.drain_events(): # itère sur les events
|
||||
if envoyer(row): buf.delete_event(row["id"])
|
||||
else: buf.mark_attempt(row["id"], "event")
|
||||
"""
|
||||
|
||||
def __init__(self, buffer_dir: Path):
|
||||
self.buffer_dir = Path(buffer_dir)
|
||||
self.buffer_dir.mkdir(parents=True, exist_ok=True)
|
||||
self.db_path = self.buffer_dir / "pending_events.db"
|
||||
self._lock = threading.Lock()
|
||||
self._init_db()
|
||||
|
||||
# ---------------------------------------------------------------
|
||||
# Initialisation / gestion corruption
|
||||
# ---------------------------------------------------------------
|
||||
|
||||
def _init_db(self):
|
||||
"""Crée les tables si elles n'existent pas.
|
||||
|
||||
En cas de fichier corrompu, on le renomme en .corrupted et on recrée
|
||||
un buffer vide. On préfère perdre un buffer non lisible plutôt que
|
||||
de crasher l'agent au démarrage.
|
||||
"""
|
||||
try:
|
||||
with self._connect() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS pending_events (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
session_id TEXT NOT NULL,
|
||||
payload TEXT NOT NULL,
|
||||
attempts INTEGER NOT NULL DEFAULT 0,
|
||||
created_at REAL NOT NULL
|
||||
)
|
||||
"""
|
||||
)
|
||||
conn.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS pending_images (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
session_id TEXT NOT NULL,
|
||||
shot_id TEXT NOT NULL,
|
||||
image_path TEXT NOT NULL,
|
||||
attempts INTEGER NOT NULL DEFAULT 0,
|
||||
created_at REAL NOT NULL
|
||||
)
|
||||
"""
|
||||
)
|
||||
conn.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_events_created "
|
||||
"ON pending_events(created_at)"
|
||||
)
|
||||
conn.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_images_created "
|
||||
"ON pending_images(created_at)"
|
||||
)
|
||||
conn.commit()
|
||||
except sqlite3.DatabaseError as e:
|
||||
logger.warning(
|
||||
f"Buffer SQLite corrompu ({e}) — renommage en .corrupted "
|
||||
f"et recréation d'un buffer vide"
|
||||
)
|
||||
try:
|
||||
corrupted = self.db_path.with_suffix(
|
||||
f".corrupted.{int(time.time())}"
|
||||
)
|
||||
os.rename(self.db_path, corrupted)
|
||||
except OSError:
|
||||
# Si le rename échoue, on tente la suppression directe
|
||||
try:
|
||||
os.remove(self.db_path)
|
||||
except OSError:
|
||||
pass
|
||||
# Nouvelle tentative (table vide)
|
||||
with self._connect() as conn:
|
||||
conn.execute(
|
||||
"CREATE TABLE IF NOT EXISTS pending_events ("
|
||||
"id INTEGER PRIMARY KEY AUTOINCREMENT, "
|
||||
"session_id TEXT NOT NULL, payload TEXT NOT NULL, "
|
||||
"attempts INTEGER NOT NULL DEFAULT 0, "
|
||||
"created_at REAL NOT NULL)"
|
||||
)
|
||||
conn.execute(
|
||||
"CREATE TABLE IF NOT EXISTS pending_images ("
|
||||
"id INTEGER PRIMARY KEY AUTOINCREMENT, "
|
||||
"session_id TEXT NOT NULL, shot_id TEXT NOT NULL, "
|
||||
"image_path TEXT NOT NULL, "
|
||||
"attempts INTEGER NOT NULL DEFAULT 0, "
|
||||
"created_at REAL NOT NULL)"
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
def _connect(self) -> sqlite3.Connection:
|
||||
"""Connexion SQLite en mode WAL (meilleure concurrence)."""
|
||||
conn = sqlite3.connect(
|
||||
str(self.db_path),
|
||||
timeout=5.0,
|
||||
check_same_thread=False,
|
||||
isolation_level=None, # autocommit — on gère les transactions
|
||||
)
|
||||
try:
|
||||
conn.execute("PRAGMA journal_mode=WAL")
|
||||
conn.execute("PRAGMA synchronous=NORMAL")
|
||||
except sqlite3.DatabaseError:
|
||||
pass
|
||||
conn.row_factory = sqlite3.Row
|
||||
return conn
|
||||
|
||||
# ---------------------------------------------------------------
|
||||
# Écriture — persiste un item
|
||||
# ---------------------------------------------------------------
|
||||
|
||||
def add_event(self, session_id: str, event: dict) -> bool:
|
||||
"""Persiste un événement. Retourne True si écrit, False sinon.
|
||||
|
||||
Si le buffer dépasse MAX_BUFFER_ITEMS, on drop l'insertion (plutôt
|
||||
que saturer le disque). On log un warning au premier dépassement.
|
||||
"""
|
||||
with self._lock:
|
||||
try:
|
||||
with self._connect() as conn:
|
||||
count = conn.execute(
|
||||
"SELECT COUNT(*) FROM pending_events"
|
||||
).fetchone()[0]
|
||||
if count >= MAX_BUFFER_ITEMS:
|
||||
logger.warning(
|
||||
f"Buffer persistant saturé ({count} events) "
|
||||
f"— event droppé"
|
||||
)
|
||||
return False
|
||||
conn.execute(
|
||||
"INSERT INTO pending_events "
|
||||
"(session_id, payload, attempts, created_at) "
|
||||
"VALUES (?, ?, 0, ?)",
|
||||
(session_id, json.dumps(event), time.time()),
|
||||
)
|
||||
return True
|
||||
except (sqlite3.DatabaseError, TypeError, ValueError) as e:
|
||||
logger.error(f"Buffer add_event échoué : {e}")
|
||||
return False
|
||||
|
||||
def add_image(
|
||||
self, session_id: str, image_path: str, shot_id: str
|
||||
) -> bool:
|
||||
"""Persiste une référence image (chemin fichier + shot_id).
|
||||
|
||||
On ne stocke PAS les bytes de l'image (risque de faire gonfler la DB) :
|
||||
uniquement le chemin. Donc l'image doit rester présente sur disque
|
||||
tant qu'elle n'a pas été envoyée avec succès au serveur.
|
||||
"""
|
||||
with self._lock:
|
||||
try:
|
||||
with self._connect() as conn:
|
||||
count = conn.execute(
|
||||
"SELECT COUNT(*) FROM pending_images"
|
||||
).fetchone()[0]
|
||||
if count >= MAX_BUFFER_ITEMS:
|
||||
logger.warning(
|
||||
f"Buffer persistant saturé ({count} images) "
|
||||
f"— image droppée"
|
||||
)
|
||||
return False
|
||||
conn.execute(
|
||||
"INSERT INTO pending_images "
|
||||
"(session_id, shot_id, image_path, attempts, created_at) "
|
||||
"VALUES (?, ?, ?, 0, ?)",
|
||||
(session_id, shot_id, image_path, time.time()),
|
||||
)
|
||||
return True
|
||||
except sqlite3.DatabaseError as e:
|
||||
logger.error(f"Buffer add_image échoué : {e}")
|
||||
return False
|
||||
|
||||
# ---------------------------------------------------------------
|
||||
# Lecture — drain dans l'ordre chronologique
|
||||
# ---------------------------------------------------------------
|
||||
|
||||
def drain_events(self, limit: int = 100) -> list:
|
||||
"""Retourne les events en attente, triés par date de création."""
|
||||
with self._lock:
|
||||
try:
|
||||
with self._connect() as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT id, session_id, payload, attempts "
|
||||
"FROM pending_events "
|
||||
"ORDER BY created_at ASC LIMIT ?",
|
||||
(limit,),
|
||||
).fetchall()
|
||||
return [dict(r) for r in rows]
|
||||
except sqlite3.DatabaseError as e:
|
||||
logger.error(f"Buffer drain_events échoué : {e}")
|
||||
return []
|
||||
|
||||
def drain_images(self, limit: int = 50) -> list:
|
||||
"""Retourne les images en attente, triées par date de création."""
|
||||
with self._lock:
|
||||
try:
|
||||
with self._connect() as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT id, session_id, shot_id, image_path, attempts "
|
||||
"FROM pending_images "
|
||||
"ORDER BY created_at ASC LIMIT ?",
|
||||
(limit,),
|
||||
).fetchall()
|
||||
return [dict(r) for r in rows]
|
||||
except sqlite3.DatabaseError as e:
|
||||
logger.error(f"Buffer drain_images échoué : {e}")
|
||||
return []
|
||||
|
||||
# ---------------------------------------------------------------
|
||||
# Marquage — succès, échec, abandon
|
||||
# ---------------------------------------------------------------
|
||||
|
||||
def delete_event(self, row_id: int):
|
||||
"""Supprime un event après envoi réussi."""
|
||||
with self._lock:
|
||||
try:
|
||||
with self._connect() as conn:
|
||||
conn.execute(
|
||||
"DELETE FROM pending_events WHERE id = ?", (row_id,)
|
||||
)
|
||||
except sqlite3.DatabaseError as e:
|
||||
logger.error(f"Buffer delete_event échoué : {e}")
|
||||
|
||||
def delete_image(self, row_id: int):
|
||||
"""Supprime une image après envoi réussi."""
|
||||
with self._lock:
|
||||
try:
|
||||
with self._connect() as conn:
|
||||
conn.execute(
|
||||
"DELETE FROM pending_images WHERE id = ?", (row_id,)
|
||||
)
|
||||
except sqlite3.DatabaseError as e:
|
||||
logger.error(f"Buffer delete_image échoué : {e}")
|
||||
|
||||
def increment_attempts(self, row_id: int, kind: str) -> int:
|
||||
"""Incrémente le compteur d'attempts. Retourne la nouvelle valeur.
|
||||
|
||||
kind : "event" ou "image"
|
||||
"""
|
||||
table = "pending_events" if kind == "event" else "pending_images"
|
||||
with self._lock:
|
||||
try:
|
||||
with self._connect() as conn:
|
||||
conn.execute(
|
||||
f"UPDATE {table} SET attempts = attempts + 1 "
|
||||
"WHERE id = ?",
|
||||
(row_id,),
|
||||
)
|
||||
row = conn.execute(
|
||||
f"SELECT attempts FROM {table} WHERE id = ?", (row_id,)
|
||||
).fetchone()
|
||||
return int(row["attempts"]) if row else MAX_ATTEMPTS
|
||||
except sqlite3.DatabaseError as e:
|
||||
logger.error(f"Buffer increment_attempts échoué : {e}")
|
||||
return MAX_ATTEMPTS
|
||||
|
||||
def abandon_exceeded(self) -> int:
|
||||
"""Supprime les items ayant dépassé MAX_ATTEMPTS.
|
||||
|
||||
Un item abandonné est logué en erreur (trace AI Act) puis supprimé.
|
||||
Retourne le nombre d'items abandonnés.
|
||||
"""
|
||||
abandoned = 0
|
||||
with self._lock:
|
||||
try:
|
||||
with self._connect() as conn:
|
||||
# Events abandonnés
|
||||
rows = conn.execute(
|
||||
"SELECT id, session_id, payload FROM pending_events "
|
||||
"WHERE attempts >= ?",
|
||||
(MAX_ATTEMPTS,),
|
||||
).fetchall()
|
||||
for r in rows:
|
||||
try:
|
||||
event_type = json.loads(r["payload"]).get(
|
||||
"type", "?"
|
||||
)
|
||||
except (ValueError, TypeError):
|
||||
event_type = "?"
|
||||
logger.error(
|
||||
f"Buffer : event abandonné après {MAX_ATTEMPTS} "
|
||||
f"tentatives — session={r['session_id']} "
|
||||
f"type={event_type}"
|
||||
)
|
||||
abandoned += 1
|
||||
conn.execute(
|
||||
"DELETE FROM pending_events WHERE attempts >= ?",
|
||||
(MAX_ATTEMPTS,),
|
||||
)
|
||||
|
||||
# Images abandonnées
|
||||
rows = conn.execute(
|
||||
"SELECT id, session_id, shot_id FROM pending_images "
|
||||
"WHERE attempts >= ?",
|
||||
(MAX_ATTEMPTS,),
|
||||
).fetchall()
|
||||
for r in rows:
|
||||
logger.error(
|
||||
f"Buffer : image abandonnée après {MAX_ATTEMPTS} "
|
||||
f"tentatives — session={r['session_id']} "
|
||||
f"shot_id={r['shot_id']}"
|
||||
)
|
||||
abandoned += 1
|
||||
conn.execute(
|
||||
"DELETE FROM pending_images WHERE attempts >= ?",
|
||||
(MAX_ATTEMPTS,),
|
||||
)
|
||||
except sqlite3.DatabaseError as e:
|
||||
logger.error(f"Buffer abandon_exceeded échoué : {e}")
|
||||
return abandoned
|
||||
|
||||
# ---------------------------------------------------------------
|
||||
# Introspection
|
||||
# ---------------------------------------------------------------
|
||||
|
||||
def counts(self) -> dict:
|
||||
"""Retourne (events_count, images_count) pour diagnostic."""
|
||||
with self._lock:
|
||||
try:
|
||||
with self._connect() as conn:
|
||||
ev = conn.execute(
|
||||
"SELECT COUNT(*) FROM pending_events"
|
||||
).fetchone()[0]
|
||||
im = conn.execute(
|
||||
"SELECT COUNT(*) FROM pending_images"
|
||||
).fetchone()[0]
|
||||
return {"events": ev, "images": im}
|
||||
except sqlite3.DatabaseError:
|
||||
return {"events": 0, "images": 0}
|
||||
|
||||
def is_empty(self) -> bool:
|
||||
c = self.counts()
|
||||
return c["events"] == 0 and c["images"] == 0
|
||||
@@ -14,18 +14,40 @@ Robustesse (P0-2) :
|
||||
- Health-check périodique (30s) pour recovery du flag _server_available
|
||||
- Compression JPEG qualité 85 pour les images (réduction ~5-10x)
|
||||
- Backpressure : queue bornée (maxsize=100), drop des heartbeat si pleine
|
||||
|
||||
Conformité AI Act (Article 12 — journalisation automatique) :
|
||||
- Purge après ACK : les screenshots locaux sont supprimés après HTTP 200
|
||||
du serveur (par défaut). Le serveur devient la source de vérité.
|
||||
- Buffer persistant : les events/images prioritaires non envoyés sont
|
||||
persistés dans un SQLite local (agent_v1/buffer/pending_events.db)
|
||||
et rejoués au démarrage et à la reconnexion.
|
||||
"""
|
||||
|
||||
import enum
|
||||
import io
|
||||
import logging
|
||||
import os
|
||||
import queue
|
||||
import threading
|
||||
import time
|
||||
from typing import Callable, Optional
|
||||
|
||||
import requests
|
||||
from PIL import Image
|
||||
|
||||
from ..config import API_TOKEN, STREAMING_ENDPOINT
|
||||
from ..config import API_TOKEN, BASE_DIR, STREAMING_ENDPOINT
|
||||
from ..core.log_safe import _title_hash
|
||||
from .persistent_buffer import MAX_ATTEMPTS, PersistentBuffer
|
||||
|
||||
|
||||
# Fix P0-E : résultat d'envoi d'image trivaleur (succès / échec réseau / fichier
|
||||
# disparu). On ne doit PAS considérer un FileNotFoundError comme un succès
|
||||
# HTTP 200 — sinon le buffer SQLite supprime l'entrée alors que le serveur n'a
|
||||
# jamais reçu l'image (perte silencieuse).
|
||||
class ImageSendResult(enum.Enum):
|
||||
OK = "ok" # HTTP 200, serveur a accusé réception
|
||||
FAILED = "failed" # Erreur réseau/serveur récupérable (retry OK)
|
||||
FILE_GONE = "file_gone" # Fichier local introuvable (abandon, pas retry)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -42,8 +64,28 @@ JPEG_QUALITY = 85
|
||||
# Taille max de la queue (backpressure)
|
||||
QUEUE_MAX_SIZE = 100
|
||||
|
||||
# Types d'événements à ne jamais dropper
|
||||
PRIORITY_EVENT_TYPES = {"click", "key", "scroll", "action", "screenshot"}
|
||||
# Types d'événements à ne jamais dropper.
|
||||
# Les noms historiques sont conservés, mais les événements réels du captor
|
||||
# Agent V1 sont mouse_click/key_combo/text_input/mouse_scroll.
|
||||
PRIORITY_EVENT_TYPES = {
|
||||
"click", "key", "scroll", "action", "screenshot",
|
||||
"mouse_click", "double_click", "key_combo", "key_press",
|
||||
"text_input", "mouse_scroll",
|
||||
}
|
||||
|
||||
# Purge locale après ACK serveur (Partie A de l'audit)
|
||||
# Activé par défaut : le serveur conserve déjà les screenshots 180 jours
|
||||
# (conformité AI Act Article 12). Désactivable via RPA_PURGE_AFTER_ACK=0
|
||||
# pour debugging local.
|
||||
PURGE_AFTER_ACK = os.environ.get("RPA_PURGE_AFTER_ACK", "1").lower() in (
|
||||
"1", "true", "yes",
|
||||
)
|
||||
|
||||
# Chemin du buffer persistant (Partie B de l'audit)
|
||||
BUFFER_DIR = BASE_DIR / "buffer"
|
||||
|
||||
# Intervalle entre deux tentatives de drain du buffer (secondes)
|
||||
BUFFER_DRAIN_INTERVAL_S = 15
|
||||
|
||||
|
||||
class TraceStreamer:
|
||||
@@ -54,8 +96,25 @@ class TraceStreamer:
|
||||
self.running = False
|
||||
self._thread = None
|
||||
self._health_thread = None
|
||||
self._drain_thread = None
|
||||
self._server_available = True # Désactivé après trop d'échecs
|
||||
|
||||
# Buffer persistant — partagé entre sessions (survit au redémarrage)
|
||||
# Initialisé paresseusement pour ne pas payer le coût SQLite en dehors
|
||||
# d'un streaming actif.
|
||||
self._buffer: PersistentBuffer | None = None
|
||||
self._on_finalize_result: Optional[Callable[[dict], None]] = None
|
||||
|
||||
def set_on_finalize_result(self, callback: Optional[Callable[[dict], None]]) -> None:
|
||||
"""Définir un callback appelé avec le payload JSON de /finalize."""
|
||||
self._on_finalize_result = callback
|
||||
|
||||
def _get_buffer(self) -> PersistentBuffer:
|
||||
"""Retourne le buffer persistant, en l'initialisant au besoin."""
|
||||
if self._buffer is None:
|
||||
self._buffer = PersistentBuffer(BUFFER_DIR)
|
||||
return self._buffer
|
||||
|
||||
@staticmethod
|
||||
def _auth_headers() -> dict:
|
||||
"""Headers d'authentification Bearer pour les requêtes API."""
|
||||
@@ -75,7 +134,12 @@ class TraceStreamer:
|
||||
target=self._health_check_loop, daemon=True
|
||||
)
|
||||
self._health_thread.start()
|
||||
logger.info(f"Streamer pour {self.session_id} démarré")
|
||||
# Thread de drain du buffer persistant (rejoue les items en attente)
|
||||
self._drain_thread = threading.Thread(
|
||||
target=self._buffer_drain_loop, daemon=True
|
||||
)
|
||||
self._drain_thread.start()
|
||||
logger.info(f"Streamer démarré")
|
||||
|
||||
def stop(self):
|
||||
"""Arrêter le streaming et finaliser la session côté serveur.
|
||||
@@ -99,8 +163,11 @@ class TraceStreamer:
|
||||
if self._health_thread:
|
||||
self._health_thread.join(timeout=2.0)
|
||||
|
||||
if self._drain_thread:
|
||||
self._drain_thread.join(timeout=2.0)
|
||||
|
||||
self._finalize_session()
|
||||
logger.info(f"Streamer pour {self.session_id} arrêté")
|
||||
logger.info(f"Streamer arrêté")
|
||||
|
||||
def push_event(self, event_data: dict):
|
||||
"""Enfile un événement pour envoi immédiat.
|
||||
@@ -126,11 +193,21 @@ class TraceStreamer:
|
||||
|
||||
Quand la queue est pleine :
|
||||
- Les événements prioritaires (click, key, action, screenshot) sont
|
||||
ajoutés en bloquant brièvement (0.5s)
|
||||
- Les heartbeat sont silencieusement droppés
|
||||
ajoutés en bloquant brièvement (0.5s). Si toujours pleine → persistés
|
||||
dans le buffer SQLite pour rejeu ultérieur.
|
||||
- Les heartbeat sont silencieusement droppés.
|
||||
- Si le serveur est marqué indisponible, on persiste immédiatement les
|
||||
items prioritaires (évite de remplir la queue inutilement).
|
||||
"""
|
||||
is_priority = self._is_priority_item(item_type, data)
|
||||
|
||||
# Serveur indisponible + item prioritaire → on persiste directement
|
||||
# sans polluer la queue RAM (qui ne sera jamais vidée tant que le
|
||||
# serveur est down).
|
||||
if is_priority and not self._server_available:
|
||||
self._persist_to_buffer(item_type, data)
|
||||
return
|
||||
|
||||
try:
|
||||
self.queue.put_nowait((item_type, data))
|
||||
except queue.Full:
|
||||
@@ -139,10 +216,18 @@ class TraceStreamer:
|
||||
try:
|
||||
self.queue.put((item_type, data), timeout=0.5)
|
||||
except queue.Full:
|
||||
logger.warning(
|
||||
f"Queue pleine — événement prioritaire droppé "
|
||||
f"(type={item_type})"
|
||||
)
|
||||
# Persistance disque (ne JAMAIS dropper un prioritaire)
|
||||
persisted = self._persist_to_buffer(item_type, data)
|
||||
if persisted:
|
||||
logger.warning(
|
||||
f"Queue pleine — événement prioritaire persisté "
|
||||
f"sur disque (type={item_type})"
|
||||
)
|
||||
else:
|
||||
logger.error(
|
||||
f"Queue pleine ET buffer saturé — événement "
|
||||
f"prioritaire perdu (type={item_type})"
|
||||
)
|
||||
else:
|
||||
# Heartbeat ou événement non-critique : on drop silencieusement
|
||||
logger.debug(
|
||||
@@ -163,6 +248,23 @@ class TraceStreamer:
|
||||
return event_type in PRIORITY_EVENT_TYPES
|
||||
return False
|
||||
|
||||
def _persist_to_buffer(self, item_type: str, data) -> bool:
|
||||
"""Persiste un item dans le buffer SQLite. Retourne True si OK.
|
||||
|
||||
Utilisé quand la queue est pleine ou le serveur indisponible.
|
||||
"""
|
||||
try:
|
||||
buf = self._get_buffer()
|
||||
if item_type == "event" and isinstance(data, dict):
|
||||
return buf.add_event(self.session_id, data)
|
||||
if item_type == "image":
|
||||
path, shot_id = data
|
||||
return buf.add_image(self.session_id, path, shot_id)
|
||||
except Exception as e:
|
||||
# On n'arrête jamais l'agent si le buffer échoue
|
||||
logger.error(f"Persistance buffer échouée : {e}")
|
||||
return False
|
||||
|
||||
# =========================================================================
|
||||
# Boucle d'envoi
|
||||
# =========================================================================
|
||||
@@ -174,16 +276,36 @@ class TraceStreamer:
|
||||
try:
|
||||
item_type, data = self.queue.get(timeout=0.5)
|
||||
success = False
|
||||
is_file_gone = False
|
||||
if item_type == "event":
|
||||
success = self._send_with_retry(self._send_event, data)
|
||||
elif item_type == "image":
|
||||
success = self._send_with_retry(self._send_image, *data)
|
||||
result = self._send_with_retry(self._send_image, *data)
|
||||
# Fix P0-E : distinguer FILE_GONE du vrai succès HTTP.
|
||||
if result is ImageSendResult.OK:
|
||||
success = True
|
||||
elif result is ImageSendResult.FILE_GONE:
|
||||
# Fichier disparu : pas de retry, pas de persistance
|
||||
# (on ne peut plus le renvoyer). On considère l'item
|
||||
# comme traité sans comptabiliser un succès réseau.
|
||||
is_file_gone = True
|
||||
success = False
|
||||
else:
|
||||
success = False
|
||||
self.queue.task_done()
|
||||
|
||||
if success:
|
||||
consecutive_failures = 0
|
||||
elif is_file_gone:
|
||||
# Fichier introuvable — déjà logué ERROR dans _send_image.
|
||||
# On ne persiste PAS dans le buffer (retry voué à échouer).
|
||||
consecutive_failures = 0
|
||||
else:
|
||||
consecutive_failures += 1
|
||||
# Après 3 retries infructueux, si l'item est prioritaire,
|
||||
# on le persiste pour ne pas le perdre définitivement.
|
||||
if self._is_priority_item(item_type, data):
|
||||
self._persist_to_buffer(item_type, data)
|
||||
if consecutive_failures >= 10:
|
||||
logger.warning(
|
||||
"10 échecs consécutifs — serveur marqué indisponible"
|
||||
@@ -200,15 +322,22 @@ class TraceStreamer:
|
||||
# Retry avec backoff exponentiel
|
||||
# =========================================================================
|
||||
|
||||
def _send_with_retry(self, send_fn, *args) -> bool:
|
||||
def _send_with_retry(self, send_fn, *args):
|
||||
"""Tente l'envoi avec retry et backoff exponentiel.
|
||||
|
||||
3 tentatives max avec délais de 1s, 2s, 4s entre chaque.
|
||||
Retourne True si l'envoi a réussi, False sinon.
|
||||
Retourne :
|
||||
- True / ImageSendResult.OK si l'envoi a réussi
|
||||
- ImageSendResult.FILE_GONE (images uniquement) — pas de retry
|
||||
- False / ImageSendResult.FAILED sinon
|
||||
"""
|
||||
# Première tentative (sans délai)
|
||||
if send_fn(*args):
|
||||
return True
|
||||
first = send_fn(*args)
|
||||
if first is ImageSendResult.OK or first is True:
|
||||
return first
|
||||
# Fix P0-E : FILE_GONE → pas de retry, l'erreur est permanente.
|
||||
if first is ImageSendResult.FILE_GONE:
|
||||
return first
|
||||
|
||||
# Retries avec backoff
|
||||
for attempt, delay in enumerate(RETRY_DELAYS, start=1):
|
||||
@@ -219,9 +348,13 @@ class TraceStreamer:
|
||||
f"Retry {attempt}/{MAX_RETRIES} dans {delay}s..."
|
||||
)
|
||||
time.sleep(delay)
|
||||
if send_fn(*args):
|
||||
result = send_fn(*args)
|
||||
if result is ImageSendResult.OK or result is True:
|
||||
logger.debug(f"Retry {attempt} réussi")
|
||||
return True
|
||||
return result
|
||||
# FILE_GONE pendant un retry — idem, on arrête
|
||||
if result is ImageSendResult.FILE_GONE:
|
||||
return result
|
||||
|
||||
logger.debug(f"Envoi échoué après {MAX_RETRIES} retries")
|
||||
return False
|
||||
@@ -260,6 +393,115 @@ class TraceStreamer:
|
||||
except Exception:
|
||||
logger.debug("Health-check échoué — serveur toujours indisponible")
|
||||
|
||||
# =========================================================================
|
||||
# Drain du buffer persistant (Partie B)
|
||||
# =========================================================================
|
||||
|
||||
def _buffer_drain_loop(self):
|
||||
"""Rejoue les items persistés en arrière-plan.
|
||||
|
||||
Tourne tant que self.running. Essaie de drainer le buffer toutes les
|
||||
BUFFER_DRAIN_INTERVAL_S secondes, mais seulement si :
|
||||
- le serveur est disponible,
|
||||
- il y a effectivement des items en attente.
|
||||
|
||||
Au premier passage (démarrage agent), on draine immédiatement pour
|
||||
rejouer tout ce qui a été persisté lors de la session précédente.
|
||||
"""
|
||||
# Au démarrage : drain immédiat (pas d'attente)
|
||||
first_pass = True
|
||||
while self.running:
|
||||
if not first_pass:
|
||||
time.sleep(BUFFER_DRAIN_INTERVAL_S)
|
||||
if not self.running:
|
||||
break
|
||||
first_pass = False
|
||||
|
||||
if not self._server_available:
|
||||
continue
|
||||
|
||||
try:
|
||||
buf = self._get_buffer()
|
||||
# Abandonner d'abord les items exceeded (évite de les retenter)
|
||||
abandoned = buf.abandon_exceeded()
|
||||
if abandoned:
|
||||
logger.warning(
|
||||
f"Buffer : {abandoned} items abandonnés "
|
||||
f"après {MAX_ATTEMPTS} tentatives"
|
||||
)
|
||||
|
||||
counts = buf.counts()
|
||||
if counts["events"] == 0 and counts["images"] == 0:
|
||||
continue
|
||||
|
||||
logger.info(
|
||||
f"Buffer drain : {counts['events']} events, "
|
||||
f"{counts['images']} images en attente — rejeu"
|
||||
)
|
||||
self._drain_buffer_once(buf)
|
||||
except Exception as e:
|
||||
logger.error(f"Buffer drain loop échoué : {e}")
|
||||
|
||||
def _drain_buffer_once(self, buf: PersistentBuffer):
|
||||
"""Une passe de drain : envoie ce qui peut l'être, incrémente le reste.
|
||||
|
||||
On arrête dès qu'un envoi échoue (serveur probablement down).
|
||||
"""
|
||||
# Events d'abord (plus légers, priorité métier AI Act)
|
||||
for row in buf.drain_events(limit=50):
|
||||
if not self._server_available:
|
||||
return
|
||||
try:
|
||||
import json as _json
|
||||
event = _json.loads(row["payload"])
|
||||
except (ValueError, TypeError):
|
||||
logger.error(
|
||||
f"Buffer : payload event #{row['id']} corrompu, suppression"
|
||||
)
|
||||
buf.delete_event(row["id"])
|
||||
continue
|
||||
if self._send_event(event):
|
||||
buf.delete_event(row["id"])
|
||||
else:
|
||||
buf.increment_attempts(row["id"], "event")
|
||||
# Serveur répond mal — on arrête la passe
|
||||
return
|
||||
|
||||
# Puis images
|
||||
for row in buf.drain_images(limit=20):
|
||||
if not self._server_available:
|
||||
return
|
||||
image_path = row["image_path"]
|
||||
shot_id = row["shot_id"]
|
||||
if not os.path.exists(image_path):
|
||||
# Fichier local disparu (purge, clean-up) — on abandonne.
|
||||
# Fix P0-E : log ERROR (pas warning) — c'est une perte de donnée.
|
||||
logger.error(
|
||||
f"Buffer : image #{row['id']} introuvable sur disque "
|
||||
f"({image_path}) — entrée abandonnée (le serveur n'a "
|
||||
f"jamais reçu cette image, session={row['session_id']}, "
|
||||
f"shot={shot_id})"
|
||||
)
|
||||
buf.delete_image(row["id"])
|
||||
continue
|
||||
result = self._send_image(image_path, shot_id)
|
||||
if result is ImageSendResult.OK or result is True:
|
||||
buf.delete_image(row["id"])
|
||||
elif result is ImageSendResult.FILE_GONE:
|
||||
# Fix P0-E : fichier disparu pendant l'envoi.
|
||||
# Ce n'est PAS un succès HTTP — ne pas considérer comme tel.
|
||||
# On supprime néanmoins l'entrée (retry voué à échouer)
|
||||
# mais avec un log ERROR explicite.
|
||||
logger.error(
|
||||
f"Buffer : image #{row['id']} disparue pendant l'envoi "
|
||||
f"({image_path}) — entrée abandonnée, pas de retry "
|
||||
f"(session={row['session_id']}, shot={shot_id})"
|
||||
)
|
||||
buf.delete_image(row["id"])
|
||||
else:
|
||||
buf.increment_attempts(row["id"], "image")
|
||||
return
|
||||
|
||||
# =========================================================================
|
||||
# Compression JPEG
|
||||
# =========================================================================
|
||||
@@ -287,6 +529,56 @@ class TraceStreamer:
|
||||
logger.warning(f"Compression JPEG échouée, envoi PNG brut: {e}")
|
||||
return None, None, None
|
||||
|
||||
# =========================================================================
|
||||
# Purge locale après ACK (Partie A)
|
||||
# =========================================================================
|
||||
|
||||
@staticmethod
|
||||
def _purge_local_image(path: str):
|
||||
"""Supprime un screenshot local après ACK 200 du serveur.
|
||||
|
||||
Ne crashe JAMAIS si le fichier est verrouillé (cas Windows) ou
|
||||
déjà supprimé : on log en debug et on continue. L'auto-cleanup
|
||||
de SessionStorage repassera plus tard.
|
||||
"""
|
||||
if not PURGE_AFTER_ACK:
|
||||
return
|
||||
try:
|
||||
os.remove(path)
|
||||
logger.debug(f"Screenshot local purgé après ACK : {path}")
|
||||
except FileNotFoundError:
|
||||
# Déjà supprimé ou chemin invalide — silencieux
|
||||
pass
|
||||
except PermissionError as e:
|
||||
# Windows verrouille parfois les fichiers (antivirus, indexation...)
|
||||
logger.debug(
|
||||
f"Purge différée (fichier verrouillé) : {path} — {e}"
|
||||
)
|
||||
except OSError as e:
|
||||
logger.debug(f"Purge échouée : {path} — {e}")
|
||||
|
||||
# =========================================================================
|
||||
# Protection redirect POST→GET (INC-7)
|
||||
# =========================================================================
|
||||
|
||||
@staticmethod
|
||||
def _check_redirect(resp, url: str):
|
||||
"""Detecter et logger une redirection sur un POST.
|
||||
|
||||
La lib requests transforme un POST en GET sur 301/302 (RFC 7231).
|
||||
Avec allow_redirects=False, on recoit le 301/302 directement.
|
||||
On log un WARNING explicite pour que l'admin corrige l'URL.
|
||||
"""
|
||||
if resp.status_code in (301, 302, 307, 308):
|
||||
location = resp.headers.get("Location", "?")
|
||||
logger.warning(
|
||||
f"Redirection {resp.status_code} detectee sur POST {url} "
|
||||
f"→ {location}. Verifiez que RPA_SERVER_URL utilise "
|
||||
f"https:// si le serveur redirige."
|
||||
)
|
||||
return True
|
||||
return False
|
||||
|
||||
# =========================================================================
|
||||
# Envois HTTP
|
||||
# =========================================================================
|
||||
@@ -294,15 +586,20 @@ class TraceStreamer:
|
||||
def _register_session(self):
|
||||
"""Enregistrer la session auprès du serveur (avec identifiant machine)."""
|
||||
try:
|
||||
url = f"{STREAMING_ENDPOINT}/register"
|
||||
resp = requests.post(
|
||||
f"{STREAMING_ENDPOINT}/register",
|
||||
url,
|
||||
params={
|
||||
"session_id": self.session_id,
|
||||
"machine_id": self.machine_id,
|
||||
},
|
||||
headers=self._auth_headers(),
|
||||
timeout=3,
|
||||
allow_redirects=False,
|
||||
)
|
||||
if self._check_redirect(resp, url):
|
||||
logger.warning("Enregistrement session échoué (redirect)")
|
||||
return
|
||||
if resp.ok:
|
||||
logger.info(
|
||||
f"Session {self.session_id} enregistrée sur le serveur "
|
||||
@@ -322,28 +619,40 @@ class TraceStreamer:
|
||||
C'est la dernière chance de sauver les données de la session.
|
||||
"""
|
||||
try:
|
||||
url = f"{STREAMING_ENDPOINT}/finalize"
|
||||
resp = requests.post(
|
||||
f"{STREAMING_ENDPOINT}/finalize",
|
||||
url,
|
||||
params={
|
||||
"session_id": self.session_id,
|
||||
"machine_id": self.machine_id,
|
||||
},
|
||||
headers=self._auth_headers(),
|
||||
timeout=30, # Le build workflow peut prendre du temps
|
||||
allow_redirects=False,
|
||||
)
|
||||
self._check_redirect(resp, url)
|
||||
if resp.ok:
|
||||
result = resp.json()
|
||||
logger.info(f"Session finalisée: {result}")
|
||||
logger.info(f"Session finalisée [status={result.get('status')}, wf_hash={_title_hash(result.get('workflow_name',''))}]")
|
||||
if self._on_finalize_result is not None:
|
||||
try:
|
||||
self._on_finalize_result(result)
|
||||
except Exception as cb_error:
|
||||
logger.warning(
|
||||
"Callback finalize ignoré après erreur: %s",
|
||||
cb_error,
|
||||
)
|
||||
else:
|
||||
logger.warning(f"Finalisation échouée: {resp.status_code}")
|
||||
except Exception as e:
|
||||
logger.debug(f"Finalisation échouée: {e}")
|
||||
logger.warning(f"Finalisation échouée: {e}")
|
||||
|
||||
def _send_event(self, event: dict) -> bool:
|
||||
"""Envoyer un événement au serveur (avec identifiant machine)."""
|
||||
if not self._server_available:
|
||||
return False
|
||||
try:
|
||||
url = f"{STREAMING_ENDPOINT}/event"
|
||||
payload = {
|
||||
"session_id": self.session_id,
|
||||
"timestamp": time.time(),
|
||||
@@ -351,24 +660,36 @@ class TraceStreamer:
|
||||
"machine_id": self.machine_id,
|
||||
}
|
||||
resp = requests.post(
|
||||
f"{STREAMING_ENDPOINT}/event",
|
||||
url,
|
||||
json=payload,
|
||||
headers=self._auth_headers(),
|
||||
timeout=2,
|
||||
allow_redirects=False,
|
||||
)
|
||||
if self._check_redirect(resp, url):
|
||||
return False
|
||||
return resp.ok
|
||||
except Exception as e:
|
||||
logger.debug(f"Streaming Event échoué: {e}")
|
||||
return False
|
||||
|
||||
def _send_image(self, path: str, shot_id: str) -> bool:
|
||||
def _send_image(self, path: str, shot_id: str):
|
||||
"""Envoyer un screenshot au serveur, compressé en JPEG.
|
||||
|
||||
Utilise un context manager pour le fallback PNG afin d'éviter
|
||||
les fuites de descripteurs de fichier.
|
||||
|
||||
Partie A (purge après ACK) : en cas de HTTP 200 confirmé, le fichier
|
||||
local est supprimé (le serveur devient la source de vérité).
|
||||
|
||||
Fix P0-E : retourne `ImageSendResult` (OK / FAILED / FILE_GONE).
|
||||
Les appelants historiques qui attendaient un bool continuent de
|
||||
fonctionner grâce à la truthiness du enum (OK → True, reste → False),
|
||||
MAIS le drain du buffer doit désormais discriminer FILE_GONE pour
|
||||
ne pas confondre "fichier disparu" avec "envoyé avec succès".
|
||||
"""
|
||||
if not self._server_available:
|
||||
return False
|
||||
return ImageSendResult.FAILED
|
||||
try:
|
||||
# Tenter la compression JPEG (réduction ~5-10x vs PNG)
|
||||
jpeg_buf, content_type, suffix = self._compress_image_to_jpeg(path)
|
||||
@@ -379,19 +700,26 @@ class TraceStreamer:
|
||||
"machine_id": self.machine_id,
|
||||
}
|
||||
|
||||
url = f"{STREAMING_ENDPOINT}/image"
|
||||
if jpeg_buf is not None:
|
||||
# Envoi du JPEG compressé (BytesIO, pas de fuite possible)
|
||||
files = {
|
||||
"file": (f"{shot_id}{suffix}", jpeg_buf, content_type)
|
||||
}
|
||||
resp = requests.post(
|
||||
f"{STREAMING_ENDPOINT}/image",
|
||||
url,
|
||||
files=files,
|
||||
params=params,
|
||||
headers=self._auth_headers(),
|
||||
timeout=5,
|
||||
allow_redirects=False,
|
||||
)
|
||||
return resp.ok
|
||||
if self._check_redirect(resp, url):
|
||||
return ImageSendResult.FAILED
|
||||
if resp.ok:
|
||||
self._purge_local_image(path)
|
||||
return ImageSendResult.OK
|
||||
return ImageSendResult.FAILED
|
||||
else:
|
||||
# Fallback : envoi PNG original avec context manager
|
||||
with open(path, "rb") as f:
|
||||
@@ -399,13 +727,29 @@ class TraceStreamer:
|
||||
"file": (f"{shot_id}.png", f, "image/png")
|
||||
}
|
||||
resp = requests.post(
|
||||
f"{STREAMING_ENDPOINT}/image",
|
||||
url,
|
||||
files=files,
|
||||
params=params,
|
||||
headers=self._auth_headers(),
|
||||
timeout=5,
|
||||
allow_redirects=False,
|
||||
)
|
||||
return resp.ok
|
||||
if self._check_redirect(resp, url):
|
||||
return ImageSendResult.FAILED
|
||||
if resp.ok:
|
||||
self._purge_local_image(path)
|
||||
return ImageSendResult.OK
|
||||
return ImageSendResult.FAILED
|
||||
except FileNotFoundError:
|
||||
# Fix P0-E : fichier local disparu. On NE doit PAS considérer ça
|
||||
# comme un succès HTTP 200. Le serveur n'a rien reçu. On signale
|
||||
# `FILE_GONE` pour que le drain du buffer supprime l'entrée
|
||||
# (pas de retry possible) tout en loguant ERROR (pas debug).
|
||||
logger.error(
|
||||
f"Image {shot_id} introuvable sur disque ({path}) — "
|
||||
f"abandon (serveur n'a rien reçu)"
|
||||
)
|
||||
return ImageSendResult.FILE_GONE
|
||||
except Exception as e:
|
||||
logger.debug(f"Streaming Image échoué: {e}")
|
||||
return False
|
||||
return ImageSendResult.FAILED
|
||||
|
||||
481
agent_v0/agent_v1/network/updater.py
Normal file
481
agent_v0/agent_v1/network/updater.py
Normal file
@@ -0,0 +1,481 @@
|
||||
# agent_v1/network/updater.py
|
||||
"""NOYAU client de la mise à jour silencieuse de Léa (DETTE-022 v2).
|
||||
|
||||
GATED — flag `RPA_AUTO_UPDATE_ENABLED` (défaut OFF). Tant qu'il est OFF,
|
||||
rien ne se déclenche : l'intégration de ce module au runtime (boucle de poll
|
||||
de `main.py`) ne fait aucune MAJ.
|
||||
|
||||
Ce module ne contient que les parties PURES / testables, sans réseau réel :
|
||||
|
||||
- `parse_version` / `is_newer` (R3) : self-contained (le bundle client
|
||||
n'embarque PAS `server_v1` — duplication assumée, même algorithme).
|
||||
- `should_update(local_version, server_response)` : décide « faut-il
|
||||
updater ? quelle version/type ? » à partir de la réponse serveur. Double
|
||||
garde semver côté client (jamais de downgrade) = défense en profondeur.
|
||||
- `download_update(plan, staging_dir, downloader)` : télécharge le ZIP via un
|
||||
`downloader` callable INJECTABLE (aucun réseau réel en test), vérifie le
|
||||
SHA256, écrit le ZIP dans le **staging** (`Lea_next\\`-like) — JAMAIS dans
|
||||
les fichiers vivants. Retourne un plan d'application.
|
||||
- `auto_update_enabled()` : lit le flag (défaut OFF).
|
||||
|
||||
⚠️ SWAP — répartition claire des responsabilités :
|
||||
`apply_update` / `write_boot_ok_marker` ci-dessous ne font que l'ARMEMENT côté
|
||||
Python (extraction vers `agent_v1_new/` + marqueurs) — ils n'écrasent JAMAIS un
|
||||
fichier vivant. Le remplacement ATOMIQUE (renames), le redémarrage et le
|
||||
rollback sont faits HORS-PROCESS par `Lea.bat` au démarrage (revu ligne à ligne).
|
||||
|
||||
Pattern d'import / résilience aligné sur `log_shipper.py` (même branche).
|
||||
|
||||
Branche feat/push-log-dgx.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from typing import Callable, Optional, Tuple
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Niveaux de livraison (R2). `code-only` par défaut = 99 % des MAJ (~500 Ko).
|
||||
VALID_UPDATE_TYPES = ("code-only", "full")
|
||||
DEFAULT_UPDATE_TYPE = "code-only"
|
||||
|
||||
_FALLBACK_VERSION: Tuple[int, ...] = (0,)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Flag d'activation — OFF par défaut (lu à chaque appel pour faciliter tests)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def auto_update_enabled() -> bool:
|
||||
"""True si la MAJ auto client est activée (flag RPA_AUTO_UPDATE_ENABLED).
|
||||
|
||||
Défaut PRUDENT = OFF. On l'active poste par poste via config.txt / variable
|
||||
d'environnement, sans rebuild de l'installateur (même esprit que
|
||||
LOG_SHIP_ENABLED).
|
||||
"""
|
||||
return os.environ.get("RPA_AUTO_UPDATE_ENABLED", "false").lower() in (
|
||||
"true", "1", "yes", "on",
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# R3 — parse_version self-contained (le bundle client n'a pas server_v1)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def parse_version(v) -> Tuple[int, ...]:
|
||||
"""Parse une version semver en tuple d'entiers. Voir server_v1/update_check.
|
||||
|
||||
"1.0.2" → (1, 0, 2) ; "1.0.10" → (1, 0, 10) ; "v1.2.3" → (1, 2, 3).
|
||||
Tolérant et SANS exception : invalide → fallback `(0,)`.
|
||||
"""
|
||||
if not isinstance(v, str):
|
||||
return _FALLBACK_VERSION
|
||||
s = v.strip().lstrip("vV").strip()
|
||||
if not s:
|
||||
return _FALLBACK_VERSION
|
||||
try:
|
||||
from packaging.version import Version
|
||||
|
||||
return tuple(Version(s).release)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
return tuple(int(x) for x in s.split("."))
|
||||
except (ValueError, AttributeError):
|
||||
return _FALLBACK_VERSION
|
||||
|
||||
|
||||
def is_newer(candidate: str, baseline: str) -> bool:
|
||||
"""True si `candidate` strictement plus récent que `baseline` (semver)."""
|
||||
return parse_version(candidate) > parse_version(baseline)
|
||||
|
||||
|
||||
def _normalize_update_type(update_type) -> str:
|
||||
if update_type in VALID_UPDATE_TYPES:
|
||||
return update_type
|
||||
return DEFAULT_UPDATE_TYPE
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Décision client : faut-il updater ?
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def should_update(local_version: str, server_response) -> Optional[dict]:
|
||||
"""Décide à partir de la réponse serveur s'il faut updater.
|
||||
|
||||
Args:
|
||||
local_version : version courante du client (config.AGENT_VERSION).
|
||||
server_response : dict renvoyé par l'endpoint serveur
|
||||
{update_available, latest_version, update_type, url, [sha256]}.
|
||||
|
||||
Returns:
|
||||
Un PLAN d'update `{target_version, update_type, url, sha256}` si une MAJ
|
||||
valide est à faire, sinon None.
|
||||
|
||||
Défense en profondeur : même si `update_available` est True, le client
|
||||
REVÉRIFIE en semver (`is_newer`) — il ne descend JAMAIS vers une version
|
||||
<= locale. Tolérant : réponse malformée → None (jamais d'exception).
|
||||
"""
|
||||
if not isinstance(server_response, dict):
|
||||
return None
|
||||
if not server_response.get("update_available"):
|
||||
return None
|
||||
|
||||
target = server_response.get("latest_version")
|
||||
url = server_response.get("url")
|
||||
if not target or not url:
|
||||
return None
|
||||
|
||||
# Double garde semver : pas de downgrade, pas d'égalité.
|
||||
if not is_newer(target, local_version):
|
||||
return None
|
||||
|
||||
return {
|
||||
"target_version": target,
|
||||
"update_type": _normalize_update_type(server_response.get("update_type")),
|
||||
"url": url,
|
||||
"sha256": server_response.get("sha256"),
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Téléchargement — downloader INJECTABLE, SHA256, staging only
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _default_downloader(url: str) -> bytes:
|
||||
"""Téléchargement réel du ZIP (best-effort, pattern streamer/log_shipper).
|
||||
|
||||
Résout l'URL relative contre SERVER_BASE, ajoute le Bearer si présent.
|
||||
INJECTABLE : remplacé par un fake en test (aucun réseau réel).
|
||||
"""
|
||||
import requests # import tardif (absent de certains envs de test)
|
||||
|
||||
full_url = url
|
||||
headers = {}
|
||||
try:
|
||||
from ..config import SERVER_BASE, API_TOKEN
|
||||
|
||||
if url.startswith("/"):
|
||||
full_url = f"{SERVER_BASE}{url}"
|
||||
if API_TOKEN:
|
||||
headers["Authorization"] = f"Bearer {API_TOKEN}"
|
||||
except Exception:
|
||||
# Hors package (test isolé) : on utilise l'URL telle quelle.
|
||||
pass
|
||||
|
||||
resp = requests.get(full_url, headers=headers, timeout=30, stream=False)
|
||||
resp.raise_for_status()
|
||||
return resp.content
|
||||
|
||||
|
||||
def download_update(
|
||||
plan: dict,
|
||||
staging_dir,
|
||||
downloader: Optional[Callable[[str], bytes]] = None,
|
||||
) -> dict:
|
||||
"""Télécharge le ZIP d'update dans le staging et vérifie son intégrité.
|
||||
|
||||
NE TOUCHE PAS aux fichiers vivants : écrit uniquement dans `staging_dir`
|
||||
(équivalent de `Lea_next\\`). L'application réelle (swap) est un stub
|
||||
réservé révision humaine (voir `apply_update`).
|
||||
|
||||
Args:
|
||||
plan : sortie de `should_update` (target_version, update_type, url, sha256).
|
||||
staging_dir : dossier de staging (créé si absent).
|
||||
downloader : callable `(url) -> bytes` INJECTABLE (défaut = HTTP réel).
|
||||
|
||||
Returns:
|
||||
Succès : {ok: True, staged_zip: str, update_type, target_version,
|
||||
sha256_verified: bool}
|
||||
Échec : {ok: False, error: str}
|
||||
Best-effort : aucune exception ne remonte ; un échec laisse le staging propre
|
||||
(pas de ZIP corrompu).
|
||||
"""
|
||||
dl = downloader if downloader is not None else _default_downloader
|
||||
staging = Path(staging_dir)
|
||||
|
||||
try:
|
||||
data = dl(plan["url"])
|
||||
except Exception as e:
|
||||
logger.warning("Téléchargement update échoué : %s", e)
|
||||
return {"ok": False, "error": f"download_failed: {e}"}
|
||||
|
||||
expected_sha = (plan.get("sha256") or "").strip().lower()
|
||||
sha256_verified = False
|
||||
if expected_sha:
|
||||
actual = hashlib.sha256(data).hexdigest()
|
||||
if actual != expected_sha:
|
||||
logger.warning(
|
||||
"SHA256 mismatch update (attendu=%s, obtenu=%s) — rejeté",
|
||||
expected_sha, actual,
|
||||
)
|
||||
return {"ok": False, "error": "sha256 mismatch — ZIP rejeté"}
|
||||
sha256_verified = True
|
||||
else:
|
||||
# Best-effort : pas de SHA fourni → on accepte mais on le signale.
|
||||
logger.info("Pas de SHA256 fourni pour l'update — intégrité non vérifiée")
|
||||
|
||||
try:
|
||||
staging.mkdir(parents=True, exist_ok=True)
|
||||
target_version = plan.get("target_version", "unknown")
|
||||
staged_zip = staging / f"lea_update_{target_version}.zip"
|
||||
staged_zip.write_bytes(data)
|
||||
except Exception as e:
|
||||
logger.warning("Écriture ZIP staging échouée : %s", e)
|
||||
return {"ok": False, "error": f"staging_write_failed: {e}"}
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
"staged_zip": str(staged_zip),
|
||||
"update_type": _normalize_update_type(plan.get("update_type")),
|
||||
"target_version": plan.get("target_version"),
|
||||
"sha256_verified": sha256_verified,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Interrogation serveur — checker INJECTABLE (GET /agents/update/check)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _default_update_checker(local_version: str, machine_id: str):
|
||||
"""Interroge le serveur : y a-t-il une MAJ ? (best-effort, INJECTABLE).
|
||||
|
||||
GET SERVER_URL/agents/update/check?current_version=..&machine_id=..
|
||||
(endpoint gated côté serveur — 503 si RPA_AUTO_UPDATE_SERVER_ENABLED OFF,
|
||||
auquel cas on renvoie None : pas de MAJ). Bearer si présent. Pattern aligné
|
||||
sur `log_shipper._default_sender`. INJECTABLE : remplacé par un fake en test.
|
||||
|
||||
Returns:
|
||||
Le dict réponse serveur (`should_update` sait le lire), ou None si
|
||||
indisponible / gated / erreur (jamais d'exception ne remonte).
|
||||
"""
|
||||
try:
|
||||
import requests # import tardif
|
||||
|
||||
headers = {}
|
||||
try:
|
||||
from ..config import SERVER_URL, API_TOKEN
|
||||
|
||||
base = SERVER_URL
|
||||
if API_TOKEN:
|
||||
headers["Authorization"] = f"Bearer {API_TOKEN}"
|
||||
except Exception:
|
||||
base = ""
|
||||
url = f"{base}/agents/update/check"
|
||||
resp = requests.get(
|
||||
url,
|
||||
params={"current_version": local_version, "machine_id": machine_id},
|
||||
headers=headers,
|
||||
timeout=10,
|
||||
allow_redirects=False,
|
||||
)
|
||||
# 503 = endpoint gated OFF côté serveur → pas de MAJ (silencieux).
|
||||
if resp.status_code == 503:
|
||||
return None
|
||||
if not resp.ok:
|
||||
logger.debug("update/check HTTP %s", resp.status_code)
|
||||
return None
|
||||
return resp.json()
|
||||
except Exception as e:
|
||||
logger.debug("update/check indisponible : %s", e)
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Orchestrateur GATED — check → décide → download (staging) → stub apply
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def run_update_cycle(
|
||||
local_version: str,
|
||||
machine_id: str,
|
||||
staging_dir,
|
||||
checker: Optional[Callable[[str, str], object]] = None,
|
||||
downloader: Optional[Callable[[str], bytes]] = None,
|
||||
app_dir=None,
|
||||
) -> dict:
|
||||
"""Un cycle complet de MAJ silencieuse — GATED, best-effort, SANS swap.
|
||||
|
||||
Enchaîne :
|
||||
1. GATE `auto_update_enabled()` (RPA_AUTO_UPDATE_ENABLED, défaut OFF) —
|
||||
si OFF, ne fait STRICTEMENT rien (aucun appel réseau).
|
||||
2. `checker(local_version, machine_id)` → réponse serveur (canary-aware).
|
||||
3. `should_update(...)` → plan (double garde semver, jamais de downgrade).
|
||||
4. `download_update(...)` → ZIP dans le STAGING + vérif SHA256. Ne touche
|
||||
JAMAIS les fichiers vivants.
|
||||
5. `apply_update` ARME le swap (extraction `agent_v1_new/` + marqueur
|
||||
UPDATE_READY) mais NE swappe PAS : le remplacement atomique + le
|
||||
redémarrage sont faits par Lea.bat au prochain démarrage. `applied`
|
||||
reste False tant que Léa n'a pas redémarré sur la nouvelle version.
|
||||
|
||||
Jamais d'exception ne remonte (ne doit JAMAIS casser Léa). Retourne un dict
|
||||
d'état pour le diagnostic / le log :
|
||||
status ∈ {disabled, check_failed, up_to_date, download_failed, staged}
|
||||
|
||||
Args:
|
||||
checker : callable `(local_version, machine_id) -> dict|None`
|
||||
INJECTABLE (défaut = HTTP réel vers l'endpoint gated).
|
||||
downloader : callable `(url) -> bytes` INJECTABLE (défaut = HTTP réel).
|
||||
"""
|
||||
if not auto_update_enabled():
|
||||
return {"status": "disabled", "applied": False}
|
||||
|
||||
chk = checker if checker is not None else _default_update_checker
|
||||
|
||||
try:
|
||||
server_response = chk(local_version, machine_id)
|
||||
except Exception as e:
|
||||
logger.warning("update check a levé : %s", e)
|
||||
return {"status": "check_failed", "applied": False, "error": str(e)}
|
||||
|
||||
plan = should_update(local_version, server_response)
|
||||
if plan is None:
|
||||
return {"status": "up_to_date", "applied": False}
|
||||
|
||||
staged = download_update(plan, staging_dir, downloader=downloader)
|
||||
if not staged.get("ok"):
|
||||
return {
|
||||
"status": "download_failed",
|
||||
"applied": False,
|
||||
"error": staged.get("error"),
|
||||
}
|
||||
|
||||
# Armement du swap : extraction du ZIP vers agent_v1_new\ + marqueur
|
||||
# UPDATE_READY. Le swap ATOMIQUE (renames) et le redémarrage sont faits
|
||||
# HORS-PROCESS par Lea.bat au prochain démarrage — JAMAIS depuis ici
|
||||
# (on n'écrase pas les fichiers d'un Léa en cours d'exécution).
|
||||
armed = apply_update(staged, app_dir=app_dir)
|
||||
|
||||
return {
|
||||
"status": "armed" if armed.get("armed") else "arm_failed",
|
||||
"applied": False, # le swap effectif est fait par Lea.bat, pas ici
|
||||
"armed": bool(armed.get("armed", False)),
|
||||
"target_version": staged.get("target_version"),
|
||||
"update_type": staged.get("update_type"),
|
||||
"staged_zip": staged.get("staged_zip"),
|
||||
"sha256_verified": staged.get("sha256_verified", False),
|
||||
"marker": armed.get("marker"),
|
||||
"error": armed.get("error"),
|
||||
}
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# SWAP — côté Python : ARMEMENT SEULEMENT (extraction + marqueurs).
|
||||
# Le remplacement ATOMIQUE des fichiers vivants + le redémarrage + le
|
||||
# rollback sont faits HORS-PROCESS par `Lea.bat` au démarrage (renames).
|
||||
# Python n'écrase JAMAIS les fichiers d'un Léa en cours d'exécution.
|
||||
# ===========================================================================
|
||||
|
||||
def _resolve_app_dir(app_dir) -> Path:
|
||||
"""Répertoire d'install (contient `agent_v1/`, `run_agent_v1.py`, `Lea.bat`).
|
||||
|
||||
INJECTABLE (tests : tmp_path). Défaut = parent du package agent_v1.
|
||||
"""
|
||||
if app_dir is not None:
|
||||
return Path(app_dir)
|
||||
try:
|
||||
from ..config import BASE_DIR # BASE_DIR = dossier du package agent_v1
|
||||
return Path(BASE_DIR).parent
|
||||
except Exception:
|
||||
return Path(__file__).resolve().parent.parent.parent
|
||||
|
||||
|
||||
def apply_update(prepared: dict, app_dir=None) -> dict:
|
||||
"""ARME le swap : extrait le ZIP staging vers `agent_v1_new/` + marqueur.
|
||||
|
||||
NE swappe PAS et NE redémarre PAS (c'est le rôle de `Lea.bat`). Écrit
|
||||
uniquement à côté des fichiers vivants (dossier neuf + marqueur), donc
|
||||
l'opération est sûre même sur un Léa en cours d'exécution.
|
||||
|
||||
1. Extrait `prepared["staged_zip"]` → `<app_dir>/agent_v1_new/`
|
||||
(nettoyé au préalable ; garde-fou zip-slip).
|
||||
2. Écrit `<app_dir>/UPDATE_READY` (JSON : version, type, chemins) que
|
||||
`Lea.bat` lira au prochain démarrage pour faire le swap atomique.
|
||||
|
||||
Best-effort : aucune exception ne remonte (ne doit jamais casser Léa).
|
||||
|
||||
Returns:
|
||||
succès : {armed: True, applied: False, target_version, update_type,
|
||||
marker, extracted_to}
|
||||
échec : {armed: False, applied: False, error}
|
||||
"""
|
||||
if not isinstance(prepared, dict):
|
||||
return {"armed": False, "applied": False, "error": "prepared invalide"}
|
||||
staged_zip = prepared.get("staged_zip")
|
||||
target_version = prepared.get("target_version", "unknown")
|
||||
update_type = _normalize_update_type(prepared.get("update_type"))
|
||||
try:
|
||||
root = _resolve_app_dir(app_dir)
|
||||
zip_path = Path(staged_zip) if staged_zip else None
|
||||
if zip_path is None or not zip_path.is_file():
|
||||
return {"armed": False, "applied": False, "error": "ZIP staging introuvable"}
|
||||
|
||||
new_dir = root / "agent_v1_new"
|
||||
if new_dir.exists():
|
||||
shutil.rmtree(new_dir, ignore_errors=True) # nettoie un staging partiel
|
||||
new_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
import zipfile
|
||||
new_root = new_dir.resolve()
|
||||
with zipfile.ZipFile(zip_path) as zf:
|
||||
for name in zf.namelist(): # garde-fou zip-slip (chemins ../)
|
||||
dest = (new_dir / name).resolve()
|
||||
if not str(dest).startswith(str(new_root)):
|
||||
shutil.rmtree(new_dir, ignore_errors=True)
|
||||
return {"armed": False, "applied": False,
|
||||
"error": f"zip-slip refusé : {name}"}
|
||||
zf.extractall(new_dir)
|
||||
|
||||
marker = root / "UPDATE_READY"
|
||||
marker.write_text(json.dumps({
|
||||
"target_version": target_version,
|
||||
"update_type": update_type,
|
||||
"extracted_to": str(new_dir),
|
||||
"staged_zip": str(zip_path),
|
||||
}), encoding="utf-8")
|
||||
|
||||
logger.info(
|
||||
"Update ARMÉ : %s (%s) → %s ; swap au prochain démarrage (Lea.bat)",
|
||||
target_version, update_type, new_dir,
|
||||
)
|
||||
return {"armed": True, "applied": False, "target_version": target_version,
|
||||
"update_type": update_type, "marker": str(marker),
|
||||
"extracted_to": str(new_dir)}
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("apply_update (armement) a échoué : %s", e)
|
||||
return {"armed": False, "applied": False, "error": f"arm_failed: {e}"}
|
||||
|
||||
|
||||
def write_boot_ok_marker(version: str, app_dir=None) -> dict:
|
||||
"""Confirme un boot sain : écrit `boot_ok_{version}` + désarme le rollback.
|
||||
|
||||
Appelé par `main.py` après ~90 s de tourne STABLE (liveness LOCALE,
|
||||
indépendante du DGX — évite un faux rollback quand le réseau est coupé).
|
||||
Retirer `PENDING_BOOT*` dit à `Lea.bat` que la nouvelle version a démarré
|
||||
correctement (sinon, au prochain lancement, Lea.bat rollback vers la
|
||||
version précédente).
|
||||
|
||||
Best-effort : aucune exception ne remonte.
|
||||
"""
|
||||
try:
|
||||
root = _resolve_app_dir(app_dir)
|
||||
marker = root / f"boot_ok_{version}"
|
||||
marker.write_text("ok", encoding="utf-8")
|
||||
cleared = []
|
||||
for p in root.glob("PENDING_BOOT*"):
|
||||
try:
|
||||
p.unlink()
|
||||
cleared.append(p.name)
|
||||
except OSError:
|
||||
pass
|
||||
logger.info("boot_ok écrit (%s) ; PENDING_BOOT retiré : %s",
|
||||
version, cleared or "aucun")
|
||||
return {"written": True, "marker": str(marker), "cleared_pending": cleared}
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("write_boot_ok_marker a échoué : %s", e)
|
||||
return {"written": False, "error": str(e)}
|
||||
@@ -3,7 +3,10 @@ mss>=9.0.1 # Capture d'écran haute performance
|
||||
pynput>=1.7.7 # Clavier/Souris Cross-plateforme
|
||||
Pillow>=10.0.0 # Crops et processing image
|
||||
requests>=2.31.0 # Streaming réseau
|
||||
httpx>=0.27 # Client HTTP orchestrateur Léa (POST /api/learn/start) — brique conversationnelle
|
||||
python-socketio[client]>=5.10,<6.0 # Bus feedback Léa 'lea:*' (compat Flask-SocketIO 5.3.x serveur)
|
||||
psutil>=5.9.0 # Monitoring CPU/RAM
|
||||
screeninfo>=0.8 # QW1 — détection des monitors physiques + offsets
|
||||
pystray>=0.19.5 # Icône Tray UI
|
||||
plyer>=2.1.0 # Notifications toast natives (remplace PyQt5)
|
||||
pywebview>=5.0 # Fenêtre de chat Léa intégrée (Edge WebView2 sur Windows)
|
||||
|
||||
0
agent_v0/agent_v1/tools/__init__.py
Normal file
0
agent_v0/agent_v1/tools/__init__.py
Normal file
88
agent_v0/agent_v1/tools/test_lea_pause_flow.py
Normal file
88
agent_v0/agent_v1/tools/test_lea_pause_flow.py
Normal file
@@ -0,0 +1,88 @@
|
||||
# agent_v1/tools/test_lea_pause_flow.py
|
||||
"""Smoke test : simuler un lea:paused localement et vérifier la bulle ChatWindow.
|
||||
|
||||
À lancer SUR WINDOWS (PC démo) :
|
||||
cd C:/rpa_vision
|
||||
.venv\\Scripts\\python.exe -m agent_v1.tools.test_lea_pause_flow
|
||||
|
||||
Ce script ouvre une ChatWindow, simule l'arrivée d'un payload paused_need_help
|
||||
avec un message LONG (350+ chars pour tester le scroll interne), puis attend
|
||||
les clics utilisateur sur Continuer/Annuler. Le test vérifie qu'il y a UN SEUL
|
||||
affichage (la bulle chat), pas de toast supplémentaire.
|
||||
|
||||
Exit code 0 si succès. Logs dans la console.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
# Configurer le logging avant tout import du package
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s [%(name)s] %(levelname)s: %(message)s',
|
||||
)
|
||||
logger = logging.getLogger("test_lea_pause_flow")
|
||||
|
||||
# Forcer le bus feedback (pour que les boutons puissent émettre vers
|
||||
# rpa-agent-chat — port 5004). Si on ne veut PAS du bus, mettre LEA_FEEDBACK_BUS=0.
|
||||
os.environ.setdefault("LEA_FEEDBACK_BUS", "1")
|
||||
os.environ.setdefault("RPA_API_TOKEN", "") # à remplir si serveur exige Bearer
|
||||
|
||||
|
||||
def main() -> int:
|
||||
try:
|
||||
from agent_v1.ui.chat_window import ChatWindow
|
||||
except ImportError as e:
|
||||
print(f"[TEST] Import ChatWindow KO : {e}")
|
||||
return 2
|
||||
|
||||
print("[TEST] Création ChatWindow...")
|
||||
cw = ChatWindow(
|
||||
server_client=None,
|
||||
on_start_callback=None,
|
||||
server_host=os.environ.get("RPA_SERVER_HOST", "192.168.1.40"),
|
||||
chat_port=5004,
|
||||
)
|
||||
# Attendre que le tk loop soit prêt
|
||||
time.sleep(2.0)
|
||||
cw.show()
|
||||
time.sleep(0.5)
|
||||
|
||||
print("[TEST] Simulation lea:paused avec long message (350 chars)...")
|
||||
long_msg = (
|
||||
"Je n'arrive pas à trouver le champ « Numéro de dossier patient » "
|
||||
"sur l'écran courant. J'ai essayé 3 stratégies de grounding visuel "
|
||||
"(template matching, OCR, VLM) sans succès. Pouvez-vous me montrer "
|
||||
"l'emplacement exact du champ, ou cliquer dessus à ma place ? "
|
||||
"Quand vous avez fini, cliquez sur Continuer pour que je reprenne."
|
||||
)
|
||||
payload = {
|
||||
"replay_id": "test_replay_pause_flow_001",
|
||||
"workflow": "Démo UHCD",
|
||||
"reason": long_msg,
|
||||
"completed": 5,
|
||||
"total": 12,
|
||||
}
|
||||
cw._add_paused_bubble(payload)
|
||||
print(f"[TEST] Bulle envoyée. Message len={len(long_msg)} chars.")
|
||||
print("[TEST] Vérifiez visuellement :")
|
||||
print(" 1) UN SEUL popup (la bulle chat dans la fenêtre Léa)")
|
||||
print(" 2) Le message long s'affiche en intégralité (scroll interne si besoin)")
|
||||
print(" 3) Boutons Continuer / Annuler visibles")
|
||||
print(" 4) Cliquez sur Annuler → bulle fermée + feedback '✗ Annulé'")
|
||||
print("[TEST] La fenêtre reste ouverte 30s. Ctrl+C pour quitter avant.")
|
||||
|
||||
try:
|
||||
for i in range(30):
|
||||
time.sleep(1.0)
|
||||
except KeyboardInterrupt:
|
||||
print("[TEST] Interruption clavier.")
|
||||
|
||||
print("[TEST] Test terminé. Vérifier visuellement les 4 points ci-dessus.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
87
agent_v0/agent_v1/tools/test_lea_toast.py
Normal file
87
agent_v0/agent_v1/tools/test_lea_toast.py
Normal file
@@ -0,0 +1,87 @@
|
||||
# agent_v1/tools/test_lea_toast.py
|
||||
"""
|
||||
Test visuel rapide du toast Léa (démo GHT 8 mai 2026).
|
||||
|
||||
Lance trois scénarios de toast successifs pour valider l'affichage Windows :
|
||||
1. Toast simple « pause supervisée »
|
||||
2. Toast avec message long (vérifier wraplength)
|
||||
3. Toast type BLOCAGE (= ce que voit l'utilisateur quand Léa est perdue)
|
||||
|
||||
Usage Windows :
|
||||
C:\\rpa_vision\\.venv\\Scripts\\python.exe C:\\rpa_vision\\agent_v1\\tools\\test_lea_toast.py
|
||||
|
||||
Le script s'attend à voir trois toasts successifs en haut-droite de l'écran
|
||||
principal, espacés de ~6 s, fond bleu Léa, autodismiss après 15 s ou clic.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def _bootstrap_path() -> None:
|
||||
"""Autoriser l'exécution directe sans -m : ajouter C:\\rpa_vision au sys.path."""
|
||||
here = Path(__file__).resolve()
|
||||
# On remonte : tools -> agent_v1 -> rpa_vision (parent du package agent_v1)
|
||||
rpa_root = here.parent.parent.parent
|
||||
if str(rpa_root) not in sys.path:
|
||||
sys.path.insert(0, str(rpa_root))
|
||||
|
||||
|
||||
def main() -> int:
|
||||
_bootstrap_path()
|
||||
|
||||
# Import après ajout du path (les deux variantes fonctionnent)
|
||||
try:
|
||||
from agent_v1.ui.paused_toast import show_paused_toast
|
||||
except Exception as e: # pragma: no cover (debug only)
|
||||
print(f"[TEST] ERREUR import agent_v1.ui.paused_toast : {e}")
|
||||
return 1
|
||||
|
||||
scenarios = [
|
||||
(
|
||||
"Toast 1/3 : pause simple",
|
||||
"Léa a besoin de votre aide",
|
||||
"Test 1/3 — Pause supervisée. Cliquez sur 'Continuer' dans la chat.",
|
||||
),
|
||||
(
|
||||
"Toast 2/3 : message long",
|
||||
"Léa — j'attends votre validation",
|
||||
(
|
||||
"Test 2/3 — J'ai trouvé 11 dossiers correspondant à vos critères "
|
||||
"(UHCD, Forfait 1, PE2). Je vais traiter le dossier de M. DUPONT "
|
||||
"Jean en premier. Pouvez-vous valider que c'est le bon ordre "
|
||||
"avant que je continue ?"
|
||||
),
|
||||
),
|
||||
(
|
||||
"Toast 3/3 : blocage cible non trouvée",
|
||||
"Léa — je ne vois pas l'élément",
|
||||
(
|
||||
"Test 3/3 — Je n'arrive pas à trouver « Examens cliniques » à "
|
||||
"l'écran. Pouvez-vous me montrer où cliquer ?"
|
||||
),
|
||||
),
|
||||
]
|
||||
|
||||
for label, title, message in scenarios:
|
||||
print(f"[TEST] {label}")
|
||||
ok = show_paused_toast(title=title, message=message)
|
||||
print(f" show_paused_toast() = {ok}")
|
||||
if not ok:
|
||||
print(f" ECHEC : {label}")
|
||||
# Espacer pour que Dom voit chaque toast distinctement
|
||||
# (rate limit interne = 3s pour message identique, mais ici les
|
||||
# messages diffèrent, le rate limit ne s'applique pas)
|
||||
time.sleep(6)
|
||||
|
||||
print("[TEST] Attente 12s supplémentaires pour laisser le dernier toast vivre...")
|
||||
time.sleep(12)
|
||||
print("[TEST] OK — fin du test. Si vous avez vu 3 toasts bleus en haut-droite,")
|
||||
print(" le mécanisme Léa pause est validé.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
53
agent_v0/agent_v1/ui/_test_paused_toast.py
Normal file
53
agent_v0/agent_v1/ui/_test_paused_toast.py
Normal file
@@ -0,0 +1,53 @@
|
||||
# agent_v1/ui/_test_paused_toast.py
|
||||
"""
|
||||
Test isolé du toast paused — à exécuter directement sur Windows.
|
||||
|
||||
Usage (sur Windows, depuis C:\\rpa_vision\\agent_v1) :
|
||||
python -m agent_v1.ui._test_paused_toast
|
||||
|
||||
OU plus simple :
|
||||
python C:\\rpa_vision\\agent_v1\\ui\\_test_paused_toast.py
|
||||
|
||||
Le toast doit s'afficher en haut à droite de l'écran principal pendant ~15s.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
import time
|
||||
|
||||
|
||||
def main() -> int:
|
||||
print("[TEST] Lancement du toast paused...")
|
||||
|
||||
try:
|
||||
# Import flexible : essai relatif puis absolu
|
||||
try:
|
||||
from .paused_toast import show_paused_toast
|
||||
except ImportError:
|
||||
from paused_toast import show_paused_toast
|
||||
except Exception as e:
|
||||
print(f"[TEST] ERREUR import : {e}")
|
||||
return 1
|
||||
|
||||
ok = show_paused_toast(
|
||||
title="Léa a besoin de votre aide",
|
||||
message=(
|
||||
"Test isolé — démo GHT 8 mai 2026.\n"
|
||||
"Si vous voyez ce toast, le mécanisme de pause supervisée "
|
||||
"fonctionne correctement."
|
||||
),
|
||||
)
|
||||
print(f"[TEST] show_paused_toast() retour = {ok}")
|
||||
|
||||
if not ok:
|
||||
print("[TEST] ÉCHEC : toast non déclenché.")
|
||||
return 2
|
||||
|
||||
print("[TEST] Toast déclenché. Attente de 18s pour le voir s'afficher puis se fermer...")
|
||||
time.sleep(18)
|
||||
print("[TEST] OK — fin du test.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
420
agent_v0/agent_v1/ui/activity_panel.py
Normal file
420
agent_v0/agent_v1/ui/activity_panel.py
Normal file
@@ -0,0 +1,420 @@
|
||||
# agent_v1/ui/activity_panel.py
|
||||
"""
|
||||
Panel d'activité temps réel de Léa.
|
||||
|
||||
Affiche à l'utilisateur ce que Léa fait *maintenant* :
|
||||
- État courant (Observe / Cherche / Agit / Vérifie / Bloquée)
|
||||
- Action en cours (ex: "Clic sur Rechercher")
|
||||
- Progression (ex: "3/15")
|
||||
- Temps écoulé depuis le début du workflow
|
||||
|
||||
Contraintes :
|
||||
- Fallback silencieux si tkinter absent (ne crash jamais)
|
||||
- Thread-safe (mises à jour depuis les threads de replay)
|
||||
- Pas de dépendance à PyQt5 (seulement tkinter, déjà utilisé par chat_window)
|
||||
|
||||
Utilisation :
|
||||
panel = ActivityPanel()
|
||||
panel.definir_workflow("Saisie patient", nb_etapes=15)
|
||||
panel.mettre_a_jour(etat=EtatLea.AGIT, action="Clic sur Valider", etape=3)
|
||||
panel.masquer()
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from enum import Enum
|
||||
from typing import Optional
|
||||
|
||||
from ..core.log_safe import _title_hash
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class EtatLea(Enum):
|
||||
"""États macroscopiques de Léa pendant un replay."""
|
||||
|
||||
INACTIVE = ("inactive", "Prête", "#808080") # Gris
|
||||
OBSERVE = ("observe", "Observe", "#4A90E2") # Bleu
|
||||
CHERCHE = ("cherche", "Cherche", "#F5A623") # Orange
|
||||
AGIT = ("agit", "Agit", "#7ED321") # Vert
|
||||
VERIFIE = ("verifie", "Vérifie", "#9013FE") # Violet
|
||||
BLOQUEE = ("bloquee", "Bloquée", "#D0021B") # Rouge
|
||||
TERMINE = ("termine", "Terminé", "#50E3C2") # Turquoise
|
||||
|
||||
def __init__(self, code: str, libelle: str, couleur: str) -> None:
|
||||
self.code = code
|
||||
self.libelle = libelle
|
||||
self.couleur = couleur
|
||||
|
||||
|
||||
@dataclass
|
||||
class EtatActivite:
|
||||
"""Instantané de l'activité courante de Léa.
|
||||
|
||||
Utilisé par le panel et exposé par `ActivityPanel.snapshot()` pour les
|
||||
tests (sans dépendre de tkinter).
|
||||
"""
|
||||
|
||||
etat: EtatLea = EtatLea.INACTIVE
|
||||
action_courante: str = ""
|
||||
nom_workflow: str = ""
|
||||
etape: int = 0
|
||||
nb_etapes: int = 0
|
||||
debut_timestamp: float = 0.0
|
||||
dernier_message: str = ""
|
||||
|
||||
def temps_ecoule_s(self) -> float:
|
||||
"""Temps écoulé depuis le début du workflow (secondes)."""
|
||||
if self.debut_timestamp <= 0:
|
||||
return 0.0
|
||||
return max(0.0, time.time() - self.debut_timestamp)
|
||||
|
||||
def progression_texte(self) -> str:
|
||||
"""Représentation textuelle de la progression (ex: '3/15')."""
|
||||
if self.nb_etapes <= 0:
|
||||
return ""
|
||||
return f"{self.etape}/{self.nb_etapes}"
|
||||
|
||||
def temps_ecoule_texte(self) -> str:
|
||||
"""Représentation humaine du temps écoulé (ex: '12s', '1m24s')."""
|
||||
s = int(self.temps_ecoule_s())
|
||||
if s < 60:
|
||||
return f"{s}s"
|
||||
return f"{s // 60}m{s % 60:02d}s"
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
"""Sérialiser pour le logging et les tests."""
|
||||
return {
|
||||
"etat": self.etat.code,
|
||||
"etat_libelle": self.etat.libelle,
|
||||
"action_courante": self.action_courante,
|
||||
"nom_workflow": self.nom_workflow,
|
||||
"etape": self.etape,
|
||||
"nb_etapes": self.nb_etapes,
|
||||
"progression": self.progression_texte(),
|
||||
"temps_ecoule_s": round(self.temps_ecoule_s(), 1),
|
||||
"dernier_message": self.dernier_message,
|
||||
}
|
||||
|
||||
|
||||
class ActivityPanel:
|
||||
"""Panel d'activité de Léa.
|
||||
|
||||
Thread-safe. Le panel tkinter est créé à la demande (lazy) et uniquement
|
||||
si tkinter est disponible. Toutes les méthodes sont safe à appeler même
|
||||
si l'UI n'est pas dispo (fallback silencieux).
|
||||
"""
|
||||
|
||||
def __init__(self, activer_ui: bool = True) -> None:
|
||||
self._lock = threading.RLock()
|
||||
self._etat = EtatActivite()
|
||||
self._activer_ui = activer_ui
|
||||
# UI tkinter (créée à la demande dans le thread UI)
|
||||
self._tk_root = None
|
||||
self._tk_labels: dict = {}
|
||||
self._ui_disponible = None # Lazy : résolu au premier usage
|
||||
self._listeners = [] # Callbacks pour les changements d'état
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# API publique (thread-safe)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def definir_workflow(self, nom: str, nb_etapes: int = 0) -> None:
|
||||
"""Démarrer le suivi d'un nouveau workflow."""
|
||||
with self._lock:
|
||||
self._etat = EtatActivite(
|
||||
etat=EtatLea.OBSERVE,
|
||||
nom_workflow=nom,
|
||||
nb_etapes=nb_etapes,
|
||||
debut_timestamp=time.time(),
|
||||
)
|
||||
self._notifier_changement()
|
||||
self._rafraichir_ui()
|
||||
logger.info(f"[ACTIVITY] Workflow démarré : [wf_hash={_title_hash(nom)}] ({nb_etapes} étapes)")
|
||||
|
||||
def mettre_a_jour(
|
||||
self,
|
||||
etat: Optional[EtatLea] = None,
|
||||
action: Optional[str] = None,
|
||||
etape: Optional[int] = None,
|
||||
message: Optional[str] = None,
|
||||
) -> None:
|
||||
"""Mettre à jour l'état affiché.
|
||||
|
||||
Tous les paramètres sont optionnels — on ne met à jour que ce qui est
|
||||
fourni. Les autres champs conservent leur valeur actuelle.
|
||||
"""
|
||||
with self._lock:
|
||||
if etat is not None:
|
||||
self._etat.etat = etat
|
||||
if action is not None:
|
||||
self._etat.action_courante = action
|
||||
if etape is not None:
|
||||
self._etat.etape = etape
|
||||
if message is not None:
|
||||
self._etat.dernier_message = message
|
||||
|
||||
self._notifier_changement()
|
||||
self._rafraichir_ui()
|
||||
|
||||
def terminer(self, succes: bool = True) -> None:
|
||||
"""Marquer le workflow comme terminé."""
|
||||
with self._lock:
|
||||
self._etat.etat = EtatLea.TERMINE if succes else EtatLea.BLOQUEE
|
||||
if not succes:
|
||||
self._etat.dernier_message = (
|
||||
self._etat.dernier_message or "Léa a rendu la main"
|
||||
)
|
||||
self._notifier_changement()
|
||||
self._rafraichir_ui()
|
||||
|
||||
def reinitialiser(self) -> None:
|
||||
"""Remettre le panel en état inactif."""
|
||||
with self._lock:
|
||||
self._etat = EtatActivite()
|
||||
self._notifier_changement()
|
||||
self._rafraichir_ui()
|
||||
|
||||
def snapshot(self) -> EtatActivite:
|
||||
"""Obtenir un instantané immuable de l'état courant (pour les tests)."""
|
||||
with self._lock:
|
||||
return EtatActivite(
|
||||
etat=self._etat.etat,
|
||||
action_courante=self._etat.action_courante,
|
||||
nom_workflow=self._etat.nom_workflow,
|
||||
etape=self._etat.etape,
|
||||
nb_etapes=self._etat.nb_etapes,
|
||||
debut_timestamp=self._etat.debut_timestamp,
|
||||
dernier_message=self._etat.dernier_message,
|
||||
)
|
||||
|
||||
def masquer(self) -> None:
|
||||
"""Masquer le panel UI si affiché."""
|
||||
if self._tk_root is not None:
|
||||
try:
|
||||
self._tk_root.withdraw()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def afficher(self) -> None:
|
||||
"""Afficher le panel UI si disponible."""
|
||||
self._creer_ui_si_besoin()
|
||||
if self._tk_root is not None:
|
||||
try:
|
||||
self._tk_root.deiconify()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def on_change(self, callback) -> None:
|
||||
"""Enregistrer un listener appelé à chaque changement d'état."""
|
||||
with self._lock:
|
||||
self._listeners.append(callback)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Gestion UI tkinter (lazy, fallback silencieux)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _creer_ui_si_besoin(self) -> None:
|
||||
"""Créer la fenêtre tkinter au premier usage (lazy)."""
|
||||
if not self._activer_ui:
|
||||
return
|
||||
if self._tk_root is not None:
|
||||
return
|
||||
if self._ui_disponible is False:
|
||||
return # Déjà testé et indisponible
|
||||
|
||||
try:
|
||||
import tkinter as tk
|
||||
except Exception as e:
|
||||
logger.debug(f"[ACTIVITY] tkinter indisponible : {e}")
|
||||
self._ui_disponible = False
|
||||
return
|
||||
|
||||
try:
|
||||
self._tk_root = tk.Toplevel() if _tk_root_existe() else tk.Tk()
|
||||
self._tk_root.title("Léa — Activité")
|
||||
self._tk_root.geometry("340x180+40+40")
|
||||
self._tk_root.attributes("-topmost", True)
|
||||
self._tk_root.resizable(False, False)
|
||||
self._tk_root.configure(bg="#1E1E1E")
|
||||
|
||||
titre = tk.Label(
|
||||
self._tk_root,
|
||||
text="Léa",
|
||||
font=("Segoe UI", 14, "bold"),
|
||||
fg="#FFFFFF",
|
||||
bg="#1E1E1E",
|
||||
)
|
||||
titre.pack(pady=(10, 2))
|
||||
|
||||
self._tk_labels["etat"] = tk.Label(
|
||||
self._tk_root,
|
||||
text="Prête",
|
||||
font=("Segoe UI", 11),
|
||||
fg="#808080",
|
||||
bg="#1E1E1E",
|
||||
)
|
||||
self._tk_labels["etat"].pack()
|
||||
|
||||
self._tk_labels["action"] = tk.Label(
|
||||
self._tk_root,
|
||||
text="",
|
||||
font=("Segoe UI", 10),
|
||||
fg="#FFFFFF",
|
||||
bg="#1E1E1E",
|
||||
wraplength=300,
|
||||
)
|
||||
self._tk_labels["action"].pack(pady=(8, 2))
|
||||
|
||||
self._tk_labels["progression"] = tk.Label(
|
||||
self._tk_root,
|
||||
text="",
|
||||
font=("Segoe UI", 9),
|
||||
fg="#B0B0B0",
|
||||
bg="#1E1E1E",
|
||||
)
|
||||
self._tk_labels["progression"].pack()
|
||||
|
||||
self._tk_labels["temps"] = tk.Label(
|
||||
self._tk_root,
|
||||
text="",
|
||||
font=("Segoe UI", 9),
|
||||
fg="#808080",
|
||||
bg="#1E1E1E",
|
||||
)
|
||||
self._tk_labels["temps"].pack(pady=(4, 0))
|
||||
|
||||
self._tk_labels["message"] = tk.Label(
|
||||
self._tk_root,
|
||||
text="",
|
||||
font=("Segoe UI", 9, "italic"),
|
||||
fg="#B0B0B0",
|
||||
bg="#1E1E1E",
|
||||
wraplength=300,
|
||||
)
|
||||
self._tk_labels["message"].pack(pady=(6, 10))
|
||||
|
||||
# Masquer par défaut : on affiche seulement pendant un workflow
|
||||
self._tk_root.withdraw()
|
||||
self._ui_disponible = True
|
||||
except Exception as e:
|
||||
logger.debug(f"[ACTIVITY] Impossible de créer l'UI : {e}")
|
||||
self._ui_disponible = False
|
||||
self._tk_root = None
|
||||
|
||||
def _rafraichir_ui(self) -> None:
|
||||
"""Mettre à jour les labels tkinter (safe si l'UI n'existe pas)."""
|
||||
if not self._activer_ui or self._ui_disponible is False:
|
||||
return
|
||||
self._creer_ui_si_besoin()
|
||||
if self._tk_root is None:
|
||||
return
|
||||
|
||||
try:
|
||||
with self._lock:
|
||||
snap = self.snapshot()
|
||||
|
||||
# Utiliser after(0) pour rester dans le thread UI tkinter
|
||||
def _update():
|
||||
try:
|
||||
self._tk_labels["etat"].config(
|
||||
text=snap.etat.libelle,
|
||||
fg=snap.etat.couleur,
|
||||
)
|
||||
if snap.action_courante:
|
||||
self._tk_labels["action"].config(text=snap.action_courante)
|
||||
else:
|
||||
self._tk_labels["action"].config(text="")
|
||||
|
||||
prog = snap.progression_texte()
|
||||
if prog and snap.nom_workflow:
|
||||
self._tk_labels["progression"].config(
|
||||
text=f"« {snap.nom_workflow} » — {prog}"
|
||||
)
|
||||
elif snap.nom_workflow:
|
||||
self._tk_labels["progression"].config(
|
||||
text=f"« {snap.nom_workflow} »"
|
||||
)
|
||||
else:
|
||||
self._tk_labels["progression"].config(text="")
|
||||
|
||||
if snap.debut_timestamp > 0:
|
||||
self._tk_labels["temps"].config(
|
||||
text=f"⏱ {snap.temps_ecoule_texte()}"
|
||||
)
|
||||
else:
|
||||
self._tk_labels["temps"].config(text="")
|
||||
|
||||
self._tk_labels["message"].config(text=snap.dernier_message)
|
||||
|
||||
# Afficher automatiquement si actif
|
||||
if snap.etat != EtatLea.INACTIVE:
|
||||
self._tk_root.deiconify()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
self._tk_root.after(0, _update)
|
||||
except Exception:
|
||||
# Si le root a été détruit
|
||||
self._tk_root = None
|
||||
self._ui_disponible = False
|
||||
except Exception as e:
|
||||
logger.debug(f"[ACTIVITY] Erreur rafraîchissement UI : {e}")
|
||||
|
||||
def _notifier_changement(self) -> None:
|
||||
"""Notifier tous les listeners du changement d'état."""
|
||||
with self._lock:
|
||||
listeners = list(self._listeners)
|
||||
snap = self.snapshot()
|
||||
|
||||
for cb in listeners:
|
||||
try:
|
||||
cb(snap)
|
||||
except Exception as e:
|
||||
logger.debug(f"[ACTIVITY] Listener erreur : {e}")
|
||||
|
||||
|
||||
def _tk_root_existe() -> bool:
|
||||
"""Vérifier si un root tkinter existe déjà (pour créer un Toplevel)."""
|
||||
try:
|
||||
import tkinter as tk
|
||||
|
||||
default_root = getattr(tk, "_default_root", None)
|
||||
return default_root is not None
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Singleton global (optionnel)
|
||||
# ============================================================================
|
||||
|
||||
|
||||
_INSTANCE_GLOBALE: Optional[ActivityPanel] = None
|
||||
_LOCK_SINGLETON = threading.Lock()
|
||||
|
||||
|
||||
def get_activity_panel(activer_ui: bool = True) -> ActivityPanel:
|
||||
"""Obtenir l'instance globale du panel d'activité (lazy)."""
|
||||
global _INSTANCE_GLOBALE
|
||||
with _LOCK_SINGLETON:
|
||||
if _INSTANCE_GLOBALE is None:
|
||||
_INSTANCE_GLOBALE = ActivityPanel(activer_ui=activer_ui)
|
||||
return _INSTANCE_GLOBALE
|
||||
|
||||
|
||||
def reset_activity_panel() -> None:
|
||||
"""Réinitialiser le singleton (utile pour les tests)."""
|
||||
global _INSTANCE_GLOBALE
|
||||
with _LOCK_SINGLETON:
|
||||
if _INSTANCE_GLOBALE is not None:
|
||||
try:
|
||||
_INSTANCE_GLOBALE.masquer()
|
||||
except Exception:
|
||||
pass
|
||||
_INSTANCE_GLOBALE = None
|
||||
@@ -3,23 +3,46 @@ Mini serveur HTTP sur l'agent Windows pour les captures d'ecran a la demande
|
||||
et les operations fichiers.
|
||||
|
||||
Ecoute sur le port 5006 (configurable via RPA_CAPTURE_PORT).
|
||||
Bind par defaut sur 127.0.0.1 (configurable via RPA_CAPTURE_BIND).
|
||||
Endpoints :
|
||||
GET /capture -> screenshot frais en base64 (JPEG)
|
||||
GET /health -> {"status": "ok"}
|
||||
GET /health -> {"status": "ok"} (pas d'auth — sonde liveness)
|
||||
POST /file-action -> operations fichiers (list, create, move, copy, sort)
|
||||
|
||||
Securite :
|
||||
- Authentification Bearer obligatoire (RPA_API_TOKEN) pour /capture et
|
||||
/file-action. Sans token configure, ces endpoints sont desactives.
|
||||
- Les tentatives non authentifiees sont loguees (WARNING) avec l'IP source.
|
||||
- Bind defaut localhost. Pour exposer sur le LAN (cas VWB backend qui
|
||||
appelle l'agent a distance), definir explicitement
|
||||
RPA_CAPTURE_BIND=0.0.0.0. L'auth reste alors la seule protection.
|
||||
"""
|
||||
import threading
|
||||
import logging
|
||||
import json
|
||||
import base64
|
||||
import hmac
|
||||
import io
|
||||
import os
|
||||
import time
|
||||
from http.server import HTTPServer, BaseHTTPRequestHandler
|
||||
|
||||
from ..core.log_safe import _path_ext
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
CAPTURE_PORT = int(os.environ.get("RPA_CAPTURE_PORT", "5006"))
|
||||
# Bind par defaut sur localhost — defense en profondeur.
|
||||
# Pour le deploiement VWB (backend Linux -> agent Windows), definir
|
||||
# RPA_CAPTURE_BIND=0.0.0.0 explicitement. L'auth par token reste requise.
|
||||
CAPTURE_BIND = os.environ.get("RPA_CAPTURE_BIND", "127.0.0.1")
|
||||
|
||||
# Token d'authentification (partage avec le streaming). Doit etre defini pour
|
||||
# que /capture et /file-action soient accessibles.
|
||||
CAPTURE_TOKEN = os.environ.get("RPA_API_TOKEN", "")
|
||||
|
||||
# Endpoints ouverts (pas d'auth requise — sondes techniques uniquement)
|
||||
_PUBLIC_PATHS = {"/health"}
|
||||
|
||||
# Floutage des données sensibles (conformité AI Act)
|
||||
BLUR_SENSITIVE = os.environ.get("RPA_BLUR_SENSITIVE", "true").lower() in ("true", "1", "yes")
|
||||
@@ -33,6 +56,8 @@ class CaptureHandler(BaseHTTPRequestHandler):
|
||||
|
||||
def do_GET(self):
|
||||
if self.path == "/capture":
|
||||
if not self._check_auth():
|
||||
return
|
||||
self._handle_capture()
|
||||
elif self.path == "/health":
|
||||
self._send_json(200, {"status": "ok"})
|
||||
@@ -41,10 +66,56 @@ class CaptureHandler(BaseHTTPRequestHandler):
|
||||
|
||||
def do_POST(self):
|
||||
if self.path == "/file-action":
|
||||
if not self._check_auth():
|
||||
return
|
||||
self._handle_file_action()
|
||||
else:
|
||||
self._send_json(404, {"error": "not found"})
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _check_auth(self) -> bool:
|
||||
"""Valide le Bearer token. Renvoie 401/503 si invalide.
|
||||
|
||||
- Si aucun token n'est configure cote serveur (RPA_API_TOKEN vide),
|
||||
on refuse toutes les requetes sensibles (503) — fail-closed.
|
||||
- Sinon, on compare en temps constant via hmac.compare_digest.
|
||||
- Les tentatives echouees sont loguees avec l'IP source.
|
||||
"""
|
||||
# Autoriser les endpoints publics
|
||||
if self.path in _PUBLIC_PATHS:
|
||||
return True
|
||||
|
||||
peer = self.client_address[0] if self.client_address else "?"
|
||||
|
||||
if not CAPTURE_TOKEN:
|
||||
logger.error(
|
||||
"Refus %s depuis %s : RPA_API_TOKEN non configure "
|
||||
"(capture server en mode fail-closed)",
|
||||
self.path, peer,
|
||||
)
|
||||
self._send_json(503, {
|
||||
"error": "capture server non configure (token manquant)",
|
||||
})
|
||||
return False
|
||||
|
||||
auth_header = self.headers.get("Authorization", "")
|
||||
token = ""
|
||||
if auth_header.startswith("Bearer "):
|
||||
token = auth_header[len("Bearer "):].strip()
|
||||
|
||||
if not token or not hmac.compare_digest(token, CAPTURE_TOKEN):
|
||||
logger.warning(
|
||||
"Tentative d'acces non autorisee a %s depuis %s "
|
||||
"(token %s)",
|
||||
self.path, peer,
|
||||
"absent" if not token else "invalide",
|
||||
)
|
||||
self._send_json(401, {"error": "unauthorized"})
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def do_OPTIONS(self):
|
||||
"""Gestion CORS preflight."""
|
||||
self.send_response(200)
|
||||
@@ -89,14 +160,25 @@ class CaptureHandler(BaseHTTPRequestHandler):
|
||||
"""Capture l'ecran principal et le renvoie en base64 JPEG."""
|
||||
t0 = time.perf_counter()
|
||||
try:
|
||||
import mss
|
||||
from PIL import Image
|
||||
from ..vision.capturer import (
|
||||
capture_foreground_window_image,
|
||||
capture_screen_image,
|
||||
)
|
||||
|
||||
with mss.mss() as sct:
|
||||
monitor = sct.monitors[1] # ecran principal
|
||||
raw = sct.grab(monitor)
|
||||
|
||||
img = Image.frombytes("RGB", raw.size, raw.bgra, "raw", "BGRX")
|
||||
_monitor, img, meta = capture_screen_image()
|
||||
if img is None:
|
||||
img, win_meta = capture_foreground_window_image()
|
||||
meta.update(win_meta)
|
||||
if img is None:
|
||||
elapsed_ms = (time.perf_counter() - t0) * 1000
|
||||
logger.error("Erreur capture : aucun backend exploitable (%s)", meta)
|
||||
self._send_json(503, {
|
||||
"error": "capture_unavailable",
|
||||
"source": meta.get("backend", "unknown"),
|
||||
"capture_ms": round(elapsed_ms),
|
||||
"diagnostics": meta,
|
||||
})
|
||||
return
|
||||
|
||||
# Floutage des données sensibles (conformité AI Act)
|
||||
if BLUR_SENSITIVE:
|
||||
@@ -111,15 +193,22 @@ class CaptureHandler(BaseHTTPRequestHandler):
|
||||
img_b64 = base64.b64encode(buf.getvalue()).decode()
|
||||
|
||||
elapsed_ms = (time.perf_counter() - t0) * 1000
|
||||
logger.info(f"Capture {img.width}x{img.height} en {elapsed_ms:.0f}ms")
|
||||
logger.info(
|
||||
"Capture %sx%s via %s en %.0fms",
|
||||
img.width,
|
||||
img.height,
|
||||
meta.get("backend", "unknown"),
|
||||
elapsed_ms,
|
||||
)
|
||||
|
||||
self._send_json(200, {
|
||||
"image": img_b64,
|
||||
"width": img.width,
|
||||
"height": img.height,
|
||||
"format": "jpeg",
|
||||
"source": "windows_live",
|
||||
"source": meta.get("backend", "windows_live"),
|
||||
"capture_ms": round(elapsed_ms),
|
||||
"diagnostics": meta,
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
@@ -225,7 +314,7 @@ class _FileActionHandlerLocal:
|
||||
})
|
||||
extensions[ext] = extensions.get(ext, 0) + 1
|
||||
|
||||
logger.info(f"Liste dossier '{path_str}' : {len(files)} fichiers")
|
||||
logger.info(f"Liste dossier [ext={_path_ext(path_str)}] : {len(files)} fichiers")
|
||||
return {"files": files, "count": len(files), "extensions": extensions, "path": path_str}
|
||||
|
||||
def _create_dir(self, params: dict) -> dict:
|
||||
@@ -241,7 +330,7 @@ class _FileActionHandlerLocal:
|
||||
target = _Path(path_str)
|
||||
existed = target.exists()
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
logger.info(f"Dossier '{path_str}' {'existait deja' if existed else 'cree'}")
|
||||
logger.info(f"Dossier [ext={_path_ext(path_str)}] {'existait deja' if existed else 'cree'}")
|
||||
return {"created": not existed, "path": path_str, "already_existed": existed}
|
||||
|
||||
def _move_file(self, params: dict) -> dict:
|
||||
@@ -263,7 +352,7 @@ class _FileActionHandlerLocal:
|
||||
|
||||
_Path(dst).parent.mkdir(parents=True, exist_ok=True)
|
||||
_shutil.move(src, dst)
|
||||
logger.info(f"Fichier deplace : '{src}' -> '{dst}'")
|
||||
logger.info(f"Fichier deplace : [ext={_path_ext(src)}] -> [ext={_path_ext(dst)}]")
|
||||
return {"moved": True, "source": src, "destination": dst}
|
||||
|
||||
def _copy_file(self, params: dict) -> dict:
|
||||
@@ -289,7 +378,7 @@ class _FileActionHandlerLocal:
|
||||
_shutil.copytree(src, dst)
|
||||
else:
|
||||
_shutil.copy2(src, dst)
|
||||
logger.info(f"Fichier copie : '{src}' -> '{dst}'")
|
||||
logger.info(f"Fichier copie : [ext={_path_ext(src)}] -> [ext={_path_ext(dst)}]")
|
||||
return {"copied": True, "source": src, "destination": dst}
|
||||
|
||||
def _sort_by_extension(self, params: dict) -> dict:
|
||||
@@ -338,7 +427,7 @@ class _FileActionHandlerLocal:
|
||||
extensions[ext] = extensions.get(ext, 0) + 1
|
||||
|
||||
logger.info(
|
||||
f"Classement par extension '{source_dir_str}' : {len(moved)} fichiers"
|
||||
f"Classement par extension [ext={_path_ext(source_dir_str)}] : {len(moved)} fichiers"
|
||||
)
|
||||
return {
|
||||
"moved": moved,
|
||||
@@ -351,21 +440,46 @@ class _FileActionHandlerLocal:
|
||||
class CaptureServer:
|
||||
"""Serveur de capture d'ecran en temps reel (thread daemon)."""
|
||||
|
||||
def __init__(self, port: int = CAPTURE_PORT):
|
||||
def __init__(self, port: int = CAPTURE_PORT, bind: str = CAPTURE_BIND):
|
||||
self._port = port
|
||||
self._bind = bind
|
||||
self._server: HTTPServer | None = None
|
||||
self._thread: threading.Thread | None = None
|
||||
|
||||
def start(self):
|
||||
"""Demarre le serveur dans un thread daemon."""
|
||||
"""Demarre le serveur dans un thread daemon.
|
||||
|
||||
Avertit si le serveur est expose sur le LAN sans token configure.
|
||||
"""
|
||||
# Defense en profondeur : refus de demarrer si expose LAN sans auth
|
||||
exposed_lan = self._bind not in ("127.0.0.1", "localhost", "::1")
|
||||
if exposed_lan and not CAPTURE_TOKEN:
|
||||
logger.error(
|
||||
"REFUS demarrage capture server : bind=%s (LAN) sans "
|
||||
"RPA_API_TOKEN. Definir le token ou RPA_CAPTURE_BIND=127.0.0.1.",
|
||||
self._bind,
|
||||
)
|
||||
print(
|
||||
f"[CAPTURE] REFUS demarrage : bind={self._bind} sans token. "
|
||||
f"Definir RPA_API_TOKEN ou RPA_CAPTURE_BIND=127.0.0.1."
|
||||
)
|
||||
return
|
||||
|
||||
try:
|
||||
self._server = HTTPServer(("0.0.0.0", self._port), CaptureHandler)
|
||||
self._server = HTTPServer((self._bind, self._port), CaptureHandler)
|
||||
self._thread = threading.Thread(
|
||||
target=self._server.serve_forever, daemon=True
|
||||
)
|
||||
self._thread.start()
|
||||
logger.info(f"Capture server demarre sur le port {self._port}")
|
||||
print(f"[CAPTURE] Serveur de capture demarre sur le port {self._port}")
|
||||
auth_mode = "token requis" if CAPTURE_TOKEN else "token absent (fail-closed)"
|
||||
logger.info(
|
||||
"Capture server demarre sur %s:%s (%s)",
|
||||
self._bind, self._port, auth_mode,
|
||||
)
|
||||
print(
|
||||
f"[CAPTURE] Serveur de capture demarre sur "
|
||||
f"{self._bind}:{self._port} ({auth_mode})"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Impossible de demarrer le capture server : {e}")
|
||||
print(f"[CAPTURE] ERREUR demarrage : {e}")
|
||||
|
||||
@@ -5,17 +5,32 @@ Fenetre de chat Lea integree au systray — version tkinter native.
|
||||
Remplace l'approche Edge browser par une vraie fenetre tkinter integree.
|
||||
Design professionnel, theme clair, ancree en bas a droite de l'ecran.
|
||||
Tourne dans son propre thread daemon pour ne pas bloquer pystray.
|
||||
|
||||
Le runtime Python embedded Windows ne contient pas toujours Tcl/Tk. Dans ce
|
||||
cas, le menu "Discuter avec Lea" ouvre le chat DGX dans le navigateur.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import math
|
||||
import threading
|
||||
import time
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse
|
||||
from typing import Any, Callable, Dict, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# FeedbackBus : import fail-safe (le ChatWindow doit tourner même si python-socketio
|
||||
# n'est pas installé sur le poste client, par exemple ancienne installation Pauline)
|
||||
try:
|
||||
from ..network.feedback_bus import FeedbackBusClient
|
||||
_HAS_FEEDBACK_BUS = True
|
||||
except Exception:
|
||||
FeedbackBusClient = None # type: ignore
|
||||
_HAS_FEEDBACK_BUS = False
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Theme — palette professionnelle claire
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -42,6 +57,25 @@ SCROLLBAR_BG = "#E5E7EB" # Fond scrollbar
|
||||
SCROLLBAR_FG = "#9CA3AF" # Curseur scrollbar
|
||||
MSG_BORDER_COLOR = "#D1D5DB" # Bordure subtile des bulles de messages
|
||||
|
||||
# Bulle paused_need_help (J3.5) — alerte non bloquante, asset démo majeur
|
||||
PAUSED_BG = "#FEF3C7" # Jaune pâle
|
||||
PAUSED_BORDER = "#F59E0B" # Orange ambré
|
||||
PAUSED_FG = "#92400E" # Brun foncé (lisible sur fond jaune)
|
||||
PAUSED_BTN_RESUME_BG = "#22C55E" # Vert
|
||||
PAUSED_BTN_RESUME_HOVER = "#16A34A"
|
||||
PAUSED_BTN_ABORT_BG = "#9CA3AF" # Gris neutre (pas dramatique)
|
||||
PAUSED_BTN_ABORT_HOVER = "#6B7280"
|
||||
|
||||
# Bulle "Léa exécute" (J3.4) — distincte des bulles chat normales
|
||||
ACTION_BG = "#F1F5F9" # Gris très clair (différencie d'une réponse chat)
|
||||
ACTION_BORDER = "#CBD5E1" # Gris pâle
|
||||
ACTION_FG = "#1E293B" # Gris foncé
|
||||
ACTION_META_FG = "#94A3B8" # Métadonnées en gris discret
|
||||
ACTION_ICON_RUN = "#3B82F6" # Bleu (en cours)
|
||||
ACTION_ICON_OK = "#22C55E" # Vert (succès)
|
||||
ACTION_ICON_ERR = "#EF4444" # Rouge (échec)
|
||||
ACTION_ICON_INFO = "#64748B" # Gris (neutre)
|
||||
|
||||
# Dimensions — confortables
|
||||
WIN_WIDTH = 600
|
||||
WIN_HEIGHT = 800
|
||||
@@ -62,6 +96,80 @@ FONT_SEND_BTN = ("Segoe UI", 13)
|
||||
FONT_RESIZE_GRIP = ("Segoe UI", 10)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Templates de bulles "Léa exécute" (J3.4)
|
||||
# Chaque template prend un payload et retourne (icon, icon_color, title).
|
||||
# Les libellés sont volontairement neutres : le contexte métier vient du
|
||||
# payload (workflow, action, message), pas de hardcoding.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _tpl_action_started(payload: Dict[str, Any]) -> tuple:
|
||||
wf = payload.get("workflow") or "?"
|
||||
return ("▶", ACTION_ICON_RUN, f"Démarrage : {wf}")
|
||||
|
||||
|
||||
def _tpl_action_progress(payload: Dict[str, Any]) -> tuple:
|
||||
cur = payload.get("current", "?")
|
||||
tot = payload.get("total", "?")
|
||||
step = payload.get("step")
|
||||
title = step if step else f"Étape {cur}/{tot}"
|
||||
return ("⋯", ACTION_ICON_RUN, str(title))
|
||||
|
||||
|
||||
def _tpl_done(payload: Dict[str, Any]) -> tuple:
|
||||
success = bool(payload.get("success", True))
|
||||
msg = payload.get("message") or ("Terminé" if success else "Échec")
|
||||
if success:
|
||||
return ("✓", ACTION_ICON_OK, str(msg))
|
||||
return ("✗", ACTION_ICON_ERR, str(msg))
|
||||
|
||||
|
||||
def _tpl_need_confirm(payload: Dict[str, Any]) -> tuple:
|
||||
action = payload.get("action") or {}
|
||||
desc = action.get("description") if isinstance(action, dict) else None
|
||||
title = desc or "J'attends ton accord avant de continuer"
|
||||
return ("?", ACTION_ICON_RUN, str(title))
|
||||
|
||||
|
||||
def _tpl_step_result(payload: Dict[str, Any]) -> tuple:
|
||||
status = (payload.get("status") or "").lower()
|
||||
msg = payload.get("message") or status or "Étape terminée"
|
||||
if status in ("ok", "success", "approved"):
|
||||
return ("✓", ACTION_ICON_OK, str(msg))
|
||||
if status in ("error", "failed"):
|
||||
return ("✗", ACTION_ICON_ERR, str(msg))
|
||||
return ("·", ACTION_ICON_INFO, str(msg))
|
||||
|
||||
|
||||
def _tpl_resumed(payload: Dict[str, Any]) -> tuple:
|
||||
return ("→", ACTION_ICON_OK, "Reprise")
|
||||
|
||||
|
||||
_ACTION_TEMPLATES = {
|
||||
"lea:action_started": _tpl_action_started,
|
||||
"lea:action_progress": _tpl_action_progress,
|
||||
"lea:done": _tpl_done,
|
||||
"lea:need_confirm": _tpl_need_confirm,
|
||||
"lea:step_result": _tpl_step_result,
|
||||
"lea:resumed": _tpl_resumed,
|
||||
}
|
||||
|
||||
|
||||
def _extract_meta(payload: Dict[str, Any]) -> str:
|
||||
"""Métadonnées techniques en pied de bulle (workflow, étape, replay_id court)."""
|
||||
parts = []
|
||||
wf = payload.get("workflow")
|
||||
if wf:
|
||||
parts.append(str(wf))
|
||||
cur, tot = payload.get("current"), payload.get("total")
|
||||
if cur is not None and tot is not None:
|
||||
parts.append(f"étape {cur}/{tot}")
|
||||
rid = payload.get("replay_id")
|
||||
if rid:
|
||||
parts.append(f"#{str(rid)[-6:]}")
|
||||
return " • ".join(parts)
|
||||
|
||||
|
||||
class ChatWindow:
|
||||
"""Fenetre de chat Lea en tkinter natif.
|
||||
|
||||
@@ -91,6 +199,8 @@ class ChatWindow:
|
||||
self._root = None
|
||||
self._ready = threading.Event()
|
||||
self._messages = [] # historique local
|
||||
self._bus: Optional[Any] = None # FeedbackBusClient (J3.3, peut rester None)
|
||||
self._active_paused_bubble: Optional[Dict[str, Any]] = None # bulle paused active (J3.5)
|
||||
|
||||
# S'abonner aux changements de l'etat partage
|
||||
if self._shared_state is not None:
|
||||
@@ -114,7 +224,10 @@ class ChatWindow:
|
||||
|
||||
def toggle(self) -> None:
|
||||
"""Afficher/masquer la fenetre de chat."""
|
||||
if self._destroyed or self._root is None:
|
||||
if self._destroyed:
|
||||
return
|
||||
if self._root is None:
|
||||
self._open_browser_fallback()
|
||||
return
|
||||
if self._visible:
|
||||
self.hide()
|
||||
@@ -123,7 +236,10 @@ class ChatWindow:
|
||||
|
||||
def show(self) -> None:
|
||||
"""Afficher la fenetre."""
|
||||
if self._destroyed or self._root is None:
|
||||
if self._destroyed:
|
||||
return
|
||||
if self._root is None:
|
||||
self._open_browser_fallback()
|
||||
return
|
||||
self._root.after(0, self._do_show)
|
||||
|
||||
@@ -152,6 +268,79 @@ class ChatWindow:
|
||||
"""Mettre a jour le client serveur (appele si cree apres la fenetre)."""
|
||||
self._server_client = server_client
|
||||
|
||||
def _chat_url(self) -> str:
|
||||
"""Retourne l'URL web du chat, derivee de la config serveur."""
|
||||
configured_url = self._chat_url_from_server_url(self._configured_server_url())
|
||||
if self._server_client is not None:
|
||||
chat_base = getattr(self._server_client, "_chat_base", None)
|
||||
if chat_base:
|
||||
chat_base = str(chat_base).rstrip("/")
|
||||
if not self._is_local_url(chat_base):
|
||||
return chat_base
|
||||
if configured_url:
|
||||
return configured_url
|
||||
|
||||
if configured_url:
|
||||
return configured_url
|
||||
|
||||
host = (self._server_host or "localhost").strip()
|
||||
if host.startswith(("http://", "https://")):
|
||||
parsed = urlparse(host)
|
||||
scheme = parsed.scheme or "http"
|
||||
hostname = parsed.hostname or "localhost"
|
||||
return f"{scheme}://{hostname}:{self._chat_port}"
|
||||
|
||||
return f"http://{host}:{self._chat_port}"
|
||||
|
||||
@staticmethod
|
||||
def _is_local_url(url: str) -> bool:
|
||||
try:
|
||||
host = urlparse(url).hostname
|
||||
except Exception:
|
||||
return False
|
||||
return host in {"localhost", "127.0.0.1", "::1"}
|
||||
|
||||
def _chat_url_from_server_url(self, server_url: Optional[str]) -> Optional[str]:
|
||||
if not server_url:
|
||||
return None
|
||||
try:
|
||||
parsed = urlparse(server_url.strip())
|
||||
except Exception:
|
||||
return None
|
||||
if not parsed.hostname or parsed.hostname in {"localhost", "127.0.0.1", "::1"}:
|
||||
return None
|
||||
scheme = parsed.scheme or "http"
|
||||
return f"{scheme}://{parsed.hostname}:{self._chat_port}"
|
||||
|
||||
def _configured_server_url(self) -> Optional[str]:
|
||||
env_url = os.environ.get("RPA_SERVER_URL", "").strip()
|
||||
if env_url:
|
||||
return env_url
|
||||
|
||||
try:
|
||||
# Installed layout: <app>/agent_v1/ui/chat_window.py.
|
||||
for parent in Path(__file__).resolve().parents:
|
||||
cfg = parent / "config.txt"
|
||||
if cfg.exists():
|
||||
for line in cfg.read_text(encoding="utf-8", errors="ignore").splitlines():
|
||||
if line.startswith("RPA_SERVER_URL="):
|
||||
return line.split("=", 1)[1].strip()
|
||||
except Exception:
|
||||
logger.debug("Lecture config.txt pour chat_url impossible", exc_info=True)
|
||||
return None
|
||||
|
||||
def _open_browser_fallback(self) -> None:
|
||||
"""Fallback POC quand tkinter est absent du Python embedded."""
|
||||
url = self._chat_url()
|
||||
try:
|
||||
import webbrowser
|
||||
if webbrowser.open(url, new=1):
|
||||
logger.info("ChatWindow indisponible, chat ouvert dans le navigateur: %s", url)
|
||||
else:
|
||||
logger.warning("ChatWindow indisponible, ouverture navigateur refusee: %s", url)
|
||||
except Exception as exc:
|
||||
logger.error("Impossible d'ouvrir le chat dans le navigateur (%s): %s", url, exc)
|
||||
|
||||
def _on_shared_state_change(self, state) -> None:
|
||||
"""Callback appele quand l'etat partage change (depuis le systray ou ailleurs).
|
||||
|
||||
@@ -266,6 +455,9 @@ class ChatWindow:
|
||||
# Signaler que la fenetre est prete
|
||||
self._ready.set()
|
||||
|
||||
# Demarrer le bus feedback Lea (events 'lea:*' temps reel)
|
||||
self._start_feedback_bus()
|
||||
|
||||
# Boucle tkinter
|
||||
root.mainloop()
|
||||
|
||||
@@ -608,6 +800,12 @@ class ChatWindow:
|
||||
|
||||
def _do_destroy(self) -> None:
|
||||
"""Detruit la fenetre (appele dans le thread tkinter)."""
|
||||
if self._bus is not None:
|
||||
try:
|
||||
self._bus.stop()
|
||||
except Exception:
|
||||
pass
|
||||
self._bus = None
|
||||
if self._root is not None:
|
||||
try:
|
||||
self._root.quit()
|
||||
@@ -617,6 +815,608 @@ class ChatWindow:
|
||||
self._root = None
|
||||
self._visible = False
|
||||
|
||||
# ======================================================================
|
||||
# FeedbackBus — bulles temps reel pendant l'execution (J3.3)
|
||||
# ======================================================================
|
||||
|
||||
def _start_feedback_bus(self) -> None:
|
||||
"""Demarrer la connexion au bus 'lea:*' si flag actif et lib disponible."""
|
||||
if not _HAS_FEEDBACK_BUS:
|
||||
logger.debug("FeedbackBus non disponible (python-socketio manquant)")
|
||||
return
|
||||
flag = os.environ.get("LEA_FEEDBACK_BUS", "0").lower()
|
||||
if flag not in ("1", "true", "yes", "on"):
|
||||
return
|
||||
try:
|
||||
url = f"http://{self._server_host}:{self._chat_port}"
|
||||
token = os.environ.get("RPA_API_TOKEN", "") or None
|
||||
self._bus = FeedbackBusClient(url, token=token, on_event=self._on_lea_event)
|
||||
self._bus.start()
|
||||
logger.info("FeedbackBus demarre : %s", url)
|
||||
except Exception:
|
||||
logger.debug("FeedbackBus init silenced", exc_info=True)
|
||||
self._bus = None
|
||||
|
||||
def _on_lea_event(self, event: str, payload: Dict[str, Any]) -> None:
|
||||
"""Callback bus → bulle Lea. Thread-safe : helpers utilisent root.after."""
|
||||
payload = payload or {}
|
||||
|
||||
# J3.5 : la pause supervisée a sa propre bulle interactive
|
||||
if event == "lea:paused":
|
||||
self._add_paused_bubble(payload)
|
||||
return
|
||||
if event in ("lea:resumed", "lea:done"):
|
||||
self._close_active_paused_bubble(reason=event)
|
||||
# on continue pour afficher la bulle d'action (cf. dispatch ci-dessous)
|
||||
|
||||
# Acks bus (resume_acked, abort_acked) : silencieux côté UI
|
||||
if event in ("lea:resume_acked", "lea:abort_acked"):
|
||||
return
|
||||
|
||||
# J3.4 : bulle "Léa exécute" stylisée (séparée des bulles chat normales)
|
||||
rendered = _ACTION_TEMPLATES.get(event)
|
||||
if rendered is None:
|
||||
# Event inconnu : on affiche en bulle d'action neutre
|
||||
self._add_action_bubble(
|
||||
icon="·", icon_color=ACTION_ICON_INFO,
|
||||
title=event.removeprefix("lea:"),
|
||||
meta=_extract_meta(payload),
|
||||
)
|
||||
return
|
||||
icon, icon_color, title = rendered(payload)
|
||||
self._add_action_bubble(
|
||||
icon=icon, icon_color=icon_color, title=title,
|
||||
meta=_extract_meta(payload),
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Bulle "Léa exécute" stylisée (J3.4)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _add_action_bubble(
|
||||
self, icon: str, icon_color: str, title: str, meta: str = "",
|
||||
) -> None:
|
||||
if self._root is None:
|
||||
return
|
||||
self._root.after(0, lambda: self._render_action_bubble(icon, icon_color, title, meta))
|
||||
|
||||
def _render_action_bubble(
|
||||
self, icon: str, icon_color: str, title: str, meta: str,
|
||||
) -> None:
|
||||
tk = self._tk
|
||||
if getattr(self, "_msg_frame", None) is None:
|
||||
return
|
||||
now = datetime.now().strftime("%H:%M")
|
||||
|
||||
container = tk.Frame(self._msg_frame, bg=BG_COLOR)
|
||||
container.pack(fill=tk.X, padx=MARGIN, pady=3)
|
||||
|
||||
inner = tk.Frame(
|
||||
container, bg=ACTION_BG, padx=10, pady=6,
|
||||
highlightbackground=ACTION_BORDER, highlightthickness=1,
|
||||
)
|
||||
inner.pack(anchor=tk.W, padx=(0, 70), fill=tk.X)
|
||||
|
||||
row = tk.Frame(inner, bg=ACTION_BG)
|
||||
row.pack(fill=tk.X, anchor=tk.W)
|
||||
|
||||
tk.Label(
|
||||
row, text=icon, bg=ACTION_BG, fg=icon_color,
|
||||
font=("Segoe UI", 13, "bold"), padx=4,
|
||||
).pack(side=tk.LEFT)
|
||||
|
||||
tk.Label(
|
||||
row, text=title, bg=ACTION_BG, fg=ACTION_FG,
|
||||
font=FONT_MSG, anchor="w", justify=tk.LEFT,
|
||||
wraplength=MSG_WRAP_WIDTH - 60,
|
||||
).pack(side=tk.LEFT, fill=tk.X, expand=True, padx=(2, 0))
|
||||
|
||||
if meta:
|
||||
tk.Label(
|
||||
inner, text=f"{meta} • {now}",
|
||||
bg=ACTION_BG, fg=ACTION_META_FG,
|
||||
font=FONT_TIMESTAMP, anchor="w",
|
||||
).pack(fill=tk.X, anchor=tk.W, pady=(2, 0))
|
||||
|
||||
# UX fix 8 mai 2026 : auto-scroll après chaque bulle d'action
|
||||
self._scroll_to_bottom()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Bulle paused_need_help interactive (J3.5)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _add_paused_bubble(self, payload: Dict[str, Any]) -> None:
|
||||
"""Ajouter une bulle paused interactive (asset démo : Léa demande de l'aide).
|
||||
|
||||
IMPORTANT (8 mai 2026, démo GHT) : par défaut la fenêtre démarre cachée
|
||||
(`root.withdraw()`). Il FAUT la rendre visible et la forcer au premier
|
||||
plan, sinon Dom ne voit jamais la bulle. On exécute dans le thread
|
||||
tkinter via `root.after(0, ...)`.
|
||||
|
||||
UX fix 8 mai 2026 : un seul affichage (la bulle chat). Plus de toast
|
||||
en double — on force juste la chat window au premier plan.
|
||||
"""
|
||||
if self._root is None:
|
||||
return
|
||||
|
||||
def _show_and_render():
|
||||
try:
|
||||
self._do_show()
|
||||
# Re-pin topmost pour passer devant les apps actives
|
||||
self._root.attributes("-topmost", True)
|
||||
self._root.lift()
|
||||
# Bell sonore léger pour attirer l'attention (Tkinter natif)
|
||||
try:
|
||||
self._root.bell()
|
||||
except Exception:
|
||||
pass
|
||||
except Exception:
|
||||
logger.debug("force-show chat_window silenced", exc_info=True)
|
||||
try:
|
||||
# UX fix mai 2026 : repartir d'un chat vide pour focaliser
|
||||
# l'attention sur la question (clear visuel uniquement,
|
||||
# self._messages reste intact pour la traçabilité debug).
|
||||
self._clear_chat_history()
|
||||
self._render_paused_bubble(payload)
|
||||
except Exception:
|
||||
logger.exception("render paused bubble failed; using fallback")
|
||||
try:
|
||||
self._clear_chat_history()
|
||||
self._render_paused_fallback_bubble(payload)
|
||||
except Exception:
|
||||
logger.debug("render paused fallback silenced", exc_info=True)
|
||||
|
||||
self._root.after(0, _show_and_render)
|
||||
|
||||
def _clear_chat_history(self) -> None:
|
||||
"""Vide la zone d'affichage du chat (widgets enfants de _msg_frame).
|
||||
|
||||
Détruit uniquement les widgets visuels — self._messages (liste mémoire)
|
||||
reste intact pour la traçabilité debug. Cohérent avec _do_remove_typing
|
||||
qui détruit aussi le widget sans toucher à l'historique.
|
||||
"""
|
||||
if getattr(self, "_msg_frame", None) is None:
|
||||
return
|
||||
try:
|
||||
for child in list(self._msg_frame.winfo_children()):
|
||||
child.destroy()
|
||||
self._active_paused_bubble = None
|
||||
if hasattr(self, "_typing_frame"):
|
||||
self._typing_frame = None
|
||||
self._scroll_to_bottom()
|
||||
except Exception:
|
||||
logger.debug("clear chat history silenced", exc_info=True)
|
||||
|
||||
@staticmethod
|
||||
def _compute_paused_bubble_height(
|
||||
reason_str: str,
|
||||
chars_per_line: int = 52,
|
||||
max_rows: int = 14,
|
||||
) -> tuple:
|
||||
"""Calcule la hauteur du Text (en lignes) + si une scrollbar est
|
||||
nécessaire pour le message d'une bulle paused.
|
||||
|
||||
Patch 22 mai 2026 — fix troncature : on prend en compte les \\n
|
||||
explicites (les `reason` serveur peuvent lister plusieurs
|
||||
candidats avec un saut de ligne par item) en plus de la longueur
|
||||
en caractères, et on active la scrollbar dès que le cap est
|
||||
atteint pour éviter que du contenu disparaisse silencieusement.
|
||||
|
||||
Retourne ``(height_lines, needs_scrollbar)``.
|
||||
"""
|
||||
if not reason_str:
|
||||
return 2, False
|
||||
text = str(reason_str)
|
||||
chars_per_line = max(24, int(chars_per_line or 52))
|
||||
estimated = 0
|
||||
for raw_line in text.splitlines() or [""]:
|
||||
estimated += max(1, math.ceil(len(raw_line) / chars_per_line))
|
||||
cap = max(2, int(max_rows or 14))
|
||||
height = max(2, min(cap, estimated))
|
||||
# Scrollbar dès que le cap est atteint OU contenu long (filet
|
||||
# textuel : ≥ 200 chars implique souvent un débordement visuel
|
||||
# même quand les lignes brutes sont peu nombreuses).
|
||||
needs_scroll = (estimated >= cap) or (len(text) > 200)
|
||||
return height, needs_scroll
|
||||
|
||||
def _paused_text_layout(self) -> tuple:
|
||||
"""Retourne ``(wrap_px, chars_per_line, max_rows)`` pour la bulle pause.
|
||||
|
||||
La fenêtre Léa est souvent redimensionnée à ~380px de large sur le
|
||||
poste Windows. Les anciennes estimations fixes calculaient trop peu
|
||||
de lignes et tronquaient le message. On part donc des dimensions
|
||||
réelles du canvas et de la métrique de la police Tk.
|
||||
"""
|
||||
canvas_w = 0
|
||||
canvas_h = 0
|
||||
try:
|
||||
canvas_w = int(self._canvas.winfo_width()) if self._canvas is not None else 0
|
||||
canvas_h = int(self._canvas.winfo_height()) if self._canvas is not None else 0
|
||||
except Exception:
|
||||
canvas_w = canvas_h = 0
|
||||
|
||||
# Marges: container + padding inner + petite marge droite. La bulle
|
||||
# de pause est une alerte critique, elle utilise donc presque toute
|
||||
# la largeur disponible sur les fenêtres étroites.
|
||||
wrap_px = max(220, canvas_w - (2 * MARGIN) - 52) if canvas_w else 360
|
||||
|
||||
avg_char = 8
|
||||
line_px = 22
|
||||
try:
|
||||
from tkinter import font as tkfont
|
||||
font = tkfont.Font(font=FONT_MSG)
|
||||
avg_char = max(6, font.measure("n"))
|
||||
line_px = max(18, font.metrics("linespace"))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
chars_per_line = max(24, int(wrap_px / avg_char))
|
||||
# Réserver titre, metadata, boutons, feedback et padding. Même sur
|
||||
# une petite fenêtre, on garde assez de lignes pour ne pas couper un
|
||||
# message d'erreur standard.
|
||||
max_rows = 14
|
||||
if canvas_h:
|
||||
max_rows = max(5, min(18, int((canvas_h - 145) / line_px)))
|
||||
return wrap_px, chars_per_line, max_rows
|
||||
|
||||
def _render_paused_bubble(self, payload: Dict[str, Any]) -> None:
|
||||
tk = self._tk
|
||||
if getattr(self, "_msg_frame", None) is None:
|
||||
return
|
||||
|
||||
replay_id = str(payload.get("replay_id", "") or "")
|
||||
workflow = payload.get("workflow", "?")
|
||||
reason = payload.get("reason") or "Action incertaine — j'ai besoin de votre validation."
|
||||
completed = payload.get("completed", 0)
|
||||
total = payload.get("total", "?")
|
||||
now = datetime.now().strftime("%H:%M")
|
||||
|
||||
container = tk.Frame(self._msg_frame, bg=BG_COLOR)
|
||||
container.pack(fill=tk.X, padx=MARGIN, pady=6)
|
||||
|
||||
inner = tk.Frame(
|
||||
container, bg=PAUSED_BG, padx=14, pady=12,
|
||||
highlightbackground=PAUSED_BORDER, highlightthickness=2,
|
||||
)
|
||||
inner.pack(anchor=tk.W, padx=(0, 12), fill=tk.X)
|
||||
|
||||
tk.Label(
|
||||
inner, text=f"⏸ Pause supervisée • {now}",
|
||||
bg=PAUSED_BG, fg=PAUSED_FG,
|
||||
font=("Segoe UI", 12, "bold"), anchor="w",
|
||||
).pack(fill=tk.X, anchor=tk.W)
|
||||
|
||||
# Message borné et scrollable : sur une fenêtre Léa étroite, une
|
||||
# bulle trop haute fait disparaître le début du diagnostic hors du
|
||||
# viewport. On garde donc la bulle compacte et on scrolle le texte.
|
||||
reason_str = str(reason)
|
||||
_wrap_px, chars_per_line, max_rows = self._paused_text_layout()
|
||||
text_rows, needs_text_scroll = self._compute_paused_bubble_height(
|
||||
reason_str,
|
||||
chars_per_line=chars_per_line,
|
||||
max_rows=max_rows,
|
||||
)
|
||||
text_frame = tk.Frame(inner, bg=PAUSED_BG)
|
||||
text_frame.pack(fill=tk.X, anchor=tk.W, pady=(6, 0))
|
||||
reason_msg = tk.Text(
|
||||
text_frame,
|
||||
height=text_rows,
|
||||
wrap=tk.WORD,
|
||||
bg=PAUSED_BG,
|
||||
fg=PAUSED_FG,
|
||||
font=FONT_MSG,
|
||||
bd=0,
|
||||
highlightthickness=0,
|
||||
relief=tk.FLAT,
|
||||
padx=0,
|
||||
pady=0,
|
||||
cursor="arrow",
|
||||
)
|
||||
reason_msg.insert("1.0", reason_str)
|
||||
reason_msg.configure(state="disabled")
|
||||
reason_msg.pack(side=tk.LEFT, fill=tk.X, expand=True)
|
||||
if needs_text_scroll:
|
||||
scrollbar = tk.Scrollbar(
|
||||
text_frame,
|
||||
orient=tk.VERTICAL,
|
||||
command=reason_msg.yview,
|
||||
width=12,
|
||||
)
|
||||
reason_msg.configure(yscrollcommand=scrollbar.set)
|
||||
scrollbar.pack(side=tk.RIGHT, fill=tk.Y, padx=(6, 0))
|
||||
|
||||
tk.Label(
|
||||
inner, text=f"{workflow} — étape {completed}/{total}",
|
||||
bg=PAUSED_BG, fg=TIMESTAMP_FG, font=FONT_TIMESTAMP, anchor="w",
|
||||
).pack(fill=tk.X, anchor=tk.W, pady=(4, 8))
|
||||
|
||||
btn_frame = tk.Frame(inner, bg=PAUSED_BG)
|
||||
btn_frame.pack(fill=tk.X, anchor=tk.W)
|
||||
|
||||
btn_resume = tk.Button(
|
||||
btn_frame, text="Continuer",
|
||||
bg=PAUSED_BTN_RESUME_BG, fg="white", font=FONT_QUICK_BTN,
|
||||
padx=14, pady=4, bd=0, cursor="hand2",
|
||||
activebackground=PAUSED_BTN_RESUME_HOVER, activeforeground="white",
|
||||
command=lambda: self._on_paused_resume(replay_id),
|
||||
)
|
||||
btn_resume.pack(side=tk.LEFT, padx=(0, 8))
|
||||
|
||||
btn_abort = tk.Button(
|
||||
btn_frame, text="Annuler",
|
||||
bg=PAUSED_BTN_ABORT_BG, fg="white", font=FONT_QUICK_BTN,
|
||||
padx=14, pady=4, bd=0, cursor="hand2",
|
||||
activebackground=PAUSED_BTN_ABORT_HOVER, activeforeground="white",
|
||||
command=lambda: self._on_paused_abort(replay_id),
|
||||
)
|
||||
btn_abort.pack(side=tk.LEFT)
|
||||
|
||||
# Zone de feedback (mise à jour après clic, avant l'ack du bus)
|
||||
feedback_label = tk.Label(
|
||||
inner, text="", bg=PAUSED_BG, fg=PAUSED_FG,
|
||||
font=FONT_TIMESTAMP, anchor="w",
|
||||
)
|
||||
feedback_label.pack(fill=tk.X, anchor=tk.W, pady=(6, 0))
|
||||
|
||||
self._active_paused_bubble = {
|
||||
"container": container, "inner": inner,
|
||||
"btn_resume": btn_resume, "btn_abort": btn_abort,
|
||||
"feedback_label": feedback_label,
|
||||
"replay_id": replay_id,
|
||||
}
|
||||
|
||||
# Scroll automatique vers la nouvelle bulle (visible immédiatement)
|
||||
self._scroll_to_bottom()
|
||||
|
||||
def _render_paused_fallback_bubble(self, payload: Dict[str, Any]) -> None:
|
||||
"""Rendu minimal de secours si la bulle riche echoue."""
|
||||
tk = self._tk
|
||||
if getattr(self, "_msg_frame", None) is None:
|
||||
return
|
||||
|
||||
replay_id = str(payload.get("replay_id", "") or "")
|
||||
workflow = payload.get("workflow", "?")
|
||||
reason = str(
|
||||
payload.get("reason")
|
||||
or "Action incertaine - j'ai besoin de votre validation."
|
||||
)
|
||||
completed = payload.get("completed", 0)
|
||||
total = payload.get("total", "?")
|
||||
now = datetime.now().strftime("%H:%M")
|
||||
|
||||
container = tk.Frame(self._msg_frame, bg=BG_COLOR)
|
||||
container.pack(fill=tk.X, padx=MARGIN, pady=6)
|
||||
|
||||
inner = tk.Frame(
|
||||
container, bg=PAUSED_BG, padx=14, pady=12,
|
||||
highlightbackground=PAUSED_BORDER, highlightthickness=2,
|
||||
)
|
||||
inner.pack(anchor=tk.W, padx=(0, 12), fill=tk.X)
|
||||
|
||||
tk.Label(
|
||||
inner, text=f"Pause supervisee - {now}",
|
||||
bg=PAUSED_BG, fg=PAUSED_FG,
|
||||
font=("Segoe UI", 12, "bold"), anchor="w",
|
||||
).pack(fill=tk.X, anchor=tk.W)
|
||||
|
||||
wrap_px = 360
|
||||
try:
|
||||
if self._canvas is not None:
|
||||
wrap_px = max(220, int(self._canvas.winfo_width()) - 80)
|
||||
except Exception:
|
||||
pass
|
||||
tk.Label(
|
||||
inner, text=reason, bg=PAUSED_BG, fg=PAUSED_FG,
|
||||
font=FONT_MSG, wraplength=wrap_px, justify=tk.LEFT,
|
||||
anchor=tk.W,
|
||||
).pack(fill=tk.X, anchor=tk.W, pady=(6, 0))
|
||||
|
||||
tk.Label(
|
||||
inner, text=f"{workflow} - etape {completed}/{total}",
|
||||
bg=PAUSED_BG, fg=TIMESTAMP_FG, font=FONT_TIMESTAMP, anchor="w",
|
||||
).pack(fill=tk.X, anchor=tk.W, pady=(4, 8))
|
||||
|
||||
btn_frame = tk.Frame(inner, bg=PAUSED_BG)
|
||||
btn_frame.pack(fill=tk.X, anchor=tk.W)
|
||||
|
||||
btn_resume = tk.Button(
|
||||
btn_frame, text="Continuer",
|
||||
bg=PAUSED_BTN_RESUME_BG, fg="white", font=FONT_QUICK_BTN,
|
||||
padx=14, pady=4, bd=0, cursor="hand2",
|
||||
activebackground=PAUSED_BTN_RESUME_HOVER, activeforeground="white",
|
||||
command=lambda: self._on_paused_resume(replay_id),
|
||||
)
|
||||
btn_resume.pack(side=tk.LEFT, padx=(0, 8))
|
||||
|
||||
btn_abort = tk.Button(
|
||||
btn_frame, text="Annuler",
|
||||
bg=PAUSED_BTN_ABORT_BG, fg="white", font=FONT_QUICK_BTN,
|
||||
padx=14, pady=4, bd=0, cursor="hand2",
|
||||
activebackground=PAUSED_BTN_ABORT_HOVER, activeforeground="white",
|
||||
command=lambda: self._on_paused_abort(replay_id),
|
||||
)
|
||||
btn_abort.pack(side=tk.LEFT)
|
||||
|
||||
feedback_label = tk.Label(
|
||||
inner, text="", bg=PAUSED_BG, fg=PAUSED_FG,
|
||||
font=FONT_TIMESTAMP, anchor="w",
|
||||
)
|
||||
feedback_label.pack(fill=tk.X, anchor=tk.W, pady=(6, 0))
|
||||
|
||||
self._active_paused_bubble = {
|
||||
"container": container, "inner": inner,
|
||||
"btn_resume": btn_resume, "btn_abort": btn_abort,
|
||||
"feedback_label": feedback_label,
|
||||
"replay_id": replay_id,
|
||||
}
|
||||
self._scroll_to_bottom()
|
||||
|
||||
def _close_active_paused_bubble(self, reason: str) -> None:
|
||||
if self._active_paused_bubble is None or self._root is None:
|
||||
return
|
||||
self._root.after(0, lambda: self._do_close_paused_bubble(reason))
|
||||
|
||||
def _do_close_paused_bubble(self, reason: str) -> None:
|
||||
bubble = self._active_paused_bubble
|
||||
if bubble is None:
|
||||
return
|
||||
try:
|
||||
bubble["btn_resume"].config(state="disabled")
|
||||
bubble["btn_abort"].config(state="disabled")
|
||||
label_text = {
|
||||
"lea:resumed": "→ Reprise",
|
||||
"lea:done": "→ Terminé",
|
||||
}.get(reason, f"→ {reason}")
|
||||
self._tk.Label(
|
||||
bubble["inner"], text=label_text,
|
||||
bg=PAUSED_BG, fg=PAUSED_FG, font=FONT_TIMESTAMP, anchor="w",
|
||||
).pack(fill="x", anchor="w", pady=(6, 0))
|
||||
except Exception:
|
||||
logger.debug("close paused bubble silenced", exc_info=True)
|
||||
self._active_paused_bubble = None
|
||||
|
||||
def _on_paused_resume(self, replay_id: str) -> None:
|
||||
"""Bouton Continuer : émettre lea:replay_resume + feedback immédiat UI.
|
||||
|
||||
UX fix 8 mai 2026 : on désactive les 2 boutons et on affiche un message
|
||||
de feedback dès le clic, sans attendre l'ack serveur. Le bus émet en
|
||||
arrière-plan ; si la connexion est tombée, on log un warning visible.
|
||||
|
||||
Fallback HTTP 22 mai 2026 : si le bus SocketIO est déconnecté, on
|
||||
retombe sur un POST direct ``/replay/{id}/resume`` via
|
||||
``server_client``. Si les deux échouent, on ré-active les boutons
|
||||
et on saute l'auto-hide pour permettre à l'utilisateur de
|
||||
réessayer manuellement (sinon le replay reste figé côté serveur).
|
||||
"""
|
||||
if not replay_id:
|
||||
self._update_paused_feedback("⚠ replay_id manquant — impossible de relancer")
|
||||
return
|
||||
emitted, channel = self._dispatch_paused_action(
|
||||
replay_id,
|
||||
bus_method="resume_replay",
|
||||
client_method="resume_replay",
|
||||
)
|
||||
self._disable_paused_buttons()
|
||||
if emitted:
|
||||
self._update_paused_feedback("→ Reprise demandée…")
|
||||
logger.info(
|
||||
"paused_bubble: replay_resume émis pour %s via %s",
|
||||
replay_id, channel,
|
||||
)
|
||||
try:
|
||||
self._root.after(500, self._do_hide)
|
||||
except Exception:
|
||||
logger.debug("auto-hide on resume silenced", exc_info=True)
|
||||
return
|
||||
# Échec sur les deux canaux : laisser l'utilisateur réessayer.
|
||||
self._update_paused_feedback("⚠ Serveur injoignable — réessayez")
|
||||
self._enable_paused_buttons()
|
||||
logger.warning(
|
||||
"paused_bubble: bus et HTTP indisponibles, resume non émis "
|
||||
"pour %s", replay_id,
|
||||
)
|
||||
|
||||
def _on_paused_abort(self, replay_id: str) -> None:
|
||||
"""Bouton Annuler : émettre lea:replay_abort + fermeture locale immédiate.
|
||||
|
||||
UX fix 8 mai 2026 : on ferme la bulle localement dès le clic (le serveur
|
||||
n'envoie pas de lea:resumed pour un abort, donc sans cette fermeture
|
||||
locale la bulle restait coincée — c'était la cause de "Annuler ne
|
||||
fonctionne pas" rapportée par Dom).
|
||||
|
||||
Fallback HTTP 22 mai 2026 : symétrique de ``_on_paused_resume`` —
|
||||
si le bus est déconnecté, POST direct ``/replay/{id}/cancel``.
|
||||
L'abort ferme la bulle localement quelle que soit l'issue (l'état
|
||||
serveur sera réconcilié au prochain poll /replay/next).
|
||||
"""
|
||||
emitted, channel = self._dispatch_paused_action(
|
||||
replay_id,
|
||||
bus_method="abort_replay",
|
||||
client_method="abort_replay",
|
||||
)
|
||||
self._disable_paused_buttons()
|
||||
if emitted:
|
||||
self._update_paused_feedback("✗ Annulé")
|
||||
logger.info(
|
||||
"paused_bubble: replay_abort émis pour %s via %s",
|
||||
replay_id, channel,
|
||||
)
|
||||
else:
|
||||
self._update_paused_feedback("✗ Annulé (serveur injoignable)")
|
||||
logger.warning(
|
||||
"paused_bubble: bus et HTTP indisponibles, abort non émis "
|
||||
"pour %s", replay_id,
|
||||
)
|
||||
# Fermer la bulle en local (l'abort n'a pas de lea:resumed associé)
|
||||
self._close_active_paused_bubble(reason="abort_local")
|
||||
# UX fix mai 2026 : minimiser la fenêtre après 500ms (cohérence
|
||||
# avec _on_paused_resume, demandé explicitement par Dom).
|
||||
try:
|
||||
self._root.after(500, self._do_hide)
|
||||
except Exception:
|
||||
logger.debug("auto-hide on abort silenced", exc_info=True)
|
||||
|
||||
def _dispatch_paused_action(
|
||||
self,
|
||||
replay_id: str,
|
||||
bus_method: str,
|
||||
client_method: str,
|
||||
) -> tuple:
|
||||
"""Envoyer une action de bulle paused via bus puis fallback HTTP.
|
||||
|
||||
Retourne ``(emitted, channel)`` où ``channel`` vaut ``"bus"``,
|
||||
``"http"`` ou ``""`` (aucun chemin n'a abouti).
|
||||
"""
|
||||
if self._bus is not None and getattr(self._bus, "connected", False):
|
||||
try:
|
||||
if getattr(self._bus, bus_method)(replay_id):
|
||||
return True, "bus"
|
||||
except Exception:
|
||||
logger.debug("paused_bubble: bus %s silenced", bus_method, exc_info=True)
|
||||
if self._server_client is not None and hasattr(self._server_client, client_method):
|
||||
try:
|
||||
if getattr(self._server_client, client_method)(replay_id):
|
||||
return True, "http"
|
||||
except Exception:
|
||||
logger.debug(
|
||||
"paused_bubble: server_client %s silenced",
|
||||
client_method, exc_info=True,
|
||||
)
|
||||
return False, ""
|
||||
|
||||
def _disable_paused_buttons(self) -> None:
|
||||
if not self._active_paused_bubble:
|
||||
return
|
||||
try:
|
||||
self._active_paused_bubble["btn_resume"].config(state="disabled")
|
||||
self._active_paused_bubble["btn_abort"].config(state="disabled")
|
||||
except Exception:
|
||||
logger.debug("disable paused buttons silenced", exc_info=True)
|
||||
|
||||
def _enable_paused_buttons(self) -> None:
|
||||
"""Ré-activer les boutons Continuer/Annuler de la bulle paused
|
||||
active. Appelé quand l'envoi a échoué sur tous les canaux —
|
||||
l'utilisateur doit pouvoir réessayer manuellement.
|
||||
"""
|
||||
if not self._active_paused_bubble:
|
||||
return
|
||||
try:
|
||||
self._active_paused_bubble["btn_resume"].config(state="normal")
|
||||
self._active_paused_bubble["btn_abort"].config(state="normal")
|
||||
except Exception:
|
||||
logger.debug("enable paused buttons silenced", exc_info=True)
|
||||
|
||||
def _update_paused_feedback(self, text: str) -> None:
|
||||
if not self._active_paused_bubble:
|
||||
return
|
||||
label = self._active_paused_bubble.get("feedback_label")
|
||||
if label is None:
|
||||
return
|
||||
try:
|
||||
label.config(text=text)
|
||||
except Exception:
|
||||
logger.debug("update paused feedback silenced", exc_info=True)
|
||||
|
||||
# ======================================================================
|
||||
# Ajout de messages dans la zone de chat
|
||||
# ======================================================================
|
||||
@@ -957,8 +1757,19 @@ class ChatWindow:
|
||||
self._add_lea_message(
|
||||
f"C'est parti ! Montrez-moi comment faire \u00ab {name} \u00bb."
|
||||
)
|
||||
|
||||
# --- P1-LEA-SHADOW : d\u00e9clencher d'abord l'orchestrateur L\u00e9a Linux ---
|
||||
# On contacte agent-chat AVANT la capture locale : si la session
|
||||
# serveur d\u00e9marre, on r\u00e9cup\u00e8re un session_id + un message d'accueil
|
||||
# de L\u00e9a qu'on affiche dans le chat. Si \u00e9chec : mode d\u00e9grad\u00e9
|
||||
# (capture locale uniquement, sans assistance conversationnelle).
|
||||
self._start_lea_orchestrator_session(name)
|
||||
|
||||
# --- Comportement historique pr\u00e9serv\u00e9 : capture locale ---
|
||||
# Le pipeline streaming (frames/\u00e9v\u00e9nements) reste pilot\u00e9 par
|
||||
# agent_v1 local. L'orchestrateur Linux ne touche PAS \u00e0 la
|
||||
# capture, il pilote uniquement le dialogue de fin de session.
|
||||
try:
|
||||
# Utiliser l'etat partage si disponible (synchronise le systray)
|
||||
if self._shared_state is not None:
|
||||
self._shared_state.start_recording(name)
|
||||
elif self._on_start_callback is not None:
|
||||
@@ -966,6 +1777,60 @@ class ChatWindow:
|
||||
except Exception as e:
|
||||
self._add_lea_message(f"Oups, un probl\u00e8me : {e}")
|
||||
|
||||
def _start_lea_orchestrator_session(self, session_name: str) -> None:
|
||||
"""Appelle POST /api/learn/start c\u00f4t\u00e9 agent-chat Linux (P1-LEA-SHADOW).
|
||||
|
||||
Fail-safe : toute erreur (config absente, httpx manquant, timeout,
|
||||
500 serveur...) bascule en mode d\u00e9grad\u00e9 sans bloquer la capture
|
||||
locale. Un message clair est affich\u00e9 dans le chat.
|
||||
"""
|
||||
try:
|
||||
from ..config import AGENT_CHAT_URL, API_TOKEN, MACHINE_ID
|
||||
from ..network.lea_orchestrator_client import (
|
||||
LeaOrchestratorError,
|
||||
start_learning_session,
|
||||
)
|
||||
except Exception as exc: # pragma: no cover (import-time)
|
||||
logger.error("Impossible de charger le client orchestrateur L\u00e9a : %s", exc)
|
||||
self._add_lea_message(
|
||||
"\u26a0 Impossible de joindre L\u00e9a serveur. "
|
||||
"L'apprentissage continue localement, mais sans assistance "
|
||||
"conversationnelle."
|
||||
)
|
||||
return
|
||||
|
||||
try:
|
||||
resp = start_learning_session(
|
||||
AGENT_CHAT_URL,
|
||||
machine_id=MACHINE_ID,
|
||||
session_name=session_name,
|
||||
api_token=API_TOKEN,
|
||||
trigger_source="windows_button",
|
||||
)
|
||||
except LeaOrchestratorError as exc:
|
||||
logger.error("Orchestrateur L\u00e9a injoignable : %s", exc)
|
||||
self._add_lea_message(
|
||||
"\u26a0 Impossible de joindre L\u00e9a serveur. "
|
||||
"L'apprentissage continue localement, mais sans assistance "
|
||||
"conversationnelle."
|
||||
)
|
||||
return
|
||||
except Exception as exc: # noqa: BLE001 \u2014 d\u00e9fensif
|
||||
logger.exception("Erreur inattendue orchestrateur L\u00e9a")
|
||||
self._add_lea_message(
|
||||
f"\u26a0 Erreur orchestrateur L\u00e9a : {exc}. "
|
||||
"L'apprentissage continue localement."
|
||||
)
|
||||
return
|
||||
|
||||
# Affichage du message d'accueil renvoy\u00e9 par L\u00e9a (si pr\u00e9sent)
|
||||
if resp.message:
|
||||
self._add_lea_message(resp.message)
|
||||
logger.info(
|
||||
"Session orchestrateur L\u00e9a OK : id=%s state=%s",
|
||||
resp.session_id, resp.state,
|
||||
)
|
||||
|
||||
def _on_quick_tasks(self) -> None:
|
||||
"""Bouton Lancer — demande ce que L\u00e9a sait faire."""
|
||||
self._add_user_message("Qu'est-ce que vous savez faire ?")
|
||||
|
||||
484
agent_v0/agent_v1/ui/message_contract.py
Normal file
484
agent_v0/agent_v1/ui/message_contract.py
Normal file
@@ -0,0 +1,484 @@
|
||||
"""Contrat de lisibilite des messages visibles par l'humain.
|
||||
|
||||
Ce module ne branche encore aucun point runtime. Il fournit une brique pure et
|
||||
testable pour que les sorties UI de Lea puissent refuser les messages trop
|
||||
generiques ou trop techniques avant affichage.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
import unicodedata
|
||||
from dataclasses import dataclass
|
||||
from typing import Iterable, Mapping
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
SUPERVISED_PAUSE_LABELS = (
|
||||
"J'essaie de",
|
||||
"J'attendais",
|
||||
"Je vois",
|
||||
"Peux-tu",
|
||||
)
|
||||
|
||||
MAX_VISIBLE_MESSAGE_CHARS = 720
|
||||
MAX_FIELD_CHARS = 180
|
||||
MIN_FIELD_CHARS = 4
|
||||
|
||||
_GENERIC_PHRASES = (
|
||||
"un element",
|
||||
"un élément",
|
||||
"l'element",
|
||||
"l'élément",
|
||||
"element inconnu",
|
||||
"élément inconnu",
|
||||
"cette action",
|
||||
"cette cible",
|
||||
"cible inconnue",
|
||||
"validation requise",
|
||||
"action requise",
|
||||
)
|
||||
|
||||
_ACTIONABLE_FRENCH_HINTS = (
|
||||
"peux-tu",
|
||||
"cliquer",
|
||||
"ouvrir",
|
||||
"selectionner",
|
||||
"sélectionner",
|
||||
"choisir",
|
||||
"saisir",
|
||||
"corriger",
|
||||
"montrer",
|
||||
"indiquer",
|
||||
"valider",
|
||||
"fermer",
|
||||
"placer",
|
||||
"mettre",
|
||||
"reprendre",
|
||||
)
|
||||
|
||||
_TECHNICAL_ENGLISH_TERMS = (
|
||||
"target_not_found",
|
||||
"target not found",
|
||||
"no_screen_change",
|
||||
"no screen change",
|
||||
"wrong_window",
|
||||
"wrong window",
|
||||
"validation required",
|
||||
"retry",
|
||||
"fallback",
|
||||
"timeout",
|
||||
"screenshot",
|
||||
"validator",
|
||||
"failure",
|
||||
"failed",
|
||||
"resolve target",
|
||||
"postcondition",
|
||||
"please",
|
||||
"click",
|
||||
"button",
|
||||
"target",
|
||||
"expected",
|
||||
"actual",
|
||||
"observed",
|
||||
)
|
||||
|
||||
_TECHNICAL_FIELD_RE = re.compile(
|
||||
r"\b(?:"
|
||||
r"action_id|replay_id|session_id|workflow_id|machine_id|target_spec|"
|
||||
r"vlm_description|resolution_method|resolution_score|retry_count|"
|
||||
r"x_pct|y_pct|screenshot_b64|expected_window_title|current_action_index"
|
||||
r")\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_TECHNICAL_IDENTIFIER_RE = re.compile(
|
||||
r"\b(?:action|replay|session|sess|workflow|node|edge|target|retry|"
|
||||
r"precheck|wait|trace|event|machine|run)_[A-Za-z0-9][A-Za-z0-9_.:-]{3,}\b"
|
||||
)
|
||||
_UUID_RE = re.compile(
|
||||
r"\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_LONG_HEX_RE = re.compile(r"\b[0-9a-f]{16,}\b", re.IGNORECASE)
|
||||
_PIXEL_TUPLE_RE = re.compile(r"\(\s*\d{2,5}\s*,\s*\d{2,5}\s*\)")
|
||||
_PIXEL_FIELD_RE = re.compile(
|
||||
r"\b(?:x|y|left|top|width|height|w|h|x_pct|y_pct)\s*[=:]\s*-?\d+(?:[.,]\d+)?",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_PX_RE = re.compile(r"\b\d{2,5}\s*px\b", re.IGNORECASE)
|
||||
_SCORE_RE = re.compile(
|
||||
r"\b(?:score|confidence|confiance|similarit[eé]|threshold|seuil|"
|
||||
r"probabilit[eé])\s*[:=]\s*\d+(?:[.,]\d+)?%?\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MessageValidationIssue:
|
||||
"""Un probleme detecte dans un message visible par l'humain."""
|
||||
|
||||
code: str
|
||||
detail: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MessageValidationResult:
|
||||
"""Resultat de validation d'un message utilisateur."""
|
||||
|
||||
issues: tuple[MessageValidationIssue, ...] = ()
|
||||
|
||||
@property
|
||||
def valid(self) -> bool:
|
||||
return not self.issues
|
||||
|
||||
def raise_for_errors(self) -> None:
|
||||
if not self.valid:
|
||||
raise MessageContractError(self)
|
||||
|
||||
|
||||
class MessageContractError(ValueError):
|
||||
"""Erreur levee quand un message ne respecte pas le contrat humain."""
|
||||
|
||||
def __init__(self, result: MessageValidationResult):
|
||||
self.result = result
|
||||
details = "; ".join(f"{issue.code}: {issue.detail}" for issue in result.issues)
|
||||
super().__init__(f"Message humain invalide: {details}")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SupervisedPauseFields:
|
||||
"""Champs obligatoires pour expliquer une pause supervisee."""
|
||||
|
||||
intention: str
|
||||
attendu: str
|
||||
vu: str
|
||||
demande: str
|
||||
|
||||
|
||||
DEFAULT_SUPERVISED_PAUSE_FIELDS = SupervisedPauseFields(
|
||||
intention="continuer une etape supervisee",
|
||||
attendu="un accord humain clair avant de continuer",
|
||||
vu="je suis sur une etape qui demande une verification humaine",
|
||||
demande="indiquer si je peux continuer ou corriger l'action attendue",
|
||||
)
|
||||
|
||||
|
||||
def format_supervised_pause_message(
|
||||
*,
|
||||
intention: str,
|
||||
attendu: str,
|
||||
vu: str,
|
||||
demande: str,
|
||||
) -> str:
|
||||
"""Formatter une pause supervisee claire et actionnable.
|
||||
|
||||
Le message retourne exactement quatre lignes. Si un champ reste vague ou
|
||||
technique, la fonction leve ``MessageContractError`` au lieu de produire un
|
||||
message degradant pour l'utilisateur.
|
||||
"""
|
||||
|
||||
fields = SupervisedPauseFields(
|
||||
intention=_one_line(intention),
|
||||
attendu=_one_line(attendu),
|
||||
vu=_one_line(vu),
|
||||
demande=_one_line(demande),
|
||||
)
|
||||
message = "\n".join(
|
||||
(
|
||||
f"J'essaie de : {fields.intention}",
|
||||
f"J'attendais : {fields.attendu}",
|
||||
f"Je vois : {fields.vu}",
|
||||
f"Peux-tu : {fields.demande}",
|
||||
)
|
||||
)
|
||||
validate_supervised_pause_message(message).raise_for_errors()
|
||||
return message
|
||||
|
||||
|
||||
def format_supervised_pause_from_mapping(payload: Mapping[str, object]) -> str:
|
||||
"""Formatter depuis un mapping runtime avec noms de champs explicites.
|
||||
|
||||
Alias acceptes pour faciliter l'integration progressive:
|
||||
``intention|trying_to``, ``attendu|expected``, ``vu|observed``,
|
||||
``demande|request``.
|
||||
"""
|
||||
|
||||
return format_supervised_pause_message(
|
||||
intention=_mapping_text(payload, "intention", "trying_to"),
|
||||
attendu=_mapping_text(payload, "attendu", "expected"),
|
||||
vu=_mapping_text(payload, "vu", "observed"),
|
||||
demande=_mapping_text(payload, "demande", "request"),
|
||||
)
|
||||
|
||||
|
||||
def coerce_supervised_pause_message(
|
||||
message: object = "",
|
||||
*,
|
||||
intention: object = "",
|
||||
attendu: object = "",
|
||||
vu: object = "",
|
||||
demande: object = "",
|
||||
) -> str:
|
||||
"""Retourner une pause supervisee valide, meme depuis un ancien message.
|
||||
|
||||
Si ``message`` respecte deja le contrat strict, il est conserve. Sinon on
|
||||
compose les quatre champs avec les valeurs explicites disponibles. Les
|
||||
valeurs trop vagues ou techniques sont remplacees par des fallbacks clairs.
|
||||
"""
|
||||
|
||||
raw_message = _one_line(message)
|
||||
if raw_message and validate_supervised_pause_message(raw_message).valid:
|
||||
return raw_message
|
||||
|
||||
defaults = DEFAULT_SUPERVISED_PAUSE_FIELDS
|
||||
candidates = SupervisedPauseFields(
|
||||
intention=_safe_field_text(intention, defaults.intention),
|
||||
attendu=_safe_field_text(attendu, defaults.attendu),
|
||||
vu=_safe_field_text(vu, defaults.vu),
|
||||
demande=_safe_field_text(demande or raw_message, defaults.demande),
|
||||
)
|
||||
|
||||
try:
|
||||
return format_supervised_pause_message(
|
||||
intention=candidates.intention,
|
||||
attendu=candidates.attendu,
|
||||
vu=candidates.vu,
|
||||
demande=candidates.demande,
|
||||
)
|
||||
except MessageContractError:
|
||||
return format_supervised_pause_message(
|
||||
intention=defaults.intention,
|
||||
attendu=defaults.attendu,
|
||||
vu=defaults.vu,
|
||||
demande=defaults.demande,
|
||||
)
|
||||
|
||||
|
||||
def warn_visible_message(
|
||||
message: object,
|
||||
*,
|
||||
source: str,
|
||||
supervised_pause: bool = False,
|
||||
) -> str:
|
||||
"""Log contract violations without modifying the visible message."""
|
||||
|
||||
text = str(message or "")
|
||||
validator = validate_supervised_pause_message if supervised_pause else validate_visible_message
|
||||
result = validator(text)
|
||||
if not result.valid:
|
||||
logger.warning(
|
||||
"[message_contract] invalid_message source=%s codes=%s",
|
||||
source,
|
||||
[issue.code for issue in result.issues],
|
||||
)
|
||||
return text
|
||||
|
||||
|
||||
def validate_supervised_pause_message(message: str) -> MessageValidationResult:
|
||||
"""Valider le contrat strict d'une pause supervisee."""
|
||||
|
||||
issues = list(validate_visible_message(message).issues)
|
||||
fields, structure_issues = _parse_supervised_pause(message)
|
||||
issues.extend(structure_issues)
|
||||
|
||||
if fields:
|
||||
for name, value in fields.items():
|
||||
if len(value) < MIN_FIELD_CHARS:
|
||||
issues.append(
|
||||
MessageValidationIssue(
|
||||
"field_too_short",
|
||||
f"{name} doit etre explicite",
|
||||
)
|
||||
)
|
||||
if len(value) > MAX_FIELD_CHARS:
|
||||
issues.append(
|
||||
MessageValidationIssue(
|
||||
"field_too_long",
|
||||
f"{name} depasse {MAX_FIELD_CHARS} caracteres",
|
||||
)
|
||||
)
|
||||
demande = fields.get("demande", "")
|
||||
if not _contains_actionable_french(demande) or len(demande.split()) < 4:
|
||||
issues.append(
|
||||
MessageValidationIssue(
|
||||
"not_actionable",
|
||||
"la demande doit contenir une action concrete en francais",
|
||||
)
|
||||
)
|
||||
|
||||
return _dedupe_issues(issues)
|
||||
|
||||
|
||||
def validate_visible_message(message: str) -> MessageValidationResult:
|
||||
"""Valider qu'un message visible n'est ni generique ni technique."""
|
||||
|
||||
text = str(message or "").strip()
|
||||
issues: list[MessageValidationIssue] = []
|
||||
|
||||
if not text:
|
||||
return MessageValidationResult(
|
||||
(MessageValidationIssue("empty_message", "message vide"),)
|
||||
)
|
||||
|
||||
if len(text) > MAX_VISIBLE_MESSAGE_CHARS:
|
||||
issues.append(
|
||||
MessageValidationIssue(
|
||||
"message_too_long",
|
||||
f"message au-dela de {MAX_VISIBLE_MESSAGE_CHARS} caracteres",
|
||||
)
|
||||
)
|
||||
|
||||
folded = _fold(text)
|
||||
seen_generic_phrases: set[str] = set()
|
||||
for phrase in _GENERIC_PHRASES:
|
||||
folded_phrase = _fold(phrase)
|
||||
if folded_phrase in seen_generic_phrases:
|
||||
continue
|
||||
seen_generic_phrases.add(folded_phrase)
|
||||
if folded_phrase in folded:
|
||||
issues.append(
|
||||
MessageValidationIssue(
|
||||
"generic_phrase",
|
||||
f"formulation trop generique: {phrase}",
|
||||
)
|
||||
)
|
||||
|
||||
for term in _TECHNICAL_ENGLISH_TERMS:
|
||||
if _fold(term) in folded:
|
||||
issues.append(
|
||||
MessageValidationIssue(
|
||||
"technical_english",
|
||||
f"anglais technique visible: {term}",
|
||||
)
|
||||
)
|
||||
|
||||
for code, pattern, detail in (
|
||||
("technical_field", _TECHNICAL_FIELD_RE, "champ technique brut"),
|
||||
("technical_identifier", _TECHNICAL_IDENTIFIER_RE, "identifiant technique brut"),
|
||||
("technical_identifier", _UUID_RE, "UUID brut"),
|
||||
("technical_identifier", _LONG_HEX_RE, "hash technique brut"),
|
||||
("raw_coordinates", _PIXEL_TUPLE_RE, "coordonnees pixel brutes"),
|
||||
("raw_coordinates", _PIXEL_FIELD_RE, "coordonnees techniques brutes"),
|
||||
("raw_coordinates", _PX_RE, "coordonnees pixel brutes"),
|
||||
("raw_score", _SCORE_RE, "score ou confiance brut"),
|
||||
):
|
||||
if pattern.search(text):
|
||||
issues.append(MessageValidationIssue(code, detail))
|
||||
|
||||
return _dedupe_issues(issues)
|
||||
|
||||
|
||||
def is_valid_visible_message(message: str) -> bool:
|
||||
"""Raccourci booleen pour les points d'integration UI."""
|
||||
|
||||
return validate_visible_message(message).valid
|
||||
|
||||
|
||||
def is_valid_supervised_pause_message(message: str) -> bool:
|
||||
"""Raccourci booleen pour les pauses supervisees."""
|
||||
|
||||
return validate_supervised_pause_message(message).valid
|
||||
|
||||
|
||||
def _parse_supervised_pause(
|
||||
message: str,
|
||||
) -> tuple[dict[str, str], list[MessageValidationIssue]]:
|
||||
lines = [line.rstrip() for line in str(message or "").splitlines() if line.strip()]
|
||||
issues: list[MessageValidationIssue] = []
|
||||
|
||||
if len(lines) != 4:
|
||||
issues.append(
|
||||
MessageValidationIssue(
|
||||
"invalid_structure",
|
||||
"une pause supervisee doit contenir exactement 4 lignes",
|
||||
)
|
||||
)
|
||||
return {}, issues
|
||||
|
||||
specs = (
|
||||
("intention", r"^J'essaie de\s*:\s*(.+)$"),
|
||||
("attendu", r"^J'attendais\s*:\s*(.+)$"),
|
||||
("vu", r"^Je vois\s*:\s*(.+)$"),
|
||||
("demande", r"^Peux-tu\s*:\s*(.+)$"),
|
||||
)
|
||||
fields: dict[str, str] = {}
|
||||
for line, (name, pattern) in zip(lines, specs):
|
||||
match = re.match(pattern, line)
|
||||
if not match:
|
||||
issues.append(
|
||||
MessageValidationIssue(
|
||||
"invalid_structure",
|
||||
f"ligne {len(fields) + 1} doit commencer par {SUPERVISED_PAUSE_LABELS[len(fields)]}",
|
||||
)
|
||||
)
|
||||
continue
|
||||
fields[name] = match.group(1).strip()
|
||||
|
||||
if len(fields) != 4:
|
||||
return {}, issues
|
||||
|
||||
return fields, issues
|
||||
|
||||
|
||||
def _contains_actionable_french(text: str) -> bool:
|
||||
folded = _fold(text)
|
||||
return any(_fold(hint) in folded for hint in _ACTIONABLE_FRENCH_HINTS)
|
||||
|
||||
|
||||
def _one_line(value: object) -> str:
|
||||
return re.sub(r"\s+", " ", str(value or "")).strip()
|
||||
|
||||
|
||||
def _mapping_text(payload: Mapping[str, object], *keys: str) -> str:
|
||||
for key in keys:
|
||||
value = payload.get(key)
|
||||
if value is not None:
|
||||
return str(value)
|
||||
return ""
|
||||
|
||||
|
||||
def _safe_field_text(value: object, fallback: str) -> str:
|
||||
text = _one_line(value)
|
||||
if len(text) < MIN_FIELD_CHARS or len(text) > MAX_FIELD_CHARS:
|
||||
return fallback
|
||||
if not validate_visible_message(text).valid:
|
||||
return fallback
|
||||
return text
|
||||
|
||||
|
||||
def _fold(text: str) -> str:
|
||||
normalized = unicodedata.normalize("NFKD", str(text or ""))
|
||||
ascii_text = "".join(ch for ch in normalized if not unicodedata.combining(ch))
|
||||
return ascii_text.casefold()
|
||||
|
||||
|
||||
def _dedupe_issues(issues: Iterable[MessageValidationIssue]) -> MessageValidationResult:
|
||||
seen: set[tuple[str, str]] = set()
|
||||
deduped: list[MessageValidationIssue] = []
|
||||
for issue in issues:
|
||||
key = (issue.code, issue.detail)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
deduped.append(issue)
|
||||
return MessageValidationResult(tuple(deduped))
|
||||
|
||||
|
||||
__all__ = [
|
||||
"MAX_FIELD_CHARS",
|
||||
"MAX_VISIBLE_MESSAGE_CHARS",
|
||||
"MessageContractError",
|
||||
"MessageValidationIssue",
|
||||
"MessageValidationResult",
|
||||
"SUPERVISED_PAUSE_LABELS",
|
||||
"SupervisedPauseFields",
|
||||
"coerce_supervised_pause_message",
|
||||
"format_supervised_pause_from_mapping",
|
||||
"format_supervised_pause_message",
|
||||
"is_valid_supervised_pause_message",
|
||||
"is_valid_visible_message",
|
||||
"validate_supervised_pause_message",
|
||||
"validate_visible_message",
|
||||
"warn_visible_message",
|
||||
]
|
||||
661
agent_v0/agent_v1/ui/messages.py
Normal file
661
agent_v0/agent_v1/ui/messages.py
Normal file
@@ -0,0 +1,661 @@
|
||||
# agent_v1/ui/messages.py
|
||||
"""
|
||||
Formatage des messages utilisateur pour Léa.
|
||||
|
||||
Convertit les codes d'erreur techniques (`target_not_found`, `no_screen_change`...)
|
||||
en phrases en français naturel, orientées action, adaptées à un utilisateur non
|
||||
technique (secrétaire médicale, TIM).
|
||||
|
||||
Trois niveaux de sévérité sont définis :
|
||||
- INFO — Léa fait son travail normalement
|
||||
- ATTENTION — Quelque chose de léger (ralentissement, retry)
|
||||
- BLOCAGE — Léa a besoin d'aide, elle rend la main
|
||||
|
||||
Le module est 100% pur (pas d'I/O, pas d'UI) : testable sans mocks lourds.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
from typing import Any, Mapping, Optional
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
# Accès paresseux au DomainContext
|
||||
# ----------------------------------------------------------------------------
|
||||
#
|
||||
# On importe le module à l'appel pour éviter toute dépendance circulaire
|
||||
# avec `agent_v0.server_v1.domain_context` (qui ne doit pas importer l'UI).
|
||||
# Si l'import échoue (contexte client sans server_v1), on retombe sur None
|
||||
# et les formatters gardent leur comportement générique historique.
|
||||
|
||||
|
||||
def _get_domain_ctx(domain_id: Optional[str]):
|
||||
"""Récupérer un DomainContext si possible, sinon None (fallback)."""
|
||||
if not domain_id:
|
||||
return None
|
||||
try:
|
||||
from agent_v0.server_v1.domain_context import get_domain_context # lazy
|
||||
return get_domain_context(domain_id)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _friendly_target(description: str, domain_id: Optional[str] = None) -> str:
|
||||
"""Transformer une description technique en langage métier si possible.
|
||||
|
||||
Ex (tim_codage) : "DP" → "diagnostic principal"
|
||||
Ex (comptabilite) : "TVA" → "montant de TVA"
|
||||
Retombe sur la description nettoyée si aucun domaine ne matche.
|
||||
"""
|
||||
base = _nettoyer_description_cible(description)
|
||||
ctx = _get_domain_ctx(domain_id)
|
||||
if ctx is None or not base:
|
||||
return base
|
||||
try:
|
||||
return ctx._apply_synonyms(base)
|
||||
except Exception:
|
||||
return base
|
||||
|
||||
|
||||
class NiveauMessage(Enum):
|
||||
"""Niveaux hiérarchiques des messages affichés à l'utilisateur."""
|
||||
|
||||
INFO = "info" # Fond vert clair, disparaît tout seul, 3-5s
|
||||
ATTENTION = "attention" # Fond orange clair, disparaît tout seul, 7s
|
||||
BLOCAGE = "blocage" # Fond rouge clair, reste affiché, 15s+
|
||||
|
||||
|
||||
# Durée d'affichage par défaut (secondes), par niveau
|
||||
DUREE_PAR_NIVEAU: dict[NiveauMessage, int] = {
|
||||
NiveauMessage.INFO: 4,
|
||||
NiveauMessage.ATTENTION: 7,
|
||||
NiveauMessage.BLOCAGE: 15,
|
||||
}
|
||||
|
||||
# Icône textuelle par niveau (compatible plyer/Windows/Linux)
|
||||
ICONE_PAR_NIVEAU: dict[NiveauMessage, str] = {
|
||||
NiveauMessage.INFO: "i",
|
||||
NiveauMessage.ATTENTION: "!",
|
||||
NiveauMessage.BLOCAGE: "?",
|
||||
}
|
||||
|
||||
# Les pauses supervisees peuvent contenir une raison precise, parfois longue
|
||||
# (fenetre observee, fenetre attendue, action en cours). On garde l'information
|
||||
# utile et on laisse les widgets UI gerer le wrap/scroll.
|
||||
MAX_TARGET_DESCRIPTION_CHARS = 1024
|
||||
MAX_GENERIC_TECHNICAL_MESSAGE_CHARS = 1024
|
||||
|
||||
|
||||
@dataclass
|
||||
class MessageUtilisateur:
|
||||
"""Un message prêt à être affiché à l'utilisateur.
|
||||
|
||||
Attributes:
|
||||
niveau: Hiérarchie (info/attention/blocage)
|
||||
titre: Titre court de la notification (≤60 caractères)
|
||||
corps: Corps du message en français naturel
|
||||
duree_s: Durée d'affichage recommandée (secondes)
|
||||
persistent: Si True, l'utilisateur doit fermer manuellement
|
||||
"""
|
||||
|
||||
niveau: NiveauMessage
|
||||
titre: str
|
||||
corps: str
|
||||
duree_s: int
|
||||
persistent: bool = False
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
"""Sérialiser le message (utile pour les tests et le logging)."""
|
||||
return {
|
||||
"niveau": self.niveau.value,
|
||||
"titre": self.titre,
|
||||
"corps": self.corps,
|
||||
"duree_s": self.duree_s,
|
||||
"persistent": self.persistent,
|
||||
}
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Helpers d'extraction
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def _extraire_nom_application(titre_fenetre: str) -> str:
|
||||
"""Extraire le nom de l'application à partir d'un titre de fenêtre.
|
||||
|
||||
Les titres Windows suivent généralement le format :
|
||||
"Document.txt – Bloc-notes"
|
||||
"Ma Page - Google Chrome"
|
||||
"Sans titre — Paint"
|
||||
|
||||
On retourne la partie après le dernier séparateur, ou le titre entier.
|
||||
"""
|
||||
if not titre_fenetre:
|
||||
return ""
|
||||
titre = titre_fenetre.strip()
|
||||
# Chercher le dernier séparateur parmi " – ", " — ", " - "
|
||||
for sep in (" – ", " — ", " - "):
|
||||
if sep in titre:
|
||||
return titre.rsplit(sep, 1)[-1].strip()
|
||||
return titre
|
||||
|
||||
|
||||
def _nettoyer_description_cible(description: str) -> str:
|
||||
"""Nettoyer la description technique d'une cible pour l'afficher.
|
||||
|
||||
Supprime les caractères techniques (guillemets inutiles, ':').
|
||||
"""
|
||||
if not description:
|
||||
return ""
|
||||
desc = description.strip()
|
||||
# Retirer les guillemets encapsulants
|
||||
desc = desc.strip("'\"`")
|
||||
# Limiter la longueur sans perdre les details utiles a la supervision.
|
||||
if len(desc) > MAX_TARGET_DESCRIPTION_CHARS:
|
||||
desc = desc[: MAX_TARGET_DESCRIPTION_CHARS - 3] + "..."
|
||||
return desc
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Formattage des messages techniques → humains
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def formatter_cible_non_trouvee(
|
||||
description_cible: str,
|
||||
titre_fenetre: Optional[str] = None,
|
||||
domain_id: Optional[str] = None,
|
||||
params: Optional[Mapping[str, Any]] = None,
|
||||
) -> MessageUtilisateur:
|
||||
"""Message quand Léa ne trouve pas un élément à cliquer.
|
||||
|
||||
Si un domaine métier est fourni, la description de la cible est
|
||||
transformée en langage métier via le DomainContext :
|
||||
- tim_codage + "DP" → "diagnostic principal"
|
||||
- comptabilite + "TVA" → "montant de TVA"
|
||||
|
||||
Exemple avant :
|
||||
target_not_found: 'bonjour' dans *bonjour, – Bloc-notes
|
||||
Exemple après :
|
||||
Léa a besoin d'aide
|
||||
Je ne trouve pas "bonjour" dans le Bloc-notes. Peux-tu cliquer
|
||||
dessus toi-même ? Je reprends ensuite.
|
||||
|
||||
Args:
|
||||
description_cible: Description brute de la cible.
|
||||
titre_fenetre: Titre de la fenêtre active (pour extraire l'app).
|
||||
domain_id: Domaine métier pour enrichir la sortie (optionnel).
|
||||
params: Paramètres du workflow (nom_patient, num_facture...)
|
||||
utilisés par les templates de clarification métier.
|
||||
"""
|
||||
cible = _friendly_target(description_cible, domain_id) or "l'élément"
|
||||
app = _extraire_nom_application(titre_fenetre or "")
|
||||
|
||||
# Si un domaine et un template de clarification existent, préférer la
|
||||
# question métier (plus pertinente que le message générique).
|
||||
ctx = _get_domain_ctx(domain_id)
|
||||
if ctx is not None and ctx.clarification_templates:
|
||||
try:
|
||||
corps = ctx.pose_clarification_question(
|
||||
{
|
||||
"blocked_on": "target_not_found",
|
||||
"target": description_cible or "",
|
||||
"app": app,
|
||||
"params": dict(params or {}),
|
||||
}
|
||||
)
|
||||
except Exception:
|
||||
corps = ""
|
||||
if corps:
|
||||
return MessageUtilisateur(
|
||||
niveau=NiveauMessage.BLOCAGE,
|
||||
titre="Léa a besoin d'aide",
|
||||
corps=corps,
|
||||
duree_s=DUREE_PAR_NIVEAU[NiveauMessage.BLOCAGE],
|
||||
persistent=True,
|
||||
)
|
||||
|
||||
if app:
|
||||
corps = (
|
||||
f"Je ne trouve pas « {cible} » dans {app}. "
|
||||
f"Peux-tu cliquer dessus toi-même ? Je reprends ensuite."
|
||||
)
|
||||
else:
|
||||
corps = (
|
||||
f"Je ne trouve pas « {cible} » à l'écran. "
|
||||
f"Peux-tu le faire toi-même ? Je reprends ensuite."
|
||||
)
|
||||
|
||||
return MessageUtilisateur(
|
||||
niveau=NiveauMessage.BLOCAGE,
|
||||
titre="Léa a besoin d'aide",
|
||||
corps=corps,
|
||||
duree_s=DUREE_PAR_NIVEAU[NiveauMessage.BLOCAGE],
|
||||
persistent=True,
|
||||
)
|
||||
|
||||
|
||||
def formatter_fenetre_incorrecte(
|
||||
titre_actuel: str,
|
||||
titre_attendu: str,
|
||||
) -> MessageUtilisateur:
|
||||
"""Message quand la fenêtre active n'est pas celle attendue.
|
||||
|
||||
Exemple avant :
|
||||
Fenêtre incorrecte: 'Program Manager' (attendu: 'Lea : Explorateur de fichiers')
|
||||
Exemple après :
|
||||
Léa attend une fenêtre
|
||||
J'attends « Explorateur de fichiers » mais c'est « Program Manager »
|
||||
qui est affiché. Peux-tu ouvrir la bonne fenêtre ?
|
||||
"""
|
||||
app_actuelle = _extraire_nom_application(titre_actuel) or "une autre fenêtre"
|
||||
app_attendue = _extraire_nom_application(titre_attendu) or titre_attendu
|
||||
|
||||
corps = (
|
||||
f"J'attends « {app_attendue} » mais c'est « {app_actuelle} » "
|
||||
f"qui est affiché. Peux-tu ouvrir la bonne fenêtre ?"
|
||||
)
|
||||
|
||||
return MessageUtilisateur(
|
||||
niveau=NiveauMessage.BLOCAGE,
|
||||
titre="Léa attend une fenêtre",
|
||||
corps=corps,
|
||||
duree_s=DUREE_PAR_NIVEAU[NiveauMessage.BLOCAGE],
|
||||
persistent=True,
|
||||
)
|
||||
|
||||
|
||||
def formatter_ecran_inchange(action_type: str = "") -> MessageUtilisateur:
|
||||
"""Message quand l'action n'a pas eu d'effet visible.
|
||||
|
||||
Exemple avant :
|
||||
Ecran inchange apres l'action
|
||||
Exemple après :
|
||||
Léa vérifie
|
||||
Mon clic n'a pas eu l'air de marcher. Je vais réessayer ou te
|
||||
rendre la main si ça ne passe pas.
|
||||
"""
|
||||
actions_fr = {
|
||||
"click": "Mon clic",
|
||||
"type": "Ma saisie",
|
||||
"key_combo": "Mon raccourci clavier",
|
||||
"scroll": "Mon défilement",
|
||||
}
|
||||
quoi = actions_fr.get(action_type, "Mon action")
|
||||
|
||||
corps = (
|
||||
f"{quoi} n'a pas eu l'air de marcher. Je vais réessayer, "
|
||||
f"ou te rendre la main si ça ne passe pas."
|
||||
)
|
||||
|
||||
return MessageUtilisateur(
|
||||
niveau=NiveauMessage.ATTENTION,
|
||||
titre="Léa vérifie",
|
||||
corps=corps,
|
||||
duree_s=DUREE_PAR_NIVEAU[NiveauMessage.ATTENTION],
|
||||
)
|
||||
|
||||
|
||||
def formatter_mode_apprentissage(
|
||||
raison: str = "",
|
||||
description_cible: str = "",
|
||||
titre_fenetre: Optional[str] = None,
|
||||
) -> MessageUtilisateur:
|
||||
"""Message quand Léa passe en mode apprentissage (pause supervisée).
|
||||
|
||||
L'utilisateur doit comprendre :
|
||||
1. Léa est bloquée et a besoin d'aide
|
||||
2. L'utilisateur doit prendre la main et montrer comment faire
|
||||
3. Ctrl+Shift+L pour signaler qu'il a fini
|
||||
|
||||
Le ton est humble, clair, actionnable. Pas technique.
|
||||
|
||||
Exemple :
|
||||
Léa a besoin d'aide
|
||||
Je n'y arrive pas, montrez-moi comment faire.
|
||||
Quand vous avez fini, appuyez sur Ctrl+Shift+L.
|
||||
"""
|
||||
cible = _nettoyer_description_cible(description_cible) if description_cible else ""
|
||||
app = _extraire_nom_application(titre_fenetre or "") if titre_fenetre else ""
|
||||
|
||||
# Construire un contexte court si disponible
|
||||
contexte = ""
|
||||
if cible and app:
|
||||
contexte = f" (« {cible} » dans {app})"
|
||||
elif cible:
|
||||
contexte = f" (« {cible} »)"
|
||||
|
||||
corps = (
|
||||
f"Je n'y arrive pas{contexte}, montrez-moi comment faire. "
|
||||
f"Quand vous avez fini, appuyez sur Ctrl+Shift+L."
|
||||
)
|
||||
|
||||
return MessageUtilisateur(
|
||||
niveau=NiveauMessage.BLOCAGE,
|
||||
titre="Léa a besoin d'aide",
|
||||
corps=corps,
|
||||
duree_s=DUREE_PAR_NIVEAU[NiveauMessage.BLOCAGE],
|
||||
persistent=True,
|
||||
)
|
||||
|
||||
|
||||
def formatter_connexion_perdue(hote_serveur: str = "") -> MessageUtilisateur:
|
||||
"""Message quand la connexion avec le serveur est perdue.
|
||||
|
||||
Rassurant : on dit qu'on va réessayer automatiquement.
|
||||
"""
|
||||
corps = (
|
||||
"J'ai perdu le lien avec le serveur. Je retente automatiquement, "
|
||||
"pas besoin d'intervenir."
|
||||
)
|
||||
|
||||
return MessageUtilisateur(
|
||||
niveau=NiveauMessage.ATTENTION,
|
||||
titre="Léa est déconnectée",
|
||||
corps=corps,
|
||||
duree_s=DUREE_PAR_NIVEAU[NiveauMessage.ATTENTION],
|
||||
)
|
||||
|
||||
|
||||
def formatter_connexion_retablie() -> MessageUtilisateur:
|
||||
"""Message quand la connexion serveur est rétablie."""
|
||||
return MessageUtilisateur(
|
||||
niveau=NiveauMessage.INFO,
|
||||
titre="Léa",
|
||||
corps="C'est bon, la connexion est revenue. Je continue.",
|
||||
duree_s=DUREE_PAR_NIVEAU[NiveauMessage.INFO],
|
||||
)
|
||||
|
||||
|
||||
def formatter_debut_workflow(nom_workflow: str, nb_etapes: int = 0) -> MessageUtilisateur:
|
||||
"""Message au démarrage d'un workflow de replay."""
|
||||
if nb_etapes > 0:
|
||||
corps = (
|
||||
f"Je démarre « {nom_workflow} » ({nb_etapes} étapes). "
|
||||
f"Je t'indique mon avancement."
|
||||
)
|
||||
else:
|
||||
corps = f"Je démarre « {nom_workflow} ». Je t'indique mon avancement."
|
||||
|
||||
return MessageUtilisateur(
|
||||
niveau=NiveauMessage.INFO,
|
||||
titre="Léa démarre",
|
||||
corps=corps,
|
||||
duree_s=DUREE_PAR_NIVEAU[NiveauMessage.INFO],
|
||||
)
|
||||
|
||||
|
||||
def formatter_etape_workflow(
|
||||
etape_actuelle: int,
|
||||
nb_etapes: int,
|
||||
description: str = "",
|
||||
) -> MessageUtilisateur:
|
||||
"""Message pour la progression d'une étape."""
|
||||
if description:
|
||||
desc = _nettoyer_description_cible(description)
|
||||
corps = f"Étape {etape_actuelle}/{nb_etapes} — {desc}"
|
||||
else:
|
||||
corps = f"Étape {etape_actuelle}/{nb_etapes}"
|
||||
|
||||
return MessageUtilisateur(
|
||||
niveau=NiveauMessage.INFO,
|
||||
titre="Léa avance",
|
||||
corps=corps,
|
||||
duree_s=3,
|
||||
)
|
||||
|
||||
|
||||
def formatter_retry(action_type: str = "", tentative: int = 2) -> MessageUtilisateur:
|
||||
"""Message quand Léa retente une action."""
|
||||
corps = (
|
||||
f"Je retente (tentative {tentative}). Ça arrive parfois, "
|
||||
f"l'écran était peut-être en cours de chargement."
|
||||
)
|
||||
return MessageUtilisateur(
|
||||
niveau=NiveauMessage.ATTENTION,
|
||||
titre="Léa retente",
|
||||
corps=corps,
|
||||
duree_s=DUREE_PAR_NIVEAU[NiveauMessage.ATTENTION],
|
||||
)
|
||||
|
||||
|
||||
def formatter_ralentissement() -> MessageUtilisateur:
|
||||
"""Message quand Léa prend plus de temps que prévu."""
|
||||
return MessageUtilisateur(
|
||||
niveau=NiveauMessage.ATTENTION,
|
||||
titre="Léa prend son temps",
|
||||
corps="Je vais plus lentement que prévu. L'écran met du temps à répondre.",
|
||||
duree_s=DUREE_PAR_NIVEAU[NiveauMessage.ATTENTION],
|
||||
)
|
||||
|
||||
|
||||
def formatter_fin_workflow(
|
||||
succes: bool,
|
||||
nom_workflow: str = "",
|
||||
nb_etapes: int = 0,
|
||||
duree_s: float = 0.0,
|
||||
domain_id: Optional[str] = None,
|
||||
items_count: int = 0,
|
||||
failed_count: int = 0,
|
||||
params: Optional[Mapping[str, Any]] = None,
|
||||
) -> MessageUtilisateur:
|
||||
"""Message à la fin d'un workflow.
|
||||
|
||||
Si un domaine métier est fourni (et qu'il expose des summary_templates),
|
||||
on utilise `DomainContext.describe_workflow_outcome` pour formuler un
|
||||
rapport en langage métier (ex: "J'ai codé 14 dossiers sur 15").
|
||||
|
||||
Args:
|
||||
succes: True si l'ensemble du workflow a réussi.
|
||||
nom_workflow: Nom du workflow.
|
||||
nb_etapes: Nombre d'étapes techniques (pour fallback générique).
|
||||
duree_s: Durée totale en secondes.
|
||||
domain_id: Domaine métier (optionnel).
|
||||
items_count: Nombre d'items métier traités (ex: 15 dossiers).
|
||||
failed_count: Nombre d'items en échec.
|
||||
params: Infos supplémentaires passées aux templates.
|
||||
"""
|
||||
ctx = _get_domain_ctx(domain_id)
|
||||
if ctx is not None and ctx.summary_templates:
|
||||
try:
|
||||
corps = ctx.describe_workflow_outcome(
|
||||
workflow_name=nom_workflow,
|
||||
success=succes,
|
||||
items_count=items_count or max(1, nb_etapes),
|
||||
failed_count=failed_count,
|
||||
elapsed_s=duree_s,
|
||||
extra=dict(params or {}),
|
||||
)
|
||||
except Exception:
|
||||
corps = ""
|
||||
if corps:
|
||||
if succes and failed_count == 0:
|
||||
return MessageUtilisateur(
|
||||
niveau=NiveauMessage.INFO,
|
||||
titre="Léa a terminé",
|
||||
corps=corps,
|
||||
duree_s=6,
|
||||
)
|
||||
if succes and failed_count > 0:
|
||||
return MessageUtilisateur(
|
||||
niveau=NiveauMessage.ATTENTION,
|
||||
titre="Léa a terminé partiellement",
|
||||
corps=corps,
|
||||
duree_s=DUREE_PAR_NIVEAU[NiveauMessage.ATTENTION],
|
||||
)
|
||||
return MessageUtilisateur(
|
||||
niveau=NiveauMessage.BLOCAGE,
|
||||
titre="Léa s'arrête",
|
||||
corps=corps,
|
||||
duree_s=DUREE_PAR_NIVEAU[NiveauMessage.BLOCAGE],
|
||||
persistent=True,
|
||||
)
|
||||
|
||||
if succes:
|
||||
if nom_workflow and nb_etapes > 0:
|
||||
corps = (
|
||||
f"C'est fait ! « {nom_workflow} » est terminé "
|
||||
f"({nb_etapes} étapes en {int(duree_s)}s)."
|
||||
)
|
||||
else:
|
||||
corps = "C'est fait ! Tout s'est bien passé."
|
||||
return MessageUtilisateur(
|
||||
niveau=NiveauMessage.INFO,
|
||||
titre="Léa a terminé",
|
||||
corps=corps,
|
||||
duree_s=6,
|
||||
)
|
||||
else:
|
||||
corps = (
|
||||
"Je n'ai pas pu terminer. Je te rends la main, "
|
||||
"tu peux continuer à partir de là où je me suis arrêtée."
|
||||
)
|
||||
return MessageUtilisateur(
|
||||
niveau=NiveauMessage.BLOCAGE,
|
||||
titre="Léa s'arrête",
|
||||
corps=corps,
|
||||
duree_s=DUREE_PAR_NIVEAU[NiveauMessage.BLOCAGE],
|
||||
persistent=True,
|
||||
)
|
||||
|
||||
|
||||
def formatter_erreur_generique(
|
||||
message_technique: str,
|
||||
domain_id: Optional[str] = None,
|
||||
params: Optional[Mapping[str, Any]] = None,
|
||||
) -> MessageUtilisateur:
|
||||
"""Formater un message d'erreur technique non catégorisé.
|
||||
|
||||
On essaie de détecter les motifs connus dans le message technique pour
|
||||
le router vers le bon formatter spécialisé, sinon on emballe le message.
|
||||
Si `domain_id` est fourni, il est propagé aux formatters spécialisés
|
||||
pour produire un message en langage métier.
|
||||
"""
|
||||
if not message_technique:
|
||||
return MessageUtilisateur(
|
||||
niveau=NiveauMessage.ATTENTION,
|
||||
titre="Léa",
|
||||
corps="J'ai rencontré un petit souci. Je continue.",
|
||||
duree_s=DUREE_PAR_NIVEAU[NiveauMessage.ATTENTION],
|
||||
)
|
||||
|
||||
msg_lower = message_technique.lower()
|
||||
|
||||
# target_not_found[:...]
|
||||
if "target_not_found" in msg_lower:
|
||||
# Essayer d'extraire la description après le ':'
|
||||
match = re.match(r"target_not_found[:\s]*(.*)", message_technique, re.IGNORECASE)
|
||||
desc = match.group(1).strip() if match else ""
|
||||
return formatter_cible_non_trouvee(desc, domain_id=domain_id, params=params)
|
||||
|
||||
# Fenêtre incorrecte: 'X' (attendu: 'Y')
|
||||
if "fenêtre incorrecte" in msg_lower or "fenetre incorrecte" in msg_lower:
|
||||
# Extraire actuel et attendu
|
||||
m_actuel = re.search(r"[:,]\s*['\"]([^'\"]+)['\"]", message_technique)
|
||||
m_attendu = re.search(r"attendu[:\s]*['\"]([^'\"]+)['\"]", message_technique)
|
||||
actuel = m_actuel.group(1) if m_actuel else ""
|
||||
attendu = m_attendu.group(1) if m_attendu else ""
|
||||
return formatter_fenetre_incorrecte(actuel, attendu)
|
||||
|
||||
# Ecran inchangé
|
||||
if "inchang" in msg_lower or "no_screen_change" in msg_lower:
|
||||
return formatter_ecran_inchange()
|
||||
|
||||
# Policy abort / supervise
|
||||
if "policy_abort" in msg_lower or "visual_resolve_failed" in msg_lower:
|
||||
return formatter_cible_non_trouvee(
|
||||
message_technique, domain_id=domain_id, params=params
|
||||
)
|
||||
|
||||
# Fallback : message technique tronqué
|
||||
msg_tronque = message_technique.strip()
|
||||
if len(msg_tronque) > MAX_GENERIC_TECHNICAL_MESSAGE_CHARS:
|
||||
msg_tronque = msg_tronque[: MAX_GENERIC_TECHNICAL_MESSAGE_CHARS - 3] + "..."
|
||||
|
||||
return MessageUtilisateur(
|
||||
niveau=NiveauMessage.ATTENTION,
|
||||
titre="Léa",
|
||||
corps=f"J'ai rencontré un souci : {msg_tronque}",
|
||||
duree_s=DUREE_PAR_NIVEAU[NiveauMessage.ATTENTION],
|
||||
)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Détection fenêtre Léa (utilisé par l'executor pour ignorer sa propre UI)
|
||||
# ============================================================================
|
||||
|
||||
|
||||
# Motifs qui identifient une fenêtre appartenant à Léa (l'agent lui-même).
|
||||
# On utilise des regex avec \b pour éviter les faux positifs sur des noms
|
||||
# contenant "lea" (ex: "cléa.txt", "leapfrog", "replay").
|
||||
_MOTIFS_FENETRE_LEA_REGEX = (
|
||||
r"\bléa\b",
|
||||
r"\blea\b(?!p)", # "lea" mot entier, pas "leapfrog"
|
||||
r"lea\s*[—–\-:]", # "Lea —", "Lea -", "Lea :"
|
||||
r"léa\s*[—–\-:]",
|
||||
r"\bassistante ia\b",
|
||||
r"\bléa ia\b",
|
||||
r"\blea ia\b",
|
||||
)
|
||||
|
||||
|
||||
def est_fenetre_lea(titre_fenetre: str) -> bool:
|
||||
"""Détecter si un titre de fenêtre appartient à l'agent Léa lui-même.
|
||||
|
||||
Utilisé pour éviter que Léa ne se considère comme une fenêtre intrusive
|
||||
dans ses propres pré-vérifications.
|
||||
|
||||
Utilise des regex avec des word boundaries pour éviter les faux positifs
|
||||
sur des noms de fichiers contenant "lea" (ex: "cléa.txt", "replay.log").
|
||||
"""
|
||||
if not titre_fenetre:
|
||||
return False
|
||||
titre_lower = titre_fenetre.lower().strip()
|
||||
return any(re.search(motif, titre_lower) for motif in _MOTIFS_FENETRE_LEA_REGEX)
|
||||
|
||||
|
||||
# Fenêtres parasites Windows à ignorer dans les pré-vérifications.
|
||||
# Ce ne sont pas des fenêtres applicatives — c'est du bruit système
|
||||
# qui prend le focus de manière imprévisible.
|
||||
_FENETRES_BRUIT_SYSTEME = (
|
||||
"fenêtre de dépassement de capacité",
|
||||
"overflow", # version anglaise systray
|
||||
"program manager",
|
||||
"barre des tâches",
|
||||
"task bar",
|
||||
"cortana",
|
||||
"action center",
|
||||
"centre de notifications",
|
||||
)
|
||||
|
||||
|
||||
def est_fenetre_bruit(titre_fenetre: str) -> bool:
|
||||
"""Détecter si un titre de fenêtre est du bruit système Windows.
|
||||
|
||||
Ces fenêtres prennent le focus de manière imprévisible (systray overflow,
|
||||
taskbar, Program Manager) et ne sont jamais la cible d'une action utilisateur.
|
||||
"""
|
||||
if not titre_fenetre:
|
||||
return True # pas de titre = bruit
|
||||
titre_lower = titre_fenetre.lower().strip()
|
||||
if titre_lower == "unknown_window":
|
||||
return True
|
||||
return any(p in titre_lower for p in _FENETRES_BRUIT_SYSTEME)
|
||||
|
||||
|
||||
# Conservé pour rétro-compatibilité avec le code qui listait MOTIFS_FENETRE_LEA
|
||||
MOTIFS_FENETRE_LEA = (
|
||||
"léa",
|
||||
"lea —",
|
||||
"léa —",
|
||||
"lea -",
|
||||
"léa -",
|
||||
"lea assistante",
|
||||
"léa assistante",
|
||||
"lea : ",
|
||||
"léa : ",
|
||||
"assistante ia",
|
||||
)
|
||||
@@ -5,6 +5,14 @@ Utilise plyer pour les notifications système, sans dépendance PyQt5.
|
||||
|
||||
Remplace les dialogues Qt par des toasts non-bloquants.
|
||||
Thread-safe avec rate limiting (1 notification / 2 secondes max).
|
||||
|
||||
Les messages utilisateur sont formatés via `agent_v1.ui.messages` qui convertit
|
||||
les codes techniques (target_not_found, etc.) en français naturel.
|
||||
|
||||
Hiérarchie des notifications (cf. messages.NiveauMessage) :
|
||||
- INFO : auto-dismiss en ~4s, rate-limité classique
|
||||
- ATTENTION : auto-dismiss en ~7s, rate-limité classique
|
||||
- BLOCAGE : persistant (15s+), bypass du rate limit
|
||||
"""
|
||||
|
||||
import logging
|
||||
@@ -12,6 +20,23 @@ import threading
|
||||
import time
|
||||
from typing import Optional
|
||||
|
||||
from .messages import (
|
||||
MessageUtilisateur,
|
||||
NiveauMessage,
|
||||
formatter_cible_non_trouvee,
|
||||
formatter_connexion_perdue,
|
||||
formatter_connexion_retablie,
|
||||
formatter_debut_workflow,
|
||||
formatter_ecran_inchange,
|
||||
formatter_erreur_generique,
|
||||
formatter_etape_workflow,
|
||||
formatter_fenetre_incorrecte,
|
||||
formatter_fin_workflow,
|
||||
formatter_mode_apprentissage,
|
||||
formatter_ralentissement,
|
||||
formatter_retry,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Import conditionnel de plyer — fallback silencieux si absent
|
||||
@@ -59,7 +84,13 @@ class NotificationManager:
|
||||
# Méthode générique
|
||||
# ------------------------------------------------------------------ #
|
||||
|
||||
def notify(self, title: str, message: str, timeout: int = 5) -> bool:
|
||||
def notify(
|
||||
self,
|
||||
title: str,
|
||||
message: str,
|
||||
timeout: int = 5,
|
||||
bypass_rate_limit: bool = False,
|
||||
) -> bool:
|
||||
"""
|
||||
Affiche une notification toast.
|
||||
|
||||
@@ -67,6 +98,8 @@ class NotificationManager:
|
||||
title: Titre de la notification.
|
||||
message: Corps du message.
|
||||
timeout: Durée d'affichage en secondes.
|
||||
bypass_rate_limit: Si True, ignore le rate limit (pour les blocages
|
||||
importants qui ne doivent pas être écrasés).
|
||||
|
||||
Returns:
|
||||
True si la notification a été envoyée, False sinon
|
||||
@@ -76,17 +109,21 @@ class NotificationManager:
|
||||
logger.debug("Notification ignorée (plyer absent) : %s", title)
|
||||
return False
|
||||
|
||||
with self._lock:
|
||||
now = time.monotonic()
|
||||
elapsed = now - self._last_notification_time
|
||||
if elapsed < RATE_LIMIT_SECONDS:
|
||||
logger.debug(
|
||||
"Notification ignorée (rate limit, %.1fs restantes) : %s",
|
||||
RATE_LIMIT_SECONDS - elapsed,
|
||||
title,
|
||||
)
|
||||
return False
|
||||
self._last_notification_time = now
|
||||
if not bypass_rate_limit:
|
||||
with self._lock:
|
||||
now = time.monotonic()
|
||||
elapsed = now - self._last_notification_time
|
||||
if elapsed < RATE_LIMIT_SECONDS:
|
||||
logger.debug(
|
||||
"Notification ignorée (rate limit, %.1fs restantes) : %s",
|
||||
RATE_LIMIT_SECONDS - elapsed,
|
||||
title,
|
||||
)
|
||||
return False
|
||||
self._last_notification_time = now
|
||||
else:
|
||||
with self._lock:
|
||||
self._last_notification_time = time.monotonic()
|
||||
|
||||
# Envoi dans un thread dédié pour ne jamais bloquer l'appelant
|
||||
thread = threading.Thread(
|
||||
@@ -97,6 +134,48 @@ class NotificationManager:
|
||||
thread.start()
|
||||
return True
|
||||
|
||||
def notify_message(self, msg: MessageUtilisateur) -> bool:
|
||||
"""Envoyer un MessageUtilisateur structuré (niveau, titre, corps).
|
||||
|
||||
Les messages BLOCAGE bypass le rate limit pour garantir que
|
||||
l'utilisateur voit qu'on a besoin de lui.
|
||||
|
||||
UX fix 8 mai 2026 (démo GHT) : la bulle ChatWindow Léa V1 (Tkinter
|
||||
topmost + bell + force-show) est désormais l'affichage canonique pour
|
||||
les BLOCAGE de pause supervisée. On NE déclenche PLUS show_paused_toast
|
||||
depuis ici — Dom rapportait 3 popups en parallèle (toast executor,
|
||||
toast bubble, toast notifications). Plyer reste actif comme
|
||||
notification système discrète. Le toast Tkinter custom est conservé
|
||||
pour les fallbacks sans ChatWindow (cf. executor.Plan B).
|
||||
"""
|
||||
bypass = msg.niveau == NiveauMessage.BLOCAGE
|
||||
# Log aussi pour tracer dans les logs fichiers
|
||||
self._log_message(msg)
|
||||
|
||||
return self.notify(
|
||||
title=msg.titre,
|
||||
message=msg.corps,
|
||||
timeout=msg.duree_s,
|
||||
bypass_rate_limit=bypass,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _log_message(msg: MessageUtilisateur) -> None:
|
||||
"""Logger un message utilisateur avec le niveau approprié.
|
||||
|
||||
Les logs agents sont plus lisibles quand on route info → INFO,
|
||||
attention → WARNING, blocage → ERROR, avec un préfixe [LEA].
|
||||
"""
|
||||
prefix = f"[LEA] {msg.titre}: {msg.corps}"
|
||||
if msg.niveau == NiveauMessage.INFO:
|
||||
logger.info(prefix)
|
||||
elif msg.niveau == NiveauMessage.ATTENTION:
|
||||
logger.warning(prefix)
|
||||
elif msg.niveau == NiveauMessage.BLOCAGE:
|
||||
logger.error(prefix)
|
||||
else:
|
||||
logger.info(prefix)
|
||||
|
||||
def _send(self, title: str, message: str, timeout: int) -> None:
|
||||
"""Envoi effectif de la notification (exécuté dans un thread dédié)."""
|
||||
try:
|
||||
@@ -180,40 +259,93 @@ class NotificationManager:
|
||||
timeout=3,
|
||||
)
|
||||
|
||||
def replay_finished(self, success: bool, workflow_name: str) -> bool:
|
||||
"""Notification de fin de replay (succès ou échec)."""
|
||||
if success:
|
||||
return self.notify(
|
||||
title=APP_NAME,
|
||||
message="C'est fait ! Tout s'est bien passé.",
|
||||
timeout=5,
|
||||
)
|
||||
else:
|
||||
return self.notify(
|
||||
title=APP_NAME,
|
||||
message="Hmm, j'ai eu un souci. Vous pouvez me remontrer ?",
|
||||
timeout=7,
|
||||
)
|
||||
def replay_target_not_found(
|
||||
self,
|
||||
target_description: str,
|
||||
window_title: Optional[str] = None,
|
||||
) -> bool:
|
||||
"""Notification quand un élément n'est pas trouvé pendant le replay.
|
||||
|
||||
def connection_changed(self, connected: bool, server_host: str) -> bool:
|
||||
Le replay est mis en pause et attend une intervention humaine.
|
||||
Utilise `messages.formatter_cible_non_trouvee` pour un message en
|
||||
français naturel.
|
||||
"""
|
||||
msg = formatter_cible_non_trouvee(target_description, window_title)
|
||||
return self.notify_message(msg)
|
||||
|
||||
def replay_wrong_window(self, current_title: str, expected_title: str) -> bool:
|
||||
"""Notification quand la fenêtre active n'est pas celle attendue."""
|
||||
msg = formatter_fenetre_incorrecte(current_title, expected_title)
|
||||
return self.notify_message(msg)
|
||||
|
||||
def replay_no_screen_change(self, action_type: str = "") -> bool:
|
||||
"""Notification quand une action n'a pas eu d'effet visible."""
|
||||
msg = formatter_ecran_inchange(action_type)
|
||||
return self.notify_message(msg)
|
||||
|
||||
def replay_learning_mode(
|
||||
self,
|
||||
raison: str = "",
|
||||
target_description: str = "",
|
||||
window_title: Optional[str] = None,
|
||||
) -> bool:
|
||||
"""Notification quand Léa passe en mode apprentissage.
|
||||
|
||||
Léa est bloquée et demande à l'utilisateur de montrer comment faire.
|
||||
Message humble et actionnable pour un utilisateur non technique.
|
||||
"""
|
||||
msg = formatter_mode_apprentissage(raison, target_description, window_title)
|
||||
return self.notify_message(msg)
|
||||
|
||||
def replay_retry(self, action_type: str = "", tentative: int = 2) -> bool:
|
||||
"""Notification quand Léa retente une action."""
|
||||
msg = formatter_retry(action_type, tentative)
|
||||
return self.notify_message(msg)
|
||||
|
||||
def replay_slow(self) -> bool:
|
||||
"""Notification quand Léa va plus lentement que prévu."""
|
||||
msg = formatter_ralentissement()
|
||||
return self.notify_message(msg)
|
||||
|
||||
def replay_finished(
|
||||
self,
|
||||
success: bool,
|
||||
workflow_name: str,
|
||||
step_count: int = 0,
|
||||
duration_s: float = 0.0,
|
||||
) -> bool:
|
||||
"""Notification de fin de replay (succès ou échec)."""
|
||||
msg = formatter_fin_workflow(success, workflow_name, step_count, duration_s)
|
||||
return self.notify_message(msg)
|
||||
|
||||
def replay_workflow_started(self, workflow_name: str, step_count: int = 0) -> bool:
|
||||
"""Notification de début de workflow (remplace `replay_started`)."""
|
||||
msg = formatter_debut_workflow(workflow_name, step_count)
|
||||
return self.notify_message(msg)
|
||||
|
||||
def replay_step_progress(
|
||||
self,
|
||||
current: int,
|
||||
total: int,
|
||||
description: str = "",
|
||||
) -> bool:
|
||||
"""Notification de progression d'une étape (niveau INFO)."""
|
||||
msg = formatter_etape_workflow(current, total, description)
|
||||
return self.notify_message(msg)
|
||||
|
||||
def connection_changed(self, connected: bool, server_host: str = "") -> bool:
|
||||
"""Notification de changement d'état de la connexion serveur."""
|
||||
if connected:
|
||||
return self.notify(
|
||||
title=APP_NAME,
|
||||
message="Connectée au serveur.",
|
||||
timeout=5,
|
||||
)
|
||||
msg = formatter_connexion_retablie()
|
||||
else:
|
||||
return self.notify(
|
||||
title=APP_NAME,
|
||||
message="J'ai perdu la connexion avec le serveur.",
|
||||
timeout=7,
|
||||
)
|
||||
msg = formatter_connexion_perdue(server_host)
|
||||
return self.notify_message(msg)
|
||||
|
||||
def error(self, message: str) -> bool:
|
||||
"""Notification d'erreur."""
|
||||
return self.notify(
|
||||
title=APP_NAME,
|
||||
message=f"Oups, un problème : {message}",
|
||||
timeout=10,
|
||||
)
|
||||
"""Notification d'erreur générique.
|
||||
|
||||
Essaie d'abord de détecter un motif technique connu et de formater
|
||||
correctement, sinon fallback sur un message générique aidant.
|
||||
"""
|
||||
msg = formatter_erreur_generique(message)
|
||||
return self.notify_message(msg)
|
||||
|
||||
290
agent_v0/agent_v1/ui/paused_toast.py
Normal file
290
agent_v0/agent_v1/ui/paused_toast.py
Normal file
@@ -0,0 +1,290 @@
|
||||
# agent_v1/ui/paused_toast.py
|
||||
"""
|
||||
Toast Tkinter custom pour la pause supervisée (« Léa a besoin de votre aide »).
|
||||
|
||||
Démo GHT 8 mai 2026 — Fallback robuste 100 % autonome quand :
|
||||
- plyer.notification est silencieux sous Windows 11 (Focus Assist, balloon tips
|
||||
bloqués par la stratégie système),
|
||||
- la ChatWindow Léa V1 est `withdraw()`-cachée par défaut (Dom ne la voit pas),
|
||||
- aucune autre UI ne peut garantir que Dom verra physiquement le message.
|
||||
|
||||
Stratégie :
|
||||
- Toplevel topmost overrideredirect en haut à droite de l'écran principal,
|
||||
- fond bleu Léa, titre + message, auto-close après TOAST_DURATION_S,
|
||||
- thread-safe : peut être appelé depuis n'importe quel thread (le polling
|
||||
replay tourne dans un daemon thread, pas le thread principal),
|
||||
- aucune dépendance externe (juste tkinter stdlib),
|
||||
- rate limit interne pour éviter le flood (1 toast / 3s minimum).
|
||||
|
||||
Si un Tk root existe déjà dans le process (ChatWindow), on attache le Toplevel
|
||||
à ce root via `root.after(0, ...)` — c'est l'idiome thread-safe officiel de
|
||||
tkinter. Sinon on crée un Tk() dédié dans un daemon thread.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from typing import Any, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Couleurs cohérentes avec le thème Léa (cf. chat_window.py)
|
||||
TOAST_BG = "#2563EB" # Bleu Léa (HEADER_BG)
|
||||
TOAST_FG = "#FFFFFF"
|
||||
TOAST_TITLE_BG = "#1E40AF" # Bleu plus foncé pour le bandeau titre
|
||||
TOAST_BORDER = "#1E3A8A"
|
||||
|
||||
TOAST_WIDTH = 380
|
||||
TOAST_PAD_X = 18
|
||||
TOAST_PAD_Y = 14
|
||||
TOAST_DURATION_MS = 15000
|
||||
TOAST_RATE_LIMIT_S = 3.0
|
||||
|
||||
_lock = threading.Lock()
|
||||
_last_shown_at: float = 0.0
|
||||
_last_message: str = ""
|
||||
|
||||
|
||||
def _resolve_existing_root() -> Optional[Any]:
|
||||
"""Tente de récupérer le Tk root déjà créé par la ChatWindow.
|
||||
|
||||
On évite tk._default_root (deprecated) et on remonte plutôt via les
|
||||
threads existants : la ChatWindow garde une référence dans son instance
|
||||
mais n'expose rien de global. On se rabat donc sur la création d'un Tk
|
||||
indépendant si on n'a rien — c'est sûr, tkinter supporte plusieurs Tk()
|
||||
concurrents tant qu'ils sont chacun dans leur propre thread.
|
||||
"""
|
||||
try:
|
||||
import tkinter as tk
|
||||
# tk._default_root est interne mais c'est le moyen le plus simple
|
||||
# de partager un mainloop existant. Si ChatWindow tourne, ce sera
|
||||
# son root.
|
||||
root = getattr(tk, "_default_root", None)
|
||||
if root is not None:
|
||||
# Vérifier qu'il est encore vivant
|
||||
try:
|
||||
root.winfo_exists()
|
||||
return root
|
||||
except Exception:
|
||||
return None
|
||||
return None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _build_toast(parent: Any, title: str, message: str) -> Any:
|
||||
"""Construit le Toplevel toast (appelé dans le thread tkinter)."""
|
||||
import tkinter as tk
|
||||
|
||||
top = tk.Toplevel(parent)
|
||||
top.withdraw() # éviter le flash pendant la construction
|
||||
top.overrideredirect(True) # pas de barre de titre
|
||||
top.attributes("-topmost", True)
|
||||
try:
|
||||
# Petit boost de visibilité Windows : alpha légèrement transparent
|
||||
top.attributes("-alpha", 0.97)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Bordure visuelle (cadre extérieur foncé)
|
||||
outer = tk.Frame(top, bg=TOAST_BORDER, padx=2, pady=2)
|
||||
outer.pack(fill="both", expand=True)
|
||||
|
||||
# Bandeau titre
|
||||
title_frame = tk.Frame(outer, bg=TOAST_TITLE_BG)
|
||||
title_frame.pack(fill="x")
|
||||
tk.Label(
|
||||
title_frame,
|
||||
text=f" ⏸ {title}",
|
||||
bg=TOAST_TITLE_BG,
|
||||
fg=TOAST_FG,
|
||||
font=("Segoe UI", 12, "bold"),
|
||||
anchor="w",
|
||||
padx=10,
|
||||
pady=8,
|
||||
).pack(fill="x")
|
||||
|
||||
# Corps du message
|
||||
body_frame = tk.Frame(outer, bg=TOAST_BG)
|
||||
body_frame.pack(fill="both", expand=True)
|
||||
tk.Label(
|
||||
body_frame,
|
||||
text=message,
|
||||
bg=TOAST_BG,
|
||||
fg=TOAST_FG,
|
||||
font=("Segoe UI", 11),
|
||||
wraplength=TOAST_WIDTH - 40,
|
||||
justify="left",
|
||||
anchor="w",
|
||||
padx=TOAST_PAD_X,
|
||||
pady=TOAST_PAD_Y,
|
||||
).pack(fill="both", expand=True)
|
||||
|
||||
# Pied de page : "Cliquez pour fermer"
|
||||
footer = tk.Label(
|
||||
outer,
|
||||
text="Cliquez pour fermer",
|
||||
bg=TOAST_BG,
|
||||
fg="#BFDBFE",
|
||||
font=("Segoe UI", 9, "italic"),
|
||||
anchor="e",
|
||||
padx=10,
|
||||
pady=4,
|
||||
)
|
||||
footer.pack(fill="x", side="bottom")
|
||||
|
||||
# Position : haut-droite de l'écran principal
|
||||
top.update_idletasks()
|
||||
height = top.winfo_reqheight()
|
||||
screen_w = top.winfo_screenwidth()
|
||||
x = screen_w - TOAST_WIDTH - 16
|
||||
y = 16
|
||||
top.geometry(f"{TOAST_WIDTH}x{height}+{x}+{y}")
|
||||
|
||||
# Click anywhere to close
|
||||
def _close(_=None):
|
||||
try:
|
||||
top.destroy()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
top.bind("<Button-1>", _close)
|
||||
for child in (outer, title_frame, body_frame, footer):
|
||||
try:
|
||||
child.bind("<Button-1>", _close)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Afficher + boost focus brut pour passer devant Focus Assist
|
||||
top.deiconify()
|
||||
top.lift()
|
||||
try:
|
||||
top.focus_force()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Re-pin topmost après 100 ms (Windows désactive parfois -topmost
|
||||
# quand le focus est pris par une autre app)
|
||||
def _repin():
|
||||
try:
|
||||
top.attributes("-topmost", True)
|
||||
top.lift()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
top.after(100, _repin)
|
||||
top.after(500, _repin)
|
||||
top.after(2000, _repin)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Auto-close
|
||||
try:
|
||||
top.after(TOAST_DURATION_MS, _close)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return top
|
||||
|
||||
|
||||
def _show_in_dedicated_thread(title: str, message: str) -> None:
|
||||
"""Crée un Tk() indépendant dans un daemon thread.
|
||||
|
||||
Utilisé en fallback quand aucun Tk root n'existe. Le thread vit le
|
||||
temps du toast (~15s) puis se termine proprement.
|
||||
"""
|
||||
def _run():
|
||||
try:
|
||||
# DPI awareness (Windows haute résolution)
|
||||
try:
|
||||
import ctypes
|
||||
ctypes.windll.shcore.SetProcessDpiAwareness(1)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
import tkinter as tk
|
||||
|
||||
root = tk.Tk()
|
||||
root.withdraw()
|
||||
try:
|
||||
dpi = root.winfo_fpixels("1i")
|
||||
root.tk.call("tk", "scaling", dpi / 72.0)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
top = _build_toast(root, title, message)
|
||||
|
||||
# Quitter mainloop quand le toast est détruit
|
||||
def _watch():
|
||||
try:
|
||||
if not top.winfo_exists():
|
||||
root.quit()
|
||||
return
|
||||
except Exception:
|
||||
root.quit()
|
||||
return
|
||||
root.after(200, _watch)
|
||||
|
||||
root.after(200, _watch)
|
||||
root.mainloop()
|
||||
try:
|
||||
root.destroy()
|
||||
except Exception:
|
||||
pass
|
||||
except Exception:
|
||||
logger.debug("paused_toast dedicated thread failed", exc_info=True)
|
||||
|
||||
t = threading.Thread(target=_run, daemon=True, name="paused-toast-tk")
|
||||
t.start()
|
||||
|
||||
|
||||
def show_paused_toast(
|
||||
title: str = "Léa a besoin de votre aide",
|
||||
message: str = "",
|
||||
) -> bool:
|
||||
"""Affiche un toast paused topmost.
|
||||
|
||||
Thread-safe, rate-limité, sans dépendance externe. Retourne True si le
|
||||
toast a été déclenché, False s'il a été ignoré (rate limit ou erreur).
|
||||
"""
|
||||
global _last_shown_at, _last_message
|
||||
|
||||
if not message:
|
||||
message = "Action en attente de votre validation."
|
||||
|
||||
# Rate limit basique : éviter qu'un poll en boucle ouvre 50 toasts
|
||||
now = time.monotonic()
|
||||
with _lock:
|
||||
same_message = (message == _last_message)
|
||||
elapsed = now - _last_shown_at
|
||||
if same_message and elapsed < TOAST_RATE_LIMIT_S:
|
||||
logger.debug(
|
||||
"paused_toast rate-limited (%.1fs since last identical)", elapsed
|
||||
)
|
||||
return False
|
||||
_last_shown_at = now
|
||||
_last_message = message
|
||||
|
||||
# Tentative 1 : utiliser le Tk root existant (ChatWindow) via after()
|
||||
root = _resolve_existing_root()
|
||||
if root is not None:
|
||||
try:
|
||||
root.after(0, lambda: _build_toast(root, title, message))
|
||||
logger.info("paused_toast scheduled on existing Tk root")
|
||||
return True
|
||||
except Exception:
|
||||
logger.debug("paused_toast existing-root path failed", exc_info=True)
|
||||
|
||||
# Tentative 2 : créer un Tk() dans un daemon thread
|
||||
try:
|
||||
_show_in_dedicated_thread(title, message)
|
||||
logger.info("paused_toast scheduled in dedicated thread")
|
||||
return True
|
||||
except Exception:
|
||||
logger.error("paused_toast dedicated-thread path failed", exc_info=True)
|
||||
return False
|
||||
|
||||
|
||||
__all__ = ["show_paused_toast"]
|
||||
197
agent_v0/agent_v1/ui/session_watchdog.py
Normal file
197
agent_v0/agent_v1/ui/session_watchdog.py
Normal file
@@ -0,0 +1,197 @@
|
||||
# agent_v1/ui/session_watchdog.py
|
||||
"""Watchdog de session interactive Windows — résilience RDP/Citrix.
|
||||
|
||||
Problème résolu (preuve poste clinique Émilie, 01/07) :
|
||||
09:46:28 [MAIN] agent.run() est sorti mais agent.running=True — probablement
|
||||
pystray sans session interactive (SSH)
|
||||
09:46:28 [MAIN] Keepalive headless actif — main thread bloque...
|
||||
|
||||
Sur les postes cliniques (tous RDP/Citrix), la session interactive
|
||||
disparaît quand l'utilisateur se déconnecte / la session bascule en
|
||||
verrouillage. `pystray.Icon.run()` sort alors immédiatement (plus de
|
||||
bureau interactif `WinSta0\\Default` pour recevoir les entrées et afficher
|
||||
l'icône). L'ancien `_headless_keepalive` bloquait le main thread *pour
|
||||
toujours* : l'icône tray + la fenêtre chat DISPARAISSAIENT et ne
|
||||
revenaient JAMAIS, même après reconnexion RDP. Les soignants croyaient
|
||||
que Léa avait planté (la capture continuait pourtant en fond).
|
||||
|
||||
Solution : un watchdog qui surveille la disponibilité du bureau
|
||||
interactif via `OpenInputDesktop()` (signal Win32 canonique — échoue quand
|
||||
la session est déconnectée/verrouillée, réussit à la reconnexion) et
|
||||
(re)lance l'UI tray dès qu'une session redevient disponible. Les threads
|
||||
de fond (heartbeat, replay poll, capture_server) NE SONT JAMAIS touchés :
|
||||
ils tournent contre `agent.running` et restent uniques. On ne relance
|
||||
JAMAIS un second `AgentV1` — seulement la couche UI (tray + chat).
|
||||
|
||||
État de l'art (recherche 01/07) :
|
||||
- `OpenInputDesktop()` échoue (ERROR_ACCESS_DENIED / ERROR_INVALID_...)
|
||||
quand le processus n'est pas rattaché au windowstation interactif
|
||||
`WinSta0` — c'est exactement le cas quand la session RDP est
|
||||
déconnectée. C'est la méthode fiable recommandée (comparer les
|
||||
*noms* de bureau via GetUserObjectInformation n'apporte rien de plus
|
||||
ici : on a juste besoin d'un booléen « input desktop dispo ? »).
|
||||
- `WTSGetActiveConsoleSessionId` renvoie une pseudo-session même sans
|
||||
login → PAS fiable pour ce besoin.
|
||||
- `pystray.Icon.run()` ne sort jamais en session interactive normale ;
|
||||
il sort immédiatement sinon → c'est notre signal de « session perdue ».
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import platform
|
||||
import threading
|
||||
from typing import Callable, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Intervalle de sondage du bureau interactif (secondes).
|
||||
# 3s = compromis : réactif à la reconnexion sans marteler l'API Win32.
|
||||
POLL_INTERVAL_S = 3.0
|
||||
|
||||
|
||||
def is_interactive_desktop_available() -> bool:
|
||||
"""Retourne True si un bureau interactif Windows est disponible.
|
||||
|
||||
Utilise `OpenInputDesktop()` : succès => le windowstation interactif
|
||||
(`WinSta0\\Default`) est accessible et peut afficher un tray. Échec =>
|
||||
session RDP/Citrix déconnectée ou verrouillée sans bureau d'entrée.
|
||||
|
||||
Hors Windows (Linux/dev/tests) : renvoie toujours True (pas de notion
|
||||
de bureau interactif verrouillable ici — on laisse l'UI tourner).
|
||||
Toute erreur d'appel Win32 est traitée comme « indisponible » (prudent)
|
||||
SAUF l'indisponibilité de l'API elle-même (pywin32 absent) → True pour
|
||||
ne pas priver un poste de son tray à cause d'une dépendance manquante.
|
||||
"""
|
||||
if platform.system() != "Windows":
|
||||
return True
|
||||
|
||||
try:
|
||||
import win32con # type: ignore
|
||||
import win32service # type: ignore
|
||||
except Exception:
|
||||
# pywin32 indisponible : on ne peut pas sonder → on suppose dispo
|
||||
# (comportement historique : tenter l'UI plutôt que la bloquer).
|
||||
logger.debug("pywin32 indisponible — sondage bureau interactif ignoré")
|
||||
return True
|
||||
|
||||
hdesk = None
|
||||
try:
|
||||
# DESKTOP_SWITCHDESKTOP (0x0100) = droit minimal, aligné sur l'usage
|
||||
# documenté pour tester la présence du bureau d'entrée.
|
||||
hdesk = win32service.OpenInputDesktop(0, False, win32con.DESKTOP_SWITCHDESKTOP)
|
||||
return hdesk is not None
|
||||
except Exception:
|
||||
# OpenInputDesktop lève quand aucun bureau d'entrée n'est accessible
|
||||
# (session déconnectée / verrouillée). C'est le cas « indisponible ».
|
||||
return False
|
||||
finally:
|
||||
if hdesk is not None:
|
||||
try:
|
||||
# PyHANDLE se ferme via .Close() (pywin32) ; fallback silencieux.
|
||||
hdesk.Close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
class InteractiveSessionWatchdog:
|
||||
"""Surveille la session interactive et (re)lance l'UI tray à la reconnexion.
|
||||
|
||||
Ne détient AUCUN état de capture. Sa seule responsabilité : garantir
|
||||
qu'il existe au plus UN tray vivant à la fois, et le ressusciter quand
|
||||
une session interactive redevient disponible. Les daemon threads de
|
||||
l'agent (heartbeat/replay/capture) sont indépendants et intacts.
|
||||
|
||||
Paramètres :
|
||||
run_ui : callable bloquant qui lance le tray (typiquement
|
||||
``agent.ui.run`` / ``agent.run``). Retourne quand le
|
||||
tray sort (normal en fin de session interactive).
|
||||
is_running : callable -> bool ; True tant que l'agent doit vivre
|
||||
(typiquement ``lambda: agent.running``).
|
||||
is_available : callable -> bool de détection de session (injectable
|
||||
pour les tests). Défaut = is_interactive_desktop_available.
|
||||
poll_interval_s : période de sondage quand la session est absente.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
run_ui: Callable[[], None],
|
||||
is_running: Callable[[], bool],
|
||||
is_available: Optional[Callable[[], bool]] = None,
|
||||
poll_interval_s: float = POLL_INTERVAL_S,
|
||||
) -> None:
|
||||
self._run_ui = run_ui
|
||||
self._is_running = is_running
|
||||
self._is_available = is_available or is_interactive_desktop_available
|
||||
self._poll_interval_s = poll_interval_s
|
||||
self._wake = threading.Event()
|
||||
# Sérialise le lancement de l'UI : jamais deux trays en parallèle.
|
||||
self._ui_lock = threading.Lock()
|
||||
|
||||
def stop(self) -> None:
|
||||
"""Réveille le watchdog pour qu'il réévalue ``is_running`` et sorte."""
|
||||
self._wake.set()
|
||||
|
||||
def _run_ui_once(self) -> None:
|
||||
"""Lance l'UI tray une fois (bloquant) sous verrou, avec garde d'erreur.
|
||||
|
||||
Le verrou empêche formellement qu'un second appel démarre un tray
|
||||
alors qu'un premier tourne encore (invariant « un seul tray »).
|
||||
"""
|
||||
with self._ui_lock:
|
||||
try:
|
||||
self._run_ui()
|
||||
except Exception:
|
||||
# Un crash du tray ne doit jamais tuer le watchdog : on log et
|
||||
# on laisse la boucle décider (retry ou sortie selon is_running).
|
||||
logger.exception("[WATCHDOG] Le tray UI a levé une exception")
|
||||
|
||||
def run(self) -> None:
|
||||
"""Boucle principale (bloque le main thread à la place du keepalive).
|
||||
|
||||
Cycle :
|
||||
1. Attendre qu'un bureau interactif soit disponible.
|
||||
2. (Re)lancer le tray — bloque jusqu'à sa sortie (déconnexion RDP).
|
||||
3. Recommencer tant que ``is_running`` est vrai.
|
||||
|
||||
Ne consomme pas de CPU en boucle serrée : sonde toutes les
|
||||
``poll_interval_s`` via un Event interruptible (réveil immédiat au stop).
|
||||
"""
|
||||
logger.info(
|
||||
"[WATCHDOG] Surveillance session interactive active "
|
||||
"(re-affichage auto du tray + chat à la reconnexion RDP/Citrix)."
|
||||
)
|
||||
first_cycle = True
|
||||
|
||||
while self._is_running():
|
||||
if not self._is_available():
|
||||
# Session absente : sonder périodiquement sans brûler le CPU.
|
||||
if first_cycle:
|
||||
logger.warning(
|
||||
"[WATCHDOG] Aucune session interactive — Léa reste active "
|
||||
"en fond (capture/heartbeat), tray masqué. En attente de "
|
||||
"reconnexion RDP/Citrix pour ré-afficher l'interface."
|
||||
)
|
||||
# Event.wait renvoie True si stop() a été appelé → on sort.
|
||||
if self._wake.wait(timeout=self._poll_interval_s):
|
||||
break
|
||||
first_cycle = False
|
||||
continue
|
||||
|
||||
# Session disponible : (re)lancer le tray.
|
||||
if not first_cycle:
|
||||
logger.info(
|
||||
"[WATCHDOG] Session interactive détectée — ré-affichage du "
|
||||
"tray et de la fenêtre chat de Léa."
|
||||
)
|
||||
first_cycle = False
|
||||
|
||||
# Bloque jusqu'à la sortie du tray (fin de session interactive).
|
||||
self._run_ui_once()
|
||||
|
||||
# Le tray est sorti. Si l'agent doit vivre, on reboucle (le
|
||||
# prochain tour re-sondera la session et re-affichera le tray).
|
||||
if not self._is_running():
|
||||
break
|
||||
|
||||
logger.info("[WATCHDOG] Arrêt de la surveillance de session interactive.")
|
||||
@@ -137,6 +137,15 @@ class SmartTrayV1:
|
||||
self._state_lock = threading.Lock()
|
||||
self._stop_event = threading.Event()
|
||||
|
||||
# Résilience RDP/Citrix : run() peut être rappelé plusieurs fois par le
|
||||
# watchdog de session (ré-affichage du tray à la reconnexion). Les
|
||||
# threads de fond (connexion, cache workflows, hotkey) et l'accueil ne
|
||||
# doivent démarrer QU'UNE fois — sinon on duplique les threads.
|
||||
self._bg_started = False
|
||||
# Signalé quand l'utilisateur a demandé Quitter : le watchdog ne doit
|
||||
# alors PAS relancer le tray.
|
||||
self._quit_requested = False
|
||||
|
||||
# Notifications
|
||||
self._notifier = NotificationManager()
|
||||
|
||||
@@ -371,7 +380,13 @@ class SmartTrayV1:
|
||||
)
|
||||
if name and name.strip():
|
||||
name = name.strip()
|
||||
# Utiliser l'etat partage si disponible
|
||||
|
||||
# --- P1-LEA-SHADOW : d\u00e9clencher d'abord l'orchestrateur L\u00e9a Linux ---
|
||||
# On contacte agent-chat AVANT la capture locale. Si \u00e9chec,
|
||||
# bascule en mode d\u00e9grad\u00e9 (capture locale sans assistance).
|
||||
self._start_lea_orchestrator_session(name)
|
||||
|
||||
# --- Comportement historique pr\u00e9serv\u00e9 : capture locale ---
|
||||
if self._shared_state is not None:
|
||||
try:
|
||||
self._shared_state.start_recording(name)
|
||||
@@ -393,6 +408,55 @@ class SmartTrayV1:
|
||||
|
||||
threading.Thread(target=_dialog, daemon=True).start()
|
||||
|
||||
def _start_lea_orchestrator_session(self, session_name: str) -> None:
|
||||
"""Appelle POST /api/learn/start côté agent-chat Linux (P1-LEA-SHADOW).
|
||||
|
||||
Fail-safe : toute erreur (config absente, httpx manquant, timeout,
|
||||
5xx serveur...) bascule en mode dégradé sans bloquer la capture
|
||||
locale. L'utilisateur est informé via le NotificationManager.
|
||||
"""
|
||||
try:
|
||||
from ..config import AGENT_CHAT_URL, API_TOKEN, MACHINE_ID
|
||||
from ..network.lea_orchestrator_client import (
|
||||
LeaOrchestratorError,
|
||||
start_learning_session,
|
||||
)
|
||||
except Exception as exc: # pragma: no cover (import-time)
|
||||
logger.error("Impossible de charger le client orchestrateur Léa : %s", exc)
|
||||
self._notifier.notify(
|
||||
"Léa",
|
||||
"Serveur injoignable — apprentissage local uniquement.",
|
||||
)
|
||||
return
|
||||
|
||||
try:
|
||||
resp = start_learning_session(
|
||||
AGENT_CHAT_URL,
|
||||
machine_id=MACHINE_ID,
|
||||
session_name=session_name,
|
||||
api_token=API_TOKEN,
|
||||
trigger_source="tray_button",
|
||||
)
|
||||
except LeaOrchestratorError as exc:
|
||||
logger.error("Orchestrateur Léa injoignable : %s", exc)
|
||||
self._notifier.notify(
|
||||
"Léa",
|
||||
"Serveur injoignable — apprentissage local uniquement.",
|
||||
)
|
||||
return
|
||||
except Exception: # noqa: BLE001 — défensif
|
||||
logger.exception("Erreur inattendue orchestrateur Léa")
|
||||
self._notifier.notify(
|
||||
"Léa",
|
||||
"Erreur orchestrateur — apprentissage local uniquement.",
|
||||
)
|
||||
return
|
||||
|
||||
logger.info(
|
||||
"Session orchestrateur Léa OK : id=%s state=%s",
|
||||
resp.session_id, resp.state,
|
||||
)
|
||||
|
||||
def _on_stop_session(self, _icon=None, _item=None) -> None:
|
||||
"""Termine la session en cours et envoie les donnees."""
|
||||
count = self.actions_count
|
||||
@@ -504,6 +568,100 @@ class SmartTrayV1:
|
||||
|
||||
threading.Thread(target=_replay, daemon=True).start()
|
||||
|
||||
def _launch_replay_request(
|
||||
self,
|
||||
replay_request: Dict[str, Any],
|
||||
replay_name: str,
|
||||
) -> None:
|
||||
"""Lance un replay direct depuis un payload `replay_request` serveur."""
|
||||
endpoint = (replay_request or {}).get("endpoint", "")
|
||||
session_id = (replay_request or {}).get("session_id", "")
|
||||
machine_id = (replay_request or {}).get("machine_id") or self.machine_id
|
||||
|
||||
if endpoint != "/api/v1/traces/stream/replay-session" or not session_id:
|
||||
logger.warning("Replay request non supporté: %s", replay_request)
|
||||
self._notifier.notify(
|
||||
"Léa",
|
||||
"Je ne peux pas lancer ce test automatique pour le moment.",
|
||||
)
|
||||
return
|
||||
|
||||
def _replay():
|
||||
if self.server_client is None:
|
||||
return
|
||||
|
||||
with self._state_lock:
|
||||
self._replay_active = True
|
||||
self._update_icon()
|
||||
self._notifier.notify(
|
||||
"Léa",
|
||||
f"Le système d'intelligence artificielle exécute la "
|
||||
f"tâche '{replay_name}' sur votre écran.",
|
||||
)
|
||||
|
||||
try:
|
||||
import requests
|
||||
auth_headers = {}
|
||||
if self.server_client is not None:
|
||||
auth_headers = self.server_client._auth_headers()
|
||||
resp = requests.post(
|
||||
f"{self.server_client._stream_base}{endpoint}",
|
||||
params={
|
||||
"session_id": session_id,
|
||||
"machine_id": machine_id,
|
||||
},
|
||||
headers=auth_headers,
|
||||
timeout=30,
|
||||
allow_redirects=False,
|
||||
)
|
||||
if resp.ok:
|
||||
logger.info(
|
||||
"Replay direct démarré pour session %s (machine=%s)",
|
||||
session_id,
|
||||
machine_id,
|
||||
)
|
||||
else:
|
||||
self._notifier.notify(
|
||||
"Léa",
|
||||
"Hmm, le serveur a refusé le test immédiat.",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error("Erreur lancement replay direct : %s", e)
|
||||
self._notifier.notify(
|
||||
"Léa",
|
||||
f"Oups, un problème : {e}",
|
||||
)
|
||||
finally:
|
||||
with self._state_lock:
|
||||
self._replay_active = False
|
||||
self._update_icon()
|
||||
|
||||
threading.Thread(target=_replay, daemon=True).start()
|
||||
|
||||
def offer_finalize_replay(
|
||||
self,
|
||||
replay_request: Dict[str, Any],
|
||||
replay_name: str,
|
||||
) -> None:
|
||||
"""Proposer à l'utilisateur de tester immédiatement la tâche apprise."""
|
||||
if not replay_request or not replay_request.get("session_id"):
|
||||
return
|
||||
|
||||
def _offer():
|
||||
self._notifier.notify(
|
||||
"Léa",
|
||||
f"J'ai compris la tâche '{replay_name}'. Voulez-vous la tester ?",
|
||||
)
|
||||
if not _ask_consent(
|
||||
"Léa — Test immédiat",
|
||||
f"J'ai compris la tâche '{replay_name}'. "
|
||||
"Voulez-vous la tester maintenant ?",
|
||||
):
|
||||
return
|
||||
self._launch_replay_request(replay_request, replay_name)
|
||||
|
||||
threading.Thread(target=_offer, daemon=True).start()
|
||||
|
||||
def _on_emergency_stop(self, _icon=None, _item=None) -> None:
|
||||
"""Arret d'urgence — stoppe TOUTES les activites de l'agent immediatement.
|
||||
|
||||
@@ -560,6 +718,11 @@ class SmartTrayV1:
|
||||
"""Arrete proprement l'agent et quitte."""
|
||||
logger.info("Arret demande par l'utilisateur")
|
||||
|
||||
# Marquer l'arret volontaire : le watchdog de session ne doit PAS
|
||||
# relancer le tray après un Quitter explicite (à distinguer d'une
|
||||
# simple déconnexion RDP où le tray doit revenir tout seul).
|
||||
self._quit_requested = True
|
||||
|
||||
# Arreter la session si en cours
|
||||
if self.is_recording:
|
||||
self.on_stop()
|
||||
@@ -736,17 +899,24 @@ class SmartTrayV1:
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def run(self) -> None:
|
||||
"""Demarre le tray, les threads de fond, et entre dans la boucle principale."""
|
||||
# Notification d'accueil — divulgation IA (Article 50, Reglement IA)
|
||||
self._notifier.greet()
|
||||
"""Demarre (ou ré-affiche) le tray et entre dans la boucle pystray.
|
||||
|
||||
# Enregistrer le hotkey global Ctrl+Shift+L (toggle chat)
|
||||
self._start_hotkey()
|
||||
Ré-entrant : le watchdog de session (session_watchdog.py) rappelle
|
||||
cette méthode à chaque reconnexion RDP/Citrix pour ré-afficher le
|
||||
tray + la fenêtre chat. Les initialisations one-shot (accueil,
|
||||
hotkey, threads de fond connexion/cache) sont protégées par
|
||||
``_bg_started`` pour ne PAS dupliquer les threads. Seule l'icône
|
||||
pystray est recréée à chaque appel (l'ancienne est morte avec la
|
||||
session précédente).
|
||||
"""
|
||||
self._start_background_once()
|
||||
|
||||
# Tooltip avec identifiant machine pour le multi-machine
|
||||
tray_title = f"Agent V1 - {self.machine_id}"
|
||||
|
||||
# Menu statique — reconstruit via _update_icon() quand l'état change
|
||||
# Menu statique — reconstruit via _update_icon() quand l'état change.
|
||||
# Nouvelle icône à chaque (ré)affichage : l'objet pystray précédent
|
||||
# est invalide une fois sa boucle sortie (session interactive perdue).
|
||||
self.icon = pystray.Icon(
|
||||
"AgentV1",
|
||||
self._current_icon(),
|
||||
@@ -754,6 +924,33 @@ class SmartTrayV1:
|
||||
menu=pystray.Menu(*self._get_menu_items()),
|
||||
)
|
||||
|
||||
# Rafraîchir les workflows au (ré)affichage — utile après reconnexion.
|
||||
if self._bg_started and self.server_client is not None:
|
||||
threading.Thread(target=self._fetch_workflows, daemon=True).start()
|
||||
|
||||
# Boucle principale pystray (bloquante). Sort quand la session
|
||||
# interactive disparaît (RDP déconnecté) OU sur _on_quit → le
|
||||
# watchdog décide alors de relancer ou non.
|
||||
logger.info("SmartTrayV1 demarre — entree dans la boucle pystray")
|
||||
self.icon.run()
|
||||
|
||||
def _start_background_once(self) -> None:
|
||||
"""Initialisations one-shot : accueil, hotkey, threads de fond.
|
||||
|
||||
Idempotent : les appels suivants (ré-affichage tray) sont des no-op.
|
||||
Garantit qu'on n'accumule pas de threads connexion/cache à chaque
|
||||
reconnexion RDP.
|
||||
"""
|
||||
if self._bg_started:
|
||||
return
|
||||
self._bg_started = True
|
||||
|
||||
# Notification d'accueil — divulgation IA (Article 50, Reglement IA)
|
||||
self._notifier.greet()
|
||||
|
||||
# Enregistrer le hotkey global Ctrl+Shift+L (toggle chat)
|
||||
self._start_hotkey()
|
||||
|
||||
# Demarrer le thread de verification connexion
|
||||
if self.server_client is not None:
|
||||
conn_thread = threading.Thread(
|
||||
@@ -775,7 +972,3 @@ class SmartTrayV1:
|
||||
threading.Thread(
|
||||
target=self._fetch_workflows, daemon=True
|
||||
).start()
|
||||
|
||||
# Boucle principale pystray (bloquante)
|
||||
logger.info("SmartTrayV1 demarre — entree dans la boucle pystray")
|
||||
self.icon.run()
|
||||
|
||||
110
agent_v0/agent_v1/vision/capture_io.py
Normal file
110
agent_v0/agent_v1/vision/capture_io.py
Normal file
@@ -0,0 +1,110 @@
|
||||
"""Politique de sauvegarde des captures — réduction du poids disque.
|
||||
|
||||
Constat : tous les shots étaient sauvés en PNG plein écran lossless
|
||||
(``img.save(path, "PNG", quality=...)`` — PNG ignore ``quality``), d'où
|
||||
~90 Go pour 13 sessions. La majorité de ce poids n'a aucune valeur de
|
||||
grounding (full + full_blurred en doublon, heartbeats plein écran).
|
||||
|
||||
Cette politique distingue le **type** de shot et écrit le format adapté :
|
||||
|
||||
- ``crop`` → PNG lossless. C'est la cible de grounding qwen3-vl ; on
|
||||
préserve chaque pixel (perte JPEG = bruit sur de petites icônes). Le crop
|
||||
fait 80×80 → poids négligeable, aucun intérêt à le dégrader.
|
||||
- ``full`` / ``window`` / ``context`` → JPEG ``quality=SCREENSHOT_QUALITY,
|
||||
optimize=True``. Ce sont des vues contextuelles / humaines : la
|
||||
compression JPEG (~5-10x) est sans impact fonctionnel.
|
||||
- ``heartbeat`` → JPEG **downscalé** (largeur max ``HEARTBEAT_MAX_WIDTH``,
|
||||
ratio préservé). C'est de la *liveness* (le serveur vérifie juste qu'un
|
||||
écran a changé), pas du grounding → la pleine résolution est du gaspillage.
|
||||
|
||||
``save_capture`` retourne le chemin RÉELLEMENT écrit, extension ajustée selon
|
||||
le format. L'appelant doit utiliser ce retour (et non un chemin ``.png``
|
||||
présumé) pour streamer / référencer le bon fichier.
|
||||
|
||||
⚠️ Contrat avec le serveur : l'extension du crop NE DOIT PAS changer (le
|
||||
serveur retrouve le crop par basename via ``vision_info.crop`` — voir
|
||||
``stream_processor._extract_crop_b64`` stratégie 1). C'est pourquoi ``crop``
|
||||
reste PNG. Les full/window/context/heartbeat sont retrouvés par
|
||||
``screenshot_id`` avec extension ``.png`` hardcodée côté serveur, mais le
|
||||
serveur réécrit toujours l'upload sous ``{shot_id}.png`` (le suffixe envoyé
|
||||
sur le fil est ignoré) → changer l'extension LOCALE de ces types est sûr.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from typing import Iterable
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from ..config import SCREENSHOT_QUALITY
|
||||
|
||||
# Types sauvés en JPEG (vue contextuelle / humaine, pas de grounding pixel).
|
||||
_JPEG_KINDS: frozenset = frozenset({"full", "window", "context"})
|
||||
|
||||
# Largeur max d'un heartbeat downscalé. 1280 px suffit largement pour de la
|
||||
# liveness (détecter qu'un écran a changé) ; on divise le poids d'un 2560 px
|
||||
# par ~4 (surface) avant compression JPEG.
|
||||
HEARTBEAT_MAX_WIDTH = 1280
|
||||
|
||||
|
||||
def _ensure_jpeg_ready(img: Image.Image) -> Image.Image:
|
||||
"""Convertit en RGB si nécessaire (JPEG ne supporte ni alpha ni palette)."""
|
||||
if img.mode in ("RGBA", "LA", "P"):
|
||||
return img.convert("RGB")
|
||||
return img
|
||||
|
||||
|
||||
def _downscale_to_width(img: Image.Image, max_width: int) -> Image.Image:
|
||||
"""Réduit l'image à ``max_width`` en préservant le ratio (no-op si plus petite)."""
|
||||
if img.width <= max_width:
|
||||
return img
|
||||
new_height = max(1, round(img.height * max_width / img.width))
|
||||
return img.resize((max_width, new_height), Image.LANCZOS)
|
||||
|
||||
|
||||
def save_capture(img: Image.Image, path_base: str, kind: str) -> str:
|
||||
"""Sauve ``img`` selon la politique du ``kind`` et retourne le chemin écrit.
|
||||
|
||||
Args:
|
||||
img: image PIL à sauvegarder.
|
||||
path_base: chemin SANS extension (ex.
|
||||
``.../shots/shot_0001_full``). L'extension finale (``.png`` ou
|
||||
``.jpg``) est ajoutée par la politique.
|
||||
kind: type de shot — ``"crop"`` | ``"full"`` | ``"window"`` |
|
||||
``"context"`` | ``"heartbeat"``.
|
||||
|
||||
Returns:
|
||||
Le chemin RÉELLEMENT écrit, avec la bonne extension.
|
||||
|
||||
Raises:
|
||||
ValueError: si ``kind`` n'est pas reconnu (fail-closed : on refuse
|
||||
d'écrire un fichier dont la politique est indéterminée).
|
||||
"""
|
||||
if kind == "crop":
|
||||
out_path = f"{path_base}.png"
|
||||
img.save(out_path, "PNG")
|
||||
return out_path
|
||||
|
||||
if kind in _JPEG_KINDS:
|
||||
out_path = f"{path_base}.jpg"
|
||||
_ensure_jpeg_ready(img).save(
|
||||
out_path, "JPEG", quality=SCREENSHOT_QUALITY, optimize=True
|
||||
)
|
||||
return out_path
|
||||
|
||||
if kind == "heartbeat":
|
||||
out_path = f"{path_base}.jpg"
|
||||
small = _downscale_to_width(_ensure_jpeg_ready(img), HEARTBEAT_MAX_WIDTH)
|
||||
small.save(out_path, "JPEG", quality=SCREENSHOT_QUALITY)
|
||||
return out_path
|
||||
|
||||
raise ValueError(
|
||||
f"kind de capture inconnu : {kind!r} "
|
||||
f"(attendu: crop, full, window, context, heartbeat)"
|
||||
)
|
||||
|
||||
|
||||
def known_kinds() -> Iterable[str]:
|
||||
"""Retourne les ``kind`` supportés (utile pour la validation appelant)."""
|
||||
return ("crop", *sorted(_JPEG_KINDS), "heartbeat")
|
||||
@@ -2,19 +2,422 @@
|
||||
"""
|
||||
Gestionnaire de vision avancé pour Agent V1.
|
||||
Optimisé pour le streaming fibre avec détection de changement.
|
||||
|
||||
Captures disponibles :
|
||||
- Plein écran (full) : contexte global 1920x1080+
|
||||
- Crop ciblé (crop) : 80x80 autour du clic (apprentissage VLM)
|
||||
- Fenêtre active (window) : image isolée de la fenêtre + métadonnées
|
||||
(titre, rect, coordonnées clic relatives) — cross-platform
|
||||
"""
|
||||
|
||||
import os
|
||||
import time
|
||||
import logging
|
||||
import hashlib
|
||||
import platform
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
from PIL import Image, ImageFilter, ImageStat
|
||||
import mss
|
||||
from ..config import TARGETED_CROP_SIZE, SCREENSHOT_QUALITY, BLUR_SENSITIVE
|
||||
from ..config import TARGETED_CROP_SIZE, BLUR_SENSITIVE
|
||||
from .blur_sensitive import blur_sensitive_regions
|
||||
from .capture_io import save_capture
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# OS courant (détecté une seule fois)
|
||||
_SYSTEM = platform.system()
|
||||
|
||||
# QW1 — détection multi-écrans (fallback gracieux si screeninfo absent)
|
||||
try:
|
||||
from screeninfo import get_monitors as _screeninfo_get_monitors
|
||||
_SCREENINFO_AVAILABLE = True
|
||||
except ImportError:
|
||||
_SCREENINFO_AVAILABLE = False
|
||||
|
||||
|
||||
def _get_monitors_geometry() -> List[Dict[str, Any]]:
|
||||
"""Retourne la liste des monitors physiques avec leurs offsets.
|
||||
|
||||
Returns:
|
||||
List[dict] : [{idx, x, y, w, h, primary}, ...]. Vide si screeninfo
|
||||
indisponible (le serveur tombera sur fallback composite).
|
||||
"""
|
||||
if not _SCREENINFO_AVAILABLE:
|
||||
return []
|
||||
try:
|
||||
monitors = _screeninfo_get_monitors()
|
||||
return [
|
||||
{
|
||||
"idx": i,
|
||||
"x": int(m.x),
|
||||
"y": int(m.y),
|
||||
"w": int(m.width),
|
||||
"h": int(m.height),
|
||||
"primary": bool(getattr(m, "is_primary", False)),
|
||||
}
|
||||
for i, m in enumerate(monitors)
|
||||
]
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
|
||||
def _get_active_monitor_index() -> Optional[int]:
|
||||
"""Retourne l'index logique du monitor où se trouve le curseur (focus actif).
|
||||
|
||||
Returns:
|
||||
int ou None si indéterminable.
|
||||
"""
|
||||
if not _SCREENINFO_AVAILABLE:
|
||||
return None
|
||||
try:
|
||||
import pyautogui # import paresseux : évite la dépendance dure
|
||||
cx, cy = pyautogui.position()
|
||||
for i, m in enumerate(_screeninfo_get_monitors()):
|
||||
if m.x <= cx < m.x + m.width and m.y <= cy < m.y + m.height:
|
||||
return i
|
||||
except Exception:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def _enrich_with_monitor_info(payload: dict) -> dict:
|
||||
"""Ajoute monitor_index et monitors_geometry au payload (in-place + return)."""
|
||||
if isinstance(payload, dict):
|
||||
payload["monitor_index"] = _get_active_monitor_index()
|
||||
payload["monitors_geometry"] = _get_monitors_geometry()
|
||||
return payload
|
||||
|
||||
|
||||
# Garde dimensions monitor (démo GHT 19 mai 2026) : mss.monitors[1] peut
|
||||
# retourner intermittemment des dims tronquées (cas observé 2560×60). Utiliser
|
||||
# ces dims pour normaliser des coords empoisonne la mémoire (TargetMemoryStore).
|
||||
MIN_MONITOR_WIDTH = 200
|
||||
MIN_MONITOR_HEIGHT = 200
|
||||
MONITOR_MAX_ATTEMPTS = 2
|
||||
MONITOR_RETRY_DELAY_S = 0.05
|
||||
BLACK_FRAME_MEAN_MAX = 1.0
|
||||
BLACK_FRAME_STDDEV_MAX = 1.0
|
||||
BLACK_FRAME_MAX_LUMA = 3
|
||||
|
||||
|
||||
def _is_monitor_sane(monitor) -> bool:
|
||||
"""True si les dims du monitor sont au-dessus du seuil de plausibilité."""
|
||||
if not isinstance(monitor, dict):
|
||||
return False
|
||||
w = monitor.get("width", 0) or 0
|
||||
h = monitor.get("height", 0) or 0
|
||||
return w >= MIN_MONITOR_WIDTH and h >= MIN_MONITOR_HEIGHT
|
||||
|
||||
|
||||
def _dim_str(monitor) -> str:
|
||||
"""Représentation courte WxH pour les logs (gère monitor=None)."""
|
||||
if not isinstance(monitor, dict):
|
||||
return "?x?"
|
||||
return f"{monitor.get('width', '?')}x{monitor.get('height', '?')}"
|
||||
|
||||
|
||||
def _acquire_safe_grab(max_attempts: int = MONITOR_MAX_ATTEMPTS,
|
||||
retry_delay_s: float = MONITOR_RETRY_DELAY_S,
|
||||
allow_secondary_fallback: bool = True):
|
||||
"""Ouvre mss et capture un monitor avec dimensions plausibles.
|
||||
|
||||
Stratégie en cascade :
|
||||
1. À chaque tentative, ouvrir un nouveau `mss.mss()` (peut rafraîchir le
|
||||
cache interne) et examiner monitors[1..n].
|
||||
2. Préférer monitors[1] (écran principal physique). Si aberrant ET
|
||||
`allow_secondary_fallback=True`, prendre le premier monitors[2..n]
|
||||
sain avec un WARNING explicite.
|
||||
3. Si `allow_secondary_fallback=False`, on n'accepte QUE monitors[1].
|
||||
Utile pour les méthodes qui reçoivent des coordonnées (x, y) en
|
||||
système écran composite : capturer un monitor secondaire produirait
|
||||
une image saine mais décalée par rapport à ces coords.
|
||||
4. Si aucune dim plausible : attendre `retry_delay_s` et retenter.
|
||||
5. Après `max_attempts` infructueuses : log ERROR et retourner
|
||||
(None, None) pour que l'appelant tombe en sortie d'erreur explicite.
|
||||
|
||||
Args:
|
||||
max_attempts: nombre de tentatives mss avant abandon.
|
||||
retry_delay_s: délai entre tentatives.
|
||||
allow_secondary_fallback: si False, refuser monitors[2..n] (fail-closed
|
||||
pour les méthodes coord-bearing).
|
||||
|
||||
Returns:
|
||||
Tuple (monitor_dict, PIL.Image) si capture saine réussie,
|
||||
(None, None) sinon.
|
||||
"""
|
||||
last_aberrant = None
|
||||
secondary_seen = False # un monitor secondaire sain a été vu mais refusé
|
||||
for attempt in range(max_attempts):
|
||||
with mss.mss() as sct:
|
||||
monitors = list(sct.monitors) if sct.monitors else []
|
||||
chosen = None
|
||||
chosen_idx = None
|
||||
for idx in range(1, len(monitors)):
|
||||
candidate = monitors[idx]
|
||||
if not _is_monitor_sane(candidate):
|
||||
last_aberrant = candidate
|
||||
logger.warning(
|
||||
"Monitor[%d] dims aberrantes (%s, seuil %dx%d) "
|
||||
"— attempt %d/%d",
|
||||
idx, _dim_str(candidate),
|
||||
MIN_MONITOR_WIDTH, MIN_MONITOR_HEIGHT,
|
||||
attempt + 1, max_attempts,
|
||||
)
|
||||
continue
|
||||
# Monitor sain trouvé
|
||||
if idx == 1 or allow_secondary_fallback:
|
||||
chosen = candidate
|
||||
chosen_idx = idx
|
||||
break
|
||||
# Sinon : sain mais secondaire interdit pour cet appelant
|
||||
secondary_seen = True
|
||||
logger.warning(
|
||||
"Monitor[%d] sain (%s) mais fallback secondaire refusé "
|
||||
"(allow_secondary_fallback=False) — capture cohérente "
|
||||
"des coords impossible",
|
||||
idx, _dim_str(candidate),
|
||||
)
|
||||
if chosen is not None:
|
||||
if chosen_idx != 1 or attempt > 0:
|
||||
logger.warning(
|
||||
"Capture fallback : monitor[%d] dim=%s, attempt=%d",
|
||||
chosen_idx, _dim_str(chosen), attempt + 1,
|
||||
)
|
||||
sct_img = sct.grab(chosen)
|
||||
img = Image.frombytes(
|
||||
"RGB", sct_img.size, sct_img.bgra, "raw", "BGRX",
|
||||
)
|
||||
return chosen, img
|
||||
if attempt < max_attempts - 1:
|
||||
time.sleep(retry_delay_s)
|
||||
if secondary_seen and not allow_secondary_fallback:
|
||||
logger.error(
|
||||
"Capture abandonnée : monitor[1] aberrant après %d tentatives "
|
||||
"(dernier vu %s) et fallback secondaire désactivé "
|
||||
"pour préserver la cohérence des coordonnées",
|
||||
max_attempts, _dim_str(last_aberrant),
|
||||
)
|
||||
else:
|
||||
logger.error(
|
||||
"Aucun monitor avec dims plausibles trouvé après %d tentatives "
|
||||
"(dernier vu : %s, seuil %dx%d) — capture abandonnée",
|
||||
max_attempts, _dim_str(last_aberrant),
|
||||
MIN_MONITOR_WIDTH, MIN_MONITOR_HEIGHT,
|
||||
)
|
||||
return None, None
|
||||
|
||||
|
||||
def _compute_luma_stats(img: Image.Image) -> Dict[str, float | int]:
|
||||
"""Retourne des stats simples de luminance pour diagnostiquer un frame noir."""
|
||||
gray = img.convert("L")
|
||||
stat = ImageStat.Stat(gray)
|
||||
min_luma, max_luma = gray.getextrema()
|
||||
return {
|
||||
"mean": round(float(stat.mean[0]) if stat.mean else 0.0, 2),
|
||||
"stddev": round(float(stat.stddev[0]) if stat.stddev else 0.0, 2),
|
||||
"min": int(min_luma),
|
||||
"max": int(max_luma),
|
||||
}
|
||||
|
||||
|
||||
def _is_effectively_black(img: Image.Image) -> bool:
|
||||
"""Heuristique fail-closed pour refuser un screenshot pratiquement noir."""
|
||||
stats = _compute_luma_stats(img)
|
||||
return (
|
||||
stats["max"] <= BLACK_FRAME_MAX_LUMA
|
||||
and stats["mean"] <= BLACK_FRAME_MEAN_MAX
|
||||
and stats["stddev"] <= BLACK_FRAME_STDDEV_MAX
|
||||
)
|
||||
|
||||
|
||||
def _capture_via_imagegrab() -> Tuple[Optional[Dict[str, int]], Optional[Image.Image], Dict[str, Any]]:
|
||||
"""Fallback Windows via Pillow/ImageGrab.
|
||||
|
||||
Utile quand `mss` retourne un frame noir alors que la session graphique
|
||||
utilisateur reste visible.
|
||||
"""
|
||||
if _SYSTEM != "Windows":
|
||||
return None, None, {"backend": "imagegrab", "error": "unsupported_platform"}
|
||||
|
||||
try:
|
||||
from PIL import ImageGrab
|
||||
except ImportError as exc:
|
||||
return None, None, {"backend": "imagegrab", "error": str(exc)}
|
||||
|
||||
try:
|
||||
img = ImageGrab.grab(all_screens=True)
|
||||
except Exception as exc:
|
||||
logger.warning("ImageGrab indisponible pour le fallback capture : %s", exc)
|
||||
return None, None, {"backend": "imagegrab", "error": str(exc)}
|
||||
|
||||
monitor = {"left": 0, "top": 0, "width": img.width, "height": img.height}
|
||||
return monitor, img, {
|
||||
"backend": "imagegrab",
|
||||
"luma": _compute_luma_stats(img),
|
||||
}
|
||||
|
||||
|
||||
def capture_screen_image(
|
||||
allow_secondary_fallback: bool = True,
|
||||
) -> Tuple[Optional[Dict[str, int]], Optional[Image.Image], Dict[str, Any]]:
|
||||
"""Capture plein écran avec diagnostic noir + fallback Windows.
|
||||
|
||||
Returns:
|
||||
(monitor, image, meta) où image peut être None si aucun backend plein
|
||||
écran n'a produit une image exploitable.
|
||||
"""
|
||||
monitor, img = _acquire_safe_grab(
|
||||
allow_secondary_fallback=allow_secondary_fallback
|
||||
)
|
||||
meta: Dict[str, Any] = {"backend": "mss"}
|
||||
|
||||
if img is not None:
|
||||
meta["luma"] = _compute_luma_stats(img)
|
||||
if not _is_effectively_black(img):
|
||||
return monitor, img, meta
|
||||
logger.warning(
|
||||
"Capture mss quasi noire (%s) — tentative de fallback",
|
||||
meta["luma"],
|
||||
)
|
||||
meta["mss_black_frame"] = True
|
||||
else:
|
||||
meta["mss_unavailable"] = True
|
||||
|
||||
fallback_monitor, fallback_img, fallback_meta = _capture_via_imagegrab()
|
||||
if fallback_img is not None:
|
||||
if not _is_effectively_black(fallback_img):
|
||||
logger.warning(
|
||||
"Capture fallback via ImageGrab (%sx%s)",
|
||||
fallback_img.width,
|
||||
fallback_img.height,
|
||||
)
|
||||
return fallback_monitor, fallback_img, fallback_meta
|
||||
logger.warning(
|
||||
"Capture ImageGrab quasi noire (%s)",
|
||||
fallback_meta.get("luma"),
|
||||
)
|
||||
meta["imagegrab_black_frame"] = True
|
||||
|
||||
meta["imagegrab_error"] = fallback_meta.get("error")
|
||||
return None, None, meta
|
||||
|
||||
|
||||
def _capture_window_image_windows(
|
||||
hwnd: int,
|
||||
width: int,
|
||||
height: int,
|
||||
) -> Tuple[Optional[Image.Image], Dict[str, Any]]:
|
||||
"""Capture une fenêtre Windows via PrintWindow.
|
||||
|
||||
Fallback utile quand la capture plein écran est noire mais que la fenêtre
|
||||
active reste imprimable par l'API Win32.
|
||||
"""
|
||||
if _SYSTEM != "Windows":
|
||||
return None, {"backend": "printwindow", "error": "unsupported_platform"}
|
||||
|
||||
try:
|
||||
import ctypes
|
||||
import win32gui
|
||||
import win32ui
|
||||
except ImportError as exc:
|
||||
return None, {"backend": "printwindow", "error": str(exc)}
|
||||
|
||||
last_error = None
|
||||
for flag in (3, 2, 0):
|
||||
wnd_dc = None
|
||||
src_dc = None
|
||||
mem_dc = None
|
||||
bmp = None
|
||||
try:
|
||||
wnd_dc = win32gui.GetWindowDC(hwnd)
|
||||
if not wnd_dc:
|
||||
raise RuntimeError("GetWindowDC a retourné 0")
|
||||
src_dc = win32ui.CreateDCFromHandle(wnd_dc)
|
||||
mem_dc = src_dc.CreateCompatibleDC()
|
||||
bmp = win32ui.CreateBitmap()
|
||||
bmp.CreateCompatibleBitmap(src_dc, width, height)
|
||||
mem_dc.SelectObject(bmp)
|
||||
result = ctypes.windll.user32.PrintWindow(
|
||||
hwnd, mem_dc.GetSafeHdc(), flag
|
||||
)
|
||||
bits = bmp.GetBitmapBits(True)
|
||||
img = Image.frombuffer(
|
||||
"RGB", (width, height), bits, "raw", "BGRX", 0, 1
|
||||
)
|
||||
luma = _compute_luma_stats(img)
|
||||
if result or not _is_effectively_black(img):
|
||||
return img, {
|
||||
"backend": f"printwindow:{flag}",
|
||||
"printwindow_result": int(result),
|
||||
"luma": luma,
|
||||
}
|
||||
except Exception as exc:
|
||||
last_error = str(exc)
|
||||
finally:
|
||||
try:
|
||||
if bmp is not None:
|
||||
win32gui.DeleteObject(bmp.GetHandle())
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
if mem_dc is not None:
|
||||
mem_dc.DeleteDC()
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
if src_dc is not None:
|
||||
src_dc.DeleteDC()
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
if wnd_dc is not None:
|
||||
win32gui.ReleaseDC(hwnd, wnd_dc)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return None, {
|
||||
"backend": "printwindow",
|
||||
"error": last_error or "no_usable_frame",
|
||||
}
|
||||
|
||||
|
||||
def capture_foreground_window_image() -> Tuple[Optional[Image.Image], Dict[str, Any]]:
|
||||
"""Capture la fenêtre au focus via API native si disponible."""
|
||||
try:
|
||||
from ..window_info_crossplatform import get_active_window_rect
|
||||
|
||||
rect_info = get_active_window_rect()
|
||||
except Exception as exc:
|
||||
return None, {"backend": "printwindow", "error": str(exc)}
|
||||
|
||||
if not rect_info:
|
||||
return None, {"backend": "printwindow", "error": "active_window_unavailable"}
|
||||
|
||||
win_w, win_h = rect_info.get("size", [0, 0])
|
||||
hwnd = rect_info.get("hwnd")
|
||||
if not hwnd or win_w <= 0 or win_h <= 0:
|
||||
return None, {
|
||||
"backend": "printwindow",
|
||||
"error": "active_window_handle_unavailable",
|
||||
"title": rect_info.get("title", "unknown_window"),
|
||||
}
|
||||
|
||||
img, meta = _capture_window_image_windows(hwnd, win_w, win_h)
|
||||
if img is None:
|
||||
return None, meta
|
||||
|
||||
meta.update(
|
||||
{
|
||||
"title": rect_info.get("title", "unknown_window"),
|
||||
"app_name": rect_info.get("app_name", "unknown_app"),
|
||||
"rect": rect_info.get("rect"),
|
||||
"window_size": rect_info.get("size"),
|
||||
"hwnd": hwnd,
|
||||
}
|
||||
)
|
||||
return img, meta
|
||||
|
||||
|
||||
class VisionCapturer:
|
||||
def __init__(self, session_dir: str):
|
||||
self.session_dir = session_dir
|
||||
@@ -23,70 +426,283 @@ class VisionCapturer:
|
||||
# On ne crée plus self.sct ici car mss n'est pas thread-safe sous Windows
|
||||
self.last_img_hash = None
|
||||
|
||||
def _ensure_shots_dir(self) -> None:
|
||||
"""Garantit l'existence de `shots/` avant toute écriture.
|
||||
|
||||
Le dossier est créé dans `__init__`, mais l'auto-cleanup de
|
||||
`SessionStorage` (`shutil.rmtree` par âge/taille) peut supprimer tout
|
||||
le dossier de session — y compris la session permanente `_background`.
|
||||
Sans ce garde, la capture suivante lève `[Errno 2] No such file or
|
||||
directory` (bug observé poste Émilie). On recrée donc le répertoire
|
||||
cible juste avant chaque sauvegarde.
|
||||
"""
|
||||
os.makedirs(self.shots_dir, exist_ok=True)
|
||||
|
||||
def capture_full_context(self, name_suffix: str, force=False) -> str:
|
||||
"""
|
||||
Capture l'écran complet.
|
||||
Si force=False, vérifie d'abord si l'écran a changé.
|
||||
|
||||
Enrichit les métadonnées avec le titre de la fenêtre active
|
||||
(utile pour le contextualisation des heartbeats côté serveur).
|
||||
"""
|
||||
try:
|
||||
with mss.mss() as sct:
|
||||
monitor = sct.monitors[1]
|
||||
sct_img = sct.grab(monitor)
|
||||
img = Image.frombytes("RGB", sct_img.size, sct_img.bgra, "raw", "BGRX")
|
||||
|
||||
# Détection de changement (pour Heartbeat)
|
||||
if not force:
|
||||
current_hash = self._compute_quick_hash(img)
|
||||
if current_hash == self.last_img_hash:
|
||||
return "" # Pas de changement, on économise la fibre
|
||||
self.last_img_hash = current_hash
|
||||
_monitor, img, meta = capture_screen_image()
|
||||
if img is None:
|
||||
img, win_meta = capture_foreground_window_image()
|
||||
if img is None:
|
||||
logger.error(
|
||||
"Capture plein contexte indisponible (meta=%s, window=%s)",
|
||||
meta,
|
||||
win_meta,
|
||||
)
|
||||
return ""
|
||||
logger.warning(
|
||||
"Capture plein contexte dégradée via fenêtre active (%s)",
|
||||
win_meta.get("backend"),
|
||||
)
|
||||
|
||||
# Floutage des données sensibles (conformité AI Act)
|
||||
if BLUR_SENSITIVE:
|
||||
blur_sensitive_regions(img)
|
||||
# Détection de changement (pour Heartbeat)
|
||||
if not force:
|
||||
current_hash = self._compute_quick_hash(img)
|
||||
if current_hash == self.last_img_hash:
|
||||
return "" # Pas de changement, on économise la fibre
|
||||
self.last_img_hash = current_hash
|
||||
|
||||
path = os.path.join(self.shots_dir, f"context_{int(time.time())}_{name_suffix}.png")
|
||||
img.save(path, "PNG", quality=SCREENSHOT_QUALITY)
|
||||
return path
|
||||
# Floutage des données sensibles (conformité AI Act)
|
||||
if BLUR_SENSITIVE:
|
||||
blur_sensitive_regions(img)
|
||||
|
||||
# Politique d'écriture : les heartbeats sont de la liveness pure
|
||||
# (le serveur vérifie juste qu'un écran a changé) → JPEG downscalé.
|
||||
# Les autres contextes (focus_change, result_of_*) → JPEG q85.
|
||||
kind = "heartbeat" if "heartbeat" in name_suffix else "context"
|
||||
self._ensure_shots_dir()
|
||||
path_base = os.path.join(
|
||||
self.shots_dir, f"context_{int(time.time())}_{name_suffix}"
|
||||
)
|
||||
return save_capture(img, path_base, kind)
|
||||
except Exception as e:
|
||||
logger.error(f"Erreur Context Capture: {e}")
|
||||
return ""
|
||||
|
||||
def capture_dual(self, x: int, y: int, screenshot_id: str, anonymize=False) -> dict:
|
||||
"""Capture duale (Full + Crop) systématique (forcée car liée à une action)."""
|
||||
def get_active_window_title(self) -> str:
|
||||
"""Retourne le titre de la fenêtre active (pour enrichir les heartbeats).
|
||||
|
||||
Fallback gracieux : retourne une chaîne vide si indisponible.
|
||||
"""
|
||||
try:
|
||||
with mss.mss() as sct:
|
||||
full_path = os.path.join(self.shots_dir, f"{screenshot_id}_full.png")
|
||||
monitor = sct.monitors[1]
|
||||
sct_img = sct.grab(monitor)
|
||||
img = Image.frombytes("RGB", sct_img.size, sct_img.bgra, "raw", "BGRX")
|
||||
from ..window_info_crossplatform import get_active_window_info
|
||||
info = get_active_window_info()
|
||||
return info.get("title", "")
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
# Capture du Crop (Cœur de l'apprentissage qwen3-vl)
|
||||
crop_path = os.path.join(self.shots_dir, f"{screenshot_id}_crop.png")
|
||||
w, h = TARGETED_CROP_SIZE
|
||||
left = max(0, x - w // 2)
|
||||
top = max(0, y - h // 2)
|
||||
crop_img = img.crop((left, top, left + w, top + h))
|
||||
|
||||
if anonymize:
|
||||
crop_img = crop_img.filter(ImageFilter.GaussianBlur(radius=4))
|
||||
def capture_dual(self, x: int, y: int, screenshot_id: str, anonymize=False) -> dict:
|
||||
"""Capture triple (Full + Crop + Fenêtre active) systématique.
|
||||
|
||||
# Floutage des données sensibles (conformité AI Act)
|
||||
if BLUR_SENSITIVE:
|
||||
blur_sensitive_regions(img)
|
||||
blur_sensitive_regions(crop_img)
|
||||
La fenêtre active est un AJOUT — en cas d'échec, le full + crop
|
||||
sont toujours retournés (fallback gracieux).
|
||||
"""
|
||||
try:
|
||||
# Coords (x, y) sont en système écran composite ; cropper depuis
|
||||
# un monitor secondaire (offset ≠ 0) produirait une image saine
|
||||
# mais décalée → fail-closed sur fallback secondaire.
|
||||
_monitor, img, meta = capture_screen_image(
|
||||
allow_secondary_fallback=False
|
||||
)
|
||||
if img is None:
|
||||
window_info = self.capture_active_window(
|
||||
x, y, screenshot_id, full_img=None
|
||||
)
|
||||
if window_info:
|
||||
result = {"window_capture": window_info}
|
||||
_enrich_with_monitor_info(result)
|
||||
logger.warning(
|
||||
"capture_dual dégradée: fenêtre active seule (%s)",
|
||||
meta,
|
||||
)
|
||||
return result
|
||||
return {}
|
||||
|
||||
img.save(full_path, "PNG", quality=SCREENSHOT_QUALITY)
|
||||
crop_img.save(crop_path, "PNG", quality=SCREENSHOT_QUALITY)
|
||||
full_base = os.path.join(self.shots_dir, f"{screenshot_id}_full")
|
||||
|
||||
# Mise à jour du hash pour le prochain heartbeat
|
||||
self.last_img_hash = self._compute_quick_hash(img)
|
||||
# Capture du Crop (Cœur de l'apprentissage qwen3-vl)
|
||||
crop_base = os.path.join(self.shots_dir, f"{screenshot_id}_crop")
|
||||
w, h = TARGETED_CROP_SIZE
|
||||
left = max(0, x - w // 2)
|
||||
top = max(0, y - h // 2)
|
||||
crop_img = img.crop((left, top, left + w, top + h))
|
||||
|
||||
return {"full": full_path, "crop": crop_path}
|
||||
if anonymize:
|
||||
crop_img = crop_img.filter(ImageFilter.GaussianBlur(radius=4))
|
||||
|
||||
# Floutage des données sensibles (conformité AI Act)
|
||||
if BLUR_SENSITIVE:
|
||||
blur_sensitive_regions(img)
|
||||
blur_sensitive_regions(crop_img)
|
||||
|
||||
# Politique d'écriture : full = vue contextuelle → JPEG q85 ;
|
||||
# crop = cible de grounding qwen3-vl → PNG lossless (contrat serveur).
|
||||
self._ensure_shots_dir()
|
||||
full_path = save_capture(img, full_base, "full")
|
||||
crop_path = save_capture(crop_img, crop_base, "crop")
|
||||
|
||||
# Mise à jour du hash pour le prochain heartbeat
|
||||
self.last_img_hash = self._compute_quick_hash(img)
|
||||
|
||||
result = {"full": full_path, "crop": crop_path}
|
||||
|
||||
# --- Capture de la fenêtre active ---
|
||||
# Ajout non-bloquant : enrichit le résultat avec l'image
|
||||
# de la fenêtre seule + métadonnées (titre, rect, clic relatif)
|
||||
window_info = self.capture_active_window(x, y, screenshot_id, full_img=img)
|
||||
if window_info:
|
||||
result["window_capture"] = window_info
|
||||
|
||||
# QW1 — enrichissement multi-écrans (additif, fallback gracieux)
|
||||
_enrich_with_monitor_info(result)
|
||||
|
||||
return result
|
||||
except Exception as e:
|
||||
logger.error(f"Erreur Dual Capture: {e}")
|
||||
return {}
|
||||
|
||||
def capture_active_window(
|
||||
self,
|
||||
x: int,
|
||||
y: int,
|
||||
screenshot_id: str,
|
||||
full_img: Optional[Image.Image] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Capture l'image de la fenêtre active seule + métadonnées.
|
||||
|
||||
Stratégie :
|
||||
1. Obtenir le rectangle de la fenêtre via l'API OS (pywin32 / xdotool / Quartz)
|
||||
2. Cropper depuis le screenshot plein écran (plus fiable que PrintWindow)
|
||||
3. Calculer les coordonnées du clic relatives à la fenêtre
|
||||
|
||||
Args:
|
||||
x, y: coordonnées du clic en pixels écran
|
||||
screenshot_id: identifiant pour le nom de fichier
|
||||
full_img: screenshot plein écran déjà capturé (optionnel, évite une
|
||||
double capture si appelé depuis capture_dual)
|
||||
|
||||
Returns:
|
||||
Dict avec window_image, window_title, window_rect, click_in_window,
|
||||
window_size — ou None si la fenêtre est introuvable.
|
||||
"""
|
||||
try:
|
||||
from ..window_info_crossplatform import get_active_window_rect
|
||||
|
||||
rect_info = get_active_window_rect()
|
||||
if not rect_info:
|
||||
logger.debug("Fenêtre active introuvable — skip capture fenêtre")
|
||||
return None
|
||||
|
||||
win_rect = rect_info["rect"] # [left, top, right, bottom]
|
||||
win_left, win_top, win_right, win_bottom = win_rect
|
||||
win_w, win_h = rect_info["size"] # [width, height]
|
||||
title = rect_info.get("title", "unknown_window")
|
||||
app_name = rect_info.get("app_name", "unknown_app")
|
||||
|
||||
# Ignorer les fenêtres trop petites (barres de tâches, popups système)
|
||||
if win_w < 50 or win_h < 50:
|
||||
logger.debug(f"Fenêtre trop petite ({win_w}x{win_h}) — skip")
|
||||
return None
|
||||
|
||||
# Coordonnées du clic relatives à la fenêtre
|
||||
click_rel_x = x - win_left
|
||||
click_rel_y = y - win_top
|
||||
|
||||
# Si le clic est en dehors de la fenêtre, on le signale mais on continue
|
||||
click_inside = (0 <= click_rel_x <= win_w and 0 <= click_rel_y <= win_h)
|
||||
|
||||
window_img = None
|
||||
|
||||
# --- Crop de la fenêtre depuis le plein écran ---
|
||||
if full_img is None:
|
||||
# Pas de screenshot fourni — en capturer un (cas standalone).
|
||||
# win_rect est en coords globales ; cropper depuis un monitor
|
||||
# secondaire produirait une image décalée → fail-closed sur
|
||||
# fallback secondaire.
|
||||
try:
|
||||
_monitor, full_img, _meta = capture_screen_image(
|
||||
allow_secondary_fallback=False
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Erreur capture plein écran pour fenêtre : {e}")
|
||||
full_img = None
|
||||
|
||||
if full_img is not None and not _is_effectively_black(full_img):
|
||||
img_w, img_h = full_img.size
|
||||
crop_left = max(0, win_left)
|
||||
crop_top = max(0, win_top)
|
||||
crop_right = min(img_w, win_right)
|
||||
crop_bottom = min(img_h, win_bottom)
|
||||
|
||||
if crop_right > crop_left and crop_bottom > crop_top:
|
||||
window_img = full_img.crop(
|
||||
(crop_left, crop_top, crop_right, crop_bottom)
|
||||
)
|
||||
else:
|
||||
logger.debug("Fenêtre hors écran — fallback natif si possible")
|
||||
elif full_img is not None:
|
||||
logger.warning(
|
||||
"capture_active_window: screenshot plein écran noir, fallback natif"
|
||||
)
|
||||
|
||||
if window_img is None and rect_info.get("hwnd"):
|
||||
window_img, native_meta = _capture_window_image_windows(
|
||||
rect_info["hwnd"], win_w, win_h
|
||||
)
|
||||
if window_img is not None:
|
||||
logger.warning(
|
||||
"capture_active_window via fallback natif (%s)",
|
||||
native_meta.get("backend"),
|
||||
)
|
||||
|
||||
if window_img is None:
|
||||
logger.debug("Fenêtre hors écran ou capture native indisponible")
|
||||
return None
|
||||
|
||||
# Floutage conformité AI Act
|
||||
if BLUR_SENSITIVE:
|
||||
blur_sensitive_regions(window_img)
|
||||
|
||||
# Sauvegarde — fenêtre = vue contextuelle → JPEG q85 (politique).
|
||||
self._ensure_shots_dir()
|
||||
window_base = os.path.join(
|
||||
self.shots_dir, f"{screenshot_id}_window"
|
||||
)
|
||||
window_path = save_capture(window_img, window_base, "window")
|
||||
|
||||
result = {
|
||||
"window_image": window_path,
|
||||
"window_title": title,
|
||||
"app_name": app_name,
|
||||
"window_rect": win_rect,
|
||||
"window_size": [win_w, win_h],
|
||||
"click_in_window": [click_rel_x, click_rel_y],
|
||||
"click_inside_window": click_inside,
|
||||
}
|
||||
|
||||
# QW1 — enrichissement multi-écrans (additif)
|
||||
_enrich_with_monitor_info(result)
|
||||
|
||||
logger.debug(
|
||||
f"Fenêtre capturée : {title} ({win_w}x{win_h}) — "
|
||||
f"clic relatif ({click_rel_x}, {click_rel_y})"
|
||||
)
|
||||
return result
|
||||
|
||||
except ImportError as e:
|
||||
logger.debug(f"Module fenêtre indisponible : {e}")
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error(f"Erreur capture fenêtre active : {e}")
|
||||
return None
|
||||
|
||||
def _compute_quick_hash(self, img: Image) -> str:
|
||||
"""Calcule un hash rapide basé sur une vignette réduite pour détecter les changements."""
|
||||
# On réduit l'image à 64x64 pour comparer les masses de couleurs (très rapide)
|
||||
|
||||
@@ -17,7 +17,9 @@ from __future__ import annotations
|
||||
|
||||
import platform
|
||||
import subprocess
|
||||
from typing import Dict, Optional
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from .core.log_safe import _title_hash
|
||||
|
||||
|
||||
def _run_cmd(cmd: list[str]) -> Optional[str]:
|
||||
@@ -36,11 +38,11 @@ def get_active_window_info() -> Dict[str, str]:
|
||||
"title": "...",
|
||||
"app_name": "..."
|
||||
}
|
||||
|
||||
|
||||
Détecte automatiquement l'OS et utilise la méthode appropriée.
|
||||
"""
|
||||
system = platform.system()
|
||||
|
||||
|
||||
if system == "Linux":
|
||||
return _get_window_info_linux()
|
||||
elif system == "Windows":
|
||||
@@ -51,6 +53,32 @@ def get_active_window_info() -> Dict[str, str]:
|
||||
return {"title": "unknown_window", "app_name": "unknown_app"}
|
||||
|
||||
|
||||
def get_active_window_rect() -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Renvoie le rectangle de la fenêtre active :
|
||||
{
|
||||
"title": "...",
|
||||
"app_name": "...",
|
||||
"rect": [left, top, right, bottom],
|
||||
"position": [left, top],
|
||||
"size": [width, height],
|
||||
"hwnd": int # Windows uniquement
|
||||
}
|
||||
|
||||
Retourne None si la fenêtre est introuvable ou minimisée.
|
||||
Détecte automatiquement l'OS et utilise la méthode appropriée.
|
||||
"""
|
||||
system = platform.system()
|
||||
|
||||
if system == "Windows":
|
||||
return _get_window_rect_windows()
|
||||
elif system == "Linux":
|
||||
return _get_window_rect_linux()
|
||||
elif system == "Darwin":
|
||||
return _get_window_rect_macos()
|
||||
return None
|
||||
|
||||
|
||||
def _get_window_info_linux() -> Dict[str, str]:
|
||||
"""
|
||||
Linux: utilise xdotool (X11)
|
||||
@@ -178,6 +206,163 @@ def _get_window_info_macos() -> Dict[str, str]:
|
||||
}
|
||||
|
||||
|
||||
def _get_window_rect_windows() -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Windows : utilise pywin32 pour obtenir le rectangle de la fenêtre active.
|
||||
|
||||
Retourne None si la fenêtre est minimisée (icônifiée) ou si pywin32 manque.
|
||||
"""
|
||||
try:
|
||||
import win32gui
|
||||
import win32process
|
||||
import psutil
|
||||
|
||||
hwnd = win32gui.GetForegroundWindow()
|
||||
if not hwnd:
|
||||
return None
|
||||
|
||||
# Ignorer les fenêtres minimisées (pas de contenu visible)
|
||||
if win32gui.IsIconic(hwnd):
|
||||
return None
|
||||
|
||||
title = win32gui.GetWindowText(hwnd) or "unknown_window"
|
||||
|
||||
# Rectangle de la fenêtre (coordonnées écran absolues)
|
||||
left, top, right, bottom = win32gui.GetWindowRect(hwnd)
|
||||
width = right - left
|
||||
height = bottom - top
|
||||
|
||||
# Ignorer les fenêtres de taille nulle ou absurde
|
||||
if width <= 0 or height <= 0:
|
||||
return None
|
||||
|
||||
# Nom du processus
|
||||
_, pid = win32process.GetWindowThreadProcessId(hwnd)
|
||||
try:
|
||||
app_name = psutil.Process(pid).name()
|
||||
except Exception:
|
||||
app_name = "unknown_app"
|
||||
|
||||
return {
|
||||
"title": title,
|
||||
"app_name": app_name,
|
||||
"rect": [left, top, right, bottom],
|
||||
"position": [left, top],
|
||||
"size": [width, height],
|
||||
"hwnd": hwnd,
|
||||
}
|
||||
|
||||
except ImportError:
|
||||
return None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _get_window_rect_linux() -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Linux (X11) : utilise xdotool + xwininfo pour obtenir le rectangle.
|
||||
|
||||
Nécessite : sudo apt-get install xdotool x11-utils
|
||||
"""
|
||||
try:
|
||||
# Identifiant de la fenêtre active
|
||||
wid = _run_cmd(["xdotool", "getactivewindow"])
|
||||
if not wid:
|
||||
return None
|
||||
|
||||
title = _run_cmd(["xdotool", "getactivewindow", "getwindowname"]) or "unknown_window"
|
||||
pid_str = _run_cmd(["xdotool", "getactivewindow", "getwindowpid"])
|
||||
app_name = "unknown_app"
|
||||
if pid_str:
|
||||
app_name = _run_cmd(["ps", "-p", pid_str.strip(), "-o", "comm="]) or "unknown_app"
|
||||
|
||||
# Géométrie via xdotool --shell (position + taille)
|
||||
geom_raw = _run_cmd(["xdotool", "getwindowgeometry", "--shell", wid])
|
||||
if not geom_raw:
|
||||
return None
|
||||
|
||||
vals: Dict[str, int] = {}
|
||||
for line in geom_raw.strip().splitlines():
|
||||
if "=" in line:
|
||||
k, v = line.split("=", 1)
|
||||
try:
|
||||
vals[k.strip()] = int(v.strip())
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
if not {"X", "Y", "WIDTH", "HEIGHT"} <= vals.keys():
|
||||
return None
|
||||
|
||||
x, y = vals["X"], vals["Y"]
|
||||
w, h = vals["WIDTH"], vals["HEIGHT"]
|
||||
|
||||
return {
|
||||
"title": title,
|
||||
"app_name": app_name,
|
||||
"rect": [x, y, x + w, y + h],
|
||||
"position": [x, y],
|
||||
"size": [w, h],
|
||||
}
|
||||
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _get_window_rect_macos() -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
macOS : utilise Quartz (CGWindowListCopyWindowInfo) pour obtenir le rectangle.
|
||||
|
||||
Nécessite : pip install pyobjc-framework-Quartz
|
||||
"""
|
||||
try:
|
||||
from AppKit import NSWorkspace
|
||||
from Quartz import (
|
||||
CGWindowListCopyWindowInfo,
|
||||
kCGWindowListOptionOnScreenOnly,
|
||||
kCGNullWindowID,
|
||||
)
|
||||
|
||||
active_app = NSWorkspace.sharedWorkspace().activeApplication()
|
||||
app_name = active_app.get("NSApplicationName", "unknown_app")
|
||||
|
||||
window_list = CGWindowListCopyWindowInfo(
|
||||
kCGWindowListOptionOnScreenOnly, kCGNullWindowID
|
||||
)
|
||||
|
||||
for window in window_list:
|
||||
owner_name = window.get("kCGWindowOwnerName", "")
|
||||
if owner_name != app_name:
|
||||
continue
|
||||
|
||||
bounds = window.get("kCGWindowBounds")
|
||||
if not bounds:
|
||||
continue
|
||||
|
||||
x = int(bounds.get("X", 0))
|
||||
y = int(bounds.get("Y", 0))
|
||||
w = int(bounds.get("Width", 0))
|
||||
h = int(bounds.get("Height", 0))
|
||||
if w <= 0 or h <= 0:
|
||||
continue
|
||||
|
||||
title = window.get("kCGWindowName", "unknown_window") or "unknown_window"
|
||||
|
||||
return {
|
||||
"title": title,
|
||||
"app_name": app_name,
|
||||
"rect": [x, y, x + w, y + h],
|
||||
"position": [x, y],
|
||||
"size": [w, h],
|
||||
}
|
||||
|
||||
except ImportError:
|
||||
return None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
return None
|
||||
|
||||
|
||||
# Test rapide
|
||||
if __name__ == "__main__":
|
||||
import time
|
||||
@@ -185,8 +370,13 @@ if __name__ == "__main__":
|
||||
print(f"OS détecté: {platform.system()}")
|
||||
print("\nTest de capture fenêtre active (5 secondes)...")
|
||||
print("Changez de fenêtre pour tester!\n")
|
||||
|
||||
|
||||
for i in range(5):
|
||||
info = get_active_window_info()
|
||||
print(f"[{i+1}] App: {info['app_name']:20s} | Title: {info['title']}")
|
||||
rect = get_active_window_rect()
|
||||
print(f"[{i+1}] App: {info['app_name']:20s} | Title: [title_hash={_title_hash(info['title'])}]")
|
||||
if rect:
|
||||
print(f" Rect: {rect['rect']} | Size: {rect['size']}")
|
||||
else:
|
||||
print(" Rect: non disponible")
|
||||
time.sleep(1)
|
||||
|
||||
@@ -43,6 +43,9 @@ class EventCaptorV1:
|
||||
|
||||
# État des touches modificatrices
|
||||
self.modifiers = set()
|
||||
self._pending_standalone_win = False
|
||||
self._suppress_release_only_win_combo = False
|
||||
self._raw_key_buffer: List[Dict[str, Any]] = []
|
||||
|
||||
# Tracking du focus fenêtre
|
||||
self.last_window = None
|
||||
@@ -91,6 +94,7 @@ class EventCaptorV1:
|
||||
# Flush du buffer texte restant avant arrêt
|
||||
self._flush_text_buffer()
|
||||
# Annuler le timer s'il est en cours
|
||||
emit_escape = False
|
||||
with self._text_lock:
|
||||
if self._text_flush_timer is not None:
|
||||
self._text_flush_timer.cancel()
|
||||
@@ -159,7 +163,80 @@ class EventCaptorV1:
|
||||
# Clavier
|
||||
# ----------------------------------------------------------------
|
||||
|
||||
@staticmethod
|
||||
def _get_key_name(key) -> Optional[str]:
|
||||
"""Convertit un objet pynput Key/KeyCode en nom lisible."""
|
||||
if isinstance(key, KeyCode):
|
||||
return key.char if key.char else None
|
||||
if isinstance(key, Key):
|
||||
return key.name
|
||||
return str(key)
|
||||
|
||||
@staticmethod
|
||||
def _encode_key(key) -> Dict[str, Any]:
|
||||
if isinstance(key, KeyCode):
|
||||
return {"kind": "vk", "vk": key.vk, "char": key.char}
|
||||
if isinstance(key, Key):
|
||||
return {"kind": "key", "name": key.name}
|
||||
return {"kind": "unknown", "str": str(key)}
|
||||
|
||||
@staticmethod
|
||||
def _raw_key_name(raw_key: Dict[str, Any]) -> Optional[str]:
|
||||
if raw_key.get("kind") == "vk":
|
||||
char = raw_key.get("char")
|
||||
if char and len(str(char)) == 1:
|
||||
return str(char).lower()
|
||||
if raw_key.get("kind") == "key":
|
||||
name = raw_key.get("name")
|
||||
return str(name).lower() if name else None
|
||||
return None
|
||||
|
||||
def _emit_release_only_windows_combo(self) -> bool:
|
||||
"""Infère Win+<touche> quand seuls les releases sont capturés."""
|
||||
with self._text_lock:
|
||||
raw_keys = list(getattr(self, "_raw_key_buffer", []))
|
||||
if len(raw_keys) < 2:
|
||||
return False
|
||||
cmd_names = {"cmd", "cmd_l", "cmd_r"}
|
||||
last = raw_keys[-1]
|
||||
if last.get("action") != "release" or self._raw_key_name(last) not in cmd_names:
|
||||
return False
|
||||
combo_key = None
|
||||
modifier_names = {
|
||||
"ctrl", "ctrl_l", "ctrl_r",
|
||||
"alt", "alt_l", "alt_r",
|
||||
"shift", "shift_l", "shift_r",
|
||||
"cmd", "cmd_l", "cmd_r",
|
||||
}
|
||||
for raw in reversed(raw_keys[:-1]):
|
||||
if raw.get("action") != "release":
|
||||
continue
|
||||
name = self._raw_key_name(raw)
|
||||
if name and name not in modifier_names:
|
||||
combo_key = name
|
||||
break
|
||||
if not combo_key:
|
||||
return False
|
||||
self._raw_key_buffer.clear()
|
||||
|
||||
event = {
|
||||
"type": "key_combo",
|
||||
"keys": ["win", combo_key],
|
||||
"raw_keys": raw_keys,
|
||||
"timestamp": time.time(),
|
||||
}
|
||||
self.on_event(event)
|
||||
return True
|
||||
|
||||
def _on_press(self, key):
|
||||
with self._text_lock:
|
||||
if not hasattr(self, "_raw_key_buffer"):
|
||||
self._raw_key_buffer = []
|
||||
self._raw_key_buffer.append({
|
||||
"action": "press",
|
||||
**self._encode_key(key),
|
||||
})
|
||||
|
||||
# Gestion des touches modificatrices
|
||||
if key in (Key.ctrl, Key.ctrl_l, Key.ctrl_r):
|
||||
self.modifiers.add("ctrl")
|
||||
@@ -167,15 +244,26 @@ class EventCaptorV1:
|
||||
self.modifiers.add("alt")
|
||||
elif key in (Key.shift, Key.shift_l, Key.shift_r):
|
||||
self.modifiers.add("shift")
|
||||
elif key in (Key.cmd, Key.cmd_l, Key.cmd_r):
|
||||
self.modifiers.add("win")
|
||||
self._pending_standalone_win = True
|
||||
|
||||
# --- Combos avec modificateur (sauf Shift seul) ---
|
||||
# Shift seul n'est pas un « vrai » modificateur pour les combos :
|
||||
# Shift+a = 'A' = saisie texte, pas un raccourci.
|
||||
# On considère un combo seulement si Ctrl ou Alt est enfoncé.
|
||||
has_real_modifier = self.modifiers & {"ctrl", "alt"}
|
||||
# On considère un combo seulement si Ctrl, Alt ou Win est enfoncé.
|
||||
has_real_modifier = self.modifiers & {"ctrl", "alt", "win"}
|
||||
if has_real_modifier:
|
||||
key_name = self._get_key_name(key)
|
||||
if key_name and key_name not in ("ctrl", "alt", "shift"):
|
||||
if key_name and key_name not in (
|
||||
"ctrl", "ctrl_l", "ctrl_r",
|
||||
"alt", "alt_l", "alt_r",
|
||||
"shift", "shift_l", "shift_r",
|
||||
"cmd", "cmd_l", "cmd_r",
|
||||
):
|
||||
self._pending_standalone_win = False
|
||||
if "win" in self.modifiers:
|
||||
self._suppress_release_only_win_combo = True
|
||||
# Un combo interrompt la saisie texte en cours
|
||||
self._flush_text_buffer()
|
||||
event = {
|
||||
@@ -205,14 +293,18 @@ class EventCaptorV1:
|
||||
self._reset_flush_timer()
|
||||
return
|
||||
|
||||
if key == Key.escape:
|
||||
escape_keys = [Key.esc]
|
||||
key_escape = getattr(Key, "escape", None)
|
||||
if key_escape is not None:
|
||||
escape_keys.append(key_escape)
|
||||
if key in escape_keys:
|
||||
# Annuler la saisie en cours
|
||||
self._text_buffer.clear()
|
||||
self._text_start_pos = None
|
||||
self._cancel_flush_timer()
|
||||
return
|
||||
emit_escape = True
|
||||
|
||||
if key in (Key.enter, Key.tab):
|
||||
elif key in (Key.enter, Key.tab):
|
||||
# Flush immédiat — on relâche le lock avant d'appeler
|
||||
# _flush_text_buffer (qui prend aussi le lock)
|
||||
pass # on sort du with et on flush après
|
||||
@@ -238,6 +330,15 @@ class EventCaptorV1:
|
||||
# Touche spéciale non gérée (F1, Insert, etc.) — on ignore
|
||||
return
|
||||
|
||||
if emit_escape:
|
||||
event = {
|
||||
"type": "key_combo",
|
||||
"keys": ["escape"],
|
||||
"timestamp": time.time(),
|
||||
}
|
||||
self.on_event(event)
|
||||
return
|
||||
|
||||
# Si on arrive ici, c'est Enter ou Tab → flush immédiat
|
||||
self._flush_text_buffer()
|
||||
|
||||
@@ -290,12 +391,46 @@ class EventCaptorV1:
|
||||
self.on_event(event)
|
||||
|
||||
def _on_release(self, key):
|
||||
with self._text_lock:
|
||||
self._raw_key_buffer.append({
|
||||
"action": "release",
|
||||
**self._encode_key(key),
|
||||
})
|
||||
|
||||
if key in (Key.cmd, Key.cmd_l, Key.cmd_r) and self._suppress_release_only_win_combo:
|
||||
with self._text_lock:
|
||||
self._raw_key_buffer.clear()
|
||||
self._pending_standalone_win = False
|
||||
self._suppress_release_only_win_combo = False
|
||||
self.modifiers.discard("win")
|
||||
return
|
||||
|
||||
if key in (Key.cmd, Key.cmd_l, Key.cmd_r) and self._emit_release_only_windows_combo():
|
||||
self._pending_standalone_win = False
|
||||
self._suppress_release_only_win_combo = False
|
||||
self.modifiers.discard("win")
|
||||
return
|
||||
|
||||
if key in (Key.cmd, Key.cmd_l, Key.cmd_r) and self._pending_standalone_win:
|
||||
event = {
|
||||
"type": "key_combo",
|
||||
"keys": ["win"],
|
||||
"timestamp": time.time(),
|
||||
}
|
||||
self.on_event(event)
|
||||
self._pending_standalone_win = False
|
||||
self._suppress_release_only_win_combo = False
|
||||
|
||||
if key in (Key.ctrl, Key.ctrl_l, Key.ctrl_r):
|
||||
self.modifiers.discard("ctrl")
|
||||
elif key in (Key.alt, Key.alt_l, Key.alt_r):
|
||||
self.modifiers.discard("alt")
|
||||
elif key in (Key.shift, Key.shift_l, Key.shift_r):
|
||||
self.modifiers.discard("shift")
|
||||
elif key in (Key.cmd, Key.cmd_l, Key.cmd_r):
|
||||
self.modifiers.discard("win")
|
||||
self._pending_standalone_win = False
|
||||
self._suppress_release_only_win_combo = False
|
||||
|
||||
def _watch_window_focus(self):
|
||||
"""Surveille proactivement le changement de fenêtre pour le stagiaire."""
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
214
agent_v0/deploy/windows_client/agent_v1/core/grounding.py
Normal file
214
agent_v0/deploy/windows_client/agent_v1/core/grounding.py
Normal file
@@ -0,0 +1,214 @@
|
||||
# agent_v1/core/grounding.py
|
||||
"""
|
||||
Module Grounding — localisation pure d'éléments UI sur l'écran.
|
||||
|
||||
Responsabilité unique : "Trouve l'élément X sur l'écran et retourne ses coordonnées."
|
||||
Ne prend AUCUNE décision. Si l'élément n'est pas trouvé → retourne NOT_FOUND.
|
||||
|
||||
Stratégies disponibles (cascade configurable) :
|
||||
1. Serveur SomEngine + VLM (GPU distant)
|
||||
2. Template matching local (CPU, ~10ms)
|
||||
3. VLM local direct (CPU/GPU local)
|
||||
|
||||
Séparé de Policy (qui décide quoi faire quand grounding échoue).
|
||||
Ref: docs/PLAN_ACTEUR_V1.md — Architecture MICRO (grounding + exécution)
|
||||
"""
|
||||
|
||||
import base64
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class GroundingResult:
|
||||
"""Résultat d'une tentative de localisation visuelle."""
|
||||
found: bool # L'élément a été trouvé
|
||||
x_pct: float = 0.0 # Position X en % (0.0-1.0)
|
||||
y_pct: float = 0.0 # Position Y en % (0.0-1.0)
|
||||
method: str = "" # Méthode utilisée (server_som, anchor_template, vlm_direct...)
|
||||
score: float = 0.0 # Confiance (0.0-1.0)
|
||||
elapsed_ms: float = 0.0 # Temps de résolution
|
||||
detail: str = "" # Info supplémentaire (label trouvé, raison échec)
|
||||
raw: Optional[Dict] = None # Données brutes du resolver (pour debug)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"found": self.found,
|
||||
"x_pct": self.x_pct,
|
||||
"y_pct": self.y_pct,
|
||||
"method": self.method,
|
||||
"score": round(self.score, 3),
|
||||
"elapsed_ms": round(self.elapsed_ms, 1),
|
||||
"detail": self.detail,
|
||||
}
|
||||
|
||||
|
||||
# Résultat singleton pour "pas trouvé"
|
||||
NOT_FOUND = GroundingResult(found=False, detail="Aucune méthode n'a trouvé l'élément")
|
||||
|
||||
|
||||
class GroundingEngine:
|
||||
"""Moteur de localisation visuelle d'éléments UI.
|
||||
|
||||
Encapsule la cascade de résolution (serveur → template → VLM local)
|
||||
avec une interface unifiée. Ne prend aucune décision — c'est le rôle
|
||||
de PolicyEngine.
|
||||
|
||||
Usage :
|
||||
engine = GroundingEngine(executor)
|
||||
result = engine.locate(screenshot_b64, target_spec, screen_w, screen_h)
|
||||
if result.found:
|
||||
click(result.x_pct, result.y_pct)
|
||||
"""
|
||||
|
||||
def __init__(self, executor):
|
||||
"""
|
||||
Args:
|
||||
executor: ActionExecutorV1 — fournit les méthodes de résolution existantes.
|
||||
"""
|
||||
self._executor = executor
|
||||
|
||||
def locate(
|
||||
self,
|
||||
server_url: str,
|
||||
target_spec: Dict[str, Any],
|
||||
fallback_x: float,
|
||||
fallback_y: float,
|
||||
screen_width: int,
|
||||
screen_height: int,
|
||||
strategies: Optional[List[str]] = None,
|
||||
) -> GroundingResult:
|
||||
"""Localiser un élément UI sur l'écran.
|
||||
|
||||
Exécute la cascade de stratégies dans l'ordre et retourne
|
||||
dès qu'une stratégie trouve l'élément.
|
||||
|
||||
Args:
|
||||
server_url: URL du serveur (SomEngine + VLM GPU)
|
||||
target_spec: Spécification de la cible (by_text, anchor, vlm_description...)
|
||||
fallback_x, fallback_y: Coordonnées de fallback (enregistrement)
|
||||
screen_width, screen_height: Résolution écran
|
||||
strategies: Liste ordonnée de stratégies à essayer.
|
||||
Par défaut : ["server", "template", "vlm_local"]
|
||||
|
||||
Returns:
|
||||
GroundingResult avec found=True et coordonnées, ou NOT_FOUND
|
||||
"""
|
||||
if strategies is None:
|
||||
strategies = ["server", "template", "vlm_local"]
|
||||
|
||||
# ── Apprentissage : réordonner les stratégies selon l'historique ──
|
||||
# Si le Learning sait quelle méthode marche pour cette cible,
|
||||
# la mettre en premier. C'est la boucle d'apprentissage.
|
||||
learned = target_spec.get("_learned_strategy", "")
|
||||
if learned:
|
||||
strategy_map = {
|
||||
"som_text_match": "server",
|
||||
"grounding_vlm": "server",
|
||||
"server_som": "server",
|
||||
"anchor_template": "template",
|
||||
"template_matching": "template",
|
||||
"hybrid_text_direct": "vlm_local",
|
||||
"hybrid_vlm_text": "vlm_local",
|
||||
"vlm_direct": "vlm_local",
|
||||
}
|
||||
preferred = strategy_map.get(learned, "")
|
||||
if preferred and preferred in strategies:
|
||||
strategies = [preferred] + [s for s in strategies if s != preferred]
|
||||
logger.info(
|
||||
f"Grounding: stratégie réordonnée par l'apprentissage → "
|
||||
f"{strategies} (learned={learned})"
|
||||
)
|
||||
|
||||
t_start = time.time()
|
||||
screenshot_b64 = self._executor._capture_screenshot_b64(max_width=0, quality=75)
|
||||
if not screenshot_b64:
|
||||
return GroundingResult(
|
||||
found=False, detail="Capture screenshot échouée",
|
||||
elapsed_ms=(time.time() - t_start) * 1000,
|
||||
)
|
||||
|
||||
for strategy in strategies:
|
||||
result = self._try_strategy(
|
||||
strategy, server_url, screenshot_b64, target_spec,
|
||||
fallback_x, fallback_y, screen_width, screen_height,
|
||||
)
|
||||
if result.found:
|
||||
result.elapsed_ms = (time.time() - t_start) * 1000
|
||||
return result
|
||||
|
||||
return GroundingResult(
|
||||
found=False,
|
||||
detail=f"Toutes les stratégies ont échoué ({', '.join(strategies)})",
|
||||
elapsed_ms=(time.time() - t_start) * 1000,
|
||||
)
|
||||
|
||||
def _try_strategy(
|
||||
self,
|
||||
strategy: str,
|
||||
server_url: str,
|
||||
screenshot_b64: str,
|
||||
target_spec: Dict[str, Any],
|
||||
fallback_x: float,
|
||||
fallback_y: float,
|
||||
screen_width: int,
|
||||
screen_height: int,
|
||||
) -> GroundingResult:
|
||||
"""Essayer une stratégie de grounding unique."""
|
||||
|
||||
if strategy == "server" and server_url:
|
||||
raw = self._executor._server_resolve_target(
|
||||
server_url, screenshot_b64, target_spec,
|
||||
fallback_x, fallback_y, screen_width, screen_height,
|
||||
)
|
||||
if raw and raw.get("resolved"):
|
||||
return GroundingResult(
|
||||
found=True,
|
||||
x_pct=raw["x_pct"],
|
||||
y_pct=raw["y_pct"],
|
||||
method=raw.get("method", "server"),
|
||||
score=raw.get("score", 0.0),
|
||||
detail=raw.get("matched_element", {}).get("label", ""),
|
||||
raw=raw,
|
||||
)
|
||||
|
||||
elif strategy == "template":
|
||||
anchor_b64 = target_spec.get("anchor_image_base64", "")
|
||||
if anchor_b64:
|
||||
raw = self._executor._template_match_anchor(
|
||||
screenshot_b64, anchor_b64, screen_width, screen_height,
|
||||
)
|
||||
if raw and raw.get("resolved"):
|
||||
return GroundingResult(
|
||||
found=True,
|
||||
x_pct=raw["x_pct"],
|
||||
y_pct=raw["y_pct"],
|
||||
method="anchor_template",
|
||||
score=raw.get("score", 0.0),
|
||||
raw=raw,
|
||||
)
|
||||
|
||||
elif strategy == "vlm_local":
|
||||
by_text = target_spec.get("by_text", "")
|
||||
vlm_desc = target_spec.get("vlm_description", "")
|
||||
if vlm_desc or by_text:
|
||||
raw = self._executor._hybrid_vlm_resolve(
|
||||
screenshot_b64, target_spec, screen_width, screen_height,
|
||||
)
|
||||
if raw and raw.get("resolved"):
|
||||
return GroundingResult(
|
||||
found=True,
|
||||
x_pct=raw["x_pct"],
|
||||
y_pct=raw["y_pct"],
|
||||
method=raw.get("method", "vlm_local"),
|
||||
score=raw.get("score", 0.0),
|
||||
detail=raw.get("matched_element", {}).get("label", ""),
|
||||
raw=raw,
|
||||
)
|
||||
|
||||
return GroundingResult(found=False, method=strategy, detail=f"{strategy}: pas trouvé")
|
||||
152
agent_v0/deploy/windows_client/agent_v1/core/policy.py
Normal file
152
agent_v0/deploy/windows_client/agent_v1/core/policy.py
Normal file
@@ -0,0 +1,152 @@
|
||||
# agent_v1/core/policy.py
|
||||
"""
|
||||
Module Policy — décisions intelligentes quand le grounding échoue.
|
||||
|
||||
Responsabilité unique : "Le Grounding dit NOT_FOUND. Que fait-on ?"
|
||||
Ne localise AUCUN élément — c'est le rôle du Grounding.
|
||||
|
||||
Décisions possibles :
|
||||
- RETRY : re-tenter le grounding (après popup fermée, par exemple)
|
||||
- SKIP : l'action n'est plus nécessaire (état déjà atteint)
|
||||
- ABORT : arrêter le workflow (état incohérent)
|
||||
- SUPERVISE : rendre la main à l'utilisateur
|
||||
|
||||
Séparé de Grounding (qui localise les éléments).
|
||||
Ref: docs/PLAN_ACTEUR_V1.md — Architecture MÉSO (acteur intelligent)
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Decision(Enum):
|
||||
"""Décisions possibles quand le grounding échoue."""
|
||||
RETRY = "retry" # Re-tenter (après correction : popup fermée, navigation...)
|
||||
SKIP = "skip" # Action inutile (état déjà atteint)
|
||||
ABORT = "abort" # Arrêter le workflow (état incohérent)
|
||||
SUPERVISE = "supervise" # Rendre la main à l'utilisateur (Léa dit "je bloque")
|
||||
CONTINUE = "continue" # Continuer malgré l'échec (action non critique)
|
||||
|
||||
|
||||
@dataclass
|
||||
class PolicyDecision:
|
||||
"""Résultat d'une décision Policy."""
|
||||
decision: Decision
|
||||
reason: str # Explication de la décision
|
||||
action_taken: str = "" # Action corrective effectuée (ex: "popup fermée")
|
||||
elapsed_ms: float = 0.0
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"decision": self.decision.value,
|
||||
"reason": self.reason,
|
||||
"action_taken": self.action_taken,
|
||||
"elapsed_ms": round(self.elapsed_ms, 1),
|
||||
}
|
||||
|
||||
|
||||
class PolicyEngine:
|
||||
"""Moteur de décision quand le grounding échoue.
|
||||
|
||||
Cascade de décision :
|
||||
1. Popup détectée ? → fermer et RETRY
|
||||
2. Acteur gemma4 → SKIP / ABORT / SUPERVISE
|
||||
3. Fallback → SUPERVISE (rendre la main)
|
||||
|
||||
Usage :
|
||||
policy = PolicyEngine(executor)
|
||||
decision = policy.decide(action, target_spec, grounding_result)
|
||||
if decision.decision == Decision.RETRY:
|
||||
# re-tenter le grounding
|
||||
elif decision.decision == Decision.SKIP:
|
||||
# marquer comme réussi, passer à la suite
|
||||
"""
|
||||
|
||||
def __init__(self, executor):
|
||||
self._executor = executor
|
||||
|
||||
def decide(
|
||||
self,
|
||||
action: Dict[str, Any],
|
||||
target_spec: Dict[str, Any],
|
||||
retry_count: int = 0,
|
||||
max_retries: int = 1,
|
||||
) -> PolicyDecision:
|
||||
"""Décider quoi faire quand le grounding a échoué.
|
||||
|
||||
Cascade :
|
||||
1. Si c'est le premier essai → tenter de fermer une popup → RETRY
|
||||
2. Si retry déjà fait → demander à l'acteur gemma4
|
||||
3. Selon gemma4 : SKIP, ABORT, ou SUPERVISE
|
||||
|
||||
Args:
|
||||
action: L'action qui a échoué
|
||||
target_spec: La cible non trouvée
|
||||
retry_count: Nombre de retries déjà faits
|
||||
max_retries: Maximum de retries autorisés
|
||||
"""
|
||||
t_start = time.time()
|
||||
|
||||
# ── Étape 1 : Tentative de fermeture popup (premier essai) ──
|
||||
if retry_count == 0:
|
||||
popup_handled = self._try_close_popup()
|
||||
if popup_handled:
|
||||
return PolicyDecision(
|
||||
decision=Decision.RETRY,
|
||||
reason="Popup détectée et fermée, re-tentative",
|
||||
action_taken="popup_closed",
|
||||
elapsed_ms=(time.time() - t_start) * 1000,
|
||||
)
|
||||
|
||||
# ── Étape 2 : Max retries atteint → acteur gemma4 ──
|
||||
if retry_count >= max_retries:
|
||||
actor_decision = self._ask_actor(action, target_spec)
|
||||
|
||||
if actor_decision == "PASSER":
|
||||
return PolicyDecision(
|
||||
decision=Decision.SKIP,
|
||||
reason="Acteur gemma4 : l'état est déjà atteint",
|
||||
elapsed_ms=(time.time() - t_start) * 1000,
|
||||
)
|
||||
elif actor_decision == "STOPPER":
|
||||
return PolicyDecision(
|
||||
decision=Decision.ABORT,
|
||||
reason="Acteur gemma4 : état incohérent, arrêt",
|
||||
elapsed_ms=(time.time() - t_start) * 1000,
|
||||
)
|
||||
else:
|
||||
# EXECUTER ou inconnu → pause supervisée
|
||||
return PolicyDecision(
|
||||
decision=Decision.SUPERVISE,
|
||||
reason=f"Acteur gemma4 : {actor_decision}, pause supervisée",
|
||||
elapsed_ms=(time.time() - t_start) * 1000,
|
||||
)
|
||||
|
||||
# ── Étape 3 : Encore des retries disponibles → RETRY ──
|
||||
return PolicyDecision(
|
||||
decision=Decision.RETRY,
|
||||
reason=f"Retry {retry_count + 1}/{max_retries}",
|
||||
elapsed_ms=(time.time() - t_start) * 1000,
|
||||
)
|
||||
|
||||
def _try_close_popup(self) -> bool:
|
||||
"""Tenter de fermer une popup via le handler VLM existant."""
|
||||
try:
|
||||
return self._executor._handle_popup_vlm()
|
||||
except Exception as e:
|
||||
logger.debug(f"Policy: popup handler échoué : {e}")
|
||||
return False
|
||||
|
||||
def _ask_actor(self, action: Dict, target_spec: Dict) -> str:
|
||||
"""Demander à gemma4 de décider (PASSER/EXECUTER/STOPPER)."""
|
||||
try:
|
||||
return self._executor._actor_decide(action, target_spec)
|
||||
except Exception as e:
|
||||
logger.debug(f"Policy: acteur gemma4 échoué : {e}")
|
||||
return "EXECUTER" # Fallback → supervisé
|
||||
294
agent_v0/deploy/windows_client/agent_v1/core/uia_helper.py
Normal file
294
agent_v0/deploy/windows_client/agent_v1/core/uia_helper.py
Normal file
@@ -0,0 +1,294 @@
|
||||
# core/workflow/uia_helper.py
|
||||
"""
|
||||
UIAHelper — Wrapper Python pour lea_uia.exe (helper Rust UI Automation).
|
||||
|
||||
Expose une API Python simple pour interroger UIA via le binaire Rust.
|
||||
Communique via subprocess + stdin/stdout JSON.
|
||||
|
||||
Pourquoi un helper Rust ?
|
||||
- 5-10x plus rapide que pywinauto (10-20ms vs 50-200ms)
|
||||
- Binaire standalone ~500 Ko, aucune dépendance runtime
|
||||
- Pas de problèmes de threading COM en Python
|
||||
- Crash-safe (le crash du helper n'affecte pas l'agent Python)
|
||||
|
||||
Architecture :
|
||||
Python executor
|
||||
↓ subprocess.run
|
||||
lea_uia.exe query --x 812 --y 436
|
||||
↓ UIA API Windows
|
||||
JSON response
|
||||
↓ stdout
|
||||
Python executor parse JSON
|
||||
|
||||
Si lea_uia.exe n'est pas disponible (Linux, binaire absent, crash) :
|
||||
toutes les méthodes retournent None → fallback vision automatique.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import platform
|
||||
import subprocess
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Timeout par défaut pour les appels UIA (en secondes)
|
||||
_DEFAULT_TIMEOUT = 5.0
|
||||
|
||||
# Masquer la fenêtre console lors du spawn de lea_uia.exe sur Windows.
|
||||
# Sans ce flag, chaque appel (à chaque clic utilisateur pendant
|
||||
# l'enregistrement) fait apparaître une fenêtre cmd noire brièvement
|
||||
# visible à l'écran → ralentit la souris et pollue les screenshots
|
||||
# capturés (le VLM peut "voir" le chemin lea_uia.exe comme texte cliqué).
|
||||
#
|
||||
# La valeur 0x08000000 correspond à CREATE_NO_WINDOW défini dans
|
||||
# l'API Windows. Sur Linux/Mac, la valeur est 0 et `creationflags`
|
||||
# est ignoré. getattr() gère le cas où Python expose déjà la constante
|
||||
# sur Windows.
|
||||
if platform.system() == "Windows":
|
||||
_SUBPROCESS_CREATION_FLAGS = getattr(subprocess, "CREATE_NO_WINDOW", 0x08000000)
|
||||
else:
|
||||
_SUBPROCESS_CREATION_FLAGS = 0
|
||||
|
||||
|
||||
@dataclass
|
||||
class UiaElement:
|
||||
"""Représentation Python d'un élément UIA."""
|
||||
name: str = ""
|
||||
control_type: str = ""
|
||||
class_name: str = ""
|
||||
automation_id: str = ""
|
||||
bounding_rect: Tuple[int, int, int, int] = (0, 0, 0, 0)
|
||||
is_enabled: bool = False
|
||||
is_offscreen: bool = True
|
||||
parent_path: List[Dict[str, str]] = field(default_factory=list)
|
||||
process_name: str = ""
|
||||
|
||||
def center(self) -> Tuple[int, int]:
|
||||
"""Retourner le centre du rectangle (pixels)."""
|
||||
x1, y1, x2, y2 = self.bounding_rect
|
||||
return ((x1 + x2) // 2, (y1 + y2) // 2)
|
||||
|
||||
def width(self) -> int:
|
||||
return self.bounding_rect[2] - self.bounding_rect[0]
|
||||
|
||||
def height(self) -> int:
|
||||
return self.bounding_rect[3] - self.bounding_rect[1]
|
||||
|
||||
def is_clickable(self) -> bool:
|
||||
"""Peut-on cliquer dessus ?"""
|
||||
return (
|
||||
self.is_enabled
|
||||
and not self.is_offscreen
|
||||
and self.width() > 0
|
||||
and self.height() > 0
|
||||
)
|
||||
|
||||
def path_signature(self) -> str:
|
||||
"""Signature du chemin parent (pour retrouver l'élément)."""
|
||||
parts = [f"{p['control_type']}[{p['name']}]" for p in self.parent_path if p.get("name")]
|
||||
parts.append(f"{self.control_type}[{self.name}]")
|
||||
return " > ".join(parts)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"name": self.name,
|
||||
"control_type": self.control_type,
|
||||
"class_name": self.class_name,
|
||||
"automation_id": self.automation_id,
|
||||
"bounding_rect": list(self.bounding_rect),
|
||||
"is_enabled": self.is_enabled,
|
||||
"is_offscreen": self.is_offscreen,
|
||||
"parent_path": self.parent_path,
|
||||
"process_name": self.process_name,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, d: Dict[str, Any]) -> "UiaElement":
|
||||
rect = d.get("bounding_rect", [0, 0, 0, 0])
|
||||
if isinstance(rect, list) and len(rect) >= 4:
|
||||
rect = tuple(rect[:4])
|
||||
else:
|
||||
rect = (0, 0, 0, 0)
|
||||
return cls(
|
||||
name=d.get("name", ""),
|
||||
control_type=d.get("control_type", ""),
|
||||
class_name=d.get("class_name", ""),
|
||||
automation_id=d.get("automation_id", ""),
|
||||
bounding_rect=rect,
|
||||
is_enabled=d.get("is_enabled", False),
|
||||
is_offscreen=d.get("is_offscreen", True),
|
||||
parent_path=d.get("parent_path", []),
|
||||
process_name=d.get("process_name", ""),
|
||||
)
|
||||
|
||||
|
||||
class UIAHelper:
|
||||
"""Wrapper Python pour lea_uia.exe."""
|
||||
|
||||
def __init__(self, helper_path: str = "", timeout: float = _DEFAULT_TIMEOUT):
|
||||
self._helper_path = helper_path or self._find_helper()
|
||||
self._timeout = timeout
|
||||
self._available = self._check_available()
|
||||
|
||||
def _find_helper(self) -> str:
|
||||
"""Trouver lea_uia.exe dans les emplacements standards."""
|
||||
candidates = [
|
||||
r"C:\Lea\helpers\lea_uia.exe",
|
||||
os.path.join(os.path.dirname(__file__), "..", "..",
|
||||
"agent_rust", "lea_uia", "target",
|
||||
"x86_64-pc-windows-gnu", "release", "lea_uia.exe"),
|
||||
"./helpers/lea_uia.exe",
|
||||
"lea_uia.exe",
|
||||
]
|
||||
for path in candidates:
|
||||
if os.path.isfile(path):
|
||||
return os.path.abspath(path)
|
||||
return ""
|
||||
|
||||
def _check_available(self) -> bool:
|
||||
"""Vérifier que le helper est utilisable (Windows + binaire + health OK)."""
|
||||
if platform.system() != "Windows":
|
||||
logger.debug("UIAHelper: Linux/Mac — helper désactivé")
|
||||
return False
|
||||
if not self._helper_path:
|
||||
logger.debug("UIAHelper: lea_uia.exe introuvable")
|
||||
return False
|
||||
if not os.path.isfile(self._helper_path):
|
||||
logger.debug(f"UIAHelper: chemin invalide {self._helper_path}")
|
||||
return False
|
||||
return True
|
||||
|
||||
@property
|
||||
def available(self) -> bool:
|
||||
return self._available
|
||||
|
||||
@property
|
||||
def helper_path(self) -> str:
|
||||
return self._helper_path
|
||||
|
||||
def _run(self, args: List[str]) -> Optional[Dict[str, Any]]:
|
||||
"""Exécuter lea_uia.exe avec les arguments et parser le JSON."""
|
||||
if not self._available:
|
||||
return None
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[self._helper_path] + args,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=self._timeout,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
creationflags=_SUBPROCESS_CREATION_FLAGS,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
logger.debug(
|
||||
f"UIAHelper: exit code {result.returncode}, "
|
||||
f"stderr: {result.stderr[:200]}"
|
||||
)
|
||||
return None
|
||||
output = result.stdout.strip()
|
||||
if not output:
|
||||
return None
|
||||
return json.loads(output)
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.debug(f"UIAHelper: timeout ({self._timeout}s) sur {args}")
|
||||
return None
|
||||
except json.JSONDecodeError as e:
|
||||
logger.debug(f"UIAHelper: JSON invalide — {e}")
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.debug(f"UIAHelper: erreur {e}")
|
||||
return None
|
||||
|
||||
def health(self) -> bool:
|
||||
"""Vérifier que UIA répond."""
|
||||
data = self._run(["health"])
|
||||
return data is not None and data.get("status") == "ok"
|
||||
|
||||
def query_at(
|
||||
self,
|
||||
x: int,
|
||||
y: int,
|
||||
with_parents: bool = True,
|
||||
) -> Optional[UiaElement]:
|
||||
"""Récupérer l'élément UIA à une position écran.
|
||||
|
||||
Args:
|
||||
x, y: Coordonnées pixel absolues
|
||||
with_parents: Inclure la hiérarchie des parents
|
||||
|
||||
Returns:
|
||||
UiaElement si trouvé, None sinon (pas d'élément ou UIA indispo)
|
||||
"""
|
||||
args = ["query", "--x", str(x), "--y", str(y)]
|
||||
if not with_parents:
|
||||
args.append("--with-parents=false")
|
||||
|
||||
data = self._run(args)
|
||||
if not data or data.get("status") != "ok":
|
||||
return None
|
||||
|
||||
elem_data = data.get("element")
|
||||
if not elem_data:
|
||||
return None
|
||||
return UiaElement.from_dict(elem_data)
|
||||
|
||||
def find_by_name(
|
||||
self,
|
||||
name: str,
|
||||
control_type: Optional[str] = None,
|
||||
automation_id: Optional[str] = None,
|
||||
window: Optional[str] = None,
|
||||
timeout_ms: int = 2000,
|
||||
) -> Optional[UiaElement]:
|
||||
"""Rechercher un élément par son nom (+ filtres optionnels).
|
||||
|
||||
Args:
|
||||
name: Nom exact de l'élément
|
||||
control_type: Type de contrôle (Button, Edit, MenuItem...)
|
||||
automation_id: ID d'automation
|
||||
window: Restreindre à une fenêtre spécifique
|
||||
timeout_ms: Timeout de recherche en millisecondes
|
||||
"""
|
||||
args = ["find", "--name", name, "--timeout-ms", str(timeout_ms)]
|
||||
if control_type:
|
||||
args.extend(["--control-type", control_type])
|
||||
if automation_id:
|
||||
args.extend(["--automation-id", automation_id])
|
||||
if window:
|
||||
args.extend(["--window", window])
|
||||
|
||||
data = self._run(args)
|
||||
if not data or data.get("status") != "ok":
|
||||
return None
|
||||
|
||||
elem_data = data.get("element")
|
||||
if not elem_data:
|
||||
return None
|
||||
return UiaElement.from_dict(elem_data)
|
||||
|
||||
def capture_focused(self, max_depth: int = 3) -> Optional[UiaElement]:
|
||||
"""Capturer l'élément ayant le focus + son contexte."""
|
||||
data = self._run(["capture", "--max-depth", str(max_depth)])
|
||||
if not data or data.get("status") != "ok":
|
||||
return None
|
||||
|
||||
elem_data = data.get("element")
|
||||
if not elem_data:
|
||||
return None
|
||||
return UiaElement.from_dict(elem_data)
|
||||
|
||||
|
||||
# Instance globale partagée (singleton léger)
|
||||
_SHARED_HELPER: Optional[UIAHelper] = None
|
||||
|
||||
|
||||
def get_shared_helper() -> UIAHelper:
|
||||
"""Retourner une instance partagée de UIAHelper."""
|
||||
global _SHARED_HELPER
|
||||
if _SHARED_HELPER is None:
|
||||
_SHARED_HELPER = UIAHelper()
|
||||
return _SHARED_HELPER
|
||||
@@ -319,7 +319,22 @@ class AgentV1:
|
||||
if img_hash != self._last_heartbeat_hash:
|
||||
self._last_heartbeat_hash = img_hash
|
||||
self.streamer.push_image(full_path, f"heartbeat_{int(time.time())}")
|
||||
self.streamer.push_event({"type": "heartbeat", "image": full_path, "timestamp": time.time(), "machine_id": self.machine_id})
|
||||
heartbeat_event = {
|
||||
"type": "heartbeat",
|
||||
"image": full_path,
|
||||
"timestamp": time.time(),
|
||||
"machine_id": self.machine_id,
|
||||
}
|
||||
# QW1 — enrichissement multi-écrans (monitor_index + monitors_geometry)
|
||||
# Additif, fallback gracieux : sans cet enrichissement, le serveur
|
||||
# ne reçoit l'info qu'au moment des clics, donc QW1 ne s'active
|
||||
# pas en continu sur poste Windows multi-écrans.
|
||||
try:
|
||||
from .vision.capturer import _enrich_with_monitor_info
|
||||
_enrich_with_monitor_info(heartbeat_event)
|
||||
except Exception as e:
|
||||
logger.debug("QW1 enrichissement heartbeat échoué: %s", e)
|
||||
self.streamer.push_event(heartbeat_event)
|
||||
except Exception as e:
|
||||
logger.error(f"Heartbeat error: {e}")
|
||||
time.sleep(5)
|
||||
|
||||
@@ -8,12 +8,73 @@ import os
|
||||
import time
|
||||
import logging
|
||||
import hashlib
|
||||
from typing import Any, Dict, List, Optional
|
||||
from PIL import Image, ImageFilter, ImageStat
|
||||
import mss
|
||||
from ..config import TARGETED_CROP_SIZE, SCREENSHOT_QUALITY
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# QW1 — détection multi-écrans (fallback gracieux si screeninfo absent)
|
||||
try:
|
||||
from screeninfo import get_monitors as _screeninfo_get_monitors
|
||||
_SCREENINFO_AVAILABLE = True
|
||||
except ImportError:
|
||||
_SCREENINFO_AVAILABLE = False
|
||||
|
||||
|
||||
def _get_monitors_geometry() -> List[Dict[str, Any]]:
|
||||
"""Retourne la liste des monitors physiques avec leurs offsets.
|
||||
|
||||
Returns:
|
||||
List[dict] : [{idx, x, y, w, h, primary}, ...]. Vide si screeninfo
|
||||
indisponible (le serveur tombera sur fallback composite).
|
||||
"""
|
||||
if not _SCREENINFO_AVAILABLE:
|
||||
return []
|
||||
try:
|
||||
monitors = _screeninfo_get_monitors()
|
||||
return [
|
||||
{
|
||||
"idx": i,
|
||||
"x": int(m.x),
|
||||
"y": int(m.y),
|
||||
"w": int(m.width),
|
||||
"h": int(m.height),
|
||||
"primary": bool(getattr(m, "is_primary", False)),
|
||||
}
|
||||
for i, m in enumerate(monitors)
|
||||
]
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
|
||||
def _get_active_monitor_index() -> Optional[int]:
|
||||
"""Retourne l'index logique du monitor où se trouve le curseur (focus actif).
|
||||
|
||||
Returns:
|
||||
int ou None si indéterminable.
|
||||
"""
|
||||
if not _SCREENINFO_AVAILABLE:
|
||||
return None
|
||||
try:
|
||||
import pyautogui # import paresseux : évite la dépendance dure
|
||||
cx, cy = pyautogui.position()
|
||||
for i, m in enumerate(_screeninfo_get_monitors()):
|
||||
if m.x <= cx < m.x + m.width and m.y <= cy < m.y + m.height:
|
||||
return i
|
||||
except Exception:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def _enrich_with_monitor_info(payload: dict) -> dict:
|
||||
"""Ajoute monitor_index et monitors_geometry au payload (in-place + return)."""
|
||||
if isinstance(payload, dict):
|
||||
payload["monitor_index"] = _get_active_monitor_index()
|
||||
payload["monitors_geometry"] = _get_monitors_geometry()
|
||||
return payload
|
||||
|
||||
class VisionCapturer:
|
||||
def __init__(self, session_dir: str):
|
||||
self.session_dir = session_dir
|
||||
@@ -72,7 +133,12 @@ class VisionCapturer:
|
||||
# Mise à jour du hash pour le prochain heartbeat
|
||||
self.last_img_hash = self._compute_quick_hash(img)
|
||||
|
||||
return {"full": full_path, "crop": crop_path}
|
||||
result = {"full": full_path, "crop": crop_path}
|
||||
|
||||
# QW1 — enrichissement multi-écrans (additif, fallback gracieux)
|
||||
_enrich_with_monitor_info(result)
|
||||
|
||||
return result
|
||||
except Exception as e:
|
||||
logger.error(f"Erreur Dual Capture: {e}")
|
||||
return {}
|
||||
|
||||
@@ -3,7 +3,9 @@ mss>=9.0.1 # Capture d'écran haute performance
|
||||
pynput>=1.7.7 # Clavier/Souris Cross-plateforme
|
||||
Pillow>=10.0.0 # Crops et processing image
|
||||
requests>=2.31.0 # Streaming réseau
|
||||
python-socketio[client]>=5.10,<6.0 # Bus feedback Léa 'lea:*' (compat Flask-SocketIO 5.3.x serveur)
|
||||
psutil>=5.9.0 # Monitoring CPU/RAM
|
||||
screeninfo>=0.8 # QW1 — détection des monitors physiques + offsets
|
||||
pystray>=0.19.5 # Icône Tray UI
|
||||
plyer>=2.1.0 # Notifications toast natives (remplace PyQt5)
|
||||
|
||||
|
||||
@@ -2,6 +2,17 @@
|
||||
"""
|
||||
deploy_windows.py — Script de packaging du client Windows pour Agent V1.
|
||||
|
||||
⚠️ OBSOLÈTE (avril 2026)
|
||||
Le build officiel du package Windows passe par ``deploy/build_package.sh``
|
||||
(à la racine du repo) qui lit directement ``agent_v0/agent_v1/`` et évite
|
||||
les clones intermédiaires. Ce script est conservé pour référence mais son
|
||||
manifeste ``FILE_MANIFEST`` est incomplet : il n'inclut pas
|
||||
``system_dialog_guard.py``, ``persistent_buffer.py``, ``recovery.py``,
|
||||
``uia_helper.py``, ``grounding.py``, ``policy.py``,
|
||||
``vision/blur_sensitive.py``, ``vision/system_info.py``,
|
||||
``ui/chat_window.py``, ``ui/capture_server.py``, ``ui/shared_state.py``.
|
||||
Ne PAS l'utiliser pour un packaging réel.
|
||||
|
||||
Copie uniquement les fichiers nécessaires au fonctionnement de l'agent
|
||||
sur le PC cible (Windows), sans le serveur ni les dépendances lourdes.
|
||||
|
||||
|
||||
@@ -21,36 +21,33 @@ from typing import Any, Callable, Dict, List, Optional
|
||||
logger = logging.getLogger("lea_ui.server_client")
|
||||
|
||||
|
||||
def _get_server_host() -> str:
|
||||
"""Recuperer l'adresse du serveur Linux.
|
||||
def _get_server_url() -> str:
|
||||
"""Recuperer l'URL du serveur RPA (avec /api/v1).
|
||||
|
||||
Ordre de resolution :
|
||||
1. Variable d'environnement RPA_SERVER_HOST
|
||||
2. Fichier de config agent_config.json (cle "server_host")
|
||||
3. Fallback localhost
|
||||
1. Import depuis agent_v1.config (source de verite unique)
|
||||
2. Variable d'environnement RPA_SERVER_URL
|
||||
3. Fallback http://localhost:5005/api/v1
|
||||
"""
|
||||
# 1. Variable d'environnement
|
||||
host = os.environ.get("RPA_SERVER_HOST", "").strip()
|
||||
if host:
|
||||
return host
|
||||
# 1. Import depuis config.py (source de verite)
|
||||
try:
|
||||
from agent_v1.config import SERVER_URL
|
||||
return SERVER_URL
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
# 2. Fichier de config
|
||||
config_paths = [
|
||||
os.path.join(os.path.dirname(__file__), "..", "agent_config.json"),
|
||||
os.path.join(os.path.dirname(__file__), "..", "..", "agent_config.json"),
|
||||
]
|
||||
for config_path in config_paths:
|
||||
try:
|
||||
with open(config_path, "r", encoding="utf-8") as f:
|
||||
cfg = json.load(f)
|
||||
host = cfg.get("server_host", "").strip()
|
||||
if host:
|
||||
return host
|
||||
except (OSError, json.JSONDecodeError):
|
||||
continue
|
||||
# 2. Variable d'environnement directe
|
||||
url = os.environ.get("RPA_SERVER_URL", "").strip().rstrip("/")
|
||||
if url:
|
||||
return url
|
||||
|
||||
# 3. Fallback
|
||||
return "localhost"
|
||||
return "http://localhost:5005/api/v1"
|
||||
|
||||
|
||||
def _get_server_base(server_url: str) -> str:
|
||||
"""Extraire la base URL (sans /api/v1) pour les routes racine (/health)."""
|
||||
return server_url.rsplit("/api/v1", 1)[0]
|
||||
|
||||
|
||||
class LeaServerClient:
|
||||
@@ -67,12 +64,23 @@ class LeaServerClient:
|
||||
chat_port: int = 5004,
|
||||
stream_port: int = 5005,
|
||||
) -> None:
|
||||
self._host = server_host or _get_server_host()
|
||||
# URL unifiée : SERVER_URL contient TOUJOURS /api/v1 (convention INC-1).
|
||||
# _stream_url = URL avec /api/v1 (pour les routes API)
|
||||
# _stream_base = URL sans /api/v1 (pour /health uniquement)
|
||||
self._stream_url = _get_server_url()
|
||||
self._stream_base = _get_server_base(self._stream_url)
|
||||
|
||||
# Extraire le host depuis l'URL pour le chat et pour l'affichage
|
||||
try:
|
||||
from urllib.parse import urlparse
|
||||
parsed = urlparse(self._stream_base)
|
||||
self._host = parsed.hostname or "localhost"
|
||||
except Exception:
|
||||
self._host = server_host or "localhost"
|
||||
|
||||
self._chat_port = chat_port
|
||||
self._stream_port = stream_port
|
||||
|
||||
self._chat_base = f"http://{self._host}:{self._chat_port}"
|
||||
self._stream_base = f"http://{self._host}:{self._stream_port}"
|
||||
|
||||
# Etat de connexion
|
||||
self._connected = False
|
||||
@@ -95,8 +103,8 @@ class LeaServerClient:
|
||||
self._api_token = os.environ.get("RPA_API_TOKEN", "")
|
||||
|
||||
logger.info(
|
||||
"LeaServerClient initialise : chat=%s, stream=%s",
|
||||
self._chat_base, self._stream_base,
|
||||
"LeaServerClient initialise : chat=%s, stream_url=%s, stream_base=%s",
|
||||
self._chat_base, self._stream_url, self._stream_base,
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -146,7 +154,11 @@ class LeaServerClient:
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def check_connection(self) -> bool:
|
||||
"""Tester la connexion au serveur streaming (port 5005)."""
|
||||
"""Tester la connexion au serveur streaming (port 5005).
|
||||
|
||||
Le health check utilise _stream_base (sans /api/v1) car la route
|
||||
/health est a la racine du serveur FastAPI, pas sous /api/v1.
|
||||
"""
|
||||
try:
|
||||
import requests
|
||||
resp = requests.get(
|
||||
@@ -219,7 +231,7 @@ class LeaServerClient:
|
||||
import requests
|
||||
headers = self._auth_headers()
|
||||
resp = requests.get(
|
||||
f"{self._stream_base}/api/v1/traces/stream/workflows",
|
||||
f"{self._stream_url}/traces/stream/workflows",
|
||||
headers=headers,
|
||||
timeout=10,
|
||||
)
|
||||
@@ -276,7 +288,7 @@ class LeaServerClient:
|
||||
while self._polling:
|
||||
try:
|
||||
resp = req_lib.get(
|
||||
f"{self._stream_base}/api/v1/traces/stream/replay/next",
|
||||
f"{self._stream_url}/traces/stream/replay/next",
|
||||
params={"session_id": self._poll_session_id},
|
||||
headers=self._auth_headers(),
|
||||
timeout=5,
|
||||
@@ -310,7 +322,7 @@ class LeaServerClient:
|
||||
try:
|
||||
import requests
|
||||
resp = requests.get(
|
||||
f"{self._stream_base}/api/v1/traces/stream/replays",
|
||||
f"{self._stream_url}/traces/stream/replays",
|
||||
headers=self._auth_headers(),
|
||||
timeout=5,
|
||||
)
|
||||
@@ -326,6 +338,50 @@ class LeaServerClient:
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def resume_replay(self, replay_id: str) -> bool:
|
||||
"""Reprendre un replay en pause supervisée via HTTP direct.
|
||||
|
||||
Fallback du chemin SocketIO (`lea:replay_resume` → agent_chat)
|
||||
utilisé quand le bus feedback est déconnecté au moment où
|
||||
l'utilisateur clique « Continuer » dans la bulle paused.
|
||||
|
||||
Retourne True si le serveur streaming a accepté la reprise.
|
||||
"""
|
||||
if not replay_id:
|
||||
return False
|
||||
try:
|
||||
import requests
|
||||
resp = requests.post(
|
||||
f"{self._stream_url}/traces/stream/replay/{replay_id}/resume",
|
||||
headers=self._auth_headers(),
|
||||
timeout=10,
|
||||
)
|
||||
return bool(resp.ok)
|
||||
except Exception:
|
||||
logger.debug("resume_replay HTTP silenced", exc_info=True)
|
||||
return False
|
||||
|
||||
def abort_replay(self, replay_id: str) -> bool:
|
||||
"""Annuler un replay en pause supervisée via HTTP direct.
|
||||
|
||||
Symétrique de ``resume_replay`` : fallback du chemin SocketIO
|
||||
(`lea:replay_abort`) quand le bus feedback est déconnecté.
|
||||
POSTe sur ``/replay/{id}/cancel`` côté serveur streaming.
|
||||
"""
|
||||
if not replay_id:
|
||||
return False
|
||||
try:
|
||||
import requests
|
||||
resp = requests.post(
|
||||
f"{self._stream_url}/traces/stream/replay/{replay_id}/cancel",
|
||||
headers=self._auth_headers(),
|
||||
timeout=10,
|
||||
)
|
||||
return bool(resp.ok)
|
||||
except Exception:
|
||||
logger.debug("abort_replay HTTP silenced", exc_info=True)
|
||||
return False
|
||||
|
||||
def report_action_result(
|
||||
self,
|
||||
session_id: str,
|
||||
@@ -338,7 +394,7 @@ class LeaServerClient:
|
||||
try:
|
||||
import requests
|
||||
requests.post(
|
||||
f"{self._stream_base}/api/v1/traces/stream/replay/result",
|
||||
f"{self._stream_url}/traces/stream/replay/result",
|
||||
json={
|
||||
"session_id": session_id,
|
||||
"action_id": action_id,
|
||||
|
||||
@@ -1,12 +1,97 @@
|
||||
# run_agent_v1.py
|
||||
import sys
|
||||
import os
|
||||
import atexit
|
||||
|
||||
# Ajout du répertoire courant au PYTHONPATH pour permettre les imports de modules
|
||||
current_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
if current_dir not in sys.path:
|
||||
sys.path.append(current_dir)
|
||||
|
||||
# ---------------------------------------------------------------
|
||||
# Verrou PID — empêche le lancement de plusieurs instances
|
||||
# Même si Lea.bat est double-cliqué ou lancé deux fois,
|
||||
# un seul agent tourne à la fois (defense-in-depth).
|
||||
# ---------------------------------------------------------------
|
||||
LOCK_FILE = os.path.join(current_dir, "lea_agent.lock")
|
||||
|
||||
|
||||
def _pid_is_alive(pid: int) -> bool:
|
||||
"""Vérifie si un processus avec ce PID existe encore (Windows + Unix)."""
|
||||
if sys.platform == "win32":
|
||||
try:
|
||||
import ctypes
|
||||
kernel32 = ctypes.windll.kernel32 # type: ignore[attr-defined]
|
||||
PROCESS_QUERY_LIMITED_INFORMATION = 0x1000
|
||||
handle = kernel32.OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, False, pid)
|
||||
if handle:
|
||||
kernel32.CloseHandle(handle)
|
||||
return True
|
||||
return False
|
||||
except Exception:
|
||||
# Fallback : tasklist
|
||||
try:
|
||||
import subprocess
|
||||
result = subprocess.run(
|
||||
["tasklist", "/FI", f"PID eq {pid}", "/NH"],
|
||||
capture_output=True, text=True, timeout=5,
|
||||
)
|
||||
return str(pid) in result.stdout
|
||||
except Exception:
|
||||
return False
|
||||
else:
|
||||
# Unix/Linux — os.kill(pid, 0) ne tue pas le process
|
||||
try:
|
||||
os.kill(pid, 0)
|
||||
return True
|
||||
except (OSError, ProcessLookupError):
|
||||
return False
|
||||
|
||||
|
||||
def _acquire_lock() -> bool:
|
||||
"""Tente d'acquérir le verrou PID. Retourne False si une autre instance tourne."""
|
||||
my_pid = os.getpid()
|
||||
|
||||
# Lire le PID existant
|
||||
if os.path.isfile(LOCK_FILE):
|
||||
try:
|
||||
with open(LOCK_FILE, "r", encoding="utf-8") as f:
|
||||
old_pid = int(f.read().strip())
|
||||
# Le PID dans le lock est-il encore vivant ?
|
||||
if old_pid != my_pid and _pid_is_alive(old_pid):
|
||||
return False # Une autre instance tourne déjà
|
||||
except (ValueError, OSError):
|
||||
pass # Fichier corrompu — on l'écrase
|
||||
|
||||
# Écrire notre PID
|
||||
try:
|
||||
with open(LOCK_FILE, "w", encoding="utf-8") as f:
|
||||
f.write(str(my_pid))
|
||||
except OSError:
|
||||
pass # Pas bloquant — on continue sans lock
|
||||
return True
|
||||
|
||||
|
||||
def _release_lock():
|
||||
"""Supprime le fichier lock au shutdown."""
|
||||
try:
|
||||
if os.path.isfile(LOCK_FILE):
|
||||
with open(LOCK_FILE, "r", encoding="utf-8") as f:
|
||||
stored_pid = int(f.read().strip())
|
||||
# Ne supprimer que si c'est bien NOTRE lock
|
||||
if stored_pid == os.getpid():
|
||||
os.remove(LOCK_FILE)
|
||||
except (ValueError, OSError):
|
||||
pass
|
||||
|
||||
|
||||
# Vérification du lock AVANT toute initialisation lourde
|
||||
if not _acquire_lock():
|
||||
# Une autre instance de Léa tourne déjà — on quitte silencieusement
|
||||
sys.exit(0)
|
||||
|
||||
atexit.register(_release_lock)
|
||||
|
||||
# Charger config.txt et .env comme variables d'environnement
|
||||
# (équivalent du `set` dans Lea.bat, mais fonctionne aussi sans le .bat)
|
||||
for config_file in ("config.txt", ".env"):
|
||||
@@ -32,7 +117,7 @@ logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s [%(name)s] %(levelname)s: %(message)s",
|
||||
)
|
||||
logging.info("=== Agent V1 démarrage — config chargée ===")
|
||||
logging.info("=== Agent V1 démarrage — config chargée (PID %d) ===", os.getpid())
|
||||
logging.info("RPA_SERVER_URL=%s", os.environ.get("RPA_SERVER_URL", "(non défini)"))
|
||||
logging.info("RPA_SERVER_HOST=%s", os.environ.get("RPA_SERVER_HOST", "(non défini)"))
|
||||
logging.info("RPA_API_TOKEN=%s", os.environ.get("RPA_API_TOKEN", "(non défini)")[:8] + "...")
|
||||
|
||||
77
agent_v0/server_v1/agent_logs_store.py
Normal file
77
agent_v0/server_v1/agent_logs_store.py
Normal file
@@ -0,0 +1,77 @@
|
||||
"""Store des logs poussés par les clients Léa (push-log-DGX).
|
||||
|
||||
Persiste les logs reçus du client, rangés par `machine_id`, pour consultation
|
||||
au dashboard (diagnostic des postes sans AnyDesk). Stockage fichier JSONL
|
||||
(un fichier par jour et par machine_id), rétention configurable.
|
||||
|
||||
DETTE-020/021 (observabilité). Branche feat/push-log-dgx.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
|
||||
# machine_id = entrée réseau → neutraliser tout caractère hors liste blanche
|
||||
# (anti path-traversal : '/', '\\', '..' ne doivent pas s'échapper du base_dir).
|
||||
_SAFE_MACHINE_ID_RE = re.compile(r"[^A-Za-z0-9._-]")
|
||||
|
||||
|
||||
class AgentLogsStore:
|
||||
"""Persiste et relit les logs clients rangés par machine_id (JSONL)."""
|
||||
|
||||
def __init__(self, base_dir: str | Path = "data/agent_logs"):
|
||||
self.base_dir = Path(base_dir)
|
||||
self.base_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def _machine_dir(self, machine_id: str) -> Path:
|
||||
safe = _SAFE_MACHINE_ID_RE.sub("_", machine_id or "").strip("._") or "unknown"
|
||||
d = self.base_dir / safe
|
||||
d.mkdir(parents=True, exist_ok=True)
|
||||
return d
|
||||
|
||||
def append(self, machine_id: str, entries: list[dict]) -> int:
|
||||
"""Ajoute un batch de logs pour un poste. Retourne le nb de lignes écrites."""
|
||||
if not entries:
|
||||
return 0
|
||||
now = datetime.now(timezone.utc)
|
||||
day_file = self._machine_dir(machine_id) / f"{now.date().isoformat()}.jsonl"
|
||||
with day_file.open("a", encoding="utf-8") as f:
|
||||
for entry in entries:
|
||||
record = dict(entry)
|
||||
record.setdefault("received_at", now.isoformat())
|
||||
f.write(json.dumps(record, ensure_ascii=False) + "\n")
|
||||
return len(entries)
|
||||
|
||||
def read(self, machine_id: str) -> list[dict]:
|
||||
"""Relit toutes les entrées d'un poste, triées par fichier (date) puis ordre d'écriture."""
|
||||
d = self._machine_dir(machine_id)
|
||||
out: list[dict] = []
|
||||
for jsonl in sorted(d.glob("*.jsonl")):
|
||||
with jsonl.open(encoding="utf-8") as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if line:
|
||||
out.append(json.loads(line))
|
||||
return out
|
||||
|
||||
def purge_old(self, retention_days: int = 30, now: datetime | None = None) -> int:
|
||||
"""Supprime les fichiers-jour antérieurs à la rétention. Retourne le nb supprimé.
|
||||
|
||||
Rétention basée sur la date encodée dans le nom du fichier (`YYYY-MM-DD.jsonl`),
|
||||
pas sur le mtime (déterministe, non altérable). `now` injectable pour les tests.
|
||||
"""
|
||||
now = now or datetime.now(timezone.utc)
|
||||
cutoff = (now - timedelta(days=retention_days)).date()
|
||||
removed = 0
|
||||
for jsonl in self.base_dir.rglob("*.jsonl"):
|
||||
try:
|
||||
file_date = datetime.strptime(jsonl.stem, "%Y-%m-%d").date()
|
||||
except ValueError:
|
||||
continue # nom inattendu → on ne touche pas
|
||||
if file_date < cutoff:
|
||||
jsonl.unlink()
|
||||
removed += 1
|
||||
return removed
|
||||
413
agent_v0/server_v1/agent_registry.py
Normal file
413
agent_v0/server_v1/agent_registry.py
Normal file
@@ -0,0 +1,413 @@
|
||||
# agent_v0/server_v1/agent_registry.py
|
||||
"""
|
||||
Registre des agents Lea enrolles sur le parc.
|
||||
|
||||
Alimente par les endpoints /api/v1/agents/enroll et /api/v1/agents/uninstall
|
||||
que l'installeur Inno Setup (`deploy/installer/Lea.iss`) appelle a
|
||||
l'installation et a la desinstallation sur chaque poste collaborateur.
|
||||
|
||||
Stockage : SQLite simple, cohabite avec rpa_data.db dans data/databases/.
|
||||
Aucune dependance GPU/LLM — ce module doit rester leger (juste sqlite3 +
|
||||
stdlib) pour pouvoir etre importe par le serveur HTTP.
|
||||
|
||||
Schema de la table `enrolled_agents` :
|
||||
id INTEGER PK AUTOINCREMENT
|
||||
machine_id TEXT UNIQUE NOT NULL — identifiant genere par l'installeur
|
||||
user_name TEXT — nom affichage collaborateur
|
||||
user_email TEXT
|
||||
user_id TEXT — identifiant metier (ex: AIVA-001)
|
||||
hostname TEXT
|
||||
os_info TEXT
|
||||
version TEXT — version du client Lea
|
||||
status TEXT DEFAULT 'active' — 'active' | 'uninstalled'
|
||||
enrolled_at TEXT NOT NULL — ISO 8601 UTC
|
||||
last_seen_at TEXT — ISO 8601 UTC (heartbeat / stream)
|
||||
uninstalled_at TEXT
|
||||
uninstall_reason TEXT
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
import logging
|
||||
import os
|
||||
import secrets
|
||||
import sqlite3
|
||||
import threading
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Verrou global : SQLite tolere plusieurs threads mais on serialise
|
||||
# les ecritures pour eviter les races sur _init_db + upserts concurrents.
|
||||
_DB_LOCK = threading.Lock()
|
||||
|
||||
|
||||
def _utc_now_iso() -> str:
|
||||
"""Horodatage ISO 8601 UTC (compatible toutes les autres tables)."""
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def _new_token() -> Tuple[str, str]:
|
||||
"""WP-C : genere un token poste (clair) et son empreinte SHA-256.
|
||||
|
||||
Le clair est retourne UNE seule fois a l'appelant (resultat de enroll) ; seul
|
||||
le hash est persiste dans `token_hash`. Le clair n'est jamais journalise ni
|
||||
stocke. L'auth runtime reste inchangee (aucun branchement ici sur la
|
||||
verification de token cote api_stream).
|
||||
"""
|
||||
clear = secrets.token_hex(32)
|
||||
token_hash = hashlib.sha256(clear.encode("utf-8")).hexdigest()
|
||||
return clear, token_hash
|
||||
|
||||
|
||||
def _fleet_enroll_locked() -> bool:
|
||||
"""WP-B : parc verrouille -> aucun NOUVEAU machine_id ne peut s'enroler.
|
||||
|
||||
Pilote par l'env `RPA_FLEET_ENROLL_LOCKED` (true/1/yes), reversible (relu a
|
||||
chaque appel). Ferme le contournement « poste revoque + nouveau machine_id +
|
||||
token global » : les machines deja connues gardent leur comportement, seul
|
||||
l'enrolement d'un machine_id inconnu est refuse quand le parc est verrouille.
|
||||
"""
|
||||
return os.getenv("RPA_FLEET_ENROLL_LOCKED", "").strip().lower() in ("1", "true", "yes")
|
||||
|
||||
|
||||
class AgentRegistry:
|
||||
"""Gestion CRUD des agents enrolles (SQLite)."""
|
||||
|
||||
def __init__(self, db_path: str | Path = "data/databases/rpa_data.db"):
|
||||
self.db_path = Path(db_path)
|
||||
self.db_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
self._init_db()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Infra SQLite
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _connect(self) -> sqlite3.Connection:
|
||||
# check_same_thread=False : on protege nous-memes via _DB_LOCK,
|
||||
# indispensable car FastAPI appelle les endpoints sur threads
|
||||
# differents (thread pool).
|
||||
conn = sqlite3.connect(str(self.db_path), check_same_thread=False)
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.execute("PRAGMA journal_mode=WAL")
|
||||
conn.execute("PRAGMA foreign_keys=ON")
|
||||
return conn
|
||||
|
||||
def _init_db(self) -> None:
|
||||
"""Cree la table et ses index si absents (idempotent)."""
|
||||
with _DB_LOCK, self._connect() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS enrolled_agents (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
machine_id TEXT NOT NULL UNIQUE,
|
||||
user_name TEXT,
|
||||
user_email TEXT,
|
||||
user_id TEXT,
|
||||
hostname TEXT,
|
||||
os_info TEXT,
|
||||
version TEXT,
|
||||
status TEXT NOT NULL DEFAULT 'active',
|
||||
enrolled_at TEXT NOT NULL,
|
||||
last_seen_at TEXT,
|
||||
uninstalled_at TEXT,
|
||||
uninstall_reason TEXT
|
||||
)
|
||||
"""
|
||||
)
|
||||
conn.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_enrolled_agents_status "
|
||||
"ON enrolled_agents(status)"
|
||||
)
|
||||
conn.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_enrolled_agents_machine "
|
||||
"ON enrolled_agents(machine_id)"
|
||||
)
|
||||
# WP-C Patch 1 : colonnes « token par poste », migration additive
|
||||
# idempotente. Inertes tant que l'auth par poste n'est pas branchée
|
||||
# (patchs WP-C ultérieurs). Voir DETTE-015.
|
||||
existing_cols = {
|
||||
row[1]
|
||||
for row in conn.execute(
|
||||
"PRAGMA table_info(enrolled_agents)"
|
||||
).fetchall()
|
||||
}
|
||||
for col in ("token_hash", "token_issued_at"):
|
||||
if col not in existing_cols:
|
||||
conn.execute(
|
||||
f"ALTER TABLE enrolled_agents ADD COLUMN {col} TEXT"
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Lecture
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def get(self, machine_id: str) -> Optional[Dict[str, Any]]:
|
||||
"""Recupere un agent par machine_id (ou None)."""
|
||||
with _DB_LOCK, self._connect() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT * FROM enrolled_agents WHERE machine_id = ?",
|
||||
(machine_id,),
|
||||
).fetchone()
|
||||
return dict(row) if row else None
|
||||
|
||||
def list_by_status(self, status: str) -> List[Dict[str, Any]]:
|
||||
"""Liste les agents par statut ('active' | 'uninstalled')."""
|
||||
with _DB_LOCK, self._connect() as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT * FROM enrolled_agents WHERE status = ? "
|
||||
"ORDER BY enrolled_at DESC",
|
||||
(status,),
|
||||
).fetchall()
|
||||
return [dict(r) for r in rows]
|
||||
|
||||
def count_by_status(self, status: str) -> int:
|
||||
with _DB_LOCK, self._connect() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT COUNT(*) AS n FROM enrolled_agents WHERE status = ?",
|
||||
(status,),
|
||||
).fetchone()
|
||||
return int(row["n"]) if row else 0
|
||||
|
||||
def verify_token(self, token: str | None) -> Optional[str]:
|
||||
"""WP-C : verifie un token poste, retourne le machine_id actif ou None.
|
||||
|
||||
Compare le SHA-256 du token presente aux `token_hash` des agents
|
||||
`status='active'` via `hmac.compare_digest` (comparaison a temps
|
||||
constant, evite les fuites par timing). Un agent desinstalle/revoque
|
||||
n'est pas 'active' donc refuse ; la rotation a l'enrolement invalide
|
||||
l'ancien token.
|
||||
|
||||
INERTE : non branchee sur l'auth runtime (le branchement derriere flag
|
||||
sera le Patch 4). Aucun appelant runtime a ce stade.
|
||||
"""
|
||||
if not token:
|
||||
return None
|
||||
token_hash = hashlib.sha256(token.encode("utf-8")).hexdigest()
|
||||
with _DB_LOCK, self._connect() as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT machine_id, token_hash FROM enrolled_agents "
|
||||
"WHERE status = 'active' AND token_hash IS NOT NULL"
|
||||
).fetchall()
|
||||
for row in rows:
|
||||
if hmac.compare_digest(str(row["token_hash"]), token_hash):
|
||||
return str(row["machine_id"])
|
||||
return None
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Ecriture
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def enroll(
|
||||
self,
|
||||
*,
|
||||
machine_id: str,
|
||||
user_name: str | None = None,
|
||||
user_email: str | None = None,
|
||||
user_id: str | None = None,
|
||||
hostname: str | None = None,
|
||||
os_info: str | None = None,
|
||||
version: str | None = None,
|
||||
allow_reactivate: bool = True,
|
||||
) -> Dict[str, Any]:
|
||||
"""Enregistre un nouvel agent ou reactive un agent desinstalle.
|
||||
|
||||
Returns:
|
||||
dict avec clefs {"created": bool, "reactivated": bool, "agent": row}
|
||||
|
||||
Raises:
|
||||
ValueError: si machine_id est vide.
|
||||
AgentAlreadyEnrolledError: si deja actif (status=active).
|
||||
"""
|
||||
if not machine_id or not machine_id.strip():
|
||||
raise ValueError("machine_id est obligatoire")
|
||||
machine_id = machine_id.strip()
|
||||
|
||||
now = _utc_now_iso()
|
||||
|
||||
with _DB_LOCK, self._connect() as conn:
|
||||
existing = conn.execute(
|
||||
"SELECT * FROM enrolled_agents WHERE machine_id = ?",
|
||||
(machine_id,),
|
||||
).fetchone()
|
||||
|
||||
if existing is not None:
|
||||
if existing["status"] == "active":
|
||||
# Deja enrolle et actif -> conflit explicit
|
||||
raise AgentAlreadyEnrolledError(dict(existing))
|
||||
|
||||
if existing["uninstall_reason"] == "admin_revoke":
|
||||
raise AgentRevokedError(dict(existing))
|
||||
|
||||
# Agent desinstalle : reactivation si autorise (defaut)
|
||||
if not allow_reactivate:
|
||||
raise AgentAlreadyEnrolledError(dict(existing))
|
||||
|
||||
# WP-C : rotation du token a chaque (re)enrolement.
|
||||
token, token_hash = _new_token()
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE enrolled_agents
|
||||
SET user_name = COALESCE(?, user_name),
|
||||
user_email = COALESCE(?, user_email),
|
||||
user_id = COALESCE(?, user_id),
|
||||
hostname = COALESCE(?, hostname),
|
||||
os_info = COALESCE(?, os_info),
|
||||
version = COALESCE(?, version),
|
||||
status = 'active',
|
||||
enrolled_at = ?,
|
||||
last_seen_at = ?,
|
||||
uninstalled_at = NULL,
|
||||
uninstall_reason = NULL,
|
||||
token_hash = ?,
|
||||
token_issued_at = ?
|
||||
WHERE machine_id = ?
|
||||
""",
|
||||
(
|
||||
user_name, user_email, user_id,
|
||||
hostname, os_info, version,
|
||||
now, now,
|
||||
token_hash, now,
|
||||
machine_id,
|
||||
),
|
||||
)
|
||||
conn.commit()
|
||||
row = conn.execute(
|
||||
"SELECT * FROM enrolled_agents WHERE machine_id = ?",
|
||||
(machine_id,),
|
||||
).fetchone()
|
||||
return {
|
||||
"created": False,
|
||||
"reactivated": True,
|
||||
"agent": dict(row),
|
||||
"token": token,
|
||||
}
|
||||
|
||||
# Nouvelle inscription — WP-B : refusee si le parc est verrouille
|
||||
if _fleet_enroll_locked():
|
||||
raise FleetEnrollLockedError(machine_id)
|
||||
# WP-C : token poste genere a la creation.
|
||||
token, token_hash = _new_token()
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO enrolled_agents (
|
||||
machine_id, user_name, user_email, user_id,
|
||||
hostname, os_info, version,
|
||||
status, enrolled_at, last_seen_at,
|
||||
token_hash, token_issued_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, 'active', ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
machine_id, user_name, user_email, user_id,
|
||||
hostname, os_info, version,
|
||||
now, now,
|
||||
token_hash, now,
|
||||
),
|
||||
)
|
||||
conn.commit()
|
||||
row = conn.execute(
|
||||
"SELECT * FROM enrolled_agents WHERE machine_id = ?",
|
||||
(machine_id,),
|
||||
).fetchone()
|
||||
return {
|
||||
"created": True,
|
||||
"reactivated": False,
|
||||
"agent": dict(row),
|
||||
"token": token,
|
||||
}
|
||||
|
||||
def uninstall(
|
||||
self,
|
||||
*,
|
||||
machine_id: str,
|
||||
reason: str | None = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Marque un agent comme desinstalle (soft delete).
|
||||
|
||||
Returns:
|
||||
Le row mis a jour, ou None si l'agent n'existe pas.
|
||||
"""
|
||||
if not machine_id or not machine_id.strip():
|
||||
raise ValueError("machine_id est obligatoire")
|
||||
machine_id = machine_id.strip()
|
||||
|
||||
now = _utc_now_iso()
|
||||
with _DB_LOCK, self._connect() as conn:
|
||||
existing = conn.execute(
|
||||
"SELECT * FROM enrolled_agents WHERE machine_id = ?",
|
||||
(machine_id,),
|
||||
).fetchone()
|
||||
if existing is None:
|
||||
return None
|
||||
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE enrolled_agents
|
||||
SET status = 'uninstalled',
|
||||
uninstalled_at = ?,
|
||||
uninstall_reason = ?
|
||||
WHERE machine_id = ?
|
||||
""",
|
||||
(now, reason, machine_id),
|
||||
)
|
||||
conn.commit()
|
||||
row = conn.execute(
|
||||
"SELECT * FROM enrolled_agents WHERE machine_id = ?",
|
||||
(machine_id,),
|
||||
).fetchone()
|
||||
return dict(row)
|
||||
|
||||
def touch_last_seen(self, machine_id: str) -> None:
|
||||
"""Met a jour last_seen_at (appel depuis le stream / heartbeat).
|
||||
|
||||
Silencieux si l'agent est inconnu (evite les erreurs sur vieux clients).
|
||||
Ne reactive jamais un agent desinstalle/revoque.
|
||||
"""
|
||||
if not machine_id:
|
||||
return
|
||||
now = _utc_now_iso()
|
||||
with _DB_LOCK, self._connect() as conn:
|
||||
conn.execute(
|
||||
"UPDATE enrolled_agents SET last_seen_at = ? "
|
||||
"WHERE machine_id = ? AND status = 'active'",
|
||||
(now, machine_id),
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
|
||||
class AgentAlreadyEnrolledError(Exception):
|
||||
"""Levee si on tente d'enrouler une machine deja active."""
|
||||
|
||||
def __init__(self, existing_row: Dict[str, Any]):
|
||||
self.existing = existing_row
|
||||
super().__init__(
|
||||
f"machine_id={existing_row.get('machine_id')} deja enrole "
|
||||
f"(status={existing_row.get('status')})"
|
||||
)
|
||||
|
||||
|
||||
class AgentRevokedError(Exception):
|
||||
"""Levee si un administrateur a revoque ce machine_id."""
|
||||
|
||||
def __init__(self, existing_row: Dict[str, Any]):
|
||||
self.existing = existing_row
|
||||
super().__init__(
|
||||
f"machine_id={existing_row.get('machine_id')} revoque "
|
||||
f"(reason={existing_row.get('uninstall_reason')})"
|
||||
)
|
||||
|
||||
|
||||
class FleetEnrollLockedError(Exception):
|
||||
"""Levee si le parc est verrouille (RPA_FLEET_ENROLL_LOCKED) et qu'on tente
|
||||
d'enroler un nouveau machine_id inconnu (WP-B)."""
|
||||
|
||||
def __init__(self, machine_id: str):
|
||||
self.machine_id = machine_id
|
||||
super().__init__(
|
||||
f"enrolement refuse : parc verrouille (RPA_FLEET_ENROLL_LOCKED), "
|
||||
f"machine_id={machine_id} inconnu"
|
||||
)
|
||||
File diff suppressed because it is too large
Load Diff
622
agent_v0/server_v1/chat_interface.py
Normal file
622
agent_v0/server_v1/chat_interface.py
Normal file
@@ -0,0 +1,622 @@
|
||||
"""
|
||||
ChatInterface — Interface de chat conversationnelle pour Léa.
|
||||
|
||||
Permet au TIM (Technicien Information Médicale) de parler à Léa en langage
|
||||
naturel :
|
||||
- "Ouvre le Bloc-notes et écris bonjour"
|
||||
- Léa comprend (TaskPlanner) et propose un plan
|
||||
- Le TIM confirme (ou refuse)
|
||||
- Léa exécute (replay) et envoie des updates de progression
|
||||
- Historique conversationnel conservé par session
|
||||
|
||||
C'est une couche LÉGÈRE au-dessus du TaskPlanner. Toute la logique de
|
||||
compréhension reste dans TaskPlanner — ChatInterface gère uniquement
|
||||
l'état conversationnel, la confirmation et le suivi d'exécution.
|
||||
|
||||
États de la session :
|
||||
idle → en attente d'un message
|
||||
planning → TaskPlanner.understand() en cours
|
||||
awaiting_confirmation → plan prêt, attend la confirmation du TIM
|
||||
executing → replay en cours
|
||||
done → dernier tour terminé (retour à idle au prochain message)
|
||||
error → erreur interne (instruction non comprise, exception…)
|
||||
|
||||
Langue : 100% français (c'est l'interface utilisateur).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Callable, Dict, List, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# =============================================================================
|
||||
# États
|
||||
# =============================================================================
|
||||
|
||||
STATE_IDLE = "idle"
|
||||
STATE_PLANNING = "planning"
|
||||
STATE_AWAITING_CONFIRMATION = "awaiting_confirmation"
|
||||
STATE_EXECUTING = "executing"
|
||||
STATE_DONE = "done"
|
||||
STATE_ERROR = "error"
|
||||
|
||||
VALID_STATES = {
|
||||
STATE_IDLE,
|
||||
STATE_PLANNING,
|
||||
STATE_AWAITING_CONFIRMATION,
|
||||
STATE_EXECUTING,
|
||||
STATE_DONE,
|
||||
STATE_ERROR,
|
||||
}
|
||||
|
||||
# Rôles de messages
|
||||
ROLE_USER = "user"
|
||||
ROLE_LEA = "lea"
|
||||
ROLE_SYSTEM = "system"
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Message
|
||||
# =============================================================================
|
||||
|
||||
@dataclass
|
||||
class ChatMessage:
|
||||
"""Un message dans l'historique d'une conversation."""
|
||||
role: str # "user", "lea", "system"
|
||||
content: str # Texte du message
|
||||
timestamp: float = field(default_factory=time.time)
|
||||
# Données contextuelles optionnelles (plan, résultat, progression…)
|
||||
meta: Dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"role": self.role,
|
||||
"content": self.content,
|
||||
"timestamp": self.timestamp,
|
||||
"meta": self.meta,
|
||||
}
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# ChatSession
|
||||
# =============================================================================
|
||||
|
||||
class ChatSession:
|
||||
"""Une conversation entre un utilisateur et Léa.
|
||||
|
||||
Maintient l'historique, l'état courant, et le dernier plan en attente
|
||||
de confirmation. Thread-safe (un lock par session).
|
||||
|
||||
Dépendances injectées (pour tester facilement) :
|
||||
- task_planner : instance de TaskPlanner (ou mock)
|
||||
- workflows_provider : callable () -> List[Dict] (liste des workflows)
|
||||
- replay_callback : callable (session_id, machine_id, params) -> replay_id
|
||||
- status_provider : callable (replay_id) -> Dict (pour suivre l'exécution)
|
||||
|
||||
Toutes ces dépendances sont optionnelles : ChatSession dégrade
|
||||
gracieusement (fallback) si gemma4 / replay indisponibles.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
session_id: str = "",
|
||||
task_planner: Any = None,
|
||||
workflows_provider: Optional[Callable[[], List[Dict[str, Any]]]] = None,
|
||||
replay_callback: Optional[Callable[..., str]] = None,
|
||||
status_provider: Optional[Callable[[str], Dict[str, Any]]] = None,
|
||||
machine_id: str = "default",
|
||||
):
|
||||
self.session_id = session_id or f"chat_{uuid.uuid4().hex[:12]}"
|
||||
self.machine_id = machine_id
|
||||
self.created_at = time.time()
|
||||
self.updated_at = self.created_at
|
||||
|
||||
self._task_planner = task_planner
|
||||
self._workflows_provider = workflows_provider
|
||||
self._replay_callback = replay_callback
|
||||
self._status_provider = status_provider
|
||||
|
||||
self._state: str = STATE_IDLE
|
||||
self._messages: List[ChatMessage] = []
|
||||
self._pending_plan: Any = None # TaskPlan en attente de confirmation
|
||||
self._active_replay_id: str = "" # Replay courant (si executing)
|
||||
self._last_progress: Dict[str, Any] = {}
|
||||
|
||||
self._lock = threading.RLock()
|
||||
|
||||
# Message d'accueil
|
||||
self._append(
|
||||
ROLE_LEA,
|
||||
"Bonjour ! Je suis Léa. Dites-moi ce que vous voulez que je fasse.",
|
||||
meta={"welcome": True},
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------
|
||||
# Accesseurs
|
||||
# ---------------------------------------------------------------------
|
||||
|
||||
@property
|
||||
def state(self) -> str:
|
||||
with self._lock:
|
||||
return self._state
|
||||
|
||||
def get_history(self) -> List[Dict[str, Any]]:
|
||||
"""Retourne l'historique complet des messages (sérialisé)."""
|
||||
with self._lock:
|
||||
return [m.to_dict() for m in self._messages]
|
||||
|
||||
def get_snapshot(self) -> Dict[str, Any]:
|
||||
"""État complet pour l'UI (historique + état + progression)."""
|
||||
with self._lock:
|
||||
return {
|
||||
"session_id": self.session_id,
|
||||
"state": self._state,
|
||||
"machine_id": self.machine_id,
|
||||
"created_at": self.created_at,
|
||||
"updated_at": self.updated_at,
|
||||
"messages": [m.to_dict() for m in self._messages],
|
||||
"pending_plan": (
|
||||
self._pending_plan.to_dict()
|
||||
if self._pending_plan is not None
|
||||
else None
|
||||
),
|
||||
"active_replay_id": self._active_replay_id,
|
||||
"progress": dict(self._last_progress),
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------
|
||||
# API publique
|
||||
# ---------------------------------------------------------------------
|
||||
|
||||
def send_message(self, text: str) -> Dict[str, Any]:
|
||||
"""Envoyer un message utilisateur.
|
||||
|
||||
Trois cas possibles selon l'état courant :
|
||||
1. awaiting_confirmation → c'est une réponse OUI/NON
|
||||
2. executing → on rafraîchit la progression
|
||||
3. idle/done/error → nouvelle instruction, on appelle TaskPlanner
|
||||
"""
|
||||
text = (text or "").strip()
|
||||
if not text:
|
||||
return {
|
||||
"ok": False,
|
||||
"error": "Message vide",
|
||||
"state": self._state,
|
||||
}
|
||||
|
||||
with self._lock:
|
||||
# Cas 1 : on attend une confirmation
|
||||
if self._state == STATE_AWAITING_CONFIRMATION:
|
||||
return self._handle_confirmation_reply(text)
|
||||
|
||||
# Cas 2 : en pleine exécution → message ajouté mais pas d'action
|
||||
if self._state == STATE_EXECUTING:
|
||||
self._append(ROLE_USER, text)
|
||||
self._append(
|
||||
ROLE_LEA,
|
||||
"Je suis en train d'exécuter le workflow. Un instant…",
|
||||
)
|
||||
return {"ok": True, "state": self._state}
|
||||
|
||||
# Cas 3 : nouvelle instruction
|
||||
self._append(ROLE_USER, text)
|
||||
self._set_state(STATE_PLANNING)
|
||||
|
||||
# Appel TaskPlanner hors du lock (peut être lent : gemma4)
|
||||
return self._plan_and_reply(text)
|
||||
|
||||
def confirm(self, confirmed: bool = True) -> Dict[str, Any]:
|
||||
"""Confirmer (ou refuser) l'exécution du plan en attente."""
|
||||
with self._lock:
|
||||
if self._state != STATE_AWAITING_CONFIRMATION:
|
||||
return {
|
||||
"ok": False,
|
||||
"error": f"Pas de plan en attente (état={self._state})",
|
||||
"state": self._state,
|
||||
}
|
||||
|
||||
if not confirmed:
|
||||
self._append(
|
||||
ROLE_LEA,
|
||||
"D'accord, j'annule. Dites-moi autre chose quand vous voulez.",
|
||||
)
|
||||
self._pending_plan = None
|
||||
self._set_state(STATE_IDLE)
|
||||
return {"ok": True, "state": self._state, "confirmed": False}
|
||||
|
||||
plan = self._pending_plan
|
||||
if plan is None:
|
||||
self._set_state(STATE_IDLE)
|
||||
return {
|
||||
"ok": False,
|
||||
"error": "Aucun plan à confirmer",
|
||||
"state": self._state,
|
||||
}
|
||||
|
||||
self._set_state(STATE_EXECUTING)
|
||||
|
||||
# Exécution hors du lock
|
||||
return self._execute_plan(plan)
|
||||
|
||||
def refresh_progress(self) -> Dict[str, Any]:
|
||||
"""Rafraîchir la progression du replay en cours.
|
||||
|
||||
Appelé par le client (polling) pour obtenir les updates d'exécution.
|
||||
Si le replay est terminé, passe l'état à done.
|
||||
"""
|
||||
with self._lock:
|
||||
if self._state != STATE_EXECUTING or not self._active_replay_id:
|
||||
return {"ok": True, "state": self._state, "progress": self._last_progress}
|
||||
|
||||
replay_id = self._active_replay_id
|
||||
provider = self._status_provider
|
||||
|
||||
if provider is None:
|
||||
return {"ok": True, "state": self._state, "progress": {}}
|
||||
|
||||
try:
|
||||
status = provider(replay_id) or {}
|
||||
except Exception as e:
|
||||
logger.warning(f"ChatSession: status_provider erreur: {e}")
|
||||
status = {}
|
||||
|
||||
with self._lock:
|
||||
self._last_progress = status
|
||||
self.updated_at = time.time()
|
||||
|
||||
# Détection de fin
|
||||
replay_status = str(status.get("status", "")).lower()
|
||||
completed = status.get("completed_actions", 0)
|
||||
total = status.get("total_actions", 0)
|
||||
|
||||
if replay_status in ("done", "completed", "finished", "success"):
|
||||
summary = (
|
||||
f"Workflow terminé ! {completed}/{total} actions réussies."
|
||||
if total
|
||||
else "Workflow terminé."
|
||||
)
|
||||
self._append(ROLE_LEA, summary, meta={"progress": dict(status)})
|
||||
self._set_state(STATE_DONE)
|
||||
self._active_replay_id = ""
|
||||
elif replay_status in ("failed", "error", "aborted"):
|
||||
err = status.get("error") or status.get("message") or "Erreur inconnue"
|
||||
self._append(
|
||||
ROLE_LEA,
|
||||
f"Le workflow a échoué : {err}",
|
||||
meta={"progress": dict(status)},
|
||||
)
|
||||
self._set_state(STATE_ERROR)
|
||||
self._active_replay_id = ""
|
||||
elif replay_status == "paused_need_help":
|
||||
self._append(
|
||||
ROLE_LEA,
|
||||
"Je suis bloquée sur une action, j'ai besoin d'aide…",
|
||||
meta={"progress": dict(status)},
|
||||
)
|
||||
# on reste en executing pour que le TIM puisse reprendre
|
||||
# else : toujours en cours, pas de message
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
"state": self._state,
|
||||
"progress": dict(self._last_progress),
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------
|
||||
# Logique interne
|
||||
# ---------------------------------------------------------------------
|
||||
|
||||
def _plan_and_reply(self, instruction: str) -> Dict[str, Any]:
|
||||
"""Appeler TaskPlanner.understand() et produire une réponse."""
|
||||
plan = None
|
||||
error_msg = ""
|
||||
|
||||
if self._task_planner is None:
|
||||
error_msg = "Planificateur indisponible"
|
||||
else:
|
||||
try:
|
||||
workflows = []
|
||||
if self._workflows_provider is not None:
|
||||
try:
|
||||
workflows = self._workflows_provider() or []
|
||||
except Exception as e:
|
||||
logger.warning(f"ChatSession: workflows_provider erreur: {e}")
|
||||
workflows = []
|
||||
|
||||
plan = self._task_planner.understand(
|
||||
instruction=instruction,
|
||||
available_workflows=workflows,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"ChatSession: TaskPlanner.understand erreur: {e}")
|
||||
error_msg = f"Erreur de compréhension : {e}"
|
||||
|
||||
# Fallback gracieux si pas de plan / gemma4 indisponible
|
||||
if plan is None:
|
||||
with self._lock:
|
||||
self._append(
|
||||
ROLE_LEA,
|
||||
f"Désolée, je n'arrive pas à comprendre pour l'instant. {error_msg}".strip(),
|
||||
meta={"error": error_msg},
|
||||
)
|
||||
self._set_state(STATE_ERROR)
|
||||
return {
|
||||
"ok": False,
|
||||
"state": self._state,
|
||||
"error": error_msg,
|
||||
}
|
||||
|
||||
# Plan non compris
|
||||
if not plan.understood:
|
||||
reason = plan.error or "je n'ai pas compris votre demande"
|
||||
with self._lock:
|
||||
self._append(
|
||||
ROLE_LEA,
|
||||
(
|
||||
f"Désolée, {reason}. "
|
||||
"Pouvez-vous reformuler ? Je connais les workflows que vous m'avez appris."
|
||||
),
|
||||
meta={"plan": plan.to_dict()},
|
||||
)
|
||||
self._set_state(STATE_ERROR)
|
||||
return {
|
||||
"ok": False,
|
||||
"state": self._state,
|
||||
"plan": plan.to_dict(),
|
||||
"error": reason,
|
||||
}
|
||||
|
||||
# Plan compris → formuler la proposition
|
||||
proposal = self._format_proposal(plan)
|
||||
|
||||
with self._lock:
|
||||
self._pending_plan = plan
|
||||
self._append(ROLE_LEA, proposal, meta={"plan": plan.to_dict()})
|
||||
self._set_state(STATE_AWAITING_CONFIRMATION)
|
||||
return {
|
||||
"ok": True,
|
||||
"state": self._state,
|
||||
"plan": plan.to_dict(),
|
||||
"message": proposal,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _format_proposal(plan: Any) -> str:
|
||||
"""Formuler une proposition en français à partir d'un TaskPlan."""
|
||||
lines = []
|
||||
lines.append(f"J'ai compris : « {plan.instruction} ».")
|
||||
|
||||
if plan.workflow_name:
|
||||
conf_pct = int(round((plan.match_confidence or 0.0) * 100))
|
||||
lines.append(
|
||||
f"Je vais utiliser le workflow « {plan.workflow_name} »"
|
||||
f" (confiance {conf_pct}%)."
|
||||
)
|
||||
elif plan.mode == "free" and plan.steps:
|
||||
lines.append(
|
||||
f"Je n'ai pas de workflow enregistré pour ça, "
|
||||
f"mais j'ai planifié {len(plan.steps)} étape(s) :"
|
||||
)
|
||||
for i, step in enumerate(plan.steps[:5], 1):
|
||||
desc = step.get("description", "") if isinstance(step, dict) else str(step)
|
||||
lines.append(f" {i}. {desc}")
|
||||
if len(plan.steps) > 5:
|
||||
lines.append(f" … et {len(plan.steps) - 5} autre(s) étape(s).")
|
||||
else:
|
||||
lines.append("Je n'ai pas de plan d'action clair pour cette demande.")
|
||||
|
||||
if plan.parameters:
|
||||
params_str = ", ".join(f"{k}={v}" for k, v in plan.parameters.items())
|
||||
lines.append(f"Paramètres détectés : {params_str}.")
|
||||
|
||||
if plan.is_loop:
|
||||
src = plan.loop_source or "éléments à traiter"
|
||||
lines.append(f"Traitement en boucle sur : {src}.")
|
||||
|
||||
lines.append("")
|
||||
lines.append("Est-ce que je peux y aller ? (oui / non)")
|
||||
return "\n".join(lines)
|
||||
|
||||
def _handle_confirmation_reply(self, text: str) -> Dict[str, Any]:
|
||||
"""Interpréter un message utilisateur comme OUI/NON."""
|
||||
self._append(ROLE_USER, text)
|
||||
yes_tokens = {"oui", "yes", "ok", "y", "go", "vas-y", "allez", "allez-y", "confirme", "confirmer", "continue"}
|
||||
no_tokens = {"non", "no", "annule", "annuler", "stop", "arrête", "arrete", "abandonne", "abandonner"}
|
||||
|
||||
t = text.strip().lower().rstrip("!.?")
|
||||
|
||||
if t in yes_tokens or any(t.startswith(tok + " ") for tok in yes_tokens):
|
||||
# Déverrouiller : sortir du lock avant d'exécuter (confirm re-prend le lock)
|
||||
pass
|
||||
elif t in no_tokens or any(t.startswith(tok + " ") for tok in no_tokens):
|
||||
self._append(
|
||||
ROLE_LEA,
|
||||
"D'accord, j'annule. Dites-moi autre chose quand vous voulez.",
|
||||
)
|
||||
self._pending_plan = None
|
||||
self._set_state(STATE_IDLE)
|
||||
return {"ok": True, "state": self._state, "confirmed": False}
|
||||
else:
|
||||
self._append(
|
||||
ROLE_LEA,
|
||||
"Je n'ai pas compris votre réponse. Répondez « oui » pour lancer ou « non » pour annuler.",
|
||||
)
|
||||
return {"ok": True, "state": self._state, "needs_clarification": True}
|
||||
|
||||
# Libérer le lock pour confirm() qui le re-prendra
|
||||
plan = self._pending_plan
|
||||
self._pending_plan = None
|
||||
self._set_state(STATE_EXECUTING)
|
||||
# Exécution hors du lock (sortie du with bloc appelant)
|
||||
# Note : _handle_confirmation_reply est appelé sous lock via send_message
|
||||
# On ne peut pas appeler _execute_plan ici sans risque de double-lock.
|
||||
# On relâche le lock via une astuce : on retourne un marqueur et send_message
|
||||
# orchestrera. Ici on appelle directement _execute_plan qui utilise RLock,
|
||||
# donc c'est safe (re-entrant).
|
||||
return self._execute_plan(plan)
|
||||
|
||||
def _execute_plan(self, plan: Any) -> Dict[str, Any]:
|
||||
"""Lancer le replay correspondant au plan."""
|
||||
if plan is None:
|
||||
with self._lock:
|
||||
self._append(ROLE_LEA, "Rien à exécuter.", meta={})
|
||||
self._set_state(STATE_IDLE)
|
||||
return {"ok": False, "state": self._state, "error": "Aucun plan"}
|
||||
|
||||
if self._replay_callback is None:
|
||||
with self._lock:
|
||||
self._append(
|
||||
ROLE_LEA,
|
||||
"Je ne peux pas exécuter : aucun moteur d'exécution n'est configuré.",
|
||||
)
|
||||
self._set_state(STATE_ERROR)
|
||||
return {
|
||||
"ok": False,
|
||||
"state": self._state,
|
||||
"error": "replay_callback non configuré",
|
||||
}
|
||||
|
||||
# Annoncer le démarrage
|
||||
with self._lock:
|
||||
self._append(
|
||||
ROLE_LEA,
|
||||
"C'est parti ! Je lance le workflow…",
|
||||
meta={"plan": plan.to_dict()},
|
||||
)
|
||||
|
||||
# Appeler le callback
|
||||
try:
|
||||
if plan.workflow_match:
|
||||
replay_id = self._replay_callback(
|
||||
session_id=plan.workflow_match,
|
||||
machine_id=self.machine_id,
|
||||
params=plan.parameters,
|
||||
)
|
||||
else:
|
||||
# Mode libre : pas encore branché côté chat (on refuse proprement)
|
||||
replay_id = ""
|
||||
raise RuntimeError(
|
||||
"Mode libre non supporté pour l'instant — "
|
||||
"entraînez un workflow pour cette tâche"
|
||||
)
|
||||
except Exception as e:
|
||||
with self._lock:
|
||||
self._append(
|
||||
ROLE_LEA,
|
||||
f"Je n'ai pas pu lancer le workflow : {e}",
|
||||
meta={"error": str(e)},
|
||||
)
|
||||
self._set_state(STATE_ERROR)
|
||||
return {"ok": False, "state": self._state, "error": str(e)}
|
||||
|
||||
with self._lock:
|
||||
self._active_replay_id = replay_id or ""
|
||||
return {
|
||||
"ok": True,
|
||||
"state": self._state,
|
||||
"replay_id": self._active_replay_id,
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------
|
||||
|
||||
def _append(self, role: str, content: str, meta: Optional[Dict[str, Any]] = None) -> None:
|
||||
"""Ajouter un message à l'historique (doit être appelé sous lock)."""
|
||||
msg = ChatMessage(role=role, content=content, meta=meta or {})
|
||||
self._messages.append(msg)
|
||||
self.updated_at = msg.timestamp
|
||||
|
||||
def _set_state(self, new_state: str) -> None:
|
||||
"""Changer d'état (doit être appelé sous lock)."""
|
||||
if new_state not in VALID_STATES:
|
||||
raise ValueError(f"État invalide : {new_state}")
|
||||
old = self._state
|
||||
self._state = new_state
|
||||
self.updated_at = time.time()
|
||||
if old != new_state:
|
||||
logger.debug(
|
||||
f"ChatSession {self.session_id}: {old} -> {new_state}"
|
||||
)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# ChatManager — registre en mémoire des sessions
|
||||
# =============================================================================
|
||||
|
||||
class ChatManager:
|
||||
"""Registre en mémoire des sessions de chat.
|
||||
|
||||
Thread-safe. Utilisé par l'API FastAPI pour gérer plusieurs
|
||||
conversations simultanées.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
task_planner: Any = None,
|
||||
workflows_provider: Optional[Callable[[], List[Dict[str, Any]]]] = None,
|
||||
replay_callback: Optional[Callable[..., str]] = None,
|
||||
status_provider: Optional[Callable[[str], Dict[str, Any]]] = None,
|
||||
):
|
||||
self._task_planner = task_planner
|
||||
self._workflows_provider = workflows_provider
|
||||
self._replay_callback = replay_callback
|
||||
self._status_provider = status_provider
|
||||
self._sessions: Dict[str, ChatSession] = {}
|
||||
self._lock = threading.RLock()
|
||||
|
||||
def create_session(self, machine_id: str = "default") -> ChatSession:
|
||||
"""Créer une nouvelle session de chat."""
|
||||
session = ChatSession(
|
||||
task_planner=self._task_planner,
|
||||
workflows_provider=self._workflows_provider,
|
||||
replay_callback=self._replay_callback,
|
||||
status_provider=self._status_provider,
|
||||
machine_id=machine_id,
|
||||
)
|
||||
with self._lock:
|
||||
self._sessions[session.session_id] = session
|
||||
logger.info(f"ChatManager: session créée {session.session_id}")
|
||||
return session
|
||||
|
||||
def get_session(self, session_id: str) -> Optional[ChatSession]:
|
||||
with self._lock:
|
||||
return self._sessions.get(session_id)
|
||||
|
||||
def list_sessions(self) -> List[Dict[str, Any]]:
|
||||
with self._lock:
|
||||
return [
|
||||
{
|
||||
"session_id": s.session_id,
|
||||
"state": s.state,
|
||||
"machine_id": s.machine_id,
|
||||
"created_at": s.created_at,
|
||||
"updated_at": s.updated_at,
|
||||
"message_count": len(s.get_history()),
|
||||
}
|
||||
for s in self._sessions.values()
|
||||
]
|
||||
|
||||
def delete_session(self, session_id: str) -> bool:
|
||||
with self._lock:
|
||||
return self._sessions.pop(session_id, None) is not None
|
||||
|
||||
def cleanup_old(self, max_age_s: float = 3600 * 24) -> int:
|
||||
"""Supprimer les sessions inactives depuis max_age_s secondes."""
|
||||
now = time.time()
|
||||
removed = 0
|
||||
with self._lock:
|
||||
to_delete = [
|
||||
sid for sid, s in self._sessions.items()
|
||||
if (now - s.updated_at) > max_age_s
|
||||
]
|
||||
for sid in to_delete:
|
||||
del self._sessions[sid]
|
||||
removed += 1
|
||||
return removed
|
||||
5
agent_v0/server_v1/core/__init__.py
Normal file
5
agent_v0/server_v1/core/__init__.py
Normal file
@@ -0,0 +1,5 @@
|
||||
"""Sous-package `core` du serveur (server_v1).
|
||||
|
||||
Sert de point de montage pour les composants serveur internes
|
||||
(par ex. `dialog/` — DialogResolver MVP R2).
|
||||
"""
|
||||
36
agent_v0/server_v1/core/dialog/__init__.py
Normal file
36
agent_v0/server_v1/core/dialog/__init__.py
Normal file
@@ -0,0 +1,36 @@
|
||||
"""DialogResolver — R2 MVP P0.
|
||||
|
||||
Centralise la résolution des modaux runtime côté serveur via un catalogue
|
||||
``KNOWN_DIALOGS`` (10 entrées P0) + un ``DialogResolver`` qui renvoie une
|
||||
politique stricte ``auto`` / ``pause`` / ``skip``.
|
||||
|
||||
Spec source : ``docs/recherche/SPEC_POPUPS_CATALOGUE.md``.
|
||||
|
||||
Périmètre P0 explicite :
|
||||
- Catalogue minimal 10 entrées (Easily save/overwrite/confirm/clinical-warning,
|
||||
Notepad unsaved, Windows save confirm, Windows file-explorer fallback, UAC,
|
||||
Hello CredUI, browser update).
|
||||
- Validateur déclaratif ``system_modals_cannot_be_overridden`` : refuse toute
|
||||
surcharge ``auto`` / ``skip`` sur un modal SYSTÈME (`windows-` / `defender-`).
|
||||
- Pas de modification d'``executor.py`` (rebranchement côté agent_v1 = P1).
|
||||
"""
|
||||
|
||||
from .catalog import KNOWN_DIALOGS, DialogPolicy, DialogSpec
|
||||
from .resolver import (
|
||||
DialogResolution,
|
||||
DialogResolver,
|
||||
DeclarativeOverride,
|
||||
SystemModalOverrideError,
|
||||
system_modals_cannot_be_overridden,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"KNOWN_DIALOGS",
|
||||
"DialogPolicy",
|
||||
"DialogSpec",
|
||||
"DialogResolver",
|
||||
"DialogResolution",
|
||||
"DeclarativeOverride",
|
||||
"SystemModalOverrideError",
|
||||
"system_modals_cannot_be_overridden",
|
||||
]
|
||||
262
agent_v0/server_v1/core/dialog/catalog.py
Normal file
262
agent_v0/server_v1/core/dialog/catalog.py
Normal file
@@ -0,0 +1,262 @@
|
||||
"""Catalogue des modaux runtime connus — R2 MVP P0.
|
||||
|
||||
Source de vérité unique (côté serveur) pour les 10 entrées P0.
|
||||
Réutilise les patterns présents dans ``agent_v1/core/executor.py``
|
||||
(``_KNOWN_RUNTIME_DIALOGS``, ``_CONTEXTUAL_RUNTIME_DIALOGS``) sans les
|
||||
dupliquer côté agent.
|
||||
|
||||
Format compact : un ``DialogSpec`` par modal, avec :
|
||||
- ``id`` — identifiant kebab-case stable (clé de ``KNOWN_DIALOGS``).
|
||||
- ``title_patterns`` — patterns à matcher dans le titre fenêtre
|
||||
(case/accent-insensitive, voir ``DialogResolver._normalize``).
|
||||
- ``evidence_texts`` — patterns secondaires requis dans l'OCR/UIA
|
||||
des textes visibles (utilisé quand le titre seul est ambigu, ex.
|
||||
Bloc-notes).
|
||||
- ``button_texts`` — labels cibles si ``policy=auto``.
|
||||
- ``policy`` — politique par défaut, trichotomie stricte
|
||||
(``auto`` / ``pause`` / ``skip``).
|
||||
- ``declarative_override`` — autorise un workflow VWB à surcharger
|
||||
``policy`` via ``expected_modal`` ? Toujours ``False`` pour SYSTÈME.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Dict, Literal, Tuple
|
||||
|
||||
# Trichotomie stricte des politiques. Tout autre valeur est interdite.
|
||||
DialogPolicy = Literal["auto", "pause", "skip"]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DialogSpec:
|
||||
"""Description compacte d'un modal connu.
|
||||
|
||||
Frozen pour éviter les mutations accidentelles (le catalogue est
|
||||
une constante globale, partagée entre threads via ``DialogResolver``).
|
||||
"""
|
||||
|
||||
id: str
|
||||
title_patterns: Tuple[str, ...]
|
||||
evidence_texts: Tuple[str, ...] = field(default_factory=tuple)
|
||||
button_texts: Tuple[str, ...] = field(default_factory=tuple)
|
||||
policy: DialogPolicy = "pause"
|
||||
declarative_override: bool = False
|
||||
description: str = ""
|
||||
|
||||
|
||||
# Préfixes d'IDs catalogue qui désignent des modaux SYSTÈME — politique
|
||||
# ``pause`` STRICTE et non surchargeable par un workflow VWB
|
||||
# (cf. SPEC_POPUPS_CATALOGUE.md §3 + validateur).
|
||||
SYSTEM_DIALOG_ID_PREFIXES: Tuple[str, ...] = ("windows-", "defender-")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 10 entrées P0 — démo Urgence_aiva + démo Bloc-notes (replay 4c38dbb8)
|
||||
# ---------------------------------------------------------------------------
|
||||
#
|
||||
# Sémantique :
|
||||
# - les `title_patterns` sont matchés en substring après normalisation
|
||||
# case/accent-insensitive ; quand `evidence_texts` est non vide, AU MOINS
|
||||
# UN pattern doit aussi se retrouver dans les textes fournis (utile pour
|
||||
# Bloc-notes / Notepad dont le titre seul est trop générique).
|
||||
# - `button_texts` n'est utilisé qu'avec `policy="auto"` ; il liste les
|
||||
# labels acceptables (priorité = ordre dans le tuple).
|
||||
#
|
||||
# Important : `windows-file-explorer` est inclus comme *fallback transition*
|
||||
# (replay 4c38dbb8 — clic "Enregistrer" → fenêtre observée
|
||||
# "rpa_vision : Explorateur de fichiers" au lieu de Bloc-notes). On le marque
|
||||
# `pause` pour laisser un humain trancher tant que le contextual matching
|
||||
# côté agent n'a pas rebranché DialogResolver (P1).
|
||||
KNOWN_DIALOGS: Dict[str, DialogSpec] = {
|
||||
"confirm-save-overwrite": DialogSpec(
|
||||
id="confirm-save-overwrite",
|
||||
title_patterns=(
|
||||
"confirmer l'enregistrement",
|
||||
"confirm save as",
|
||||
),
|
||||
button_texts=("Oui", "Yes", "Remplacer", "Replace"),
|
||||
policy="auto",
|
||||
declarative_override=True,
|
||||
description=(
|
||||
"Windows/Easily — confirmation d'écrasement de fichier "
|
||||
"(`Voulez-vous le remplacer ?`)."
|
||||
),
|
||||
),
|
||||
"notepad-unsaved-changes": DialogSpec(
|
||||
id="notepad-unsaved-changes",
|
||||
title_patterns=("bloc-notes", "notepad"),
|
||||
evidence_texts=(
|
||||
"ne pas enregistrer",
|
||||
"don't save",
|
||||
"voulez-vous enregistrer",
|
||||
"do you want to save",
|
||||
),
|
||||
button_texts=("Enregistrer", "Save"),
|
||||
policy="auto",
|
||||
declarative_override=True,
|
||||
description=(
|
||||
"Bloc-notes / Notepad — `Voulez-vous enregistrer les modifications ?` "
|
||||
"Titre seul ambigu → exige une evidence visuelle."
|
||||
),
|
||||
),
|
||||
"windows-file-explorer": DialogSpec(
|
||||
id="windows-file-explorer",
|
||||
title_patterns=(
|
||||
"explorateur de fichiers",
|
||||
"file explorer",
|
||||
),
|
||||
# Pas de button_texts : aucune action auto en P0.
|
||||
policy="pause",
|
||||
declarative_override=True,
|
||||
description=(
|
||||
"Fenêtre Explorateur de fichiers détectée comme premier plan "
|
||||
"alors qu'on attendait Bloc-notes (cas replay 4c38dbb8). "
|
||||
"Fallback `pause` pour escalade humaine en attendant le "
|
||||
"contextual matching côté agent_v1 (P1)."
|
||||
),
|
||||
),
|
||||
"easily-save-unconfirmed": DialogSpec(
|
||||
id="easily-save-unconfirmed",
|
||||
title_patterns=(
|
||||
"easily assure",
|
||||
"easily assure - confirmation",
|
||||
),
|
||||
evidence_texts=(
|
||||
"voulez-vous enregistrer",
|
||||
"enregistrer les modifications",
|
||||
"do you want to save",
|
||||
"unsaved changes",
|
||||
),
|
||||
button_texts=("Enregistrer", "Save"),
|
||||
policy="auto",
|
||||
declarative_override=True,
|
||||
description=(
|
||||
"Easily Assure — Confirmation d'enregistrement avant fermeture "
|
||||
"(placeholder : signature OCR à affiner sur capture réelle)."
|
||||
),
|
||||
),
|
||||
"easily-overwrite-file": DialogSpec(
|
||||
id="easily-overwrite-file",
|
||||
title_patterns=(
|
||||
"confirmer l'enregistrement",
|
||||
"confirm save as",
|
||||
),
|
||||
evidence_texts=(
|
||||
"existe déjà",
|
||||
"voulez-vous le remplacer",
|
||||
"already exists",
|
||||
"overwrite",
|
||||
),
|
||||
button_texts=("Oui", "Yes"),
|
||||
policy="auto",
|
||||
declarative_override=True,
|
||||
description=(
|
||||
"Easily Assure — popup d'écrasement de fichier "
|
||||
"(placeholder : signature OCR à affiner)."
|
||||
),
|
||||
),
|
||||
"easily-confirm-action": DialogSpec(
|
||||
id="easily-confirm-action",
|
||||
title_patterns=("confirmer", "confirm"),
|
||||
evidence_texts=(
|
||||
"êtes-vous sûr",
|
||||
"are you sure",
|
||||
"confirmer l'enregistrement",
|
||||
),
|
||||
button_texts=("Oui", "Yes"),
|
||||
policy="auto",
|
||||
declarative_override=True,
|
||||
description=(
|
||||
"Easily Assure — confirmation générique d'une action métier "
|
||||
"(placeholder)."
|
||||
),
|
||||
),
|
||||
"easily-clinical-warning": DialogSpec(
|
||||
id="easily-clinical-warning",
|
||||
title_patterns=(
|
||||
"avertissement clinique",
|
||||
"easily assure - avertissement",
|
||||
"clinical alert",
|
||||
),
|
||||
evidence_texts=(
|
||||
"attention",
|
||||
"avertissement clinique",
|
||||
"allergie",
|
||||
"contre-indication",
|
||||
"warning",
|
||||
),
|
||||
# Pas de button_texts : la décision est clinique, humaine, par design.
|
||||
policy="pause",
|
||||
declarative_override=False,
|
||||
description=(
|
||||
"Easily Assure — avertissement clinique (allergie, contre-indication). "
|
||||
"Décision médicale OBLIGATOIRE — `pause` non surchargeable."
|
||||
),
|
||||
),
|
||||
"windows-uac": DialogSpec(
|
||||
id="windows-uac",
|
||||
title_patterns=(
|
||||
"contrôle de compte d'utilisateur",
|
||||
"user account control",
|
||||
),
|
||||
evidence_texts=(
|
||||
"voulez-vous autoriser cette application",
|
||||
"do you want to allow this app",
|
||||
),
|
||||
policy="pause",
|
||||
declarative_override=False,
|
||||
description=(
|
||||
"Windows UAC — élévation de privilèges. JAMAIS auto-accept en "
|
||||
"healthtech. `pause` STRICT, non surchargeable par déclaratif workflow."
|
||||
),
|
||||
),
|
||||
"windows-hello-credui": DialogSpec(
|
||||
id="windows-hello-credui",
|
||||
title_patterns=(
|
||||
"sécurité windows",
|
||||
"windows security",
|
||||
),
|
||||
evidence_texts=(
|
||||
"windows hello",
|
||||
"saisissez votre code pin",
|
||||
"enter your pin",
|
||||
"touchez le capteur",
|
||||
"fingerprint",
|
||||
"connectez-vous à votre compte",
|
||||
"sign in to your account",
|
||||
),
|
||||
policy="pause",
|
||||
declarative_override=False,
|
||||
description=(
|
||||
"Windows Hello / CredUI — identification physique requise par "
|
||||
"construction (PIN, empreinte, MFA). `pause` STRICT."
|
||||
),
|
||||
),
|
||||
"edge-update": DialogSpec(
|
||||
id="edge-update",
|
||||
title_patterns=(
|
||||
"microsoft edge",
|
||||
"microsoft edge a été mis à jour",
|
||||
"google chrome",
|
||||
),
|
||||
evidence_texts=(
|
||||
"a été mis à jour",
|
||||
"redémarrer",
|
||||
"relancer",
|
||||
"was updated",
|
||||
"relaunch",
|
||||
),
|
||||
policy="skip",
|
||||
declarative_override=True,
|
||||
description=(
|
||||
"Edge / Chrome — bulle de mise à jour non bloquante "
|
||||
"(ignore par défaut, ne casse pas le workflow)."
|
||||
),
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def is_system_dialog(modal_id: str) -> bool:
|
||||
"""Vrai si le modal appartient à la catégorie SYSTÈME (Windows/Defender)."""
|
||||
return modal_id.startswith(SYSTEM_DIALOG_ID_PREFIXES)
|
||||
264
agent_v0/server_v1/core/dialog/resolver.py
Normal file
264
agent_v0/server_v1/core/dialog/resolver.py
Normal file
@@ -0,0 +1,264 @@
|
||||
"""DialogResolver — R2 MVP P0.
|
||||
|
||||
Match titre + evidence → ``DialogResolution`` (policy stricte + action).
|
||||
Réutilise la normalisation case/accent-insensitive développée pour
|
||||
``ActionExecutorV1._normalize_loose_text`` (executor.py).
|
||||
|
||||
Pas de dépendance Windows : pur Python, testable hors VM.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Dict, Iterable, Mapping, Optional, Sequence
|
||||
|
||||
from .catalog import (
|
||||
KNOWN_DIALOGS,
|
||||
DialogPolicy,
|
||||
DialogSpec,
|
||||
SYSTEM_DIALOG_ID_PREFIXES,
|
||||
is_system_dialog,
|
||||
)
|
||||
|
||||
|
||||
_TRANSLATION_TABLE = str.maketrans(
|
||||
{
|
||||
"’": "'",
|
||||
"‘": "'",
|
||||
"`": "'",
|
||||
"´": "'",
|
||||
"–": "-",
|
||||
"—": "-",
|
||||
"−": "-",
|
||||
"\xa0": " ",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _normalize(value: Optional[str]) -> str:
|
||||
"""Casefold + dé-ambiguïse apostrophes/tirets/non-breaking-space.
|
||||
|
||||
Logique alignée sur ``ActionExecutorV1._normalize_loose_text``
|
||||
(agent_v1/core/executor.py) pour rester cohérent côté agent.
|
||||
"""
|
||||
if not value:
|
||||
return ""
|
||||
normalized = str(value).casefold().translate(_TRANSLATION_TABLE)
|
||||
return " ".join(normalized.split())
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DialogResolution:
|
||||
"""Résultat d'une résolution. Sérialisable JSON via ``to_dict``.
|
||||
|
||||
- ``matched`` : True si un modal du catalogue a été identifié.
|
||||
- ``dialog_id`` : ID catalogue (``""`` si pas de match).
|
||||
- ``policy`` : politique stricte appliquée (``"auto" | "pause" | "skip"``).
|
||||
Quand aucun match : ``"pause"`` par défaut (politique conservative
|
||||
healthtech, cf. SPEC §1.1 règle d'or n°4).
|
||||
- ``action`` : dict décrivant le geste à effectuer si ``policy=="auto"``,
|
||||
``None`` sinon.
|
||||
- ``reason`` : message FR court pour audit / bulle Léa.
|
||||
"""
|
||||
|
||||
matched: bool
|
||||
dialog_id: str
|
||||
policy: DialogPolicy
|
||||
action: Optional[Dict[str, Any]] = None
|
||||
reason: str = ""
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"matched": self.matched,
|
||||
"dialog_id": self.dialog_id,
|
||||
"policy": self.policy,
|
||||
"action": self.action,
|
||||
"reason": self.reason,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DeclarativeOverride:
|
||||
"""Surcharge déclarative remontée par un workflow VWB (``expected_modal``).
|
||||
|
||||
Le ``DialogResolver`` ne consomme cette structure que si la spec de base
|
||||
autorise ``declarative_override=True``. Les modaux SYSTÈME sont rejetés
|
||||
en amont par :func:`system_modals_cannot_be_overridden`.
|
||||
"""
|
||||
|
||||
dialog_id: str
|
||||
policy: DialogPolicy
|
||||
button_label: Optional[str] = None
|
||||
confirm: bool = False
|
||||
|
||||
|
||||
class SystemModalOverrideError(ValueError):
|
||||
"""Lève en cas de tentative de surcharger un modal SYSTÈME en auto/skip."""
|
||||
|
||||
|
||||
def system_modals_cannot_be_overridden(override: DeclarativeOverride) -> DeclarativeOverride:
|
||||
"""Validateur déclaratif (à brancher côté VWB schema + côté API).
|
||||
|
||||
Toute déclaration ``expected_modal`` qui cible un modal SYSTÈME
|
||||
(préfixes ``windows-`` / ``defender-``) ET tente une politique
|
||||
différente de ``"pause"`` est rejetée par construction.
|
||||
|
||||
Cf. SPEC_POPUPS_CATALOGUE.md §3 — règle d'or n°1.
|
||||
"""
|
||||
if is_system_dialog(override.dialog_id) and override.policy != "pause":
|
||||
raise SystemModalOverrideError(
|
||||
f"expected_modal.policy='{override.policy}' interdit pour "
|
||||
f"'{override.dialog_id}' (catégorie SYSTÈME — toujours 'pause' "
|
||||
f"en healthtech)."
|
||||
)
|
||||
return override
|
||||
|
||||
|
||||
class DialogResolver:
|
||||
"""Résolveur de modaux runtime — P0.
|
||||
|
||||
Stateless : peut être instancié une fois côté serveur et appelé en
|
||||
concurrence. La méthode :meth:`resolve` n'effectue aucun I/O.
|
||||
"""
|
||||
|
||||
def __init__(self, catalog: Optional[Mapping[str, DialogSpec]] = None) -> None:
|
||||
# Copie défensive — le caller peut injecter un sous-ensemble pour
|
||||
# les tests sans muter ``KNOWN_DIALOGS``.
|
||||
self._catalog: Dict[str, DialogSpec] = dict(catalog or KNOWN_DIALOGS)
|
||||
|
||||
@property
|
||||
def catalog(self) -> Mapping[str, DialogSpec]:
|
||||
return self._catalog
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# API publique
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def resolve(
|
||||
self,
|
||||
current_title: str,
|
||||
evidence_texts: Optional[Sequence[str]] = None,
|
||||
declarative_override: Optional[DeclarativeOverride] = None,
|
||||
) -> DialogResolution:
|
||||
"""Identifier un modal et calculer sa politique effective.
|
||||
|
||||
- ``current_title`` : titre fenêtre courante (Windows ``GetWindowText``
|
||||
/ Linux ``xdotool getactivewindow getwindowname``).
|
||||
- ``evidence_texts`` : tableau de textes secondaires (OCR/UIA) — sert
|
||||
à lever l'ambiguïté quand le titre seul ne suffit pas (Bloc-notes).
|
||||
- ``declarative_override`` : surcharge VWB. Doit avoir été validée
|
||||
en amont par :func:`system_modals_cannot_be_overridden` ; on
|
||||
le revalide ici par sécurité (défense en profondeur).
|
||||
|
||||
Retourne toujours une ``DialogResolution`` (jamais ``None``).
|
||||
Sans match, politique conservative ``pause``.
|
||||
"""
|
||||
norm_title = _normalize(current_title)
|
||||
norm_evidences = tuple(_normalize(t) for t in (evidence_texts or ()))
|
||||
|
||||
spec = self._find_matching_spec(norm_title, norm_evidences)
|
||||
if spec is None:
|
||||
return DialogResolution(
|
||||
matched=False,
|
||||
dialog_id="",
|
||||
policy="pause",
|
||||
action=None,
|
||||
reason=(
|
||||
"Aucun modal connu n'a matché ce titre/evidence — "
|
||||
"pause conservative (healthtech)."
|
||||
),
|
||||
)
|
||||
|
||||
effective_policy = spec.policy
|
||||
applied_override = False
|
||||
|
||||
if declarative_override and declarative_override.dialog_id == spec.id:
|
||||
# Garde-fou systémique : on rejette toute surcharge SYSTÈME même
|
||||
# si appelée directement sur ``resolve`` (défense en profondeur).
|
||||
system_modals_cannot_be_overridden(declarative_override)
|
||||
if spec.declarative_override:
|
||||
effective_policy = declarative_override.policy
|
||||
applied_override = True
|
||||
|
||||
action = self._build_action(spec, effective_policy, declarative_override if applied_override else None)
|
||||
reason = self._build_reason(spec, effective_policy, applied_override)
|
||||
|
||||
return DialogResolution(
|
||||
matched=True,
|
||||
dialog_id=spec.id,
|
||||
policy=effective_policy,
|
||||
action=action,
|
||||
reason=reason,
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Internes
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _find_matching_spec(
|
||||
self,
|
||||
norm_title: str,
|
||||
norm_evidences: Iterable[str],
|
||||
) -> Optional[DialogSpec]:
|
||||
if not norm_title:
|
||||
return None
|
||||
evidences = tuple(norm_evidences)
|
||||
for spec in self._catalog.values():
|
||||
if not self._title_matches(spec, norm_title):
|
||||
continue
|
||||
if spec.evidence_texts:
|
||||
if not self._evidence_matches(spec, evidences):
|
||||
continue
|
||||
return spec
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _title_matches(spec: DialogSpec, norm_title: str) -> bool:
|
||||
for pattern in spec.title_patterns:
|
||||
norm_pattern = _normalize(pattern)
|
||||
if norm_pattern and norm_pattern in norm_title:
|
||||
return True
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _evidence_matches(spec: DialogSpec, norm_evidences: Sequence[str]) -> bool:
|
||||
for pattern in spec.evidence_texts:
|
||||
norm_pattern = _normalize(pattern)
|
||||
if not norm_pattern:
|
||||
continue
|
||||
for ev in norm_evidences:
|
||||
if norm_pattern in ev:
|
||||
return True
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _build_action(
|
||||
spec: DialogSpec,
|
||||
policy: DialogPolicy,
|
||||
override: Optional[DeclarativeOverride],
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
if policy != "auto":
|
||||
return None
|
||||
# Bouton cible : surcharge déclarative > premier button_text catalogue.
|
||||
button_label = None
|
||||
if override and override.button_label:
|
||||
button_label = override.button_label
|
||||
elif spec.button_texts:
|
||||
button_label = spec.button_texts[0]
|
||||
|
||||
return {
|
||||
"type": "click_button",
|
||||
"button_label": button_label,
|
||||
"fallback_button_labels": list(spec.button_texts),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _build_reason(
|
||||
spec: DialogSpec,
|
||||
policy: DialogPolicy,
|
||||
applied_override: bool,
|
||||
) -> str:
|
||||
base = f"Modal '{spec.id}' identifié — policy={policy}"
|
||||
if applied_override:
|
||||
base += " (surcharge workflow)"
|
||||
return base
|
||||
@@ -3,35 +3,83 @@
|
||||
Contexte métier pour les appels VLM — rend Léa experte du domaine.
|
||||
|
||||
Chaque workflow est associé à un domaine métier (médical, comptable, etc.)
|
||||
qui enrichit TOUS les prompts VLM (Observer, Critic, acteur, enrichissement).
|
||||
qui enrichit TOUS les prompts VLM (Observer, Critic, acteur, enrichissement)
|
||||
ET la personnalité de Léa (résumés, questions de clarification, rapports).
|
||||
|
||||
Un gemma4 qui sait qu'il regarde un DPI et que l'utilisateur fait du codage
|
||||
CIM-10 prend des décisions bien meilleures qu'un VLM générique.
|
||||
CIM-10 prend des décisions bien meilleures qu'un VLM générique. Et Léa qui
|
||||
dit "J'ai codé 14 dossiers sur 15" plutôt que "J'ai exécuté 112 clics" est
|
||||
bien plus utile pour un TIM.
|
||||
|
||||
Premier domaine : TIM (Technicien d'Information Médicale)
|
||||
- Logiciels DPI/DMS (dossier patient informatisé)
|
||||
- Codage CIM-10 / CCAM / GHM
|
||||
- Lecture de comptes rendus médicaux
|
||||
- Validation des séjours / RSS / RSA
|
||||
Domaines pré-configurés :
|
||||
- tim_codage : TIM, codage CIM-10 / CCAM / PMSI, DPI
|
||||
- comptabilite : factures, TVA, OCR, plans comptables
|
||||
- rh_paie : fiches de paie, employés, charges sociales
|
||||
- stocks_logistique : bons, commandes, réceptions, inventaires
|
||||
- generic : fallback bureautique
|
||||
|
||||
Usage :
|
||||
Usage basique :
|
||||
ctx = get_domain_context("tim_codage")
|
||||
prompt = f"{ctx.system_prompt}\n\n{user_prompt}"
|
||||
prompt = ctx.enrich_prompt(user_prompt, role="actor")
|
||||
|
||||
Usage langage métier :
|
||||
ctx = get_domain_context("tim_codage")
|
||||
phrase = ctx.summarize_action("click", {"target": "DP"})
|
||||
# → "saisir le diagnostic principal"
|
||||
|
||||
question = ctx.pose_clarification_question(
|
||||
{"blocked_on": "target_not_found", "target": "Fichier patient",
|
||||
"params": {"nom_patient": "Mme Durand"}}
|
||||
)
|
||||
# → "Je ne trouve pas le dossier de Mme Durand..."
|
||||
|
||||
rapport = ctx.describe_workflow_outcome(
|
||||
workflow_name="Codage séjours janvier",
|
||||
success=True,
|
||||
items_count=15,
|
||||
failed_count=1,
|
||||
)
|
||||
# → "J'ai codé 14 dossiers sur 15..."
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import unicodedata
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Dict, List, Optional
|
||||
from typing import Any, Dict, List, Mapping, Optional
|
||||
|
||||
from core.detection import vlm_config
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _strip_accents(s: str) -> str:
|
||||
"""Supprimer les accents pour les comparaisons insensibles aux diacritiques."""
|
||||
if not s:
|
||||
return ""
|
||||
nkfd = unicodedata.normalize("NFKD", s)
|
||||
return "".join(c for c in nkfd if not unicodedata.combining(c))
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Data class
|
||||
# =========================================================================
|
||||
|
||||
|
||||
@dataclass
|
||||
class DomainContext:
|
||||
"""Contexte métier pour un domaine spécifique."""
|
||||
domain_id: str # Identifiant unique (tim_codage, comptabilite, etc.)
|
||||
name: str # Nom lisible (Codage médical TIM)
|
||||
description: str # Description courte du métier
|
||||
"""Contexte métier pour un domaine spécifique.
|
||||
|
||||
Contient à la fois les hints pour les prompts VLM et les éléments de
|
||||
personnalité de Léa (langage métier, questions, rapports).
|
||||
"""
|
||||
|
||||
domain_id: str # tim_codage, comptabilite, ...
|
||||
name: str # Nom lisible
|
||||
description: str # Description courte
|
||||
|
||||
# Prompt système injecté dans TOUS les appels VLM
|
||||
system_prompt: str = ""
|
||||
@@ -39,18 +87,47 @@ class DomainContext:
|
||||
# Vocabulaire métier (termes que le VLM doit connaître)
|
||||
vocabulary: List[str] = field(default_factory=list)
|
||||
|
||||
# Applications connues (noms de logiciels que le VLM peut rencontrer)
|
||||
# Applications connues
|
||||
known_apps: List[str] = field(default_factory=list)
|
||||
|
||||
# Écrans types (descriptions des écrans courants du métier)
|
||||
# Écrans types
|
||||
screen_patterns: Dict[str, str] = field(default_factory=dict)
|
||||
|
||||
# --- Personnalité Léa -------------------------------------------------
|
||||
|
||||
# Mapping d'actions techniques (click/type/key_combo) vers description métier,
|
||||
# indexé par un mot-clé lisible trouvé dans la cible/texte.
|
||||
# Format : { (action_type, keyword_lower) : "description métier" }
|
||||
# Exemple : ("click", "dp") → "saisir le diagnostic principal"
|
||||
common_actions: Dict[str, str] = field(default_factory=dict)
|
||||
|
||||
# Synonymes métier : technique → forme lisible
|
||||
# Exemple : {"dp": "diagnostic principal", "das": "diagnostics associés"}
|
||||
vocabulary_synonyms: Dict[str, str] = field(default_factory=dict)
|
||||
|
||||
# Templates de questions de clarification (selon la raison de blocage).
|
||||
# Clé = identifiant de blocage ("target_not_found", "ambiguous_field", ...)
|
||||
# Valeur = template f-string (champs: {target}, {app}, {nom_patient}, ...)
|
||||
clarification_templates: Dict[str, str] = field(default_factory=dict)
|
||||
|
||||
# Templates de résumés de fin de workflow.
|
||||
# Clés attendues :
|
||||
# - "success" : tout a marché
|
||||
# - "partial" : succès partiel (failed_count > 0)
|
||||
# - "failure" : échec complet
|
||||
# - "success_one" : cas 1 élément (pour éviter "1 dossiers")
|
||||
# - "item_singular" : libellé d'un item ("dossier")
|
||||
# - "item_plural" : libellé au pluriel ("dossiers")
|
||||
summary_templates: Dict[str, str] = field(default_factory=dict)
|
||||
|
||||
# ------------------------------------------------------------------ API
|
||||
|
||||
def enrich_prompt(self, prompt: str, role: str = "") -> str:
|
||||
"""Enrichir un prompt avec le contexte métier.
|
||||
|
||||
Args:
|
||||
prompt: Le prompt original
|
||||
role: Le rôle du VLM (observer, critic, actor, enrichment)
|
||||
role: Le rôle du VLM (observer, critic, actor, enrichment)
|
||||
"""
|
||||
parts = []
|
||||
|
||||
@@ -65,6 +142,313 @@ class DomainContext:
|
||||
parts.append(prompt)
|
||||
return "\n\n".join(parts)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Personnalité : résumé d'action en langage métier
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def summarize_action(
|
||||
self,
|
||||
action: str,
|
||||
params: Optional[Mapping[str, Any]] = None,
|
||||
) -> str:
|
||||
"""Résumer une action technique en langage métier.
|
||||
|
||||
Args:
|
||||
action: Type d'action ("click", "type", "key_combo", "wait", "scroll")
|
||||
params: Paramètres de l'action (target, text, keys, ...)
|
||||
|
||||
Returns:
|
||||
Phrase en français orientée métier. Fallback générique si aucun
|
||||
mapping ne correspond.
|
||||
|
||||
Exemples (domaine tim_codage) :
|
||||
click sur "DP" → "saisir le diagnostic principal"
|
||||
type "E11.9" → "saisir le code CIM-10 E11.9"
|
||||
click sur "Valider" → "valider le codage"
|
||||
"""
|
||||
params = dict(params or {})
|
||||
target = str(params.get("target") or params.get("description") or "").strip()
|
||||
text = str(params.get("text") or "").strip()
|
||||
keys = params.get("keys") or []
|
||||
|
||||
haystack = _strip_accents(f"{target} {text}".lower())
|
||||
|
||||
# 1) Essayer un match mot-clé dans common_actions.
|
||||
# Clés sous la forme "click:mot" ou "type:mot".
|
||||
# Comparaison insensible à la casse ET aux accents.
|
||||
for key, label in self.common_actions.items():
|
||||
if ":" not in key:
|
||||
continue
|
||||
k_action, k_word = key.split(":", 1)
|
||||
if k_action != action:
|
||||
continue
|
||||
k_word_norm = _strip_accents(k_word.lower())
|
||||
if k_word_norm and k_word_norm in haystack:
|
||||
return label
|
||||
|
||||
# 2) Essayer une substitution via vocabulary_synonyms dans la cible.
|
||||
friendly_target = self._apply_synonyms(target)
|
||||
|
||||
if action == "click":
|
||||
if friendly_target:
|
||||
return f"cliquer sur {friendly_target}"
|
||||
return "cliquer"
|
||||
|
||||
if action == "type":
|
||||
if text and friendly_target:
|
||||
return f"saisir « {text} » dans {friendly_target}"
|
||||
if text:
|
||||
return f"saisir « {text} »"
|
||||
return "saisir du texte"
|
||||
|
||||
if action == "key_combo":
|
||||
if isinstance(keys, (list, tuple)) and keys:
|
||||
return f"utiliser le raccourci {'+'.join(str(k) for k in keys)}"
|
||||
return "utiliser un raccourci clavier"
|
||||
|
||||
if action == "wait":
|
||||
return "attendre le chargement de l'écran"
|
||||
|
||||
if action == "scroll":
|
||||
return "faire défiler l'écran"
|
||||
|
||||
# Fallback ultime
|
||||
return f"effectuer l'action {action}"
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Personnalité : question de clarification
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def pose_clarification_question(
|
||||
self,
|
||||
context: Optional[Mapping[str, Any]] = None,
|
||||
) -> str:
|
||||
"""Générer une question pertinente quand Léa bloque.
|
||||
|
||||
Cherche un template dans clarification_templates selon :
|
||||
- context["blocked_on"] (ex: "target_not_found", "ambiguous_field")
|
||||
- context["target"] (la cible visée)
|
||||
- paramètres du workflow (params) disponibles pour substitution
|
||||
|
||||
Args:
|
||||
context: Dictionnaire libre contenant au minimum 'blocked_on' ou
|
||||
'target'. Peut contenir 'params' pour la substitution.
|
||||
|
||||
Returns:
|
||||
Question en français. Fallback générique si aucun template ne
|
||||
correspond.
|
||||
"""
|
||||
ctx = dict(context or {})
|
||||
blocked_on = str(ctx.get("blocked_on") or "").strip()
|
||||
target = str(ctx.get("target") or "").strip()
|
||||
params = dict(ctx.get("params") or {})
|
||||
|
||||
# Dictionnaire de substitution : champs du context + params + target
|
||||
subs: Dict[str, Any] = {
|
||||
"target": target,
|
||||
"target_friendly": self._apply_synonyms(target) or target or "cet élément",
|
||||
"app": ctx.get("app", ""),
|
||||
}
|
||||
subs.update(params)
|
||||
|
||||
# 1) Essai par clé exacte de blocage
|
||||
template = self.clarification_templates.get(blocked_on, "")
|
||||
|
||||
# 2) Essai par cible (si la cible matche un mot-clé connu)
|
||||
if not template and target:
|
||||
low = target.lower()
|
||||
for key, tpl in self.clarification_templates.items():
|
||||
if key.startswith("target:") and key.split(":", 1)[1].lower() in low:
|
||||
template = tpl
|
||||
break
|
||||
|
||||
# 3) Template générique du domaine
|
||||
if not template:
|
||||
template = self.clarification_templates.get("default", "")
|
||||
|
||||
if template:
|
||||
try:
|
||||
return template.format_map(_SafeDict(subs))
|
||||
except Exception as e: # pragma: no cover - format inattendu
|
||||
logger.warning("clarification template format error: %s", e)
|
||||
|
||||
# 4) Fallback ultime cross-domaine
|
||||
friendly = subs["target_friendly"]
|
||||
return (
|
||||
f"Je ne trouve pas {friendly}. "
|
||||
f"Peux-tu me le montrer ou me confirmer que c'est le bon écran ?"
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Personnalité : rapport final
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def describe_workflow_outcome(
|
||||
self,
|
||||
workflow_name: str = "",
|
||||
success: bool = True,
|
||||
items_count: int = 1,
|
||||
failed_count: int = 0,
|
||||
elapsed_s: float = 0.0,
|
||||
extra: Optional[Mapping[str, Any]] = None,
|
||||
use_llm: bool = False,
|
||||
) -> str:
|
||||
"""Générer un rapport de fin de workflow en langage métier.
|
||||
|
||||
Args:
|
||||
workflow_name: Nom du workflow ("Codage janvier").
|
||||
success: True si le workflow a globalement réussi.
|
||||
items_count: Nombre d'items traités (ex: 15 dossiers). 1 par défaut.
|
||||
failed_count: Nombre d'items en échec.
|
||||
elapsed_s: Durée totale (secondes).
|
||||
extra: Infos supplémentaires (hint pour le LLM).
|
||||
use_llm: Si True, tenter un appel à gemma4 pour produire le
|
||||
rapport. Fallback sur les templates en cas d'échec.
|
||||
|
||||
Returns:
|
||||
Rapport en français. Toujours une chaîne, jamais None.
|
||||
"""
|
||||
extra = dict(extra or {})
|
||||
done = max(0, items_count - failed_count)
|
||||
|
||||
item_sg = self.summary_templates.get("item_singular", "élément")
|
||||
item_pl = self.summary_templates.get("item_plural", "éléments")
|
||||
item_word = item_sg if done <= 1 else item_pl
|
||||
|
||||
# Données disponibles pour les templates
|
||||
subs = {
|
||||
"workflow_name": workflow_name or "le workflow",
|
||||
"items_count": items_count,
|
||||
"done": done,
|
||||
"failed": failed_count,
|
||||
"item_singular": item_sg,
|
||||
"item_plural": item_pl,
|
||||
"item_word": item_word,
|
||||
"elapsed_s": int(elapsed_s),
|
||||
}
|
||||
subs.update(extra)
|
||||
|
||||
# Choisir le template adéquat
|
||||
if not success and failed_count >= items_count:
|
||||
key = "failure"
|
||||
elif failed_count > 0:
|
||||
key = "partial"
|
||||
elif items_count == 1:
|
||||
key = "success_one" if "success_one" in self.summary_templates else "success"
|
||||
else:
|
||||
key = "success"
|
||||
|
||||
template = self.summary_templates.get(key, "")
|
||||
|
||||
# Optionnel : raffiner via gemma4
|
||||
if use_llm:
|
||||
llm_text = self._llm_refine_summary(template, subs, success)
|
||||
if llm_text:
|
||||
return llm_text
|
||||
|
||||
if template:
|
||||
try:
|
||||
return template.format_map(_SafeDict(subs))
|
||||
except Exception as e: # pragma: no cover
|
||||
logger.warning("summary template format error: %s", e)
|
||||
|
||||
# Fallback générique
|
||||
if success:
|
||||
if items_count <= 1:
|
||||
return f"C'est fait, j'ai terminé « {workflow_name or 'le workflow'} »."
|
||||
return (
|
||||
f"J'ai traité {done} {item_word} sur {items_count}"
|
||||
+ (f", {failed_count} en échec." if failed_count else ".")
|
||||
)
|
||||
return (
|
||||
f"Je n'ai pas pu terminer « {workflow_name or 'le workflow'} ». "
|
||||
f"Je te rends la main."
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Helpers internes
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _apply_synonyms(self, text: str) -> str:
|
||||
"""Remplacer les sigles/termes techniques par leur forme métier.
|
||||
|
||||
Cherche mots entiers (word boundaries) en insensible à la casse.
|
||||
"""
|
||||
if not text or not self.vocabulary_synonyms:
|
||||
return text
|
||||
result = text
|
||||
for short, full in self.vocabulary_synonyms.items():
|
||||
if not short:
|
||||
continue
|
||||
pattern = r"\b" + re.escape(short) + r"\b"
|
||||
result = re.sub(pattern, full, result, flags=re.IGNORECASE)
|
||||
return result
|
||||
|
||||
def _llm_refine_summary(
|
||||
self,
|
||||
template: str,
|
||||
subs: Dict[str, Any],
|
||||
success: bool,
|
||||
) -> str:
|
||||
"""Tenter un raffinement du rapport via gemma4.
|
||||
|
||||
Appel best-effort : toute erreur retourne "" et le caller retombe sur
|
||||
le template brut. Isolé dans une méthode pour pouvoir le monkey-patcher
|
||||
dans les tests.
|
||||
"""
|
||||
try:
|
||||
import requests as _requests
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
# Endpoint VLM : piloté par config (Ollama local ou tunnel DGX = 11434).
|
||||
# GEMMA4_PORT conservé comme override legacy (ancien conteneur Docker 11435).
|
||||
_default_port = vlm_config.DEFAULT_OLLAMA_ENDPOINT.rsplit(":", 1)[-1]
|
||||
port = os.environ.get("GEMMA4_PORT", _default_port)
|
||||
url = f"http://localhost:{port}/api/chat"
|
||||
|
||||
base = ""
|
||||
if template:
|
||||
try:
|
||||
base = template.format_map(_SafeDict(subs))
|
||||
except Exception:
|
||||
base = ""
|
||||
|
||||
prompt = (
|
||||
f"Tu es Léa, une assistante RPA dans le domaine : {self.name}.\n"
|
||||
f"Tu viens de terminer un workflow. Résume en UNE à DEUX phrases "
|
||||
f"en langage métier, chaleureux mais professionnel, en français.\n\n"
|
||||
f"Données :\n"
|
||||
f"- workflow : {subs.get('workflow_name', '')}\n"
|
||||
f"- items traités : {subs.get('done', 0)} / {subs.get('items_count', 0)}\n"
|
||||
f"- échecs : {subs.get('failed', 0)}\n"
|
||||
f"- succès global : {'oui' if success else 'non'}\n"
|
||||
f"- durée : {subs.get('elapsed_s', 0)}s\n\n"
|
||||
f"Base suggérée (tu peux la reformuler) : {base or '(aucune)'}\n\n"
|
||||
f"Ta phrase :"
|
||||
)
|
||||
|
||||
try:
|
||||
resp = _requests.post(
|
||||
url,
|
||||
json={
|
||||
"model": vlm_config.get_vlm_model(),
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
"stream": False,
|
||||
"options": {"temperature": 0.3, "num_predict": 200},
|
||||
},
|
||||
timeout=30,
|
||||
)
|
||||
if not resp.ok:
|
||||
return ""
|
||||
content = resp.json().get("message", {}).get("content", "").strip()
|
||||
# Nettoyage basique : supprimer guillemets typographiques en bord
|
||||
content = content.strip("\"' \n")
|
||||
return content
|
||||
except Exception as e:
|
||||
logger.debug("gemma4 refine summary failed: %s", e)
|
||||
return ""
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"domain_id": self.domain_id,
|
||||
@@ -72,9 +456,24 @@ class DomainContext:
|
||||
"description": self.description,
|
||||
"known_apps": self.known_apps,
|
||||
"vocabulary_count": len(self.vocabulary),
|
||||
"common_actions_count": len(self.common_actions),
|
||||
"has_clarification_templates": bool(self.clarification_templates),
|
||||
"has_summary_templates": bool(self.summary_templates),
|
||||
}
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Utilitaires
|
||||
# =========================================================================
|
||||
|
||||
|
||||
class _SafeDict(dict):
|
||||
"""dict pour str.format_map qui retourne "" pour les clés manquantes."""
|
||||
|
||||
def __missing__(self, key): # type: ignore[override]
|
||||
return ""
|
||||
|
||||
|
||||
# Hints par rôle VLM — adaptés au contexte métier
|
||||
_ROLE_HINTS = {
|
||||
"observer": (
|
||||
@@ -100,6 +499,7 @@ _ROLE_HINTS = {
|
||||
# Domaines pré-configurés
|
||||
# =========================================================================
|
||||
|
||||
|
||||
_TIM_CODAGE = DomainContext(
|
||||
domain_id="tim_codage",
|
||||
name="Codage médical TIM",
|
||||
@@ -156,8 +556,405 @@ _TIM_CODAGE = DomainContext(
|
||||
"recherche_code": "Recherche de code CIM-10 ou CCAM (champ de recherche + arborescence)",
|
||||
"validation_ghm": "Écran de validation du groupage avec GHM calculé et valorisation",
|
||||
},
|
||||
vocabulary_synonyms={
|
||||
"DP": "diagnostic principal",
|
||||
"DAS": "diagnostics associés",
|
||||
"CMA": "complication associée",
|
||||
"UM": "unité médicale",
|
||||
"CR": "compte rendu",
|
||||
"RSS": "résumé de sortie",
|
||||
"RSA": "résumé anonymisé",
|
||||
"GHM": "groupe homogène de malades",
|
||||
"IPP": "identifiant patient",
|
||||
},
|
||||
common_actions={
|
||||
"click:dp": "saisir le diagnostic principal",
|
||||
"click:diagnostic principal": "saisir le diagnostic principal",
|
||||
"click:das": "ajouter un diagnostic associé",
|
||||
"click:ccam": "saisir un acte CCAM",
|
||||
"click:valider": "valider le codage",
|
||||
"click:valider le codage": "valider le codage",
|
||||
"click:grouper": "calculer le GHM",
|
||||
"click:ghm": "consulter le groupage GHM",
|
||||
"click:dossier patient": "ouvrir le dossier patient",
|
||||
"click:fiche patient": "ouvrir la fiche patient",
|
||||
"click:compte rendu": "consulter le compte rendu",
|
||||
"click:cr": "consulter le compte rendu",
|
||||
"click:rechercher": "rechercher un code CIM-10",
|
||||
"type:cim": "saisir un code CIM-10",
|
||||
},
|
||||
clarification_templates={
|
||||
"default": (
|
||||
"Je ne trouve pas {target_friendly}. "
|
||||
"Tu peux me montrer où il se trouve dans le dossier ?"
|
||||
),
|
||||
"target_not_found": (
|
||||
"Je ne trouve pas {target_friendly}. "
|
||||
"Le dossier de {nom_patient} est peut-être déjà codé ou archivé ?"
|
||||
),
|
||||
"target:fichier patient": (
|
||||
"Je ne trouve pas le dossier de {nom_patient}. "
|
||||
"Il est peut-être archivé ? Tu peux me le montrer ?"
|
||||
),
|
||||
"target:dossier": (
|
||||
"Je ne trouve pas le dossier de {nom_patient}. "
|
||||
"Il est peut-être archivé ? Tu peux me le montrer ?"
|
||||
),
|
||||
"ambiguous_code": (
|
||||
"Le compte rendu mentionne plusieurs codes possibles. "
|
||||
"Est-ce le code CIM-10 {code_a} ou {code_b} que tu préfères ?"
|
||||
),
|
||||
"no_cr": (
|
||||
"Je ne trouve pas de compte rendu pour {nom_patient}. "
|
||||
"Tu veux que je saute ce dossier ou que je continue sans ?"
|
||||
),
|
||||
},
|
||||
summary_templates={
|
||||
"item_singular": "dossier",
|
||||
"item_plural": "dossiers",
|
||||
"success_one": (
|
||||
"J'ai codé le dossier de {nom_patient} en {elapsed_s}s. "
|
||||
"Tu peux vérifier le groupage GHM."
|
||||
),
|
||||
"success": (
|
||||
"J'ai codé {done} dossiers sur {items_count}. "
|
||||
"Tout est passé sans erreur, tu peux valider le groupage."
|
||||
),
|
||||
"partial": (
|
||||
"J'ai codé {done} dossiers sur {items_count}. "
|
||||
"{failed} sont en attente — codes CIM-10 ambigus, à valider manuellement."
|
||||
),
|
||||
"failure": (
|
||||
"Je n'ai pas pu coder les dossiers de {workflow_name}. "
|
||||
"Je te rends la main, les comptes rendus sont peut-être inaccessibles."
|
||||
),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
_COMPTABILITE = DomainContext(
|
||||
domain_id="comptabilite",
|
||||
name="Comptabilité",
|
||||
description=(
|
||||
"Comptable : saisie de factures fournisseurs et clients, lettrage, "
|
||||
"rapprochement bancaire, déclarations de TVA, bilans, immobilisations."
|
||||
),
|
||||
system_prompt=(
|
||||
"Tu es un assistant expert en comptabilité d'entreprise. "
|
||||
"L'utilisateur est un comptable qui utilise un logiciel de saisie comptable "
|
||||
"(Sage, Cegid, EBP, Quadra, Isacompta) pour saisir des factures, faire "
|
||||
"les rapprochements bancaires, préparer la TVA et les bilans.\n\n"
|
||||
"Vocabulaire du métier :\n"
|
||||
"- Facture : justificatif de vente ou d'achat (numéro, date, HT, TVA, TTC)\n"
|
||||
"- HT/TVA/TTC : montants hors taxes, taxe, toutes taxes\n"
|
||||
"- Compte comptable : numéro du plan comptable général (PCG), ex 401 (fournisseurs), 411 (clients)\n"
|
||||
"- Journal : journal de saisie (achats, ventes, banque, OD)\n"
|
||||
"- Lettrage : association d'une facture avec son paiement\n"
|
||||
"- Rapprochement : comparaison compte comptable / relevé bancaire\n"
|
||||
"- OCR / LAD : reconnaissance automatique des factures scannées\n"
|
||||
"- Écriture : ligne comptable (débit/crédit)\n"
|
||||
"- Exercice : période comptable annuelle\n"
|
||||
"- Bilan / compte de résultat : états financiers\n"
|
||||
"- CA : chiffre d'affaires\n\n"
|
||||
"Écrans courants :\n"
|
||||
"- Saisie d'écritures (numéro de compte, libellé, débit, crédit)\n"
|
||||
"- Import OCR de factures fournisseurs\n"
|
||||
"- Lettrage / rapprochement\n"
|
||||
"- Brouillard / journal\n"
|
||||
"- Balance / grand livre"
|
||||
),
|
||||
vocabulary=[
|
||||
"facture", "HT", "TVA", "TTC", "compte", "journal", "lettrage",
|
||||
"rapprochement", "OCR", "LAD", "écriture", "débit", "crédit",
|
||||
"exercice", "bilan", "compte de résultat", "CA", "immobilisation",
|
||||
"fournisseur", "client", "PCG", "plan comptable",
|
||||
],
|
||||
known_apps=[
|
||||
"Sage", "Cegid", "EBP", "Quadra", "Isacompta", "Ciel Compta",
|
||||
"Odoo", "Pennylane", "Dext", "Agicap",
|
||||
],
|
||||
screen_patterns={
|
||||
"saisie_ecriture": "Saisie d'écriture comptable (compte, libellé, débit, crédit)",
|
||||
"ocr_facture": "Import OCR : zone image + champs extraits (numéro, date, HT, TVA, TTC, fournisseur)",
|
||||
"lettrage": "Liste d'écritures à lettrer (débit vs crédit)",
|
||||
"rapprochement": "Comparaison compte banque / relevé",
|
||||
"balance": "Balance comptable (comptes agrégés avec soldes)",
|
||||
},
|
||||
vocabulary_synonyms={
|
||||
"HT": "montant hors taxes",
|
||||
"TVA": "montant de TVA",
|
||||
"TTC": "montant toutes taxes",
|
||||
"CA": "chiffre d'affaires",
|
||||
"PCG": "plan comptable général",
|
||||
"OD": "opération diverse",
|
||||
},
|
||||
common_actions={
|
||||
"click:valider": "valider l'écriture",
|
||||
"click:enregistrer": "enregistrer la saisie",
|
||||
"click:lettrer": "lettrer les écritures",
|
||||
"click:rapprocher": "rapprocher avec la banque",
|
||||
"click:ocr": "lancer la reconnaissance OCR",
|
||||
"click:facture": "ouvrir la facture",
|
||||
"click:compte": "sélectionner le compte comptable",
|
||||
"type:ht": "saisir le montant hors taxes",
|
||||
"type:tva": "saisir le montant de TVA",
|
||||
"type:ttc": "saisir le montant toutes taxes",
|
||||
},
|
||||
clarification_templates={
|
||||
"default": (
|
||||
"Je ne trouve pas {target_friendly}. "
|
||||
"C'est bien la facture {num_facture} que tu veux saisir ?"
|
||||
),
|
||||
"target_not_found": (
|
||||
"Je ne trouve pas le champ {target_friendly}. "
|
||||
"C'est bien la facture {num_facture} qui doit être saisie ?"
|
||||
),
|
||||
"target:montant": (
|
||||
"Je ne trouve pas le champ « Montant HT ». "
|
||||
"C'est bien la facture {num_facture} que tu veux saisir ?"
|
||||
),
|
||||
"target:tva": (
|
||||
"Je ne trouve pas le champ TVA. Est-ce une facture à taux {taux_tva} % ?"
|
||||
),
|
||||
"ambiguous_account": (
|
||||
"Je ne sais pas sur quel compte imputer : {compte_a} ou {compte_b} ?"
|
||||
),
|
||||
},
|
||||
summary_templates={
|
||||
"item_singular": "facture",
|
||||
"item_plural": "factures",
|
||||
"success_one": (
|
||||
"J'ai saisi la facture {num_facture} en {elapsed_s}s."
|
||||
),
|
||||
"success": (
|
||||
"J'ai saisi {done} factures sur {items_count}. "
|
||||
"Tout est en brouillard, tu peux valider."
|
||||
),
|
||||
"partial": (
|
||||
"J'ai saisi {done} factures sur {items_count}. "
|
||||
"{failed} factures sont en attente — imputation comptable à vérifier."
|
||||
),
|
||||
"failure": (
|
||||
"Je n'ai pas pu saisir les factures de {workflow_name}. "
|
||||
"L'OCR n'a peut-être pas fonctionné, je te rends la main."
|
||||
),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
_RH_PAIE = DomainContext(
|
||||
domain_id="rh_paie",
|
||||
name="Ressources humaines et paie",
|
||||
description=(
|
||||
"Gestionnaire RH / paie : fiches employés, contrats, bulletins de salaire, "
|
||||
"déclarations sociales (DSN), charges, congés, absences."
|
||||
),
|
||||
system_prompt=(
|
||||
"Tu es un assistant expert en gestion RH et paie française. "
|
||||
"L'utilisateur est un gestionnaire RH ou de paie qui utilise un logiciel "
|
||||
"(Silae, Sage Paie, Cegid, ADP, PayFit) pour éditer des bulletins de salaire, "
|
||||
"gérer les contrats, les absences, et envoyer les DSN.\n\n"
|
||||
"Vocabulaire du métier :\n"
|
||||
"- Bulletin de paie : fiche de salaire mensuelle\n"
|
||||
"- DSN : Déclaration Sociale Nominative (mensuelle, transmise à l'URSSAF)\n"
|
||||
"- Brut / Net : salaire avant et après charges\n"
|
||||
"- Charges sociales / patronales : cotisations employeur et salarié\n"
|
||||
"- CDI / CDD : types de contrats\n"
|
||||
"- Période de paie : mois concerné par le bulletin\n"
|
||||
"- SMIC : salaire minimum\n"
|
||||
"- IJSS : indemnités journalières sécurité sociale\n"
|
||||
"- Congés payés : solde de congés\n"
|
||||
"- RTT : réduction du temps de travail\n"
|
||||
"- Saisie sur salaire : retenue judiciaire\n"
|
||||
"- Solde de tout compte : dernier bulletin d'un salarié qui part\n\n"
|
||||
"Écrans courants :\n"
|
||||
"- Fiche employé (identité, contrat, poste, salaire)\n"
|
||||
"- Saisie des variables (heures, absences, primes)\n"
|
||||
"- Bulletin de paie (aperçu avant validation)\n"
|
||||
"- Déclaration DSN\n"
|
||||
"- Gestion des absences / congés"
|
||||
),
|
||||
vocabulary=[
|
||||
"bulletin", "salaire", "brut", "net", "charges sociales", "DSN",
|
||||
"CDI", "CDD", "congés", "RTT", "SMIC", "IJSS", "URSSAF",
|
||||
"employé", "salarié", "contrat", "prime", "heures supplémentaires",
|
||||
"absence", "solde de tout compte", "STC",
|
||||
],
|
||||
known_apps=[
|
||||
"Silae", "Sage Paie", "Cegid Paie", "ADP", "PayFit", "Nibelis",
|
||||
"Cegedim SRH", "Lucca", "HR Access",
|
||||
],
|
||||
screen_patterns={
|
||||
"fiche_employe": "Fiche employé avec identité, contrat, poste",
|
||||
"saisie_variables": "Saisie des variables de paie (heures, absences, primes)",
|
||||
"apercu_bulletin": "Aperçu du bulletin de paie avant validation",
|
||||
"dsn": "Écran DSN (déclaration sociale nominative)",
|
||||
"conges": "Gestion des absences et congés",
|
||||
},
|
||||
vocabulary_synonyms={
|
||||
"DSN": "déclaration sociale",
|
||||
"RTT": "réduction du temps de travail",
|
||||
"STC": "solde de tout compte",
|
||||
"IJSS": "indemnités journalières",
|
||||
"CP": "congés payés",
|
||||
},
|
||||
common_actions={
|
||||
"click:valider": "valider le bulletin",
|
||||
"click:editer": "éditer le bulletin",
|
||||
"click:bulletin": "ouvrir le bulletin de paie",
|
||||
"click:employe": "ouvrir la fiche employé",
|
||||
"click:dsn": "lancer la DSN",
|
||||
"click:conges": "gérer les congés",
|
||||
"click:absence": "saisir une absence",
|
||||
"type:heures": "saisir les heures travaillées",
|
||||
"type:prime": "saisir une prime",
|
||||
},
|
||||
clarification_templates={
|
||||
"default": (
|
||||
"Je ne trouve pas {target_friendly} pour {nom_employe}. "
|
||||
"Tu peux me confirmer la période de paie ?"
|
||||
),
|
||||
"target_not_found": (
|
||||
"Je ne trouve pas {target_friendly} dans la fiche de {nom_employe}. "
|
||||
"Le contrat est peut-être clôturé ?"
|
||||
),
|
||||
"target:employe": (
|
||||
"Je ne trouve pas {nom_employe} dans la liste. "
|
||||
"Est-il encore actif dans l'entreprise ?"
|
||||
),
|
||||
"ambiguous_period": (
|
||||
"Est-ce la période {periode_a} ou {periode_b} que tu veux traiter ?"
|
||||
),
|
||||
},
|
||||
summary_templates={
|
||||
"item_singular": "bulletin",
|
||||
"item_plural": "bulletins",
|
||||
"success_one": (
|
||||
"J'ai édité le bulletin de {nom_employe} en {elapsed_s}s."
|
||||
),
|
||||
"success": (
|
||||
"J'ai édité {done} bulletins sur {items_count}. "
|
||||
"La paie est prête pour validation."
|
||||
),
|
||||
"partial": (
|
||||
"J'ai édité {done} bulletins sur {items_count}. "
|
||||
"{failed} sont en attente — variables de paie à compléter."
|
||||
),
|
||||
"failure": (
|
||||
"Je n'ai pas pu éditer les bulletins de {workflow_name}. "
|
||||
"Il y a peut-être un blocage côté logiciel de paie."
|
||||
),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
_STOCKS_LOGISTIQUE = DomainContext(
|
||||
domain_id="stocks_logistique",
|
||||
name="Stocks et logistique",
|
||||
description=(
|
||||
"Gestionnaire de stocks / logistique : bons de commande, bons de livraison, "
|
||||
"réceptions, inventaires, mouvements de stock, expéditions."
|
||||
),
|
||||
system_prompt=(
|
||||
"Tu es un assistant expert en gestion de stocks et logistique. "
|
||||
"L'utilisateur utilise un ERP ou WMS (SAP, Dynamics, Odoo, Sage, Divalto) "
|
||||
"pour gérer les commandes, les réceptions, les expéditions et les inventaires.\n\n"
|
||||
"Vocabulaire du métier :\n"
|
||||
"- BC : Bon de Commande (achat ou vente)\n"
|
||||
"- BL : Bon de Livraison\n"
|
||||
"- BR : Bon de Réception\n"
|
||||
"- Article / Référence / SKU : produit en stock\n"
|
||||
"- Emplacement : localisation physique (allée, rayon, emplacement)\n"
|
||||
"- Mouvement de stock : entrée, sortie, transfert\n"
|
||||
"- Inventaire : comptage physique pour recaler le stock théorique\n"
|
||||
"- FIFO / LIFO : ordre de sortie des stocks\n"
|
||||
"- ERP : progiciel de gestion intégré\n"
|
||||
"- WMS : Warehouse Management System\n"
|
||||
"- Picking : préparation de commande\n"
|
||||
"- Quantité en stock / disponible / réservée\n\n"
|
||||
"Écrans courants :\n"
|
||||
"- Saisie de bon de commande / réception\n"
|
||||
"- Liste des articles (avec photo, quantité, emplacement)\n"
|
||||
"- Inventaire (comptage)\n"
|
||||
"- Mouvements de stock\n"
|
||||
"- Picking list (liste de préparation)"
|
||||
),
|
||||
vocabulary=[
|
||||
"bon de commande", "BC", "bon de livraison", "BL", "bon de réception", "BR",
|
||||
"article", "référence", "SKU", "emplacement", "stock", "inventaire",
|
||||
"mouvement", "entrée", "sortie", "picking", "FIFO", "LIFO", "ERP", "WMS",
|
||||
"fournisseur", "client", "quantité", "disponible", "réservé",
|
||||
],
|
||||
known_apps=[
|
||||
"SAP", "Dynamics", "Odoo", "Sage X3", "Divalto", "Cegid",
|
||||
"Oracle NetSuite", "Reflex WMS", "Infolog",
|
||||
],
|
||||
screen_patterns={
|
||||
"bon_commande": "Saisie de bon de commande (fournisseur, lignes d'articles, quantités)",
|
||||
"reception": "Bon de réception (rapprochement avec la commande)",
|
||||
"inventaire": "Saisie d'inventaire (article, emplacement, quantité comptée)",
|
||||
"picking": "Liste de préparation avec articles et emplacements",
|
||||
"mouvement": "Mouvement de stock (entrée/sortie/transfert)",
|
||||
},
|
||||
vocabulary_synonyms={
|
||||
"BC": "bon de commande",
|
||||
"BL": "bon de livraison",
|
||||
"BR": "bon de réception",
|
||||
"SKU": "référence produit",
|
||||
"WMS": "gestion d'entrepôt",
|
||||
"ERP": "progiciel de gestion",
|
||||
},
|
||||
common_actions={
|
||||
"click:valider": "valider le bon",
|
||||
"click:commande": "ouvrir le bon de commande",
|
||||
"click:livraison": "ouvrir le bon de livraison",
|
||||
"click:reception": "saisir la réception",
|
||||
"click:inventaire": "démarrer l'inventaire",
|
||||
"click:article": "sélectionner un article",
|
||||
"click:picking": "démarrer la préparation",
|
||||
"type:quantite": "saisir la quantité",
|
||||
"type:reference": "saisir la référence article",
|
||||
},
|
||||
clarification_templates={
|
||||
"default": (
|
||||
"Je ne trouve pas {target_friendly}. "
|
||||
"C'est bien la commande {num_bc} qu'on traite ?"
|
||||
),
|
||||
"target_not_found": (
|
||||
"Je ne trouve pas {target_friendly}. "
|
||||
"La commande {num_bc} est peut-être déjà clôturée ?"
|
||||
),
|
||||
"target:article": (
|
||||
"Je ne trouve pas l'article {ref_article}. "
|
||||
"Il est peut-être archivé ou mal référencé ?"
|
||||
),
|
||||
"quantity_mismatch": (
|
||||
"La quantité reçue ({qte_recue}) ne correspond pas à la commande "
|
||||
"({qte_commandee}). Je saisis un écart ou tu vérifies ?"
|
||||
),
|
||||
},
|
||||
summary_templates={
|
||||
"item_singular": "bon",
|
||||
"item_plural": "bons",
|
||||
"success_one": (
|
||||
"J'ai traité le bon {num_bc} en {elapsed_s}s."
|
||||
),
|
||||
"success": (
|
||||
"J'ai traité {done} bons sur {items_count}. "
|
||||
"Les mouvements de stock sont validés."
|
||||
),
|
||||
"partial": (
|
||||
"J'ai traité {done} bons sur {items_count}. "
|
||||
"{failed} bons sont en attente — écarts de quantité à vérifier."
|
||||
),
|
||||
"failure": (
|
||||
"Je n'ai pas pu traiter les bons de {workflow_name}. "
|
||||
"L'ERP a peut-être refusé une ligne, je te rends la main."
|
||||
),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
_GENERIC = DomainContext(
|
||||
domain_id="generic",
|
||||
name="Bureautique générale",
|
||||
@@ -166,11 +963,37 @@ _GENERIC = DomainContext(
|
||||
"Tu es un assistant RPA qui observe des applications bureautiques. "
|
||||
"Décris précisément ce que tu vois à l'écran."
|
||||
),
|
||||
summary_templates={
|
||||
"item_singular": "action",
|
||||
"item_plural": "actions",
|
||||
"success_one": "C'est fait, j'ai terminé « {workflow_name} » en {elapsed_s}s.",
|
||||
"success": (
|
||||
"J'ai terminé « {workflow_name} » : {done} {item_word} exécutées "
|
||||
"sur {items_count}."
|
||||
),
|
||||
"partial": (
|
||||
"J'ai terminé « {workflow_name} » partiellement : "
|
||||
"{done} {item_word} sur {items_count} ({failed} en échec)."
|
||||
),
|
||||
"failure": (
|
||||
"Je n'ai pas pu terminer « {workflow_name} ». Je te rends la main."
|
||||
),
|
||||
},
|
||||
clarification_templates={
|
||||
"default": (
|
||||
"Je ne trouve pas {target_friendly} à l'écran. "
|
||||
"Tu peux me le montrer ?"
|
||||
),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
# Registre des domaines disponibles
|
||||
_DOMAINS: Dict[str, DomainContext] = {
|
||||
"tim_codage": _TIM_CODAGE,
|
||||
"comptabilite": _COMPTABILITE,
|
||||
"rh_paie": _RH_PAIE,
|
||||
"stocks_logistique": _STOCKS_LOGISTIQUE,
|
||||
"generic": _GENERIC,
|
||||
}
|
||||
|
||||
@@ -179,7 +1002,8 @@ def get_domain_context(domain_id: str = "generic") -> DomainContext:
|
||||
"""Récupérer le contexte métier par ID.
|
||||
|
||||
Args:
|
||||
domain_id: Identifiant du domaine (tim_codage, generic, etc.)
|
||||
domain_id: Identifiant du domaine (tim_codage, comptabilite, rh_paie,
|
||||
stocks_logistique, generic, etc.)
|
||||
|
||||
Returns:
|
||||
DomainContext correspondant, ou generic si non trouvé.
|
||||
|
||||
373
agent_v0/server_v1/execution_plan_runner.py
Normal file
373
agent_v0/server_v1/execution_plan_runner.py
Normal file
@@ -0,0 +1,373 @@
|
||||
# agent_v0/server_v1/execution_plan_runner.py
|
||||
"""
|
||||
ExecutionPlanRunner — Adaptateur ExecutionPlan → actions replay.
|
||||
|
||||
Pièce d'intégration du pipeline V4 :
|
||||
RawTrace → IRBuilder → WorkflowIR → ExecutionCompiler → ExecutionPlan → Runtime
|
||||
|
||||
Ce module convertit un `ExecutionPlan` (plan pré-compilé, déterministe) en
|
||||
liste d'actions au format attendu par l'executor replay actuel (clé x_pct,
|
||||
y_pct, target_spec, etc.), puis les injecte dans `_replay_queues`.
|
||||
|
||||
L'ancien chemin `build_replay_from_raw_events()` dans stream_processor.py
|
||||
reste inchangé — les deux chemins coexistent pendant la transition.
|
||||
|
||||
Format d'action produit (compatible executor existant) :
|
||||
{
|
||||
"action_id": "act_...",
|
||||
"type": "click",
|
||||
"x_pct": 0.5,
|
||||
"y_pct": 0.3,
|
||||
"visual_mode": True,
|
||||
"target_spec": {
|
||||
"by_text": "...",
|
||||
"window_title": "...",
|
||||
"vlm_description": "...",
|
||||
"anchor_image_base64": "...",
|
||||
},
|
||||
"expected_window_title": "...",
|
||||
}
|
||||
|
||||
Auteur: Dom, Alice - Avril 2026
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
import threading
|
||||
import uuid
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from core.workflow.execution_plan import (
|
||||
ExecutionNode,
|
||||
ExecutionPlan,
|
||||
ResolutionStrategy,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Substitution de variables
|
||||
# =========================================================================
|
||||
# Le WorkflowIR utilise la syntaxe `{var}` dans les champs texte.
|
||||
# Ici on supporte les deux : `{var}` (IR natif) et `${var}` (replay legacy).
|
||||
_VARIABLE_RE_CURLY = re.compile(r"\{(\w+)\}")
|
||||
_VARIABLE_RE_DOLLAR = re.compile(r"\$\{(\w+)\}")
|
||||
|
||||
|
||||
def substitute_variables(text: str, variables: Dict[str, Any]) -> str:
|
||||
"""Remplacer `{var}` et `${var}` par leurs valeurs.
|
||||
|
||||
Priorité : variables fournies > placeholder brut (inchangé si inconnu).
|
||||
"""
|
||||
if not text or not variables:
|
||||
return text
|
||||
|
||||
def replacer(match: "re.Match[str]") -> str:
|
||||
var_name = match.group(1)
|
||||
if var_name in variables:
|
||||
return str(variables[var_name])
|
||||
return match.group(0)
|
||||
|
||||
text = _VARIABLE_RE_DOLLAR.sub(replacer, text)
|
||||
text = _VARIABLE_RE_CURLY.sub(replacer, text)
|
||||
return text
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Conversion ExecutionNode → action replay
|
||||
# =========================================================================
|
||||
|
||||
|
||||
def _strategy_to_target_spec(
|
||||
strategy: Optional[ResolutionStrategy],
|
||||
fallbacks: Optional[List[ResolutionStrategy]] = None,
|
||||
intent: str = "",
|
||||
) -> Dict[str, Any]:
|
||||
"""Construire un `target_spec` depuis les stratégies de résolution.
|
||||
|
||||
Fusionne la primaire et les fallbacks pour donner un maximum d'indices
|
||||
au resolve_engine :
|
||||
- OCR → by_text
|
||||
- template → anchor_image_base64 (depuis anchor_b64)
|
||||
- VLM → vlm_description
|
||||
|
||||
Règle V4 : la stratégie primaire dicte la méthode préférée.
|
||||
Le champ `resolve_order` liste les méthodes dans l'ordre à essayer.
|
||||
Le resolve_engine honore cet ordre au lieu de sa cascade par défaut.
|
||||
|
||||
resolve_order est la clé du "zéro VLM au runtime" :
|
||||
- ["ocr", "template", "vlm"] → V4 typique (OCR rapide)
|
||||
- ["template", "ocr", "vlm"] → apprentissage : template marche mieux
|
||||
- ["vlm"] → éléments sans texte (icônes)
|
||||
"""
|
||||
spec: Dict[str, Any] = {}
|
||||
|
||||
all_strategies: List[ResolutionStrategy] = []
|
||||
if strategy is not None:
|
||||
all_strategies.append(strategy)
|
||||
if fallbacks:
|
||||
all_strategies.extend(fallbacks)
|
||||
|
||||
by_text_candidate = ""
|
||||
anchor_candidate = ""
|
||||
vlm_candidate = ""
|
||||
uia_data: Dict[str, Any] = {}
|
||||
dom_data: Dict[str, Any] = {}
|
||||
resolve_order: List[str] = []
|
||||
seen_methods: set = set()
|
||||
|
||||
for strat in all_strategies:
|
||||
if not strat:
|
||||
continue
|
||||
if strat.method == "ocr" and strat.target_text and not by_text_candidate:
|
||||
by_text_candidate = strat.target_text
|
||||
elif strat.method == "template":
|
||||
if strat.anchor_b64 and not anchor_candidate:
|
||||
anchor_candidate = strat.anchor_b64
|
||||
if strat.target_text and not by_text_candidate:
|
||||
by_text_candidate = strat.target_text
|
||||
elif strat.method == "vlm" and strat.vlm_description and not vlm_candidate:
|
||||
vlm_candidate = strat.vlm_description
|
||||
elif strat.method == "uia" and strat.uia_name and not uia_data:
|
||||
uia_data = {
|
||||
"name": strat.uia_name,
|
||||
"control_type": strat.uia_control_type,
|
||||
"automation_id": strat.uia_automation_id,
|
||||
"parent_path": strat.uia_parent_path,
|
||||
}
|
||||
elif strat.method == "dom" and strat.dom_selector and not dom_data:
|
||||
dom_data = {
|
||||
"selector": strat.dom_selector,
|
||||
"xpath": strat.dom_xpath,
|
||||
"url_pattern": strat.dom_url_pattern,
|
||||
}
|
||||
|
||||
# Construire l'ordre des méthodes (dans l'ordre primaire → fallbacks)
|
||||
if strat.method and strat.method not in seen_methods:
|
||||
resolve_order.append(strat.method)
|
||||
seen_methods.add(strat.method)
|
||||
|
||||
if by_text_candidate:
|
||||
spec["by_text"] = by_text_candidate
|
||||
if anchor_candidate:
|
||||
spec["anchor_image_base64"] = anchor_candidate
|
||||
if vlm_candidate:
|
||||
spec["vlm_description"] = vlm_candidate
|
||||
elif intent and "vlm_description" not in spec:
|
||||
# L'intention métier devient le prompt VLM de dernier recours
|
||||
spec["vlm_description"] = intent
|
||||
|
||||
# Données UIA — consommées par l'agent Windows via lea_uia.exe
|
||||
if uia_data:
|
||||
spec["uia_target"] = uia_data
|
||||
|
||||
# Données DOM — consommées par l'agent Windows via CDP (futur)
|
||||
if dom_data:
|
||||
spec["dom_target"] = dom_data
|
||||
|
||||
# Ordre de résolution pré-compilé — c'est LA pièce centrale du V4
|
||||
if resolve_order:
|
||||
spec["resolve_order"] = resolve_order
|
||||
|
||||
return spec
|
||||
|
||||
|
||||
def execution_node_to_action(
|
||||
node: ExecutionNode,
|
||||
variables: Optional[Dict[str, Any]] = None,
|
||||
id_prefix: str = "act_plan",
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Convertir un `ExecutionNode` en action replay.
|
||||
|
||||
Retourne `None` si le nœud n'est pas exécutable (type inconnu).
|
||||
|
||||
Args:
|
||||
node: Le nœud à convertir.
|
||||
variables: Dictionnaire de variables pour substituer les {var}.
|
||||
id_prefix: Préfixe pour l'action_id générée.
|
||||
"""
|
||||
variables = variables or {}
|
||||
|
||||
action: Dict[str, Any] = {
|
||||
"action_id": f"{id_prefix}_{uuid.uuid4().hex[:8]}",
|
||||
"plan_node_id": node.node_id,
|
||||
}
|
||||
|
||||
if node.intent:
|
||||
action["intention"] = node.intent
|
||||
if node.step_id:
|
||||
action["plan_step_id"] = node.step_id
|
||||
if node.is_optional:
|
||||
action["is_optional"] = True
|
||||
|
||||
# Métadonnées d'exécution utiles au runtime
|
||||
if node.timeout_ms:
|
||||
action["timeout_ms"] = node.timeout_ms
|
||||
if node.max_retries:
|
||||
action["max_retries"] = node.max_retries
|
||||
if node.recovery_action:
|
||||
action["recovery_action"] = node.recovery_action
|
||||
if node.success_condition:
|
||||
action["success_condition"] = node.success_condition.to_dict()
|
||||
|
||||
action_type = node.action_type
|
||||
|
||||
if action_type == "click":
|
||||
action["type"] = "click"
|
||||
|
||||
strategy = node.strategy_primary
|
||||
fallbacks = node.strategy_fallbacks or []
|
||||
|
||||
# ── Déduction des coordonnées depuis la stratégie primaire ──
|
||||
# - OCR : pas de coordonnées (le runtime trouve via OCR)
|
||||
# - template : l'anchor sera utilisé au runtime
|
||||
# - VLM : la description sera utilisée au runtime
|
||||
# Dans tous les cas le resolve_engine retrouve les pixels au replay.
|
||||
# On expose néanmoins un centre (0.5, 0.5) neutre pour rester
|
||||
# compatible avec les validations de queue existantes.
|
||||
action["x_pct"] = 0.5
|
||||
action["y_pct"] = 0.5
|
||||
action["visual_mode"] = True
|
||||
|
||||
target_spec = _strategy_to_target_spec(
|
||||
strategy=strategy,
|
||||
fallbacks=fallbacks,
|
||||
intent=node.intent,
|
||||
)
|
||||
|
||||
# Titre fenêtre attendu AVANT (pré-vérif stricte)
|
||||
# Si absent, aucune pré-vérif → l'action s'exécute quel que soit l'écran
|
||||
if node.expected_window_before:
|
||||
action["expected_window_before"] = node.expected_window_before
|
||||
target_spec["window_title"] = node.expected_window_before
|
||||
|
||||
# Titre fenêtre attendu APRÈS (post-vérif stricte)
|
||||
# C'est la garantie de passage à l'action suivante
|
||||
if node.success_condition and node.success_condition.expected_title:
|
||||
action["expected_window_title"] = node.success_condition.expected_title
|
||||
action["success_strict"] = (
|
||||
node.success_condition.method == "title_match"
|
||||
)
|
||||
if "window_title" not in target_spec:
|
||||
target_spec["window_title"] = node.success_condition.expected_title
|
||||
|
||||
if target_spec:
|
||||
action["target_spec"] = target_spec
|
||||
|
||||
elif action_type == "type":
|
||||
action["type"] = "type"
|
||||
text = node.text or ""
|
||||
# Substituer les variables avant d'envoyer (ex: {patient} → "DUPONT")
|
||||
action["text"] = substitute_variables(text, variables)
|
||||
if node.variable_name:
|
||||
action["variable_name"] = node.variable_name
|
||||
|
||||
elif action_type in ("key_combo", "key_press"):
|
||||
action["type"] = "key_combo"
|
||||
keys = list(node.keys or [])
|
||||
if not keys:
|
||||
return None
|
||||
action["keys"] = keys
|
||||
|
||||
elif action_type == "wait":
|
||||
action["type"] = "wait"
|
||||
duration = node.duration_ms or 1000
|
||||
action["duration_ms"] = int(duration)
|
||||
|
||||
elif action_type == "scroll":
|
||||
action["type"] = "scroll"
|
||||
# Les stratégies peuvent contenir une zone — pas exploitée ici,
|
||||
# le scroll est implicitement sur la fenêtre active.
|
||||
action["delta"] = -3
|
||||
|
||||
else:
|
||||
logger.debug("execution_node_to_action: type inconnu '%s' ignoré", action_type)
|
||||
return None
|
||||
|
||||
return action
|
||||
|
||||
|
||||
def execution_plan_to_actions(
|
||||
plan: ExecutionPlan,
|
||||
variables: Optional[Dict[str, Any]] = None,
|
||||
id_prefix: str = "act_plan",
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Convertir un `ExecutionPlan` complet en liste d'actions replay.
|
||||
|
||||
Les variables passées en argument écrasent celles du plan.
|
||||
"""
|
||||
merged_vars: Dict[str, Any] = dict(plan.variables or {})
|
||||
if variables:
|
||||
merged_vars.update(variables)
|
||||
|
||||
actions: List[Dict[str, Any]] = []
|
||||
for node in plan.nodes:
|
||||
action = execution_node_to_action(
|
||||
node=node,
|
||||
variables=merged_vars,
|
||||
id_prefix=id_prefix,
|
||||
)
|
||||
if action is not None:
|
||||
actions.append(action)
|
||||
|
||||
logger.info(
|
||||
"execution_plan_to_actions(%s) : %d nœuds → %d actions replay "
|
||||
"(vars=%d)",
|
||||
plan.plan_id, plan.total_nodes, len(actions), len(merged_vars),
|
||||
)
|
||||
return actions
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Injection dans la queue de replay
|
||||
# =========================================================================
|
||||
|
||||
|
||||
def inject_plan_into_queue(
|
||||
plan: ExecutionPlan,
|
||||
session_id: str,
|
||||
replay_queues: Dict[str, List[Dict[str, Any]]],
|
||||
variables: Optional[Dict[str, Any]] = None,
|
||||
lock: Optional[threading.Lock] = None,
|
||||
replace: bool = True,
|
||||
id_prefix: str = "act_plan",
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Injecter un `ExecutionPlan` dans la queue de replay d'une session.
|
||||
|
||||
Args:
|
||||
plan: Le plan à exécuter.
|
||||
session_id: La session Agent V1 cible.
|
||||
replay_queues: Le dict global `_replay_queues` partagé par le serveur.
|
||||
variables: Variables à substituer dans les actions.
|
||||
lock: Verrou optionnel à acquérir avant d'écrire (threadsafe).
|
||||
replace: Si True (défaut), remplace la queue existante. Sinon, append.
|
||||
id_prefix: Préfixe pour les action_id générés.
|
||||
|
||||
Returns:
|
||||
La liste des actions injectées (après substitution).
|
||||
"""
|
||||
actions = execution_plan_to_actions(
|
||||
plan=plan, variables=variables, id_prefix=id_prefix,
|
||||
)
|
||||
|
||||
def _write() -> None:
|
||||
if replace:
|
||||
replay_queues[session_id] = list(actions)
|
||||
else:
|
||||
replay_queues[session_id].extend(actions)
|
||||
|
||||
if lock is not None:
|
||||
with lock:
|
||||
_write()
|
||||
else:
|
||||
_write()
|
||||
|
||||
logger.info(
|
||||
"inject_plan_into_queue(%s) : %d actions injectées dans la queue "
|
||||
"de la session '%s' (replace=%s)",
|
||||
plan.plan_id, len(actions), session_id, replace,
|
||||
)
|
||||
return actions
|
||||
@@ -17,6 +17,20 @@ from typing import Any, Dict, List, Optional
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _infer_machine_id_from_session_id(session_id: str, fallback: str = "default") -> str:
|
||||
"""Déduire le machine_id depuis un session_id spécial si possible.
|
||||
|
||||
Les heartbeats de fond de Léa utilisent `bg_<machine_id>` comme
|
||||
identifiant de session. Lors d'un redémarrage serveur, ces sessions
|
||||
peuvent être restaurées depuis la persistance JSON avec `machine_id`
|
||||
resté à `default`. On rétablit ici l'information machine pour que les
|
||||
replays ciblés retrouvent bien la session de fond active.
|
||||
"""
|
||||
if session_id.startswith("bg_") and len(session_id) > 3:
|
||||
return session_id[3:]
|
||||
return fallback
|
||||
|
||||
|
||||
@dataclass
|
||||
class LiveSessionState:
|
||||
"""État d'une session active en mémoire."""
|
||||
@@ -65,7 +79,8 @@ class LiveSessionState:
|
||||
class LiveSessionManager:
|
||||
"""Gère les sessions live en mémoire côté serveur avec persistance disque."""
|
||||
|
||||
def __init__(self, persist_dir: str = "data/streaming_sessions"):
|
||||
def __init__(self, persist_dir: str = "data/streaming_sessions",
|
||||
live_sessions_dir: Optional[str] = None):
|
||||
self._sessions: Dict[str, LiveSessionState] = {}
|
||||
self._lock = threading.Lock()
|
||||
self._persist_dir = Path(persist_dir)
|
||||
@@ -74,17 +89,29 @@ class LiveSessionManager:
|
||||
self._persist_counter = 0 # Compteur pour limiter la fréquence de persistance
|
||||
self._persist_interval = 10 # Persister toutes les N modifications
|
||||
|
||||
# Dossier des sessions live (JSONL + screenshots)
|
||||
self._live_sessions_dir = Path(live_sessions_dir) if live_sessions_dir else None
|
||||
|
||||
# Charger les sessions persistées au démarrage
|
||||
self._load_persisted_sessions()
|
||||
# Reconstruire les sessions depuis les live_events.jsonl sur disque
|
||||
self._discover_sessions_from_disk()
|
||||
|
||||
def _load_persisted_sessions(self):
|
||||
"""Charger les sessions sauvegardées au démarrage."""
|
||||
"""Charger les sessions sauvegardées au démarrage (JSON state files)."""
|
||||
count = 0
|
||||
for session_file in sorted(self._persist_dir.glob("sess_*.json")):
|
||||
session_files = sorted(self._persist_dir.glob("sess_*.json"))
|
||||
session_files += sorted(self._persist_dir.glob("bg_*.json"))
|
||||
for session_file in session_files:
|
||||
try:
|
||||
with open(session_file, 'r', encoding='utf-8') as f:
|
||||
data = json.load(f)
|
||||
session = LiveSessionState.from_dict(data)
|
||||
if session.machine_id == "default":
|
||||
session.machine_id = _infer_machine_id_from_session_id(
|
||||
session.session_id,
|
||||
fallback=session.machine_id,
|
||||
)
|
||||
self._sessions[session.session_id] = session
|
||||
count += 1
|
||||
except Exception as e:
|
||||
@@ -92,6 +119,66 @@ class LiveSessionManager:
|
||||
if count:
|
||||
logger.info(f"{count} session(s) restaurée(s) depuis {self._persist_dir}")
|
||||
|
||||
def _discover_sessions_from_disk(self):
|
||||
"""Découvrir les sessions depuis les live_events.jsonl sur disque.
|
||||
|
||||
Reconstruit les sessions manquantes du session_manager en scannant :
|
||||
- live_sessions/sess_*/live_events.jsonl (sessions racine)
|
||||
- live_sessions/{machine_id}/sess_*/live_events.jsonl (multi-machine)
|
||||
|
||||
Ne touche pas aux sessions déjà chargées depuis le JSON persist.
|
||||
"""
|
||||
if self._live_sessions_dir is None:
|
||||
return
|
||||
live_dir = self._live_sessions_dir
|
||||
if not live_dir.exists():
|
||||
return
|
||||
|
||||
discovered = 0
|
||||
for jsonl_file in sorted(live_dir.glob("**/live_events.jsonl")):
|
||||
session_dir = jsonl_file.parent
|
||||
session_id = session_dir.name
|
||||
if not (session_id.startswith("sess_") or session_id.startswith("bg_")):
|
||||
continue
|
||||
if session_id in self._sessions:
|
||||
continue
|
||||
|
||||
# Déduire le machine_id depuis le chemin parent
|
||||
parent_name = session_dir.parent.name
|
||||
if parent_name == live_dir.name:
|
||||
machine_id = _infer_machine_id_from_session_id(session_id)
|
||||
else:
|
||||
machine_id = parent_name
|
||||
|
||||
# Compter events et screenshots
|
||||
events_count = 0
|
||||
try:
|
||||
with open(jsonl_file, 'r', encoding='utf-8') as f:
|
||||
for _ in f:
|
||||
events_count += 1
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
shots_dir = session_dir / "shots"
|
||||
shots_count = len(list(shots_dir.glob("shot_*_full.png"))) if shots_dir.exists() else 0
|
||||
|
||||
# Créer la session en mémoire
|
||||
session = LiveSessionState(
|
||||
session_id=session_id,
|
||||
machine_id=machine_id,
|
||||
finalized=False,
|
||||
)
|
||||
# Stocker le nombre d'events/shots dans les métadonnées
|
||||
session.shot_paths = {f"shot_{i:04d}": "" for i in range(shots_count)}
|
||||
self._sessions[session_id] = session
|
||||
discovered += 1
|
||||
|
||||
if discovered:
|
||||
logger.info(
|
||||
f"{discovered} session(s) découverte(s) depuis {live_dir} "
|
||||
f"(total: {len(self._sessions)} sessions en mémoire)"
|
||||
)
|
||||
|
||||
def _persist_session(self, session_id: str):
|
||||
"""Sauvegarder une session sur disque (appelé périodiquement)."""
|
||||
session = self._sessions.get(session_id)
|
||||
@@ -102,7 +189,7 @@ class LiveSessionManager:
|
||||
with open(filepath, 'w', encoding='utf-8') as f:
|
||||
json.dump(session.to_dict(), f, ensure_ascii=False)
|
||||
except Exception as e:
|
||||
logger.debug(f"Erreur persistance session {session_id}: {e}")
|
||||
logger.warning(f"Erreur persistance session {session_id}: {e}")
|
||||
|
||||
def _maybe_persist(self, session_id: str):
|
||||
"""Persister si le compteur atteint l'intervalle."""
|
||||
@@ -180,6 +267,31 @@ class LiveSessionManager:
|
||||
if meta_val is not None:
|
||||
info[meta_key] = meta_val
|
||||
session.last_window_info = info
|
||||
# Exploiter window_capture (envoyé par l'agent avec la capture fenêtre)
|
||||
# pour enrichir last_window_info avec le titre précis de la fenêtre cliquée
|
||||
window_capture = event_data.get("window_capture")
|
||||
if window_capture and isinstance(window_capture, dict):
|
||||
wc_title = window_capture.get("title", "").strip()
|
||||
wc_app = window_capture.get("app_name", "").strip()
|
||||
if wc_title:
|
||||
session.last_window_info["title"] = wc_title
|
||||
if wc_app:
|
||||
session.last_window_info["app_name"] = wc_app
|
||||
# QW1 — propager monitor_index et monitors_geometry depuis window_capture
|
||||
if "monitor_index" in window_capture:
|
||||
session.last_window_info["monitor_index"] = window_capture["monitor_index"]
|
||||
if "monitors_geometry" in window_capture:
|
||||
session.last_window_info["monitors_geometry"] = window_capture["monitors_geometry"]
|
||||
|
||||
# QW1 — propager monitor_index/monitors_geometry du payload event
|
||||
# (cas heartbeat enrichi sans window/window_title). Toujours
|
||||
# rafraîchir le focus actif (change souvent) et la géométrie
|
||||
# (l'utilisateur peut brancher/débrancher un écran).
|
||||
if "monitor_index" in event_data:
|
||||
session.last_window_info["monitor_index"] = event_data["monitor_index"]
|
||||
if "monitors_geometry" in event_data and event_data["monitors_geometry"]:
|
||||
session.last_window_info["monitors_geometry"] = event_data["monitors_geometry"]
|
||||
|
||||
# Accumuler les titres/apps pour le nommage automatique
|
||||
title = session.last_window_info.get("title", "").strip()
|
||||
app_name = session.last_window_info.get("app_name", "").strip()
|
||||
@@ -221,18 +333,41 @@ class LiveSessionManager:
|
||||
import socket
|
||||
|
||||
# Construire les événements au format RawSession
|
||||
# Important : copier TOUTES les données de l'événement (pos, text, keys, button...)
|
||||
# car Event.from_dict() met tout sauf t/type/window/screenshot_id dans event.data,
|
||||
# et le GraphBuilder utilise event.data pour construire les actions.
|
||||
events = []
|
||||
for evt in session.events:
|
||||
window_info = {
|
||||
"title": evt.get("window_title", session.last_window_info.get("title", "")),
|
||||
"app_name": evt.get("app_name", session.last_window_info.get("app_name", "unknown")),
|
||||
}
|
||||
events.append({
|
||||
# Extraire window info (plusieurs formats possibles)
|
||||
window_raw = evt.get("window")
|
||||
if isinstance(window_raw, dict):
|
||||
window_info = {
|
||||
"title": window_raw.get("title", session.last_window_info.get("title", "")),
|
||||
"app_name": window_raw.get("app_name", session.last_window_info.get("app_name", "unknown")),
|
||||
}
|
||||
else:
|
||||
window_info = {
|
||||
"title": evt.get("window_title", session.last_window_info.get("title", "")),
|
||||
"app_name": evt.get("app_name", session.last_window_info.get("app_name", "unknown")),
|
||||
}
|
||||
|
||||
raw_event = {
|
||||
"t": evt.get("timestamp", 0),
|
||||
"type": evt.get("type", "unknown"),
|
||||
"window": window_info,
|
||||
"screenshot_id": evt.get("screenshot_id"),
|
||||
})
|
||||
}
|
||||
|
||||
# Copier les données spécifiques au type d'événement
|
||||
# (pos, button, text, keys, etc.) — indispensable pour le replay
|
||||
_skip_keys = {"type", "timestamp", "window", "window_title",
|
||||
"app_name", "screenshot_id", "machine_id",
|
||||
"screen_metadata", "vision_info"}
|
||||
for key, value in evt.items():
|
||||
if key not in _skip_keys and key not in raw_event:
|
||||
raw_event[key] = value
|
||||
|
||||
events.append(raw_event)
|
||||
|
||||
# Construire les screenshots au format RawSession
|
||||
screenshots = []
|
||||
|
||||
154
agent_v0/server_v1/loop_detector.py
Normal file
154
agent_v0/server_v1/loop_detector.py
Normal file
@@ -0,0 +1,154 @@
|
||||
# agent_v0/server_v1/loop_detector.py
|
||||
"""LoopDetector composite — détection de stagnation de Léa pendant un replay (QW2).
|
||||
|
||||
Trois signaux indépendants :
|
||||
- screen_static : N captures consécutives avec CLIP similarity > seuil
|
||||
- action_repeat : N actions consécutives identiques (type + coords)
|
||||
- retry_threshold : nombre de retries cumulés >= seuil
|
||||
|
||||
Un seul signal positif → verdict.detected=True. Le serveur bascule alors le
|
||||
replay en paused_need_help avec pause_reason explicite.
|
||||
|
||||
Désactivable via env var RPA_LOOP_DETECTOR_ENABLED=0.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class LoopVerdict:
|
||||
detected: bool = False
|
||||
reason: str = ""
|
||||
signal: str = "" # "screen_static" | "action_repeat" | "retry_threshold" | ""
|
||||
evidence: Dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
def _env_int(name: str, default: int) -> int:
|
||||
try:
|
||||
return int(os.environ.get(name, default))
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
def _env_float(name: str, default: float) -> float:
|
||||
try:
|
||||
return float(os.environ.get(name, default))
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
def _env_bool_enabled(name: str) -> bool:
|
||||
val = os.environ.get(name, "1").strip().lower()
|
||||
return val not in ("0", "false", "no", "off", "")
|
||||
|
||||
|
||||
def _cosine_similarity(a, b) -> float:
|
||||
"""Similarité cosine entre deux vecteurs (listes ou np.array). Robuste vecteur nul."""
|
||||
import numpy as np
|
||||
av = np.asarray(a, dtype=np.float32).flatten()
|
||||
bv = np.asarray(b, dtype=np.float32).flatten()
|
||||
na, nb = float(np.linalg.norm(av)), float(np.linalg.norm(bv))
|
||||
if na < 1e-8 or nb < 1e-8:
|
||||
return 0.0
|
||||
return float(np.dot(av, bv) / (na * nb))
|
||||
|
||||
|
||||
class LoopDetector:
|
||||
def __init__(self, clip_embedder=None):
|
||||
self.clip_embedder = clip_embedder
|
||||
|
||||
def evaluate(
|
||||
self,
|
||||
state: Dict[str, Any],
|
||||
screenshots: List[Any],
|
||||
actions: List[Dict[str, Any]],
|
||||
) -> LoopVerdict:
|
||||
"""Évalue les 3 signaux. Retourne le premier déclenché.
|
||||
|
||||
Args:
|
||||
state: replay_state (utilisé pour retried_actions)
|
||||
screenshots: anneau d'embeddings CLIP (les N derniers)
|
||||
actions: anneau des N dernières actions exécutées
|
||||
"""
|
||||
if not _env_bool_enabled("RPA_LOOP_DETECTOR_ENABLED"):
|
||||
return LoopVerdict(detected=False)
|
||||
|
||||
# Signal A : screen_static
|
||||
verdict = self._check_screen_static(screenshots)
|
||||
if verdict.detected:
|
||||
return verdict
|
||||
|
||||
# Signal B : action_repeat
|
||||
verdict = self._check_action_repeat(actions)
|
||||
if verdict.detected:
|
||||
return verdict
|
||||
|
||||
# Signal C : retry_threshold
|
||||
verdict = self._check_retry_threshold(state)
|
||||
if verdict.detected:
|
||||
return verdict
|
||||
|
||||
return LoopVerdict(detected=False)
|
||||
|
||||
def _check_screen_static(self, screenshots: List[Any]) -> LoopVerdict:
|
||||
n_required = _env_int("RPA_LOOP_SCREEN_STATIC_N", 4)
|
||||
threshold = _env_float("RPA_LOOP_SCREEN_STATIC_THRESHOLD", 0.99)
|
||||
|
||||
if self.clip_embedder is None or len(screenshots) < n_required:
|
||||
return LoopVerdict()
|
||||
|
||||
try:
|
||||
recent = screenshots[-n_required:]
|
||||
# Embed chaque capture via le CLIP embedder (peut lever)
|
||||
embeddings = [self.clip_embedder.embed_image(img) for img in recent]
|
||||
sims = [_cosine_similarity(embeddings[i], embeddings[i + 1])
|
||||
for i in range(len(embeddings) - 1)]
|
||||
min_sim = min(sims)
|
||||
if min_sim > threshold:
|
||||
return LoopVerdict(
|
||||
detected=True,
|
||||
reason="loop_detected",
|
||||
signal="screen_static",
|
||||
evidence={"min_similarity": round(min_sim, 4),
|
||||
"n_captures": n_required,
|
||||
"threshold": threshold},
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("LoopDetector signal_A erreur (%s) — signal inerte ce tick", e)
|
||||
return LoopVerdict()
|
||||
|
||||
def _check_action_repeat(self, actions: List[Dict[str, Any]]) -> LoopVerdict:
|
||||
n_required = _env_int("RPA_LOOP_ACTION_REPEAT_N", 3)
|
||||
if len(actions) < n_required:
|
||||
return LoopVerdict()
|
||||
recent = actions[-n_required:]
|
||||
|
||||
def _signature(a: Dict[str, Any]) -> tuple:
|
||||
return (a.get("type"), a.get("x_pct"), a.get("y_pct"))
|
||||
|
||||
sigs = [_signature(a) for a in recent]
|
||||
if all(s == sigs[0] for s in sigs):
|
||||
return LoopVerdict(
|
||||
detected=True,
|
||||
reason="loop_detected",
|
||||
signal="action_repeat",
|
||||
evidence={"signature": sigs[0], "count": n_required},
|
||||
)
|
||||
return LoopVerdict()
|
||||
|
||||
def _check_retry_threshold(self, state: Dict[str, Any]) -> LoopVerdict:
|
||||
threshold = _env_int("RPA_LOOP_RETRY_THRESHOLD", 3)
|
||||
retried = int(state.get("retried_actions", 0))
|
||||
if retried >= threshold:
|
||||
return LoopVerdict(
|
||||
detected=True,
|
||||
reason="loop_detected",
|
||||
signal="retry_threshold",
|
||||
evidence={"retried_actions": retried, "threshold": threshold},
|
||||
)
|
||||
return LoopVerdict()
|
||||
99
agent_v0/server_v1/monitor_router.py
Normal file
99
agent_v0/server_v1/monitor_router.py
Normal file
@@ -0,0 +1,99 @@
|
||||
# agent_v0/server_v1/monitor_router.py
|
||||
"""MonitorRouter — résolution de l'écran cible pour le replay (QW1).
|
||||
|
||||
Stratégie en cascade :
|
||||
1. action.monitor_index (hérité de la session source) → cible cet écran
|
||||
2. session.last_focused_monitor (focus actif vu en dernier heartbeat) → fallback
|
||||
3. composite (offset 0, 0) → backward compat
|
||||
|
||||
Émet sur le bus lea:* l'event monitor_routed avec la source de la décision.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class MonitorTarget:
|
||||
"""Représente l'écran cible résolu pour une action de replay."""
|
||||
idx: int
|
||||
offset_x: int
|
||||
offset_y: int
|
||||
w: int
|
||||
h: int
|
||||
source: str # "action" | "focus" | "composite_fallback"
|
||||
|
||||
|
||||
_COMPOSITE_FALLBACK = MonitorTarget(
|
||||
idx=-1,
|
||||
offset_x=0,
|
||||
offset_y=0,
|
||||
w=0,
|
||||
h=0,
|
||||
source="composite_fallback",
|
||||
)
|
||||
|
||||
|
||||
def _find_monitor(geometry: List[Dict[str, Any]], idx: int) -> Optional[Dict[str, Any]]:
|
||||
"""Retourne le monitor d'index donné, ou None si absent."""
|
||||
for m in geometry:
|
||||
if m.get("idx") == idx:
|
||||
return m
|
||||
return None
|
||||
|
||||
|
||||
def _to_target(monitor: Dict[str, Any], source: str) -> MonitorTarget:
|
||||
return MonitorTarget(
|
||||
idx=int(monitor["idx"]),
|
||||
offset_x=int(monitor.get("x", 0)),
|
||||
offset_y=int(monitor.get("y", 0)),
|
||||
w=int(monitor.get("w", 0)),
|
||||
h=int(monitor.get("h", 0)),
|
||||
source=source,
|
||||
)
|
||||
|
||||
|
||||
def resolve_target_monitor(
|
||||
action: Dict[str, Any],
|
||||
session_state: Dict[str, Any],
|
||||
) -> MonitorTarget:
|
||||
"""Résout l'écran cible d'une action de replay.
|
||||
|
||||
Args:
|
||||
action: Dict de l'action (peut contenir `monitor_index`).
|
||||
session_state: État de la session (doit contenir `monitors_geometry`
|
||||
et `last_focused_monitor`).
|
||||
|
||||
Returns:
|
||||
MonitorTarget avec l'offset à appliquer aux coordonnées de grounding.
|
||||
"""
|
||||
geometry: List[Dict[str, Any]] = session_state.get("monitors_geometry") or []
|
||||
|
||||
# 1. Cible explicite via action
|
||||
explicit_idx = action.get("monitor_index")
|
||||
if explicit_idx is not None and geometry:
|
||||
m = _find_monitor(geometry, int(explicit_idx))
|
||||
if m is not None:
|
||||
return _to_target(m, source="action")
|
||||
# Index invalide → on tombe sur le fallback focus
|
||||
logger.warning(
|
||||
"[BUS] lea:monitor_invalid_index requested=%d available_idx=%s",
|
||||
int(explicit_idx), [g.get("idx") for g in geometry],
|
||||
)
|
||||
|
||||
# 2. Fallback focus actif
|
||||
focused_idx = session_state.get("last_focused_monitor")
|
||||
if focused_idx is not None and geometry:
|
||||
m = _find_monitor(geometry, int(focused_idx))
|
||||
if m is not None:
|
||||
return _to_target(m, source="focus")
|
||||
logger.warning(
|
||||
"[BUS] lea:monitor_unavailable focused_idx=%d available_idx=%s",
|
||||
int(focused_idx), [g.get("idx") for g in geometry],
|
||||
)
|
||||
|
||||
# 3. Fallback composite (backward compat — comportement actuel mss.monitors[0])
|
||||
return _COMPOSITE_FALLBACK
|
||||
273
agent_v0/server_v1/pii_sanitizer.py
Normal file
273
agent_v0/server_v1/pii_sanitizer.py
Normal file
@@ -0,0 +1,273 @@
|
||||
"""Assainissement PII des données capturées (titres de fenêtre, texte saisi, OCR).
|
||||
|
||||
Côté serveur. Remplace la PII par des **tokens typés et cohérents**
|
||||
(`[IPP_1]`, `[AGE_1]`, `[NOM_1]`…) : on protège la donnée **et** on garde la
|
||||
structure (champ de type NOM/IPP) utile à l'apprentissage des variables.
|
||||
|
||||
Couche 1 (ce module, sans modèle) : filet **regex** sur la PII structurée
|
||||
(IPP, NIR, téléphone, email, âge) + règles **structurelles** des titres
|
||||
cliniques (`NOM (NAISSANCE) Prénom`, `[Nom Prénom]` des fenêtres PACS). Regex
|
||||
réutilisées du projet `anonymisation`.
|
||||
Couche 2 (à venir) : NER CamemBERT-bio (ONNX) pour les noms libres que la
|
||||
couche 1 ne capte pas — branchée plus tard, ce module marche sans.
|
||||
|
||||
Branche feat/push-log-dgx — assainissement PII clinique.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import re
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
# --- Filet regex (réutilisé de anonymisation/anonymizer_core_refactored_onnx.py) ---
|
||||
RE_IPP = re.compile(r"\b(?:I\.?P\.?P\.?|IPP|N°\s*Ipp)\s*[:\-]?\s*([A-Za-z0-9]{6,})\b", re.IGNORECASE)
|
||||
RE_NIR = re.compile(r"(?<!\d)[12]\s?\d{2}\s?\d{2}\s?\d{2}\s?\d{3}\s?\d{3}\s?\d{2}(?!\d)")
|
||||
RE_EMAIL = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}")
|
||||
RE_TEL = re.compile(r"(?<!\d)(?:\+33\s?|0)\d(?:[ .\-]?\d){8}(?!\d)")
|
||||
# Âge format « titre » (« 90 ans »), plus large que le regex prose de anonymisation.
|
||||
RE_AGE = re.compile(r"\b(\d{1,3})\s*ans\b", re.IGNORECASE)
|
||||
|
||||
_MAJ = r"A-ZÉÈÀÂÊÎÔÛÄËÏÖÜÇ"
|
||||
_MIN = r"a-zàâäéèêëïîôöùûüç"
|
||||
# Format clinique « NOM (NOM_NAISSANCE) Prénom » (ex. « ROSSIGNOL (SOUBIE) Pierrette »).
|
||||
RE_NOM_NAISSANCE = re.compile(
|
||||
rf"\b[{_MAJ}][{_MAJ}\-']+\s+\([{_MAJ}][{_MAJ}\-']+\)\s+[{_MAJ}][{_MIN}\-']+\b"
|
||||
)
|
||||
# Patient entre crochets des fenêtres PACS (ex. « [DATTIN Alix] »), ≥ 2 tokens capitalisés.
|
||||
RE_NOM_BRACKET = re.compile(
|
||||
rf"\[((?:[{_MAJ}][\w{_MIN}'\-]*\s+){{1,3}}[{_MAJ}][\w{_MIN}'\-]*)\]"
|
||||
)
|
||||
# « Prénom NOM » inversé, sans parenthèses ni crochets (ex. « Alix DATTIN »).
|
||||
# 2e mot tout en MAJUSCULES → faible risque de FP (« Mozilla Firefox » ne matche pas).
|
||||
RE_PRENOM_NOM = re.compile(rf"\b[{_MAJ}][{_MIN}]+\s+[{_MAJ}][{_MAJ}\-']+\b")
|
||||
|
||||
# GXD5 Diagnostics : numéro de dossier + nom patient tout-majuscules.
|
||||
# Format réel : « GXD5 Diagnostics - 128008 - BENVENISTE MARIE-LAURENCE »
|
||||
# Le numéro (128008) = ID dossier patient (PII). Le nom = PII.
|
||||
# 2 groupes de capture : (1)=numéro, (2)=nom complet.
|
||||
RE_GXD5_DIAG = re.compile(
|
||||
rf"GXD5\s+Diagnostics\s*-\s*(\d+)\s*-\s*([{_MAJ}][{_MAJ}\-' ]+)"
|
||||
)
|
||||
|
||||
# Ordre = priorité ; group = portion à remplacer (0 = match entier).
|
||||
_DETECTORS: List[Tuple[re.Pattern, str, int]] = [
|
||||
(RE_NOM_NAISSANCE, "NOM", 0),
|
||||
(RE_NOM_BRACKET, "NOM", 0),
|
||||
(RE_GXD5_DIAG, "DOSSIER", 1), # numéro de dossier
|
||||
(RE_PRENOM_NOM, "NOM", 0),
|
||||
(RE_EMAIL, "EMAIL", 0),
|
||||
(RE_NIR, "NIR", 0),
|
||||
(RE_IPP, "IPP", 1),
|
||||
(RE_TEL, "TEL", 0),
|
||||
(RE_AGE, "AGE", 0),
|
||||
]
|
||||
# GXD5 nom (groupe 2) traité séparément — même regex, priorité juste après.
|
||||
_DETECTORS.append((RE_GXD5_DIAG, "NOM", 2))
|
||||
|
||||
# Anti-faux-positifs : termes logiciels/UI à ne jamais prendre pour un nom.
|
||||
# (Sous-ensemble inline ; les gazetteers complets arrivent avec la couche NER.)
|
||||
_SOFTWARE_BLACKLIST = {
|
||||
"FIREFOX", "MOZILLA", "CHROME", "EDGE", "EXPERT", "SANTE", "SANTÉ", "PACS",
|
||||
"CIM", "ARES", "EASILY", "CONSULTATION", "URGENCES", "SAISIE", "COURRIER",
|
||||
"DOSSIER", "PATIENT", "FENETRE", "FENÊTRE", "GXD", "WINDOWS", "CITRIX",
|
||||
}
|
||||
|
||||
|
||||
def _normalize(etype: str, value: str) -> str:
|
||||
"""Clé de cohérence : même entité -> même token."""
|
||||
if etype in ("IPP", "NIR", "TEL"):
|
||||
return re.sub(r"\s+", "", value)
|
||||
if etype == "EMAIL":
|
||||
return value.lower()
|
||||
return re.sub(r"\s+", " ", value).strip().upper()
|
||||
|
||||
|
||||
def _is_blacklisted_name(value: str) -> bool:
|
||||
toks = [t for t in re.split(r"[^\wÀ-ÿ]+", value) if t]
|
||||
return bool(toks) and all(t.upper() in _SOFTWARE_BLACKLIST for t in toks)
|
||||
|
||||
|
||||
def _assign_token(mapping: Dict, etype: str, norm: str) -> str:
|
||||
key = (etype, norm)
|
||||
if key in mapping:
|
||||
return mapping[key]
|
||||
n = 1 + sum(1 for k in mapping if isinstance(k, tuple) and k[0] == etype)
|
||||
token = f"[{etype}_{n}]"
|
||||
mapping[key] = token
|
||||
return token
|
||||
|
||||
|
||||
def anonymize_text(
|
||||
text: str, *, mapping: Optional[Dict] = None
|
||||
) -> Tuple[str, List[Dict]]:
|
||||
"""Remplace la PII de `text` par des tokens typés cohérents.
|
||||
|
||||
`mapping` : table de cohérence partagée (ex. à l'échelle d'une session) —
|
||||
la même valeur PII reçoit le même token d'un appel à l'autre. Mutée en place ;
|
||||
si None, une table locale est utilisée.
|
||||
|
||||
Retourne `(texte_assaini, entités)` où chaque entité =
|
||||
`{"type", "original", "token", "start", "end"}` (positions dans le texte source).
|
||||
"""
|
||||
if not text:
|
||||
return text, []
|
||||
if mapping is None:
|
||||
mapping = {}
|
||||
|
||||
# 1) collecte des candidats (start, end, type, valeur)
|
||||
spans: List[Tuple[int, int, str, str]] = []
|
||||
for pattern, etype, group in _DETECTORS:
|
||||
for m in pattern.finditer(text):
|
||||
start, end = m.span(group)
|
||||
if start == end:
|
||||
continue
|
||||
value = m.group(group)
|
||||
if etype == "NOM" and _is_blacklisted_name(value):
|
||||
continue
|
||||
spans.append((start, end, etype, value))
|
||||
|
||||
# 2) résolution des chevauchements (priorité = rang détecteur, puis -longueur)
|
||||
# _DETECTORS est ordonné par priorité ; le rang dans cette liste détermine
|
||||
# qui gagne quand deux patterns chevauchent. Plus prioritaire + plus long
|
||||
# = accepté en premier, les plus courts/moins prioritaires sont éliminés.
|
||||
# Fix FN « Dossier VIOLA (VIOLA) Liliane » : RE_PRENOM_NOM captait
|
||||
# « Dossier VIOLA » (rang 2) et bloquait RE_NOM_NAISSANCE « VIOLA (VIOLA)
|
||||
# Liliane » (rang 0, plus prioritaire et plus long).
|
||||
det_rank = {p: i for i, (p, _, _) in enumerate(_DETECTORS)}
|
||||
spans.sort(key=lambda s: (det_rank.get(s[2], 999), -(s[1] - s[0]), s[0]))
|
||||
occupied: List[Tuple[int, int]] = []
|
||||
accepted: List[Tuple[int, int, str, str]] = []
|
||||
for start, end, etype, value in spans:
|
||||
if all(start >= oe or end <= os for os, oe in occupied):
|
||||
accepted.append((start, end, etype, value))
|
||||
occupied.append((start, end))
|
||||
|
||||
# 3) substitution (de droite à gauche pour préserver les indices)
|
||||
entities: List[Dict] = []
|
||||
out = text
|
||||
for start, end, etype, value in sorted(accepted, key=lambda s: s[0], reverse=True):
|
||||
token = _assign_token(mapping, etype, _normalize(etype, value))
|
||||
out = out[:start] + token + out[end:]
|
||||
entities.append(
|
||||
{"type": etype, "original": value, "token": token, "start": start, "end": end}
|
||||
)
|
||||
entities.reverse()
|
||||
return out, entities
|
||||
|
||||
|
||||
# Clés portant un titre de fenêtre, où qu'elles soient imbriquées dans l'event
|
||||
# (top-level `active_window_title`, `window/to/from.title`, et surtout
|
||||
# `vision_info.window_capture.window_title` — blind spot signalé par Qwen).
|
||||
_TITLE_KEYS = ("title", "window_title", "active_window_title")
|
||||
_PLACEHOLDER_SAISIE = "[SAISIE]"
|
||||
|
||||
|
||||
def _walk_titles(obj, mapping: Dict) -> None:
|
||||
"""Parcourt récursivement l'event et assainit toute valeur de titre de fenêtre."""
|
||||
if isinstance(obj, dict):
|
||||
for k, v in obj.items():
|
||||
if k in _TITLE_KEYS and isinstance(v, str):
|
||||
obj[k] = anonymize_text(v, mapping=mapping)[0]
|
||||
else:
|
||||
_walk_titles(v, mapping)
|
||||
elif isinstance(obj, list):
|
||||
for item in obj:
|
||||
_walk_titles(item, mapping)
|
||||
|
||||
|
||||
def sanitize_event(event: Dict, *, mapping: Optional[Dict] = None) -> Dict:
|
||||
"""Assainit un event capturé avant persistance (copie, ne mute pas l'original).
|
||||
|
||||
Principe « Léa apprend l'interface, pas la donnée » (décision Dom 28/06) :
|
||||
- `text_input` : le **contenu tapé** (`text`, `raw_keys`) = donnée de santé →
|
||||
remplacé par `[SAISIE]` (on garde le champ, pas la valeur — option b) ;
|
||||
- **titres de fenêtre** (`active_window_title`, et `title` dans `window`/`to`/
|
||||
`from`) : l'**identité patient** est tokenisée, l'app/écran est gardé
|
||||
(contexte d'apprentissage), via `anonymize_text` + `mapping` partagé (cohérence).
|
||||
"""
|
||||
if mapping is None:
|
||||
mapping = {}
|
||||
ev = copy.deepcopy(event)
|
||||
|
||||
# text_input : on ne garde pas le contenu
|
||||
if ev.get("type") == "text_input":
|
||||
for k in ("text", "raw_keys"):
|
||||
if ev.get(k) not in (None, ""):
|
||||
ev[k] = _PLACEHOLDER_SAISIE
|
||||
|
||||
# tous les titres de fenêtre, où qu'ils soient imbriqués
|
||||
# (active_window_title, window/to/from.title, vision_info.window_capture.window_title…)
|
||||
_walk_titles(ev, mapping)
|
||||
|
||||
return ev
|
||||
|
||||
|
||||
def sanitize_log_entries(
|
||||
entries: List[Dict], *, mapping: Optional[Dict] = None
|
||||
) -> List[Dict]:
|
||||
"""Assainit un batch de log-entries reçues d'un client Léa avant persistance.
|
||||
|
||||
Pour chaque entrée, renvoie une **copie** où les champs texte porteurs de PII
|
||||
sont passés par `anonymize_text` :
|
||||
- `message` (str) : assaini par `anonymize_text`.
|
||||
- `logger` (str) : assaini de la même façon (peut porter un chemin patient).
|
||||
- `ts` et `level` : préservés à l'identique, jamais touchés.
|
||||
|
||||
Un `mapping` partagé est utilisé pour **toutes** les entrées du batch afin de
|
||||
garantir la cohérence des tokens (même PII → même token). Si `mapping` est
|
||||
None, un mapping local est créé et partagé entre toutes les entrées du batch.
|
||||
|
||||
Tolère les valeurs absentes, None ou non-str sans lever d'exception.
|
||||
N'utilise que `anonymize_text` — aucune regex supplémentaire.
|
||||
"""
|
||||
if not entries:
|
||||
return []
|
||||
if mapping is None:
|
||||
mapping = {}
|
||||
|
||||
result: List[Dict] = []
|
||||
for entry in entries:
|
||||
item = copy.copy(entry) # copie superficielle suffit (valeurs scalaires)
|
||||
for field in ("message", "logger"):
|
||||
v = item.get(field)
|
||||
if isinstance(v, str):
|
||||
item[field] = anonymize_text(v, mapping=mapping)[0]
|
||||
result.append(item)
|
||||
return result
|
||||
|
||||
|
||||
# Clés d'un workflow core portant du texte potentiellement PII : cible OCR
|
||||
# (`by_text`), noms d'écrans/labels dérivés des titres. Le contenu saisi est
|
||||
# déjà neutralisé à la source (sanitize_event → [SAISIE]).
|
||||
_WORKFLOW_TEXT_KEYS = ("by_text", "name", "label")
|
||||
|
||||
|
||||
def _walk_workflow_text(obj, mapping: Dict) -> None:
|
||||
"""Parcourt un workflow core et tokenise la PII des champs texte (cibles, noms)."""
|
||||
if isinstance(obj, dict):
|
||||
for k, v in obj.items():
|
||||
if k in _WORKFLOW_TEXT_KEYS and isinstance(v, str) and v:
|
||||
obj[k] = anonymize_text(v, mapping=mapping)[0]
|
||||
else:
|
||||
_walk_workflow_text(v, mapping)
|
||||
elif isinstance(obj, list):
|
||||
for item in obj:
|
||||
_walk_workflow_text(item, mapping)
|
||||
|
||||
|
||||
def sanitize_workflow_dict(workflow_dict: Dict, *, mapping: Optional[Dict] = None) -> Dict:
|
||||
"""Assainit un workflow core (JSON appris) avant import/persistance en DB VWB.
|
||||
|
||||
Tokenise la PII des champs texte (cible OCR `by_text`, noms d'écrans, labels)
|
||||
via `anonymize_text`, en gardant l'interface intacte (« Léa apprend
|
||||
l'interface, pas la donnée »). Copie — l'original n'est pas muté.
|
||||
|
||||
Limite (couche 1) : ne capte que la PII structurée (IPP, NOM clinique…) ;
|
||||
les noms libres relèvent de la couche 2 NER.
|
||||
"""
|
||||
if mapping is None:
|
||||
mapping = {}
|
||||
wf = copy.deepcopy(workflow_dict)
|
||||
_walk_workflow_text(wf, mapping)
|
||||
return wf
|
||||
3142
agent_v0/server_v1/replay_engine.py
Normal file
3142
agent_v0/server_v1/replay_engine.py
Normal file
File diff suppressed because it is too large
Load Diff
143
agent_v0/server_v1/replay_failure_logger.py
Normal file
143
agent_v0/server_v1/replay_failure_logger.py
Normal file
@@ -0,0 +1,143 @@
|
||||
# agent_v0/server_v1/replay_failure_logger.py
|
||||
"""
|
||||
Logger des echecs de replay pour l'apprentissage futur.
|
||||
|
||||
Chaque echec de resolution visuelle (target_not_found) est sauvegarde dans un
|
||||
fichier JSONL par session, avec le screenshot de ce que l'agent voit au moment
|
||||
de l'echec. Ces donnees alimentent le learning loop : re-entrainement des
|
||||
embeddings, ajustement des seuils, enrichissement des target_spec.
|
||||
|
||||
Structure :
|
||||
data/training/replay_failures/{replay_id}/failures.jsonl
|
||||
data/training/replay_failures/{replay_id}/screenshots/{action_id}.jpg
|
||||
"""
|
||||
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
logger = logging.getLogger("replay_failure_logger")
|
||||
|
||||
# Repertoire racine des echecs de replay
|
||||
_FAILURES_BASE_DIR = Path("data/training/replay_failures")
|
||||
|
||||
# Lock pour les ecritures concurrentes
|
||||
_write_lock = threading.Lock()
|
||||
|
||||
|
||||
def log_replay_failure(
|
||||
replay_id: str,
|
||||
action_id: str,
|
||||
target_spec: Optional[Dict[str, Any]],
|
||||
screenshot_b64: Optional[str],
|
||||
resolution_attempts: Optional[List[Dict[str, Any]]] = None,
|
||||
error: str = "target_not_found",
|
||||
extra: Optional[Dict[str, Any]] = None,
|
||||
) -> Optional[str]:
|
||||
"""Sauvegarder un echec de replay pour l'apprentissage futur.
|
||||
|
||||
Args:
|
||||
replay_id: Identifiant du replay en cours
|
||||
action_id: Identifiant de l'action echouee
|
||||
target_spec: Specification de la cible recherchee
|
||||
screenshot_b64: Screenshot JPEG base64 de ce que l'agent voit
|
||||
resolution_attempts: Liste des tentatives de resolution (methode, score, etc.)
|
||||
error: Type d'erreur (defaut: "target_not_found")
|
||||
extra: Champs supplementaires a stocker
|
||||
|
||||
Returns:
|
||||
Chemin du fichier JSONL cree, ou None en cas d'erreur.
|
||||
"""
|
||||
try:
|
||||
# Creer le repertoire de la session
|
||||
session_dir = _FAILURES_BASE_DIR / replay_id
|
||||
session_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Sauvegarder le screenshot si fourni
|
||||
screenshot_path = None
|
||||
if screenshot_b64:
|
||||
screenshots_dir = session_dir / "screenshots"
|
||||
screenshots_dir.mkdir(exist_ok=True)
|
||||
screenshot_path = str(screenshots_dir / f"{action_id}.jpg")
|
||||
try:
|
||||
img_bytes = base64.b64decode(screenshot_b64)
|
||||
with open(screenshot_path, "wb") as f:
|
||||
f.write(img_bytes)
|
||||
except Exception as e:
|
||||
logger.warning(f"Impossible de sauvegarder le screenshot : {e}")
|
||||
screenshot_path = None
|
||||
|
||||
# Construire l'entree JSONL
|
||||
entry = {
|
||||
"timestamp": time.strftime("%Y-%m-%dT%H:%M:%S"),
|
||||
"replay_id": replay_id,
|
||||
"action_id": action_id,
|
||||
"target_spec": _sanitize_target_spec(target_spec) if target_spec else None,
|
||||
"screenshot_path": screenshot_path,
|
||||
"resolution_attempts": resolution_attempts or [],
|
||||
"error": error,
|
||||
}
|
||||
if extra:
|
||||
entry.update(extra)
|
||||
|
||||
# Ecrire dans le fichier JSONL (thread-safe)
|
||||
jsonl_path = session_dir / "failures.jsonl"
|
||||
with _write_lock:
|
||||
with open(jsonl_path, "a", encoding="utf-8") as f:
|
||||
f.write(json.dumps(entry, ensure_ascii=False) + "\n")
|
||||
|
||||
logger.info(
|
||||
f"Echec replay loggue : replay={replay_id} action={action_id} "
|
||||
f"error={error} -> {jsonl_path}"
|
||||
)
|
||||
return str(jsonl_path)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Impossible de logger l'echec replay : {e}")
|
||||
return None
|
||||
|
||||
|
||||
def _sanitize_target_spec(target_spec: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Nettoyer le target_spec pour le stockage (retirer les images base64 volumineuses)."""
|
||||
cleaned = {}
|
||||
for key, value in target_spec.items():
|
||||
# Ne pas stocker les images base64 (trop volumineux pour le JSONL)
|
||||
if key.endswith("_base64") or key.endswith("_b64"):
|
||||
cleaned[key] = f"<{len(str(value))} chars>" if value else None
|
||||
else:
|
||||
cleaned[key] = value
|
||||
return cleaned
|
||||
|
||||
|
||||
def get_failure_count(replay_id: str) -> int:
|
||||
"""Compter le nombre d'echecs pour un replay donne."""
|
||||
jsonl_path = _FAILURES_BASE_DIR / replay_id / "failures.jsonl"
|
||||
if not jsonl_path.exists():
|
||||
return 0
|
||||
try:
|
||||
with open(jsonl_path, "r", encoding="utf-8") as f:
|
||||
return sum(1 for _ in f)
|
||||
except Exception:
|
||||
return 0
|
||||
|
||||
|
||||
def get_failures(replay_id: str) -> List[Dict[str, Any]]:
|
||||
"""Lire tous les echecs pour un replay donne."""
|
||||
jsonl_path = _FAILURES_BASE_DIR / replay_id / "failures.jsonl"
|
||||
if not jsonl_path.exists():
|
||||
return []
|
||||
failures = []
|
||||
try:
|
||||
with open(jsonl_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if line:
|
||||
failures.append(json.loads(line))
|
||||
except Exception as e:
|
||||
logger.warning(f"Erreur lecture echecs replay {replay_id} : {e}")
|
||||
return failures
|
||||
@@ -175,6 +175,101 @@ class ReplayLearner:
|
||||
|
||||
self.record(outcome)
|
||||
|
||||
def record_human_correction(
|
||||
self,
|
||||
session_id: str,
|
||||
action: Dict[str, Any],
|
||||
correction: Dict[str, Any],
|
||||
) -> None:
|
||||
"""Enregistrer une correction humaine (mode apprentissage supervisé).
|
||||
|
||||
L'humain a montré à Léa où cliquer. On stocke cette correction
|
||||
dans target_memory.db pour que la prochaine fois, Léa sache.
|
||||
"""
|
||||
target_spec = action.get("target_spec", {})
|
||||
by_text = target_spec.get("by_text", "")
|
||||
window_title = (
|
||||
target_spec.get("window_title", "")
|
||||
or action.get("window_title", "")
|
||||
or target_spec.get("expected_window_before", "")
|
||||
or (target_spec.get("context_hints") or {}).get("window_title", "")
|
||||
)
|
||||
x_pct = correction.get("x_pct")
|
||||
y_pct = correction.get("y_pct")
|
||||
last_click = correction.get("last_click")
|
||||
if (x_pct is None or y_pct is None) and isinstance(last_click, dict):
|
||||
x_pct = last_click.get("x_pct")
|
||||
y_pct = last_click.get("y_pct")
|
||||
|
||||
try:
|
||||
x_pct_f = float(x_pct)
|
||||
y_pct_f = float(y_pct)
|
||||
except (TypeError, ValueError):
|
||||
logger.warning(
|
||||
"[APPRENTISSAGE] Correction humaine non persistée : "
|
||||
"aucune coordonnée clic exploitable pour '%s'",
|
||||
by_text,
|
||||
)
|
||||
return
|
||||
|
||||
if not (0.0 < x_pct_f <= 1.0 and 0.0 < y_pct_f <= 1.0):
|
||||
logger.warning(
|
||||
"[APPRENTISSAGE] Correction humaine non persistée : "
|
||||
"coordonnées hors bornes pour '%s' (%.4f, %.4f)",
|
||||
by_text,
|
||||
x_pct_f,
|
||||
y_pct_f,
|
||||
)
|
||||
return
|
||||
|
||||
# Enregistrer dans le JSONL d'apprentissage
|
||||
outcome = ActionOutcome(
|
||||
session_id=session_id,
|
||||
action_id=action.get("action_id", ""),
|
||||
action_type="click",
|
||||
target_description=by_text,
|
||||
window_title=window_title,
|
||||
resolution_method="human_supervised",
|
||||
resolution_score=1.0, # Confiance maximale — l'humain a montré
|
||||
success=True,
|
||||
)
|
||||
self.record(outcome)
|
||||
|
||||
# Stocker dans target_memory.db pour le lookup futur
|
||||
try:
|
||||
from .replay_memory import memory_record_success
|
||||
stored = False
|
||||
if window_title:
|
||||
stored = memory_record_success(
|
||||
window_title=window_title,
|
||||
target_spec=target_spec,
|
||||
x_pct=x_pct_f,
|
||||
y_pct=y_pct_f,
|
||||
method="human_supervised",
|
||||
confidence=1.0,
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
"[APPRENTISSAGE] Correction humaine non persistée : "
|
||||
"window_title absent pour '%s'",
|
||||
by_text,
|
||||
)
|
||||
|
||||
if stored:
|
||||
logger.info(
|
||||
f"[APPRENTISSAGE] Correction stockée dans target_memory : "
|
||||
f"'{by_text}' → ({x_pct:.4f}, {y_pct:.4f})"
|
||||
)
|
||||
elif window_title:
|
||||
logger.warning(
|
||||
"[APPRENTISSAGE] Correction humaine non persistée : "
|
||||
"échec memory_record_success pour '%s' dans '%s'",
|
||||
by_text,
|
||||
window_title,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Learning: échec stockage target_memory: {e}")
|
||||
|
||||
def query_similar(
|
||||
self,
|
||||
target_description: str = "",
|
||||
|
||||
487
agent_v0/server_v1/replay_memory.py
Normal file
487
agent_v0/server_v1/replay_memory.py
Normal file
@@ -0,0 +1,487 @@
|
||||
# agent_v0/server_v1/replay_memory.py
|
||||
"""
|
||||
replay_memory — Greffe de TargetMemoryStore (Fiche #18) sur le pipeline V4.
|
||||
|
||||
Phase 1 du plan apprentissage Léa (docs/PLAN_APPRENTISSAGE_LEA.md).
|
||||
|
||||
Le runtime V4 appelle :
|
||||
- `memory_lookup()` AVANT la cascade coûteuse (OCR/template/VLM)
|
||||
- `memory_record_success()` APRÈS validation post-condition (`title_match` strict)
|
||||
- `memory_record_failure()` sur les échecs
|
||||
|
||||
Fingerprint léger V4 : les coordonnées clic (x_pct, y_pct) sont stockées dans
|
||||
les deux premières valeurs de `TargetFingerprint.bbox`, et la méthode de
|
||||
résolution ayant réussi dans le champ `etype`.
|
||||
|
||||
Signature d'écran V4 : `sha256(normalize(window_title))[:16]`. Simple et
|
||||
robuste aux données variables car les titres de fenêtre restent stables.
|
||||
Les faux positifs (même titre, écrans différents) sont rattrapés par la
|
||||
post-condition qui décrémentera la fiabilité via `record_failure()`.
|
||||
|
||||
Critère de fiabilité : 2 succès minimum et < 30% d'échecs pour déclencher
|
||||
un hit (paramètres de `TargetMemoryStore.lookup`). C'est exactement la
|
||||
cristallisation par répétition que l'on veut — Léa est un stagiaire qui
|
||||
apprend de l'observation.
|
||||
|
||||
Auteur : Dom, Alice — avril 2026
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
import os
|
||||
import unicodedata
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# =========================================================================
|
||||
# Singleton du store persistant
|
||||
# =========================================================================
|
||||
|
||||
_MEMORY_SINGLETON: Optional[Any] = None
|
||||
_MEMORY_DISABLED = False
|
||||
|
||||
_GENERIC_BUTTON_TEXTS = {
|
||||
"annuler",
|
||||
"cancel",
|
||||
"enregistrer",
|
||||
"non",
|
||||
"no",
|
||||
"ok",
|
||||
"oui",
|
||||
"ouvrir",
|
||||
"open",
|
||||
"remplacer",
|
||||
"replace",
|
||||
"save",
|
||||
"yes",
|
||||
}
|
||||
|
||||
|
||||
def get_memory_store():
|
||||
"""Retourne le `TargetMemoryStore` partagé, ou None si indisponible.
|
||||
|
||||
Lazy-init : le store n'est créé qu'au premier appel, ce qui évite
|
||||
d'importer `core.learning.target_memory_store` à l'import du module
|
||||
(et donc d'éviter les effets de bord sur le démarrage du serveur).
|
||||
"""
|
||||
global _MEMORY_SINGLETON, _MEMORY_DISABLED
|
||||
|
||||
if _MEMORY_DISABLED:
|
||||
return None
|
||||
if _MEMORY_SINGLETON is not None:
|
||||
return _MEMORY_SINGLETON
|
||||
|
||||
try:
|
||||
from core.learning.target_memory_store import TargetMemoryStore
|
||||
|
||||
base_path = os.environ.get("RPA_LEARNING_DIR", "data/learning")
|
||||
_MEMORY_SINGLETON = TargetMemoryStore(base_path=base_path)
|
||||
logger.info(
|
||||
"replay_memory: TargetMemoryStore initialisé (base=%s)", base_path,
|
||||
)
|
||||
return _MEMORY_SINGLETON
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"replay_memory: TargetMemoryStore indisponible (%s) — "
|
||||
"l'apprentissage persistant est désactivé", exc,
|
||||
)
|
||||
_MEMORY_DISABLED = True
|
||||
return None
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Normalisation de texte et hash
|
||||
# =========================================================================
|
||||
|
||||
|
||||
def _norm_text(s: str) -> str:
|
||||
"""Normalise un texte pour un hash stable (accents, casse, NBSP, espaces)."""
|
||||
if not s:
|
||||
return ""
|
||||
s = s.replace("\u00A0", " ").strip().lower()
|
||||
s = unicodedata.normalize("NFKD", s)
|
||||
s = "".join(ch for ch in s if not unicodedata.combining(ch))
|
||||
return " ".join(s.split())
|
||||
|
||||
|
||||
def _memory_lookup_skip_reason(target_spec: Dict[str, Any]) -> str:
|
||||
"""Retourne la raison pour laquelle la mémoire ne doit pas court-circuiter.
|
||||
|
||||
Les clics qui changent de fenêtre doivent être résolus visuellement à
|
||||
l'instant T : une coordonnée apprise peut être une bonne piste, mais pas
|
||||
une décision finale. Pour les boutons très génériques, on exige au moins
|
||||
un contexte de fenêtre/interaction dans la clé mémoire afin d'éviter les
|
||||
collisions entre « Enregistrer », « OK », « Oui », etc.
|
||||
"""
|
||||
if not isinstance(target_spec, dict):
|
||||
return ""
|
||||
|
||||
hints = target_spec.get("context_hints") or {}
|
||||
if bool(hints.get("requires_window_transition")):
|
||||
return "window_transition_requires_visual_confirmation"
|
||||
|
||||
button_text = _norm_text(str(target_spec.get("by_text") or ""))
|
||||
if button_text not in _GENERIC_BUTTON_TEXTS:
|
||||
return ""
|
||||
|
||||
before = (
|
||||
hints.get("expected_window_before")
|
||||
or hints.get("button_expected_before_window")
|
||||
or hints.get("window_title")
|
||||
or target_spec.get("window_title")
|
||||
)
|
||||
after = (
|
||||
hints.get("expected_window_after")
|
||||
or hints.get("button_expected_after_window")
|
||||
or hints.get("expected_after_window")
|
||||
)
|
||||
interaction = hints.get("interaction") or hints.get("foreground_dialog_id")
|
||||
role = target_spec.get("by_role")
|
||||
if not (before and role and (after or interaction)):
|
||||
return "generic_button_missing_context"
|
||||
return ""
|
||||
|
||||
|
||||
def compute_screen_sig(window_title: str) -> str:
|
||||
"""Calcule la signature d'écran V4 à partir du titre de fenêtre.
|
||||
|
||||
Le `window_title` est strict depuis la phase "controle des étapes"
|
||||
(post-condition `title_match` obligatoire). C'est notre clé naturelle.
|
||||
"""
|
||||
norm = _norm_text(window_title)
|
||||
if not norm:
|
||||
return ""
|
||||
return hashlib.sha256(norm.encode("utf-8")).hexdigest()[:16]
|
||||
|
||||
|
||||
def _round_float_list(values: Any, precision: int = 4) -> Optional[tuple[float, ...]]:
|
||||
"""Normaliser une liste de coordonnées flottantes pour le hash mémoire."""
|
||||
if not isinstance(values, (list, tuple)):
|
||||
return None
|
||||
out = []
|
||||
for value in values:
|
||||
try:
|
||||
out.append(round(float(value), precision))
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return tuple(out)
|
||||
|
||||
|
||||
def _int_pair(values: Any) -> Optional[tuple[int, int]]:
|
||||
"""Extraire une paire entière stable pour les hints spatiaux."""
|
||||
if not isinstance(values, (list, tuple)) or len(values) < 2:
|
||||
return None
|
||||
try:
|
||||
return int(values[0]), int(values[1])
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _should_reuse_recorded_window_relative_coords(fp: Any) -> bool:
|
||||
"""Décider si on doit remplacer la mémoire apprise par la position source.
|
||||
|
||||
Cette réécriture n'est légitime que pour les entrées faibles de type
|
||||
`position_fallback`/`v4_unknown`, où la mémoire ne contient pas une vraie
|
||||
localisation visuelle robuste mais seulement un clic écran dépendant de la
|
||||
résolution. Pour les méthodes visuelles apprises (template, SoM, OCR...),
|
||||
réinjecter un vieux `click_relative` source crée des collisions et des
|
||||
dérives sur des boutons homonymes (`Enregistrer`, `OK`, etc.).
|
||||
"""
|
||||
method = str(getattr(fp, "etype", "") or "").strip().lower()
|
||||
return method in {"position_fallback", "v4_unknown"}
|
||||
|
||||
|
||||
class _TargetSpecLike:
|
||||
"""Adaptateur dict → objet pour `TargetMemoryStore._hash_target_spec()`.
|
||||
|
||||
Le hash interne de TargetMemoryStore utilise `getattr(spec, "by_role", ...)`
|
||||
qui ne fonctionne pas avec un dict brut. On expose les attributs nécessaires.
|
||||
|
||||
On intègre aussi `resolve_order`, `vlm_description` et des indices
|
||||
spatiaux (SoM, click_relative) dans `context_hints` pour qu'ils entrent
|
||||
dans le hash. Sinon, deux actions `Enregistrer` dans la même fenêtre
|
||||
mais à des emplacements différents collisionnent.
|
||||
"""
|
||||
|
||||
__slots__ = ("by_role", "by_text", "by_position", "context_hints")
|
||||
|
||||
def __init__(self, d: Dict[str, Any]):
|
||||
self.by_role = d.get("by_role", "") or ""
|
||||
self.by_text = d.get("by_text", "") or ""
|
||||
self.by_position = d.get("by_position")
|
||||
|
||||
hints = dict(d.get("context_hints") or {})
|
||||
resolve_order = d.get("resolve_order")
|
||||
if resolve_order:
|
||||
hints["_resolve_order"] = "|".join(resolve_order) if isinstance(
|
||||
resolve_order, list
|
||||
) else str(resolve_order)
|
||||
if d.get("vlm_description"):
|
||||
hints["_vlm_desc"] = str(d["vlm_description"])
|
||||
if d.get("anchor_hint"):
|
||||
hints["_anchor_hint"] = str(d["anchor_hint"])
|
||||
|
||||
som_element = d.get("som_element") or {}
|
||||
som_bbox = _round_float_list(som_element.get("bbox_norm"))
|
||||
if som_bbox:
|
||||
hints["_som_bbox"] = som_bbox
|
||||
som_center = _round_float_list(som_element.get("center_norm"), precision=5)
|
||||
if som_center:
|
||||
hints["_som_center"] = som_center
|
||||
|
||||
window_capture = d.get("window_capture") or {}
|
||||
click_relative = _int_pair(window_capture.get("click_relative"))
|
||||
window_size = _int_pair(window_capture.get("window_size"))
|
||||
if click_relative and window_size:
|
||||
hints["_window_rel"] = f"{click_relative[0]},{click_relative[1]}@{window_size[0]}x{window_size[1]}"
|
||||
|
||||
self.context_hints = hints
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Lookup — consulté AVANT la cascade coûteuse
|
||||
# =========================================================================
|
||||
|
||||
|
||||
def memory_lookup(
|
||||
window_title: str,
|
||||
target_spec: Dict[str, Any],
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Cherche une résolution apprise pour cette cible sur cet écran.
|
||||
|
||||
Returns:
|
||||
Dict compatible avec le format de sortie de `_resolve_target_sync`
|
||||
(resolved, method, x_pct, y_pct, score, ...) si une entrée fiable
|
||||
est trouvée. None sinon.
|
||||
"""
|
||||
skip_reason = _memory_lookup_skip_reason(target_spec)
|
||||
if skip_reason:
|
||||
logger.info("memory_lookup SKIP : %s", skip_reason)
|
||||
return None
|
||||
|
||||
store = get_memory_store()
|
||||
if store is None:
|
||||
return None
|
||||
|
||||
screen_sig = compute_screen_sig(window_title)
|
||||
if not screen_sig:
|
||||
return None
|
||||
|
||||
try:
|
||||
spec_shim = _TargetSpecLike(target_spec)
|
||||
fp = store.lookup(screen_sig, spec_shim)
|
||||
except Exception as exc:
|
||||
logger.debug("memory_lookup: erreur lookup (%s)", exc)
|
||||
return None
|
||||
|
||||
if fp is None:
|
||||
return None
|
||||
|
||||
# Fingerprint léger : bbox = (x_pct, y_pct, 0, 0)
|
||||
try:
|
||||
x_pct = float(fp.bbox[0])
|
||||
y_pct = float(fp.bbox[1])
|
||||
except (TypeError, IndexError, ValueError):
|
||||
logger.debug("memory_lookup: fingerprint bbox invalide")
|
||||
return None
|
||||
|
||||
# Quand l'entrée mémoire provient d'un simple `position_fallback`, les
|
||||
# coordonnées stockées reflètent surtout la géométrie écran source. Dans
|
||||
# ce cas précis, réutiliser la position relative enregistrée dans la
|
||||
# fenêtre source reste préférable si elle existe.
|
||||
#
|
||||
# En revanche, pour une méthode visuelle réellement apprise
|
||||
# (`anchor_template`, `som_*`, `hybrid_text_direct`, ...), remplacer les
|
||||
# coords mémorisées par un vieux `click_relative` crée des dérives sur
|
||||
# des cibles textuelles homonymes. On garde donc les coords apprises.
|
||||
window_capture = target_spec.get("window_capture") or {}
|
||||
click_relative = window_capture.get("click_relative")
|
||||
window_size = window_capture.get("window_size")
|
||||
if (
|
||||
_should_reuse_recorded_window_relative_coords(fp)
|
||||
and (
|
||||
isinstance(click_relative, (list, tuple))
|
||||
and len(click_relative) >= 2
|
||||
and isinstance(window_size, (list, tuple))
|
||||
and len(window_size) >= 2
|
||||
)
|
||||
):
|
||||
try:
|
||||
rel_x = float(click_relative[0])
|
||||
rel_y = float(click_relative[1])
|
||||
win_w = float(window_size[0])
|
||||
win_h = float(window_size[1])
|
||||
if win_w > 1 and win_h > 1:
|
||||
x_pct = rel_x / win_w
|
||||
y_pct = rel_y / win_h
|
||||
logger.info(
|
||||
"memory_lookup: coords fenêtre source réutilisées "
|
||||
"(click_relative=%s, window_size=%s) -> (%.4f, %.4f)",
|
||||
click_relative,
|
||||
window_size,
|
||||
x_pct,
|
||||
y_pct,
|
||||
)
|
||||
except (TypeError, ValueError, ZeroDivisionError):
|
||||
logger.debug("memory_lookup: window_capture invalide, fallback bbox")
|
||||
|
||||
# Sanity check : les pourcentages doivent être dans [0, 1]
|
||||
if not (0.0 <= x_pct <= 1.0 and 0.0 <= y_pct <= 1.0):
|
||||
logger.warning(
|
||||
"memory_lookup: coords invalides (%.3f, %.3f) pour sig=%s — "
|
||||
"entrée ignorée",
|
||||
x_pct, y_pct, screen_sig,
|
||||
)
|
||||
return None
|
||||
|
||||
method = fp.etype or "memory"
|
||||
confidence = float(getattr(fp, "confidence", 0.9) or 0.9)
|
||||
|
||||
logger.info(
|
||||
"memory_lookup HIT : sig=%s method=%s coords=(%.4f, %.4f) conf=%.2f "
|
||||
"target='%s'",
|
||||
screen_sig, method, x_pct, y_pct, confidence,
|
||||
(target_spec.get("by_text") or "")[:60],
|
||||
)
|
||||
|
||||
return {
|
||||
"resolved": True,
|
||||
"method": f"memory_{method}",
|
||||
"x_pct": x_pct,
|
||||
"y_pct": y_pct,
|
||||
"score": confidence,
|
||||
"from_memory": True,
|
||||
"screen_sig": screen_sig,
|
||||
}
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Record — appelé APRÈS validation post-condition
|
||||
# =========================================================================
|
||||
|
||||
|
||||
def memory_record_success(
|
||||
window_title: str,
|
||||
target_spec: Dict[str, Any],
|
||||
x_pct: float,
|
||||
y_pct: float,
|
||||
method: str,
|
||||
confidence: float = 0.9,
|
||||
) -> bool:
|
||||
"""Enregistre une résolution réussie dans la mémoire persistante.
|
||||
|
||||
À appeler APRÈS validation de la post-condition (`title_match` strict).
|
||||
"""
|
||||
store = get_memory_store()
|
||||
if store is None:
|
||||
return False
|
||||
|
||||
screen_sig = compute_screen_sig(window_title)
|
||||
if not screen_sig:
|
||||
return False
|
||||
|
||||
# Sanity check : coordonnées dans [0, 1]
|
||||
try:
|
||||
x_pct = float(x_pct)
|
||||
y_pct = float(y_pct)
|
||||
except (TypeError, ValueError):
|
||||
logger.debug("memory_record_success: coords non numériques, skip")
|
||||
return False
|
||||
if not (0.0 <= x_pct <= 1.0 and 0.0 <= y_pct <= 1.0):
|
||||
logger.warning(
|
||||
"memory_record_success: coords hors [0,1] (%.3f, %.3f), skip — "
|
||||
"probable input parasite (target='%s' method=%s)",
|
||||
x_pct, y_pct,
|
||||
(target_spec.get("by_text") or "")[:60], method,
|
||||
)
|
||||
return False
|
||||
# Rejeter (0.0, 0.0) exact : coin haut-gauche = signature de bruit
|
||||
# (curseur NoMachine, événement OS parasite, listener pynput sans clic
|
||||
# humain réel). Cf. bug observé replay_sess_63a1313b 2026-05-24 18:31-18:32.
|
||||
if x_pct == 0.0 and y_pct == 0.0:
|
||||
logger.warning(
|
||||
"memory_record_success: coords (0.0, 0.0) rejetées — "
|
||||
"signature de bruit (target='%s' method=%s)",
|
||||
(target_spec.get("by_text") or "")[:60], method,
|
||||
)
|
||||
return False
|
||||
|
||||
try:
|
||||
from core.learning.target_memory_store import TargetFingerprint
|
||||
|
||||
# Stripper les préfixes "memory_" empilés pour ne garder que
|
||||
# la méthode de résolution originale (ex: template_matching).
|
||||
# Sans ça, le cycle lookup → record → lookup empile "memory_"
|
||||
# indéfiniment : memory_memory_memory_template_matching.
|
||||
method_clean = method or "v4_unknown"
|
||||
while method_clean.startswith("memory_"):
|
||||
method_clean = method_clean[len("memory_"):]
|
||||
method_clean = method_clean or "v4_unknown"
|
||||
fingerprint = TargetFingerprint(
|
||||
element_id=f"v4_{method_clean}",
|
||||
bbox=(x_pct, y_pct, 0.0, 0.0),
|
||||
role=target_spec.get("by_role", "") or None,
|
||||
etype=method_clean,
|
||||
label=(target_spec.get("by_text") or "")[:200] or None,
|
||||
confidence=float(confidence),
|
||||
)
|
||||
|
||||
spec_shim = _TargetSpecLike(target_spec)
|
||||
store.record_success(
|
||||
screen_signature=screen_sig,
|
||||
target_spec=spec_shim,
|
||||
fingerprint=fingerprint,
|
||||
strategy_used=method_clean,
|
||||
confidence=float(confidence),
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"memory_record_success: sig=%s method=%s coords=(%.4f, %.4f) "
|
||||
"target='%s'",
|
||||
screen_sig, method_clean, x_pct, y_pct,
|
||||
(target_spec.get("by_text") or "")[:60],
|
||||
)
|
||||
return True
|
||||
|
||||
except Exception as exc:
|
||||
logger.warning("memory_record_success: échec (%s)", exc)
|
||||
return False
|
||||
|
||||
|
||||
def memory_record_failure(
|
||||
window_title: str,
|
||||
target_spec: Dict[str, Any],
|
||||
error_message: str,
|
||||
) -> bool:
|
||||
"""Incrémente le `fail_count` pour cette (signature, target).
|
||||
|
||||
Appelé quand l'action échoue OU quand la post-condition n'est pas
|
||||
satisfaite. Le `TargetMemoryStore.lookup()` ignorera cette entrée
|
||||
si le ratio d'échecs dépasse 30%.
|
||||
"""
|
||||
store = get_memory_store()
|
||||
if store is None:
|
||||
return False
|
||||
|
||||
screen_sig = compute_screen_sig(window_title)
|
||||
if not screen_sig:
|
||||
return False
|
||||
|
||||
try:
|
||||
spec_shim = _TargetSpecLike(target_spec)
|
||||
store.record_failure(
|
||||
screen_signature=screen_sig,
|
||||
target_spec=spec_shim,
|
||||
error_message=(error_message or "unknown")[:200],
|
||||
)
|
||||
logger.debug(
|
||||
"memory_record_failure: sig=%s error='%s'",
|
||||
screen_sig, (error_message or "")[:80],
|
||||
)
|
||||
return True
|
||||
except Exception as exc:
|
||||
logger.debug("memory_record_failure: échec (%s)", exc)
|
||||
return False
|
||||
@@ -20,6 +20,8 @@ import time
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from core.detection import vlm_config
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Seuils de détection configurables
|
||||
@@ -328,10 +330,11 @@ class ReplayVerifier:
|
||||
),
|
||||
)
|
||||
|
||||
# Cas 4 : Pas de changement (key_combo, wait)
|
||||
# Pour les raccourcis clavier et attentes, l'absence de changement
|
||||
# n'est pas forcément un problème (ex: Ctrl+C ne change pas l'écran)
|
||||
if action_type in ("key_combo", "wait"):
|
||||
# Cas 4 : Pas de changement (key_combo, wait, verify_screen)
|
||||
# `verify_screen` côté agent n'est qu'une temporisation de stabilisation.
|
||||
# Il ne doit pas exiger un NOUVEAU changement visuel sinon le setup
|
||||
# boucle inutilement une fois l'application déjà ouverte.
|
||||
if action_type in ("key_combo", "wait", "verify_screen"):
|
||||
return VerificationResult(
|
||||
verified=True,
|
||||
confidence=0.4,
|
||||
@@ -433,7 +436,7 @@ class ReplayVerifier:
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Appeler le VLM pour évaluer sémantiquement le résultat de l'action.
|
||||
|
||||
Utilise gemma4 en mode texte+images (Docker port 11435) pour analyser
|
||||
Utilise le VLM (résolu via vlm_config) en mode texte+images pour analyser
|
||||
les screenshots avant/après et dire si le résultat attendu est atteint.
|
||||
|
||||
Sur Citrix (image plate), c'est la SEULE façon de vérifier intelligemment
|
||||
@@ -448,7 +451,10 @@ class ReplayVerifier:
|
||||
if not screenshot_after:
|
||||
return None
|
||||
|
||||
gemma4_port = os.environ.get("GEMMA4_PORT", "11435")
|
||||
# Endpoint VLM : piloté par config (Ollama local ou tunnel DGX = 11434).
|
||||
# GEMMA4_PORT conservé comme override legacy (ancien conteneur Docker 11435).
|
||||
_default_port = vlm_config.DEFAULT_OLLAMA_ENDPOINT.rsplit(":", 1)[-1]
|
||||
gemma4_port = os.environ.get("GEMMA4_PORT", _default_port)
|
||||
gemma4_url = f"http://localhost:{gemma4_port}/api/chat"
|
||||
|
||||
# Construire le prompt Critic
|
||||
@@ -496,7 +502,7 @@ class ReplayVerifier:
|
||||
resp = _requests.post(
|
||||
gemma4_url,
|
||||
json={
|
||||
"model": "gemma4:e4b",
|
||||
"model": vlm_config.get_vlm_model(),
|
||||
"messages": messages,
|
||||
"stream": False,
|
||||
"think": True,
|
||||
|
||||
329
agent_v0/server_v1/replay_watchdog.py
Normal file
329
agent_v0/server_v1/replay_watchdog.py
Normal file
@@ -0,0 +1,329 @@
|
||||
"""Replay orphan watchdog for in-flight replay actions.
|
||||
|
||||
This module watches `_retry_pending` and re-pushes actions that were
|
||||
dispatched by the server but never acknowledged by the Windows agent.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from typing import Any, Callable, Dict, List, Optional, Tuple
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _env_bool(name: str, default: str) -> bool:
|
||||
return os.environ.get(name, default).strip().lower() in {
|
||||
"1",
|
||||
"true",
|
||||
"yes",
|
||||
"on",
|
||||
}
|
||||
|
||||
|
||||
def _env_float(name: str, default: float) -> float:
|
||||
try:
|
||||
return float(os.environ.get(name, str(default)))
|
||||
except (TypeError, ValueError):
|
||||
logger.warning("Watchdog: invalid env %s, fallback=%s", name, default)
|
||||
return default
|
||||
|
||||
|
||||
def _env_int(name: str, default: int) -> int:
|
||||
try:
|
||||
return int(os.environ.get(name, str(default)))
|
||||
except (TypeError, ValueError):
|
||||
logger.warning("Watchdog: invalid env %s, fallback=%s", name, default)
|
||||
return default
|
||||
|
||||
|
||||
def _env_max_resends(default: int) -> int:
|
||||
raw = os.environ.get("RPA_WATCHDOG_MAX_RESENDS")
|
||||
if raw is None or not str(raw).strip():
|
||||
raw = os.environ.get("RPA_WATCHDOG_MAX_RETRIES")
|
||||
try:
|
||||
return int(raw) if raw is not None else default
|
||||
except (TypeError, ValueError):
|
||||
logger.warning("Watchdog: invalid max resend env, fallback=%s", default)
|
||||
return default
|
||||
|
||||
|
||||
WATCHDOG_ENABLED = _env_bool("RPA_WATCHDOG_ENABLED", "1")
|
||||
WATCHDOG_SCAN_INTERVAL_S = _env_float("RPA_WATCHDOG_SCAN_INTERVAL_S", 10.0)
|
||||
WATCHDOG_ORPHAN_TIMEOUT_S = _env_float("RPA_WATCHDOG_ORPHAN_TIMEOUT_S", 45.0)
|
||||
WATCHDOG_MAX_RESENDS = _env_max_resends(2)
|
||||
WATCHDOG_REPUSH_POSITION = (
|
||||
os.environ.get("RPA_WATCHDOG_REPUSH_POSITION", "head").strip().lower()
|
||||
)
|
||||
|
||||
|
||||
_metrics_lock = asyncio.Lock()
|
||||
_metrics: Dict[str, Any] = {
|
||||
"orphans_detected_total": 0,
|
||||
"orphans_resent_total": 0,
|
||||
"orphans_giveup_total": 0,
|
||||
"scans_total": 0,
|
||||
"scans_failed_total": 0,
|
||||
"last_scan_ts": 0.0,
|
||||
"last_scan_duration_ms": 0.0,
|
||||
"current_in_flight_count": 0,
|
||||
"current_orphan_count": 0,
|
||||
}
|
||||
|
||||
|
||||
async def _bump(key: str, delta: int = 1) -> None:
|
||||
async with _metrics_lock:
|
||||
_metrics[key] = _metrics.get(key, 0) + delta
|
||||
|
||||
|
||||
def get_metrics_snapshot() -> Dict[str, Any]:
|
||||
return dict(_metrics)
|
||||
|
||||
|
||||
SseNotifier = Callable[[str, str], None]
|
||||
|
||||
|
||||
class ReplayWatchdog:
|
||||
"""Background coroutine that re-pushes orphaned replay actions."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
retry_pending: Dict[str, Dict[str, Any]],
|
||||
replay_queues: Dict[str, List[Dict[str, Any]]],
|
||||
async_lock_factory: Callable[[], Any],
|
||||
sse_notifier: Optional[SseNotifier] = None,
|
||||
) -> None:
|
||||
self._retry_pending = retry_pending
|
||||
self._replay_queues = replay_queues
|
||||
self._async_lock = async_lock_factory
|
||||
self._sse_notifier = sse_notifier
|
||||
self._task: Optional[asyncio.Task] = None
|
||||
self._stopped = asyncio.Event()
|
||||
|
||||
async def start(self) -> None:
|
||||
if not WATCHDOG_ENABLED:
|
||||
logger.info("[WATCHDOG] disabled via RPA_WATCHDOG_ENABLED=0")
|
||||
return
|
||||
if self._task is not None and not self._task.done():
|
||||
logger.warning("[WATCHDOG] already started")
|
||||
return
|
||||
self._stopped.clear()
|
||||
self._task = asyncio.create_task(self._run(), name="replay_watchdog")
|
||||
logger.info(
|
||||
"[WATCHDOG] started scan=%.1fs orphan_timeout=%.1fs max_resends=%d repush=%s",
|
||||
WATCHDOG_SCAN_INTERVAL_S,
|
||||
WATCHDOG_ORPHAN_TIMEOUT_S,
|
||||
WATCHDOG_MAX_RESENDS,
|
||||
WATCHDOG_REPUSH_POSITION,
|
||||
)
|
||||
|
||||
async def stop(self, timeout_s: float = 5.0) -> None:
|
||||
if self._task is None:
|
||||
return
|
||||
self._stopped.set()
|
||||
self._task.cancel()
|
||||
try:
|
||||
await asyncio.wait_for(self._task, timeout=timeout_s)
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
except asyncio.TimeoutError:
|
||||
logger.warning("[WATCHDOG] stop timeout after %.1fs", timeout_s)
|
||||
except Exception:
|
||||
logger.exception("[WATCHDOG] unexpected stop error")
|
||||
self._task = None
|
||||
logger.info("[WATCHDOG] stopped")
|
||||
|
||||
async def _run(self) -> None:
|
||||
try:
|
||||
while not self._stopped.is_set():
|
||||
try:
|
||||
await asyncio.wait_for(
|
||||
self._stopped.wait(),
|
||||
timeout=WATCHDOG_SCAN_INTERVAL_S,
|
||||
)
|
||||
break
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
|
||||
try:
|
||||
await self._scan_once()
|
||||
except Exception:
|
||||
await _bump("scans_failed_total")
|
||||
logger.exception("[WATCHDOG] scan failed")
|
||||
except asyncio.CancelledError:
|
||||
logger.info("[WATCHDOG] cancelled")
|
||||
raise
|
||||
finally:
|
||||
logger.info("[WATCHDOG] loop terminated")
|
||||
|
||||
async def _scan_once(self) -> Dict[str, int]:
|
||||
t0 = time.time()
|
||||
await _bump("scans_total")
|
||||
|
||||
resent = 0
|
||||
gaveup = 0
|
||||
skipped = 0
|
||||
in_flight = 0
|
||||
orphans = 0
|
||||
|
||||
orphan_targets: List[Tuple[str, Dict[str, Any]]] = []
|
||||
async with self._async_lock():
|
||||
for action_id, info in list(self._retry_pending.items()):
|
||||
dispatched_at = info.get("dispatched_at", 0.0) or 0.0
|
||||
if dispatched_at <= 0:
|
||||
skipped += 1
|
||||
continue
|
||||
age = t0 - dispatched_at
|
||||
in_flight += 1
|
||||
if age < WATCHDOG_ORPHAN_TIMEOUT_S:
|
||||
continue
|
||||
orphans += 1
|
||||
orphan_targets.append((action_id, dict(info)))
|
||||
|
||||
for action_id, info in orphan_targets:
|
||||
await _bump("orphans_detected_total")
|
||||
resent_count = int(info.get("resent_count", 0) or 0)
|
||||
|
||||
if resent_count >= WATCHDOG_MAX_RESENDS:
|
||||
async with self._async_lock():
|
||||
self._retry_pending.pop(action_id, None)
|
||||
age_total = t0 - float(info.get("first_dispatched_at", t0) or t0)
|
||||
logger.error(
|
||||
"[BUS] lea:dispatch_orphan_giveup action_id=%s resent=%d age_total=%.1fs "
|
||||
"session=%s machine=%s replay=%s",
|
||||
action_id,
|
||||
resent_count,
|
||||
age_total,
|
||||
info.get("session_id", "?"),
|
||||
info.get("machine_id", "?"),
|
||||
info.get("replay_id", "?"),
|
||||
)
|
||||
gaveup += 1
|
||||
await _bump("orphans_giveup_total")
|
||||
continue
|
||||
|
||||
session_id = info.get("session_id")
|
||||
machine_id = info.get("machine_id", "default")
|
||||
action = info.get("dispatched_action") or info.get("action")
|
||||
if not session_id or not isinstance(action, dict):
|
||||
logger.warning(
|
||||
"[WATCHDOG] invalid schema for %s session_id=%r action_type=%s",
|
||||
action_id,
|
||||
session_id,
|
||||
type(action).__name__,
|
||||
)
|
||||
async with self._async_lock():
|
||||
self._retry_pending.pop(action_id, None)
|
||||
continue
|
||||
|
||||
async with self._async_lock():
|
||||
existing = self._retry_pending.get(action_id)
|
||||
if existing is None:
|
||||
logger.debug(
|
||||
"[WATCHDOG] %s acked between snapshot and resend; skip",
|
||||
action_id,
|
||||
)
|
||||
continue
|
||||
queue = self._replay_queues.setdefault(session_id, [])
|
||||
if WATCHDOG_REPUSH_POSITION == "tail":
|
||||
queue.append(dict(action))
|
||||
else:
|
||||
queue.insert(0, dict(action))
|
||||
existing["resent_count"] = resent_count + 1
|
||||
existing["last_resent_at"] = time.time()
|
||||
existing["dispatched_at"] = 0.0
|
||||
|
||||
age_total = t0 - float(info.get("first_dispatched_at", t0) or t0)
|
||||
logger.warning(
|
||||
"[BUS] lea:dispatch_orphan_resent action_id=%s resent=%d/%d age=%.1fs "
|
||||
"session=%s machine=%s replay=%s",
|
||||
action_id,
|
||||
resent_count + 1,
|
||||
WATCHDOG_MAX_RESENDS,
|
||||
age_total,
|
||||
session_id,
|
||||
machine_id,
|
||||
info.get("replay_id", "?"),
|
||||
)
|
||||
resent += 1
|
||||
await _bump("orphans_resent_total")
|
||||
|
||||
if self._sse_notifier is not None:
|
||||
try:
|
||||
self._sse_notifier(session_id, machine_id)
|
||||
except Exception as exc:
|
||||
logger.debug("[WATCHDOG] sse notifier failed: %s", exc)
|
||||
|
||||
elapsed_ms = (time.time() - t0) * 1000.0
|
||||
async with _metrics_lock:
|
||||
_metrics["last_scan_ts"] = t0
|
||||
_metrics["last_scan_duration_ms"] = elapsed_ms
|
||||
_metrics["current_in_flight_count"] = in_flight
|
||||
_metrics["current_orphan_count"] = orphans
|
||||
scans_total = _metrics["scans_total"]
|
||||
|
||||
if orphans or gaveup:
|
||||
logger.info(
|
||||
"[METRIC] watchdog scan=%d orphans=%d resent=%d gaveup=%d "
|
||||
"in_flight=%d skipped=%d elapsed_ms=%.1f",
|
||||
scans_total,
|
||||
orphans,
|
||||
resent,
|
||||
gaveup,
|
||||
in_flight,
|
||||
skipped,
|
||||
elapsed_ms,
|
||||
)
|
||||
|
||||
return {
|
||||
"orphans": orphans,
|
||||
"resent": resent,
|
||||
"gaveup": gaveup,
|
||||
"skipped": skipped,
|
||||
"in_flight": in_flight,
|
||||
}
|
||||
|
||||
|
||||
_singleton: Optional[ReplayWatchdog] = None
|
||||
|
||||
|
||||
def get_or_create_watchdog(
|
||||
retry_pending: Dict[str, Dict[str, Any]],
|
||||
replay_queues: Dict[str, List[Dict[str, Any]]],
|
||||
async_lock_factory: Callable[[], Any],
|
||||
sse_notifier: Optional[SseNotifier] = None,
|
||||
) -> ReplayWatchdog:
|
||||
global _singleton
|
||||
if _singleton is None:
|
||||
_singleton = ReplayWatchdog(
|
||||
retry_pending=retry_pending,
|
||||
replay_queues=replay_queues,
|
||||
async_lock_factory=async_lock_factory,
|
||||
sse_notifier=sse_notifier,
|
||||
)
|
||||
return _singleton
|
||||
|
||||
|
||||
@contextlib.asynccontextmanager
|
||||
async def watchdog_lifespan(
|
||||
retry_pending: Dict[str, Dict[str, Any]],
|
||||
replay_queues: Dict[str, List[Dict[str, Any]]],
|
||||
async_lock_factory: Callable[[], Any],
|
||||
sse_notifier: Optional[SseNotifier] = None,
|
||||
):
|
||||
watchdog = get_or_create_watchdog(
|
||||
retry_pending=retry_pending,
|
||||
replay_queues=replay_queues,
|
||||
async_lock_factory=async_lock_factory,
|
||||
sse_notifier=sse_notifier,
|
||||
)
|
||||
await watchdog.start()
|
||||
try:
|
||||
yield watchdog
|
||||
finally:
|
||||
await watchdog.stop()
|
||||
3172
agent_v0/server_v1/resolve_engine.py
Normal file
3172
agent_v0/server_v1/resolve_engine.py
Normal file
File diff suppressed because it is too large
Load Diff
@@ -25,6 +25,7 @@ Le worker :
|
||||
5. Se suspend quand un replay est actif (libère le GPU)
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import signal
|
||||
@@ -67,6 +68,7 @@ class VLMWorker:
|
||||
self._running = False
|
||||
self._processor = None # Initialisé au premier besoin (lazy loading GPU)
|
||||
self._current_session: Optional[str] = None
|
||||
self._started_at: str = datetime.now().isoformat()
|
||||
|
||||
# Stats
|
||||
self._stats: Dict[str, int] = {
|
||||
@@ -83,7 +85,10 @@ class VLMWorker:
|
||||
if self._processor is None:
|
||||
logger.info("Initialisation du StreamProcessor (chargement GPU)...")
|
||||
from .stream_processor import StreamProcessor
|
||||
self._processor = StreamProcessor(data_dir=str(LIVE_SESSIONS_DIR))
|
||||
self._processor = StreamProcessor(
|
||||
data_dir=str(DATA_DIR),
|
||||
enable_vlm=True,
|
||||
)
|
||||
logger.info("StreamProcessor initialisé.")
|
||||
return self._processor
|
||||
|
||||
@@ -98,6 +103,11 @@ class VLMWorker:
|
||||
logger.info(" Sessions dir : %s", LIVE_SESSIONS_DIR)
|
||||
logger.info(" Poll interval : %ds", POLL_INTERVAL)
|
||||
|
||||
# N2 + N3 : santé initiale + signal READY systemd dès le démarrage
|
||||
# (avant tout chargement GPU, pour ne pas dépasser le timeout de start).
|
||||
self._write_health("healthy")
|
||||
self._sd_notify("READY=1")
|
||||
|
||||
while self._running:
|
||||
try:
|
||||
# Vérifier si un replay est actif
|
||||
@@ -110,6 +120,7 @@ class VLMWorker:
|
||||
if session_id:
|
||||
self._process_session(session_id)
|
||||
else:
|
||||
self._write_health("healthy") # N2 : cycle idle
|
||||
time.sleep(POLL_INTERVAL)
|
||||
|
||||
except KeyboardInterrupt:
|
||||
@@ -119,6 +130,7 @@ class VLMWorker:
|
||||
logger.error("Erreur dans la boucle principale : %s", e, exc_info=True)
|
||||
time.sleep(5) # Éviter une boucle d'erreurs rapide
|
||||
|
||||
self._write_health("stopped") # N2 : santé finale
|
||||
logger.info("VLM Worker arrêté.")
|
||||
|
||||
def stop(self):
|
||||
@@ -126,6 +138,103 @@ class VLMWorker:
|
||||
self._running = False
|
||||
logger.info("Arrêt demandé.")
|
||||
|
||||
# =========================================================================
|
||||
# N2 — Health file (_worker_health.json)
|
||||
# =========================================================================
|
||||
#
|
||||
# Garde-fou anti-blocage silencieux : expose l'état de santé du worker sur
|
||||
# disque pour qu'un superviseur (humain, dashboard, watchdog) détecte un
|
||||
# worker dégradé sans avoir à fouiller les logs. Écriture atomique.
|
||||
#
|
||||
# CONFIDENTIALITÉ (HDS) : n'écrit AUCUNE donnée patient — uniquement des
|
||||
# identifiants techniques (session_id), des compteurs et des booléens de
|
||||
# composants. Jamais d'OCR, de noms de fichiers screenshots, ni de contenu
|
||||
# de session.
|
||||
|
||||
def _sd_notify(self, state: str) -> bool:
|
||||
"""Notifie systemd via $NOTIFY_SOCKET, sans dépendance `systemd.daemon`.
|
||||
|
||||
Implémentation pure socket (AF_UNIX SOCK_DGRAM) : fonctionne sous systemd
|
||||
`Type=notify` pour `READY=1` et le heartbeat `WATCHDOG=1`. No-op silencieux
|
||||
hors systemd (variable absente) ou en cas d'erreur — jamais bloquant.
|
||||
Retourne True si le message a été émis.
|
||||
"""
|
||||
addr = os.environ.get("NOTIFY_SOCKET")
|
||||
if not addr:
|
||||
return False
|
||||
try:
|
||||
import socket
|
||||
|
||||
# Namespace abstrait systemd : '@' → octet nul de préfixe
|
||||
connect_addr = "\0" + addr[1:] if addr.startswith("@") else addr
|
||||
with socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) as sock:
|
||||
sock.connect(connect_addr)
|
||||
sock.sendall(state.encode("utf-8"))
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.debug("sd_notify(%s) échoué : %s", state, e)
|
||||
return False
|
||||
|
||||
def _health_components(self) -> Dict[str, bool]:
|
||||
"""Statut booléen de chaque composant lourd, dérivé du processor."""
|
||||
proc = self._processor
|
||||
return {
|
||||
"screen_analyzer": proc is not None and getattr(proc, "_screen_analyzer", None) is not None,
|
||||
"clip_embedder": proc is not None and getattr(proc, "_clip_embedder", None) is not None,
|
||||
"faiss_manager": proc is not None and getattr(proc, "_faiss_manager", None) is not None,
|
||||
"state_embedding_builder": proc is not None and getattr(proc, "_state_embedding_builder", None) is not None,
|
||||
}
|
||||
|
||||
def _write_health(self, status: str) -> None:
|
||||
"""Écrit data/training/_worker_health.json de façon atomique.
|
||||
|
||||
`status` attendu : healthy | busy | degraded | stopped. Si le worker
|
||||
tourne en mode VLM mais que ScreenAnalyzer est absent, le statut est
|
||||
forcé à 'degraded' quelle que soit la valeur demandée.
|
||||
"""
|
||||
try:
|
||||
components = self._health_components()
|
||||
|
||||
proc = self._processor
|
||||
vlm_mode = proc is not None and getattr(proc, "_enable_vlm", False)
|
||||
if vlm_mode and not components["screen_analyzer"]:
|
||||
status = "degraded"
|
||||
|
||||
queue_path = DATA_DIR / "_worker_queue.txt"
|
||||
try:
|
||||
queue_length = len(
|
||||
[ln for ln in queue_path.read_text(encoding="utf-8").splitlines() if ln.strip()]
|
||||
) if queue_path.exists() else 0
|
||||
except Exception:
|
||||
queue_length = 0
|
||||
|
||||
payload = {
|
||||
"pid": os.getpid(),
|
||||
"started_at": self._started_at,
|
||||
"last_cycle": datetime.now().isoformat(),
|
||||
"current_session": self._current_session,
|
||||
"queue_length": queue_length,
|
||||
"components": components,
|
||||
"stats": dict(self._stats),
|
||||
"status": status,
|
||||
}
|
||||
|
||||
health_path = DATA_DIR / "_worker_health.json"
|
||||
tmp_path = health_path.with_suffix(".json.tmp")
|
||||
tmp_path.write_text(
|
||||
json.dumps(payload, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
tmp_path.rename(health_path)
|
||||
except Exception as e:
|
||||
# Le health file est un garde-fou, jamais un point de défaillance.
|
||||
logger.warning("Écriture health file échouée : %s", e)
|
||||
|
||||
# N3 : chaque écriture santé sert aussi de heartbeat watchdog systemd
|
||||
# (sauf à l'arrêt). No-op hors systemd.
|
||||
if status != "stopped":
|
||||
self._sd_notify("WATCHDOG=1")
|
||||
|
||||
# =========================================================================
|
||||
# Queue management (fichier _worker_queue.txt)
|
||||
# =========================================================================
|
||||
@@ -206,6 +315,9 @@ class VLMWorker:
|
||||
REPLAY_WAIT_TIMEOUT,
|
||||
)
|
||||
break
|
||||
# N3 : heartbeat pendant la pause replay (peut durer jusqu'à 120s,
|
||||
# sinon le watchdog tuerait un worker pourtant sain et en attente).
|
||||
self._sd_notify("WATCHDOG=1")
|
||||
time.sleep(REPLAY_CHECK_INTERVAL)
|
||||
|
||||
elapsed = time.time() - start
|
||||
@@ -220,6 +332,7 @@ class VLMWorker:
|
||||
"""Traite une session complète (analyse VLM + construction workflow)."""
|
||||
self._current_session = session_id
|
||||
logger.info("=== Début traitement session %s ===", session_id)
|
||||
self._write_health("busy") # N2 : début de session
|
||||
start_time = time.time()
|
||||
|
||||
try:
|
||||
@@ -331,6 +444,7 @@ class VLMWorker:
|
||||
|
||||
finally:
|
||||
self._current_session = None
|
||||
self._write_health("healthy") # N2 : fin de session (ou degraded auto)
|
||||
|
||||
logger.info("=== Fin traitement session %s ===", session_id)
|
||||
|
||||
@@ -347,6 +461,8 @@ class VLMWorker:
|
||||
f" ({shot_id})" if shot_id else "",
|
||||
)
|
||||
|
||||
self._write_health("busy") # N2 : heartbeat à chaque screenshot
|
||||
|
||||
# Vérifier si un replay est devenu actif pendant le traitement
|
||||
if self._is_replay_active():
|
||||
logger.info(
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user