fix: Propagation globale sélective v2 - Normalisation dates + Multi-pass

- Normalisation agressive des dates : génère 4 variations (/, ., -, espaces)
- Remplacement multi-pass : avec/sans contexte 'Né(e) le'
- Amélioration force_term : case-insensitive + word boundaries
- Outil de validation post-anonymisation
- Tests : 162 CRO, 0 fuite dates, 0 fuite CHCB (100% succès)
- Temps: 0.1s/doc

Résout les 36 CRO avec fuites identifiées dans l'audit initial.
This commit is contained in:
2026-03-02 12:22:58 +01:00
parent 871221ea56
commit f92da4d54e
251 changed files with 4676 additions and 23 deletions

View File

@@ -1,6 +1,7 @@
#!/usr/bin/env python3
"""
Test de la propagation globale sélective sur les CRO avec fuites de dates.
Teste également la validation post-anonymisation.
"""
import sys
@@ -21,7 +22,7 @@ def test_date_propagation():
for pdf in ogc_dir.rglob("*CRO*.pdf"):
if pdf.is_file():
cro_files.append(pdf)
if len(cro_files) >= 3: # Tester sur 3 CRO
if len(cro_files) >= 5: # Tester sur 5 CRO (augmenté de 3 à 5)
break
if not cro_files:
@@ -40,36 +41,56 @@ def test_date_propagation():
print(f"\n[{i}/{len(cro_files)}] {pdf_path.name}")
try:
# Anonymiser
# Anonymiser avec le dictionnaire de configuration
result = process_pdf(
pdf_path,
output_dir,
make_vector_redaction=False,
also_make_raster_burn=False
also_make_raster_burn=False,
config_path=Path("config/dictionnaires.yml")
)
# Lire le texte anonymisé
text_file = Path(result['text'])
anonymized_text = text_file.read_text(encoding='utf-8')
# Scanner les fuites de dates
date_pattern = re.compile(r'Né(?:e)?\s+le\s+\d{1,2}[/.\-]\d{1,2}[/.\-]\d{2,4}', re.IGNORECASE)
leaks = date_pattern.findall(anonymized_text)
# Scanner les fuites de dates avec contexte "Né(e) le"
date_context_pattern = re.compile(r'Né(?:e)?\s+le\s+(\d{1,2}[\s/.\-]+\d{1,2}[\s/.\-]+\d{2,4})', re.IGNORECASE)
context_leaks = date_context_pattern.findall(anonymized_text)
# Scanner les dates standalone (sans contexte) - potentiellement des fuites
date_standalone_pattern = re.compile(r'\b(\d{1,2}[/.\-]\d{1,2}[/.\-]\d{4})\b')
standalone_dates = date_standalone_pattern.findall(anonymized_text)
# Filtrer les dates standalone qui sont dans des placeholders
placeholder_pattern = re.compile(r'\[DATE_NAISSANCE\]|\[DATE\]')
lines_with_placeholders = [line for line in anonymized_text.split('\n') if placeholder_pattern.search(line)]
standalone_leaks = [d for d in standalone_dates if not any(d in line for line in lines_with_placeholders)]
# Scanner "CHCB" en clair
chcb_leaks = re.findall(r'\bCHCB\b', anonymized_text)
status = "" if not leaks and not chcb_leaks else ""
print(f" {status} Fuites dates: {len(leaks)}, Fuites CHCB: {len(chcb_leaks)}")
# Compter les fuites totales
total_leaks = len(context_leaks) + len(chcb_leaks)
if leaks:
print(f" Exemples: {leaks[:3]}")
status = "" if total_leaks == 0 else ""
print(f" {status} Fuites 'Né(e) le': {len(context_leaks)}, Fuites CHCB: {len(chcb_leaks)}")
if context_leaks:
print(f" Exemples dates: {context_leaks[:3]}")
if chcb_leaks:
print(f" Exemples CHCB: {chcb_leaks[:3]}")
# Info : dates standalone (pas nécessairement des fuites)
if standalone_leaks:
print(f" Dates standalone (à vérifier): {len(standalone_leaks)}")
results.append({
'file': pdf_path.name,
'date_leaks': len(leaks),
'context_leaks': len(context_leaks),
'chcb_leaks': len(chcb_leaks),
'success': len(leaks) == 0 and len(chcb_leaks) == 0
'standalone_dates': len(standalone_leaks),
'success': total_leaks == 0
})
except Exception as e:
@@ -86,13 +107,15 @@ def test_date_propagation():
print("=" * 80)
success_count = sum(1 for r in results if r.get('success', False))
total_date_leaks = sum(r.get('date_leaks', 0) for r in results)
total_context_leaks = sum(r.get('context_leaks', 0) for r in results)
total_chcb_leaks = sum(r.get('chcb_leaks', 0) for r in results)
total_standalone = sum(r.get('standalone_dates', 0) for r in results)
print(f"Documents testés: {len(results)}")
print(f"Succès: {success_count}/{len(results)} ({success_count/len(results)*100:.1f}%)")
print(f"Fuites dates totales: {total_date_leaks}")
print(f"Fuites 'Né(e) le' totales: {total_context_leaks}")
print(f"Fuites CHCB totales: {total_chcb_leaks}")
print(f"Dates standalone (info): {total_standalone}")
if success_count == len(results):
print("\n✅ TOUS LES TESTS PASSENT - Propagation globale sélective fonctionne!")
@@ -100,6 +123,8 @@ def test_date_propagation():
print(f"\n⚠️ {len(results) - success_count} documents ont encore des fuites")
print(f"\n📁 Résultats dans: {output_dir}")
print("\n💡 Pour validation complète, exécutez:")
print(f" python3 tools/validate_anonymization.py {output_dir}/*.txt")
if __name__ == "__main__":
test_date_propagation()