fix(core): renforcer detection PII et FINESS Corse
Couvre les corrections PII batch A/A-2, le NIR multi-ligne en flux reel, le gazetteer FINESS Corse derive depuis la base locale, et les tests de regression associes. Aucun build ni diffusion.
This commit is contained in:
@@ -91,6 +91,20 @@ class TestAdresseContextuelle:
|
||||
assert PLACEHOLDERS["ADRESSE"] in out, f"non masqué: {adresse!r} -> {out!r}"
|
||||
assert reste_visible not in out, f"fuite résiduelle: {adresse!r} -> {out!r}"
|
||||
|
||||
@pytest.mark.parametrize("adresse", [
|
||||
"15 à 35 rue Claude Boucher Bordeaux Cedex",
|
||||
"15 a 35 rue Claude Boucher Bordeaux Cedex",
|
||||
"15-35 rue Claude Boucher Bordeaux Cedex",
|
||||
])
|
||||
def test_adresse_plage_numero_etablissement(self, adresse):
|
||||
"""Cas Dom 2026-06-16 : les adresses d'établissement FINESS avec plage
|
||||
de numéros doivent être masquées sans laisser le préfixe de plage."""
|
||||
out, _ = _mask_line(adresse)
|
||||
assert out.strip() == PLACEHOLDERS["ADRESSE"], f"masquage partiel: {adresse!r} -> {out!r}"
|
||||
assert "Claude" not in out
|
||||
assert "Boucher" not in out
|
||||
assert "15" not in out
|
||||
|
||||
@pytest.mark.parametrize("ligne_clinique", [
|
||||
"3 mg/L de CRP",
|
||||
"TA 12/8 mmHg",
|
||||
@@ -150,6 +164,7 @@ class TestContexteDate:
|
||||
|
||||
def test_date_naissance_variantes_contexte(self):
|
||||
for line in ("Date de naissance : 01/02/1944",
|
||||
"Date naissance : 19/09/1972",
|
||||
"DDN 1/2/1944",
|
||||
"Née le 2 mars 1944"):
|
||||
out, _ = _mask_line(line)
|
||||
@@ -162,6 +177,13 @@ class TestContexteDate:
|
||||
assert PLACEHOLDERS["DATE_NAISSANCE"] not in out
|
||||
assert "14/03/2025" in out
|
||||
|
||||
def test_date_ancienne_sans_contexte_naissance_preservee(self):
|
||||
"""L'année ancienne seule ne suffit pas : une date clinique historique
|
||||
hors contexte naissance doit rester visible."""
|
||||
out, _ = _mask_line("Intervention réalisée le 19/09/1972")
|
||||
assert PLACEHOLDERS["DATE_NAISSANCE"] not in out
|
||||
assert "19/09/1972" in out
|
||||
|
||||
def test_date_tableau_clinique_preservee(self):
|
||||
out, _ = _mask_line("08:00 | 120/80 | 37.1 | 12/03/2024")
|
||||
assert PLACEHOLDERS["DATE_NAISSANCE"] not in out
|
||||
|
||||
220
tests/unit/test_pii_fort_a2.py
Normal file
220
tests/unit/test_pii_fort_a2.py
Normal file
@@ -0,0 +1,220 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Corrections PII FORT — batch A-2 (rectificatif Qwen 2026-06-17 11:15).
|
||||
|
||||
Nouvelles lacunes : X-L1 ADELI, X-L2 rescan ADHERENT/OGC/FAX/ADELI, #9 FAX,
|
||||
#11/#12 NIR label/no-key/multiline, X-L3 RIB/BIC, X-L5 DDN variantes.
|
||||
|
||||
Valeurs FICTIVES. Cas positif + anti-FP pour chaque, dont #12 NIR multiline
|
||||
dans le flux documentaire réel.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from anonymizer_core_refactored_onnx import (
|
||||
PLACEHOLDERS,
|
||||
RE_BARE_9DIGITS,
|
||||
RE_BIC,
|
||||
anonymise_document_regex,
|
||||
_FINESS_NUMBERS,
|
||||
_mask_admin_label,
|
||||
_mask_line_by_regex,
|
||||
load_dictionaries,
|
||||
selective_rescan,
|
||||
)
|
||||
|
||||
CFG = load_dictionaries(None)
|
||||
|
||||
|
||||
def _mask(line: str):
|
||||
audit: list = []
|
||||
out = _mask_line_by_regex(line, audit, 0, CFG)
|
||||
return out, audit
|
||||
|
||||
|
||||
# --- X-L1 ADELI ---------------------------------------------------------------
|
||||
|
||||
def test_adeli_alphanum():
|
||||
out, _ = _mask("ADELI : 9ABCDE12")
|
||||
assert PLACEHOLDERS["ADELI"] in out
|
||||
assert "9ABCDE12" not in out
|
||||
|
||||
|
||||
def test_adeli_num_label():
|
||||
out, _ = _mask("N° ADELI : 123456")
|
||||
assert PLACEHOLDERS["ADELI"] in out
|
||||
|
||||
|
||||
def test_adeli_anti_fp_no_value():
|
||||
line = "Le référentiel ADELI est ancien"
|
||||
out, _ = _mask(line)
|
||||
assert PLACEHOLDERS["ADELI"] not in out
|
||||
|
||||
|
||||
# --- #9 FAX -------------------------------------------------------------------
|
||||
|
||||
def test_fax_label_masks_as_fax():
|
||||
out, _ = _mask("Fax : 05 56 00 00 00")
|
||||
assert PLACEHOLDERS["FAX"] in out
|
||||
assert "05 56 00 00 00" not in out
|
||||
|
||||
|
||||
def test_telecopie_label_masks_as_fax():
|
||||
out, _ = _mask("Télécopie : 05 56 00 00 00")
|
||||
assert PLACEHOLDERS["FAX"] in out
|
||||
|
||||
|
||||
def test_phone_without_fax_label_stays_tel():
|
||||
out, _ = _mask("Tél : 05 56 00 00 00")
|
||||
assert PLACEHOLDERS["TEL"] in out
|
||||
assert PLACEHOLDERS["FAX"] not in out
|
||||
|
||||
|
||||
def test_fax_anti_fp_initial_no_number():
|
||||
# "F." initiale sans numéro ne doit pas produire [FAX]
|
||||
out, _ = _mask("Compte rendu rédigé")
|
||||
assert PLACEHOLDERS["FAX"] not in out
|
||||
|
||||
|
||||
# --- #11 NIR 13 chiffres avec label ------------------------------------------
|
||||
|
||||
def test_nir_no_key_with_label():
|
||||
out, _ = _mask("NIR : 2840556123456")
|
||||
assert PLACEHOLDERS["NIR"] in out
|
||||
assert "2840556123456" not in out
|
||||
|
||||
|
||||
def test_nir_no_key_label_secu():
|
||||
out, _ = _mask("N° sécurité sociale : 2840556123456")
|
||||
assert PLACEHOLDERS["NIR"] in out
|
||||
|
||||
|
||||
def test_nir_anti_fp_bare_13_digits():
|
||||
line = "Référence dossier 2840556123456 archivée"
|
||||
out, _ = _mask(line)
|
||||
assert "2840556123456" in out # pas de label NIR → pas de masque
|
||||
|
||||
|
||||
# --- X-L3 RIB / BIC → [IBAN] -------------------------------------------------
|
||||
|
||||
def test_bic_label():
|
||||
out, _ = _mask("BIC : BNPAFRPP")
|
||||
assert PLACEHOLDERS["IBAN"] in out
|
||||
|
||||
|
||||
def test_swift_label():
|
||||
out, _ = _mask("SWIFT : BNPAFRPPXXX")
|
||||
assert PLACEHOLDERS["IBAN"] in out
|
||||
|
||||
|
||||
def test_rib_label():
|
||||
out, _ = _mask("RIB : 12345 67890 12345678901 12")
|
||||
assert PLACEHOLDERS["IBAN"] in out
|
||||
|
||||
|
||||
def test_bic_anti_fp_no_label():
|
||||
# code type BIC sans label « BIC/SWIFT » ne doit pas matcher (anti-FP acronymes).
|
||||
# Vérifié au niveau regex pour isoler de tout autre masquage du pipeline.
|
||||
assert RE_BIC.search("Le service BNPAFRPP n'existe pas") is None
|
||||
|
||||
|
||||
# --- X-L5 DDN variantes (Né en / Né(e) : / Née la) ---------------------------
|
||||
|
||||
def test_ddn_ne_en_annee():
|
||||
out, _ = _mask("Né en 1972")
|
||||
assert PLACEHOLDERS["DATE_NAISSANCE"] in out
|
||||
assert "1972" not in out
|
||||
|
||||
|
||||
def test_ddn_nee_colon_sans_le():
|
||||
out, _ = _mask("Né(e) : 19/09/1972")
|
||||
assert PLACEHOLDERS["DATE_NAISSANCE"] in out
|
||||
|
||||
|
||||
def test_ddn_nee_la():
|
||||
out, _ = _mask("Née la 19/09/1972")
|
||||
assert PLACEHOLDERS["DATE_NAISSANCE"] in out
|
||||
|
||||
|
||||
def test_ddn_anti_fp_ne_a_lieu():
|
||||
# "Né à Bordeaux" : pas de date → pas de masque DDN
|
||||
out, _ = _mask("Né à Bordeaux")
|
||||
assert PLACEHOLDERS["DATE_NAISSANCE"] not in out
|
||||
|
||||
|
||||
def test_ddn_anti_fp_vu_en_annee():
|
||||
# "vu en 2020" : pas de contexte naissance → année non masquée DDN
|
||||
out, _ = _mask("Patient vu en 2020")
|
||||
assert PLACEHOLDERS["DATE_NAISSANCE"] not in out
|
||||
|
||||
|
||||
# --- X-L2 rescan : ADHERENT / OGC / FAX / ADELI propagés ---------------------
|
||||
|
||||
def test_rescan_masks_adherent():
|
||||
out = selective_rescan("Mutuelle : 123456", CFG)
|
||||
assert "123456" not in out
|
||||
|
||||
|
||||
def test_rescan_masks_adeli():
|
||||
out = selective_rescan("ADELI : 9ABCDE12", CFG)
|
||||
assert "9ABCDE12" not in out
|
||||
|
||||
|
||||
def test_rescan_masks_fax():
|
||||
out = selective_rescan("Fax : 05 56 00 00 00", CFG)
|
||||
assert "05 56 00 00 00" not in out
|
||||
|
||||
|
||||
# --- #12 NIR multiline en flux réel ------------------------------------------
|
||||
|
||||
def test_nir_multiline_real_document_flow():
|
||||
# Le pipeline masque ligne par ligne ; le cas multi-ligne doit donc passer
|
||||
# par la phase globale, pas seulement par _mask_line_by_regex.
|
||||
anon = anonymise_document_regex(["NIR :\n2840556123456"], [[]], CFG)
|
||||
assert "2840556123456" not in anon.text_out
|
||||
assert PLACEHOLDERS["NIR"] in anon.text_out
|
||||
|
||||
|
||||
def test_nir_multiline_anti_fp_without_label():
|
||||
anon = anonymise_document_regex(["Référence locale :\n2840556123456"], [[]], CFG)
|
||||
assert "2840556123456" in anon.text_out
|
||||
assert PLACEHOLDERS["NIR"] not in anon.text_out
|
||||
|
||||
|
||||
# --- X-L4 FINESS Corse : base source OK, gazetteer dérivé nécessaire ----------
|
||||
|
||||
def test_finess_bare_regex_accepts_corse_identifier():
|
||||
assert RE_BARE_9DIGITS.search("2A0000030") is not None
|
||||
assert RE_BARE_9DIGITS.search("2B0006415") is not None
|
||||
|
||||
|
||||
def test_finess_bare_corse_masks_only_when_known(monkeypatch):
|
||||
monkeypatch.setattr("anonymizer_core_refactored_onnx._FINESS_NUMBERS", {"2A0000030"})
|
||||
audit: list = []
|
||||
out = _mask_admin_label("Code établissement 2A0000030", audit, 0, CFG)
|
||||
assert PLACEHOLDERS["FINESS"] in out
|
||||
assert "2A0000030" not in out
|
||||
assert audit and audit[0].kind == "FINESS"
|
||||
|
||||
|
||||
def test_finess_bare_corse_anti_fp_when_unknown(monkeypatch):
|
||||
monkeypatch.setattr("anonymizer_core_refactored_onnx._FINESS_NUMBERS", set())
|
||||
audit: list = []
|
||||
out = _mask_admin_label("Référence locale 2A9999999", audit, 0, CFG)
|
||||
assert "2A9999999" in out
|
||||
assert PLACEHOLDERS["FINESS"] not in out
|
||||
assert not audit
|
||||
|
||||
|
||||
def test_finess_corse_source_csv_is_loaded_in_gazetteer():
|
||||
# Ces identifiants existent dans data/finess/finess_etablissements.csv.
|
||||
assert "2A0000030" in _FINESS_NUMBERS
|
||||
assert "2B0006415" in _FINESS_NUMBERS
|
||||
|
||||
|
||||
def test_finess_builder_accepts_corse_identifiers():
|
||||
from scripts.build_finess_gazetteers import RE_FINESS_IDENTIFIER
|
||||
|
||||
assert RE_FINESS_IDENTIFIER.match("2A0000030")
|
||||
assert RE_FINESS_IDENTIFIER.match("2B0006415")
|
||||
assert RE_FINESS_IDENTIFIER.match("330056123")
|
||||
196
tests/unit/test_pii_fort_corrections.py
Normal file
196
tests/unit/test_pii_fort_corrections.py
Normal file
@@ -0,0 +1,196 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Corrections PII FORT (audit Qwon 2026-06-17, 13 lacunes FORT validées Dom).
|
||||
|
||||
Batch A — extensions additives de regex déjà câblées dans le pipeline
|
||||
``_mask_line_by_regex`` (+ ``RE_FINESS``). Fichier de test DÉDIÉ pour ne pas
|
||||
entrer en collision avec la WIP hotfix sur les tests P0.
|
||||
|
||||
Toutes les valeurs sont FICTIVES. Chaque correction a un cas positif ET un
|
||||
contrôle anti-faux-positif (ne pas sur-masquer du texte clinique générique).
|
||||
|
||||
#9 (FAX) et #11/#12 (NIR avec label / multiline) nécessitent un nouveau
|
||||
placeholder / hook de masquage : marqués xfail (RED documenté) en attendant
|
||||
le batch A-2.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from anonymizer_core_refactored_onnx import (
|
||||
PLACEHOLDERS,
|
||||
RE_FINESS,
|
||||
_mask_line_by_regex,
|
||||
load_dictionaries,
|
||||
)
|
||||
|
||||
CFG = load_dictionaries(None)
|
||||
|
||||
|
||||
def _mask(line: str):
|
||||
audit: list = []
|
||||
out = _mask_line_by_regex(line, audit, 0, CFG)
|
||||
return out, audit
|
||||
|
||||
|
||||
# --- #1 NOMS : Mlle / Mademoiselle dans le contexte personne -----------------
|
||||
|
||||
def test_mlle_masks_following_name():
|
||||
out, _ = _mask("Mlle DUPONT convoquée")
|
||||
assert "DUPONT" not in out
|
||||
|
||||
|
||||
def test_mademoiselle_masks_following_name():
|
||||
out, _ = _mask("Mademoiselle Lefevre présente")
|
||||
assert "Lefevre" not in out
|
||||
|
||||
|
||||
def test_mlle_anti_fp_generic_sentence():
|
||||
line = "La consultation est programmée demain"
|
||||
out, _ = _mask(line)
|
||||
assert out == line # aucun titre → aucun masque
|
||||
|
||||
|
||||
# --- #2 NOMS : "Fait par" comme contexte personne ----------------------------
|
||||
|
||||
def test_fait_par_masks_name():
|
||||
out, _ = _mask("Fait par MARTIN")
|
||||
assert "MARTIN" not in out
|
||||
|
||||
|
||||
def test_fait_par_colon_masks_name():
|
||||
out, _ = _mask("Fait par : DURAND")
|
||||
assert "DURAND" not in out
|
||||
|
||||
|
||||
# --- #3 DDN : mois abrégés ----------------------------------------------------
|
||||
|
||||
def test_ddn_abbreviated_month_sept():
|
||||
out, _ = _mask("Né le 19 sept. 1972")
|
||||
assert PLACEHOLDERS["DATE_NAISSANCE"] in out
|
||||
assert "1972" not in out
|
||||
|
||||
|
||||
def test_ddn_abbreviated_month_janv():
|
||||
out, _ = _mask("Née le 3 janv. 1980")
|
||||
assert PLACEHOLDERS["DATE_NAISSANCE"] in out
|
||||
|
||||
|
||||
def test_ddn_full_month_still_works():
|
||||
out, _ = _mask("Né le 19 septembre 1972")
|
||||
assert PLACEHOLDERS["DATE_NAISSANCE"] in out
|
||||
|
||||
|
||||
# --- #4 DDN : labels enrichis (Naissance / DN / Nées le) ---------------------
|
||||
|
||||
def test_ddn_label_naissance_standalone():
|
||||
out, _ = _mask("Naissance : 19/09/1972")
|
||||
assert PLACEHOLDERS["DATE_NAISSANCE"] in out
|
||||
|
||||
|
||||
def test_ddn_label_dn():
|
||||
out, _ = _mask("DN : 19/09/1972")
|
||||
assert PLACEHOLDERS["DATE_NAISSANCE"] in out
|
||||
|
||||
|
||||
def test_ddn_label_nees_le():
|
||||
out, _ = _mask("Nées le 19/09/1972")
|
||||
assert PLACEHOLDERS["DATE_NAISSANCE"] in out
|
||||
|
||||
|
||||
def test_ddn_label_date_de_naissance_still_works():
|
||||
out, _ = _mask("Date de naissance : 19/09/1972")
|
||||
assert PLACEHOLDERS["DATE_NAISSANCE"] in out
|
||||
|
||||
|
||||
def test_ddn_anti_fp_clinical_date_preserved():
|
||||
# date clinique hors contexte naissance : NE DOIT PAS être masquée DDN
|
||||
out, _ = _mask("Intervention réalisée le 19/09/1972")
|
||||
assert PLACEHOLDERS["DATE_NAISSANCE"] not in out
|
||||
assert "19/09/1972" in out
|
||||
|
||||
|
||||
def test_ddn_anti_fp_lieu_de_naissance_textuel():
|
||||
# "Lieu de naissance : Bordeaux" — pas une date → pas de masque DATE_NAISSANCE
|
||||
out, _ = _mask("Lieu de naissance : Bordeaux")
|
||||
assert PLACEHOLDERS["DATE_NAISSANCE"] not in out
|
||||
|
||||
|
||||
# --- #5 FINESS Corse 2A/2B ----------------------------------------------------
|
||||
|
||||
def test_finess_corse_2a():
|
||||
assert RE_FINESS.search("FINESS : 2A0000001") is not None
|
||||
|
||||
|
||||
def test_finess_corse_2b():
|
||||
assert RE_FINESS.search("N° FINESS 2B0123456") is not None
|
||||
|
||||
|
||||
def test_finess_standard_still_matches():
|
||||
assert RE_FINESS.search("FINESS : 330056123") is not None
|
||||
|
||||
|
||||
def test_finess_anti_fp_unlabelled_number():
|
||||
# 9 chiffres sans label FINESS ne doivent pas matcher
|
||||
assert RE_FINESS.search("Total facture 123456789 euros") is None
|
||||
|
||||
|
||||
# --- #7 ADRESSES : types de voie supplémentaires -----------------------------
|
||||
|
||||
def test_adresse_villa():
|
||||
out, _ = _mask("15 villa des Nympheas")
|
||||
assert PLACEHOLDERS["ADRESSE"] in out
|
||||
|
||||
|
||||
def test_adresse_faubourg():
|
||||
out, _ = _mask("12 faubourg Saint-Honore")
|
||||
assert PLACEHOLDERS["ADRESSE"] in out
|
||||
|
||||
|
||||
def test_adresse_existing_rue_still_works():
|
||||
out, _ = _mask("35 rue Claude Boucher")
|
||||
assert PLACEHOLDERS["ADRESSE"] in out
|
||||
|
||||
|
||||
# --- #10 + #13 MUTUELLE / AMC / CSS → [ADHERENT] -----------------------------
|
||||
|
||||
def test_adherent_mutuelle_number():
|
||||
out, _ = _mask("Mutuelle : 123456")
|
||||
assert "123456" not in out
|
||||
|
||||
|
||||
def test_adherent_amc_number():
|
||||
out, _ = _mask("AMC : 1234567")
|
||||
assert "1234567" not in out
|
||||
|
||||
|
||||
def test_adherent_existing_label_still_works():
|
||||
out, _ = _mask("N° adhérent : 123456789")
|
||||
assert "123456789" not in out
|
||||
|
||||
|
||||
def test_adherent_anti_fp_short_mutuelle_name():
|
||||
# "MGEN" (4 chars) n'est pas un numéro → ne doit pas être capté comme [ADHERENT]
|
||||
out, _ = _mask("Mutuelle : MGEN")
|
||||
assert "MGEN" in out
|
||||
|
||||
|
||||
# --- #11/#12 NIR : 13 chiffres avec label + multiline (batch A-2, RED) --------
|
||||
|
||||
def test_nir_13_digits_with_label():
|
||||
# implémenté en batch A-2 (RE_NIR_NO_KEY, label-ancré)
|
||||
out, _ = _mask("NIR : 2840556123456")
|
||||
assert "2840556123456" not in out
|
||||
|
||||
|
||||
def test_nir_anti_fp_bare_13_digits_not_masked():
|
||||
# 13 chiffres SANS label NIR ne doivent jamais être masqués (anti-FP fort)
|
||||
line = "Référence dossier 2840556123456"
|
||||
out, _ = _mask(line)
|
||||
assert "2840556123456" in out
|
||||
|
||||
|
||||
# --- #9 FAX : placeholder [FAX] (batch A-2, RED) -----------------------------
|
||||
|
||||
def test_fax_label_masked():
|
||||
# implémenté en batch A-2 (RE_FAX + placeholder [FAX], appliqué avant RE_TEL)
|
||||
out, _ = _mask("Fax : 05 56 00 00 00")
|
||||
assert PLACEHOLDERS["FAX"] in out
|
||||
assert "05 56 00 00 00" not in out
|
||||
@@ -109,6 +109,31 @@ def test_ogc_pdf_redaction_does_not_mask_numeric_substrings(tmp_path):
|
||||
assert "142 : La facturation" in text
|
||||
|
||||
|
||||
def test_pdf_redaction_directly_masks_finess_address_range(tmp_path):
|
||||
"""Cas Dom 2026-06-16 : une adresse d'établissement visible dans le PDF
|
||||
doit être caviardée même si l'audit n'a pas fourni le hit exact."""
|
||||
if fitz is None:
|
||||
return
|
||||
|
||||
source = tmp_path / "finess_address_range.pdf"
|
||||
output = tmp_path / "finess_address_range.redacted.pdf"
|
||||
doc = fitz.open()
|
||||
page = doc.new_page()
|
||||
page.insert_text((72, 72), "15 à 35 rue Claude Boucher Bordeaux Cedex")
|
||||
page.insert_text((72, 108), "Motif d'hospitalisation : contrôle clinique.")
|
||||
doc.save(source)
|
||||
doc.close()
|
||||
|
||||
redact_pdf_vector(source, [], output)
|
||||
|
||||
redacted = fitz.open(output)
|
||||
text = redacted[0].get_text()
|
||||
redacted.close()
|
||||
assert "Claude Boucher" not in text
|
||||
assert "15 à 35" not in text
|
||||
assert "Motif d'hospitalisation" in text
|
||||
|
||||
|
||||
def test_crop_epi_header_name_is_masked():
|
||||
cfg = load_dictionaries(None)
|
||||
text = (
|
||||
|
||||
Reference in New Issue
Block a user