feat: parallélisation pipeline --workers N (ThreadPoolExecutor)

- Fix thread-safety FAISS index (Lock + double-check sur _loaded)
- Fix thread-safety reranker (Lock + double-check sur _reranker_model)
- main.py : flag --workers, extraction _process_group(), ThreadPoolExecutor
- benchmark_quality.py : flag --workers, subprocess en parallèle
- Validé sur 10 dossiers gold standard --workers 3 : 0 crash, codes identiques

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

scripts/benchmark_quality.py

@@ -594,6 +594,7 @@ def main():
     parser.add_argument("--no-reprocess", action="store_true", help="Analyser les outputs existants sans relancer le pipeline")
     parser.add_argument("--clean", action="store_true", help="Supprimer les outputs avant retraitement")
     parser.add_argument("--seed", type=int, default=42, help="Seed pour la sélection aléatoire")
+    parser.add_argument("--workers", type=int, default=1, help="Nombre de dossiers traités en parallèle")
     args = parser.parse_args()
 
     # Sélection dossiers
@@ -632,8 +633,40 @@ def main():
 
     # Traitement
     per_dossier = []
+    total = len(dossiers)
+
+    if args.workers > 1 and not args.no_reprocess:
+        # Mode parallèle : exécuter les pipelines en parallèle puis analyser
+        from concurrent.futures import ThreadPoolExecutor, as_completed
+        print(f"  Mode parallèle : {args.workers} workers")
+        pipeline_results: dict[str, tuple[float, bool]] = {}
+        done = 0
+        with ThreadPoolExecutor(max_workers=args.workers) as executor:
+            futures = {
+                executor.submit(run_pipeline, dossier_id, args.clean): dossier_id
+                for dossier_id in dossiers
+            }
+            for future in as_completed(futures):
+                dossier_id = futures[future]
+                try:
+                    duration, success = future.result()
+                except Exception as e:
+                    print(f"    EXCEPTION {dossier_id}: {e}")
+                    duration, success = 0.0, False
+                pipeline_results[dossier_id] = (duration, success)
+                done += 1
+                mark = "✓" if success else "✗"
+                print(f"  [{done}/{total}] {dossier_id} — {duration:.1f}s {mark}")
+
+        # Analyse séquentielle (ordre stable)
+        for dossier_id in dossiers:
+            duration, success = pipeline_results[dossier_id]
+            metrics = analyze_dossier(dossier_id, cim10, duration)
+            per_dossier.append(metrics)
+    else:
+        # Mode séquentiel (ou --no-reprocess)
         for i, dossier_id in enumerate(dossiers, 1):
-        print(f"  [{i}/{len(dossiers)}] {dossier_id}", end="", flush=True)
+            print(f"  [{i}/{total}] {dossier_id}", end="", flush=True)
 
             if args.no_reprocess:
                 duration = 0.0

src/main.py

@@ -399,6 +399,12 @@ def main(input_path: str | None = None) -> None:
         metavar="PATH",
         help="Fichier Excel de contrôle CPAM (enrichit les dossiers avec contre-argumentation)",
     )
+    parser.add_argument(
+        "--workers",
+        type=int,
+        default=1,
+        help="Nombre de dossiers traités en parallèle (défaut: 1)",
+    )
     args = parser.parse_args()
 
     if args.build_dict:
@@ -501,7 +507,8 @@ def main(input_path: str | None = None) -> None:
 
     logger.info("Traitement de %d PDF(s)...", total)
 
-    for pdfs, subdir in groups:
+    def _process_group(pdfs: list[Path], subdir: str | None) -> None:
+        """Traite un groupe de PDFs (un dossier patient)."""
         if subdir:
             logger.info("--- Dossier %s (%d PDFs) ---", subdir, len(pdfs))
 
@@ -633,6 +640,24 @@ def main(input_path: str | None = None) -> None:
             except Exception:
                 logger.exception("Erreur écriture dossier fusionné %s", subdir)
 
+    # Exécution séquentielle ou parallèle selon --workers
+    if args.workers > 1:
+        from concurrent.futures import ThreadPoolExecutor, as_completed
+        logger.info("Mode parallèle : %d workers", args.workers)
+        with ThreadPoolExecutor(max_workers=args.workers) as executor:
+            futures = {
+                executor.submit(_process_group, pdfs, subdir): subdir
+                for pdfs, subdir in groups
+            }
+            for future in as_completed(futures):
+                try:
+                    future.result()
+                except Exception:
+                    logger.exception("Erreur groupe %s", futures[future])
+    else:
+        for pdfs, subdir in groups:
+            _process_group(pdfs, subdir)
+
     logger.info("Terminé.")
 
 

src/medical/rag_index.py

@@ -14,6 +14,7 @@ from __future__ import annotations
 import json
 import logging
 import re
+import threading
 from dataclasses import dataclass, asdict
 from pathlib import Path
 from typing import Optional
@@ -26,6 +27,7 @@ logger = logging.getLogger(__name__)
 
 # Singletons pour les index chargés en mémoire
 _loaded: dict[str, tuple] = {}
+_loaded_lock = threading.Lock()
 
 
 @dataclass
@@ -574,6 +576,11 @@ def get_index(kind: str = "ref") -> tuple | None:
     """
     kind = (kind or "ref").lower()
 
+    if kind in _loaded:
+        return _loaded[kind]
+
+    with _loaded_lock:
+        # Double-check après acquisition du lock
         if kind in _loaded:
             return _loaded[kind]
 
@@ -800,4 +807,5 @@ def add_chunks_to_index(chunks: list[Chunk]) -> int:
 
 def reset_index() -> None:
     """Invalide les singletons FAISS pour forcer le rechargement au prochain accès."""
+    with _loaded_lock:
         _loaded.clear()

src/medical/rag_search.py

@@ -28,6 +28,7 @@ _embed_failed = False  # Sentinelle pour éviter les retries infinis
 
 # Singleton pour le cross-encoder de re-ranking (CPU uniquement)
 _reranker_model = None
+_reranker_lock = threading.Lock()
 
 # Score minimum de similarité FAISS pour retenir un résultat
 _MIN_SCORE = 0.3
@@ -84,12 +85,17 @@ def _get_embed_model():
 
 
 def _get_reranker():
-    """Charge le cross-encoder de re-ranking (singleton, CPU uniquement).
+    """Charge le cross-encoder de re-ranking (singleton thread-safe, CPU uniquement).
 
     Forcé sur CPU pour ne pas interférer avec Ollama sur GPU.
     """
     global _reranker_model
-    if _reranker_model is None:
+    if _reranker_model is not None:
+        return _reranker_model
+    with _reranker_lock:
+        # Double-check après acquisition du lock
+        if _reranker_model is not None:
+            return _reranker_model
         from sentence_transformers import CrossEncoder
         logger.info("Chargement du cross-encoder de re-ranking (cpu)...")
         _reranker_model = CrossEncoder(RERANKER_MODEL, device="cpu")