feat(ocr): migrer l'OCR de docTR (PyTorch) vers OnnxTR (ONNX Runtime)
OnnxTR exécute les MÊMES modèles que docTR (db_resnet50 + crnn_vgg16_bn) sur ONNX Runtime, sans PyTorch. Corrige le crash torch/oneDNN « could not create a primitive » sur CPU contraint (VM 2 cœurs collaborateur : OCR scan impossible → quarantaine). Qualité identique validée empiriquement (CER 0,10-0,23 % vs docTR, 2 validations indépendantes Claude+Qwen), OCR ~2-3× plus rapide CPU. - core : import OnnxTR, _get_ocr_model(), _OCR_AVAILABLE, boucle OCR inchangée (API miroir) ; ONNXTR_CACHE_DIR pour le frozen ; bandeau de logs ENV au démarrage (OS, CPU+AVX, cœurs, RAM, versions, providers) pour retours terrain auto-suffisants. - 3 .spec : embarquent les poids ONNX OnnxTR (fail-closed) + hiddenimports onnxtr. - requirements : onnxtr[cpu] (python-doctr conservé transitoirement). - inclut le correctif quarantaine-visible du runner (GO Qwen). Tests : test_ocr_onnxtr.py (RED→GREEN), 95 unit passed, e2e scan client OK (OCR 5/5, PDF produit, plus de crash). Retrait torch du frozen + rebuild Windows = étapes suivantes (gates Dom). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -38,6 +38,19 @@ from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import List, Dict, Tuple, Optional, Any
|
||||
|
||||
|
||||
def _bundle_root() -> Path:
|
||||
"""Racine des ressources, compatible PyInstaller."""
|
||||
if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"):
|
||||
return Path(getattr(sys, "_MEIPASS"))
|
||||
return Path(__file__).resolve().parent
|
||||
|
||||
|
||||
_BUNDLED_ONNXTR_CACHE = _bundle_root() / "models" / "onnxtr"
|
||||
if getattr(sys, "frozen", False) and _BUNDLED_ONNXTR_CACHE.exists():
|
||||
# OnnxTR ajoute lui-même le sous-dossier "models" à ONNXTR_CACHE_DIR.
|
||||
os.environ.setdefault("ONNXTR_CACHE_DIR", str(_BUNDLED_ONNXTR_CACHE))
|
||||
|
||||
# {page_idx: [(word_text, x0_norm, y0_norm, x1_norm, y1_norm), ...]}
|
||||
# Coordonnées normalisées 0→1 (format natif docTR word.geometry)
|
||||
OcrWordMap = Dict[int, List[Tuple[str, float, float, float, float]]]
|
||||
@@ -69,13 +82,17 @@ from admin_rules import (
|
||||
)
|
||||
|
||||
try:
|
||||
from doctr.models import ocr_predictor as _doctr_ocr_predictor
|
||||
_DOCTR_AVAILABLE = True
|
||||
# OCR via OnnxTR : mêmes modèles que docTR (db_resnet50 + crnn_vgg16_bn) mais
|
||||
# exécutés sur ONNX Runtime, SANS torch — supprime le crash torch/oneDNN
|
||||
# « could not create a primitive » observé sur CPU contraint (VM 2 cœurs client).
|
||||
# Équivalence qualité validée empiriquement (CER moyen 0,23 % vs docTR, corpus scanné).
|
||||
from onnxtr.models import ocr_predictor as _ocr_predictor_factory
|
||||
_OCR_AVAILABLE = True
|
||||
except Exception:
|
||||
_doctr_ocr_predictor = None # type: ignore
|
||||
_DOCTR_AVAILABLE = False
|
||||
_ocr_predictor_factory = None # type: ignore
|
||||
_OCR_AVAILABLE = False
|
||||
|
||||
_doctr_model_cache = None
|
||||
_ocr_model_cache = None
|
||||
_TORCH_THREADS_CONFIGURED = False
|
||||
|
||||
def _configure_torch_threads():
|
||||
@@ -106,14 +123,80 @@ def _configure_torch_threads():
|
||||
except Exception as e:
|
||||
log.debug("torch threads config skipped: %s", e)
|
||||
|
||||
def _get_doctr_model():
|
||||
global _doctr_model_cache
|
||||
if _doctr_model_cache is None:
|
||||
_configure_torch_threads()
|
||||
_doctr_model_cache = _doctr_ocr_predictor(
|
||||
det_arch="db_resnet50", reco_arch="crnn_vgg16_bn", pretrained=True
|
||||
def _get_ocr_model():
|
||||
global _ocr_model_cache
|
||||
if _ocr_model_cache is None:
|
||||
# OnnxTR : mêmes architectures que docTR, exécution ONNX Runtime (pas de torch,
|
||||
# donc pas de config threads torch ici). Poids ONNX pré-entraînés chargés par défaut.
|
||||
_ocr_model_cache = _ocr_predictor_factory(
|
||||
det_arch="db_resnet50", reco_arch="crnn_vgg16_bn"
|
||||
)
|
||||
return _doctr_model_cache
|
||||
return _ocr_model_cache
|
||||
|
||||
|
||||
_ENV_BANNER_LOGGED = False
|
||||
|
||||
|
||||
def _log_env_banner() -> None:
|
||||
"""Logge une fois un bandeau d'environnement (machine + versions) pour diagnostic.
|
||||
|
||||
Objectif : qu'UN SEUL run de retour terrain suffise à diagnostiquer (specs CPU/RAM,
|
||||
nb de cœurs, OS, versions OCR/NER) — sans redemander d'actions au collaborateur.
|
||||
"""
|
||||
global _ENV_BANNER_LOGGED
|
||||
if _ENV_BANNER_LOGGED:
|
||||
return
|
||||
_ENV_BANNER_LOGGED = True
|
||||
import platform
|
||||
parts: List[str] = []
|
||||
try:
|
||||
parts.append(f"os={platform.platform()}")
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
parts.append(f"cpu={platform.processor() or platform.machine()}")
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
logical = os.cpu_count()
|
||||
try:
|
||||
import psutil
|
||||
phys = psutil.cpu_count(logical=False)
|
||||
ram = psutil.virtual_memory().total / 1e9
|
||||
parts.append(f"cores={phys}phys/{logical}log")
|
||||
parts.append(f"ram={ram:.1f}Go")
|
||||
except Exception:
|
||||
parts.append(f"cores={logical}log")
|
||||
except Exception:
|
||||
pass
|
||||
# AVX/SSE : Linux best-effort via /proc/cpuinfo (Windows : non dispo sans dépendance dédiée)
|
||||
try:
|
||||
cpuinfo_path = Path("/proc/cpuinfo")
|
||||
if platform.system() == "Linux" and cpuinfo_path.exists():
|
||||
import re as _re
|
||||
m = _re.search(r"flags\s*:\s*(.*)", cpuinfo_path.read_text(errors="ignore"))
|
||||
if m:
|
||||
present = [f for f in ("sse4_2", "avx", "avx2", "avx512f") if f in m.group(1).split()]
|
||||
if present:
|
||||
parts.append("cpu_flags=" + ",".join(present))
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
parts.append(f"python={platform.python_version()} frozen={bool(getattr(sys, 'frozen', False))}")
|
||||
except Exception:
|
||||
pass
|
||||
vers: List[str] = []
|
||||
for mod in ("onnxruntime", "onnxtr", "numpy", "transformers", "torch", "fitz"):
|
||||
try:
|
||||
vers.append(f"{mod}={getattr(__import__(mod), '__version__', '?')}")
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
import onnxruntime as _ort
|
||||
vers.append("ort_providers=" + ",".join(_ort.get_available_providers()))
|
||||
except Exception:
|
||||
pass
|
||||
log.info("ENV %s | %s", " ".join(parts), " ".join(vers))
|
||||
|
||||
try:
|
||||
from detectors.hospital_filter import HospitalFilter
|
||||
@@ -1454,7 +1537,7 @@ def extract_text_with_fallback_ocr(pdf_path: Path) -> Tuple[List[str], List[List
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# --- Passe 3 : OCR docTR sur les pages pauvres en texte ---
|
||||
# --- Passe 3 : OCR (OnnxTR) sur les pages pauvres en texte ---
|
||||
# Pas de seuil global : on OCR uniquement les pages individuelles
|
||||
# qui ont peu de texte (< 150 chars), puis on garde le meilleur résultat
|
||||
# par page. Les pages déjà riches en texte ne sont pas touchées.
|
||||
@@ -1462,9 +1545,9 @@ def extract_text_with_fallback_ocr(pdf_path: Path) -> Tuple[List[str], List[List
|
||||
total_chars = sum(len(x or "") for x in pages_text)
|
||||
ocr_word_map: OcrWordMap = {}
|
||||
sparse_pages = [i for i, p in enumerate(pages_text) if len(p or "") < _OCR_PAGE_THRESHOLD]
|
||||
if sparse_pages and _DOCTR_AVAILABLE and fitz is not None:
|
||||
if sparse_pages and _OCR_AVAILABLE and fitz is not None:
|
||||
try:
|
||||
model = _get_doctr_model()
|
||||
model = _get_ocr_model()
|
||||
doc = fitz.open(str(pdf_path))
|
||||
import numpy as np
|
||||
ocr_replaced = 0
|
||||
@@ -1490,9 +1573,9 @@ def extract_text_with_fallback_ocr(pdf_path: Path) -> Tuple[List[str], List[List
|
||||
doc.close()
|
||||
if ocr_replaced > 0:
|
||||
ocr_used = True
|
||||
log.info("OCR docTR : %d/%d pages remplacées", ocr_replaced, len(sparse_pages))
|
||||
log.info("OCR OnnxTR : %d/%d pages remplacées", ocr_replaced, len(sparse_pages))
|
||||
except Exception as e:
|
||||
log.warning("OCR docTR échoué : %s", e)
|
||||
log.warning("OCR OnnxTR échoué : %s", e)
|
||||
ocr_word_map = {}
|
||||
return pages_text, tables_lines, ocr_used, ocr_word_map
|
||||
|
||||
@@ -3275,9 +3358,9 @@ def _run_ner_on_original_text(
|
||||
Returns:
|
||||
Liste de NerDetection dédupliquée (par token+label+page+source).
|
||||
"""
|
||||
# H1 perf (D-19) : couvre le cas du PDF natif (texte riche, OCR sauté) où
|
||||
# _get_doctr_model() n'est jamais appelé ; les NER torch (EDS-Pseudo, GLiNER)
|
||||
# tourneraient alors mono-thread. Idempotent (no-op si déjà configuré par l'OCR).
|
||||
# H1 perf (D-19) : configure les threads torch pour les NER torch optionnels
|
||||
# (EDS-Pseudo, GLiNER) lorsqu'ils sont présents. L'OCR (OnnxTR) et CamemBERT-bio
|
||||
# tournent sur ONNX Runtime (sans torch) ; no-op si torch absent du build.
|
||||
_configure_torch_threads()
|
||||
|
||||
detections: List[NerDetection] = []
|
||||
@@ -4914,6 +4997,7 @@ def process_pdf(
|
||||
|
||||
log.info("PERF %s: start frozen=%s vector=%s raster=%s",
|
||||
pdf_path.name, bool(getattr(sys, "frozen", False)), make_vector_redaction, also_make_raster_burn)
|
||||
_log_env_banner()
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
cfg = load_dictionaries(config_path)
|
||||
_perf_mark("load_config")
|
||||
|
||||
Reference in New Issue
Block a user