60 lines
1.2 KiB
YAML
60 lines
1.2 KiB
YAML
# OMOP Pipeline Configuration
|
|
|
|
# Database Configuration
|
|
database:
|
|
host: localhost
|
|
port: 5432
|
|
database: omop_cdm
|
|
user: dom
|
|
password: loli
|
|
pool_size: 10
|
|
max_overflow: 20
|
|
pool_timeout: 30
|
|
pool_recycle: 3600
|
|
|
|
# ETL Configuration
|
|
etl:
|
|
batch_size: 1000
|
|
num_workers: 8
|
|
max_retries: 3
|
|
retry_delay: 5 # seconds
|
|
checkpoint_interval: 10000 # records
|
|
|
|
# Mapping Configuration
|
|
mapping:
|
|
cache_size: 10000
|
|
use_custom_mappings: true
|
|
unmapped_concept_id: 0
|
|
|
|
# Validation Configuration
|
|
validation:
|
|
min_completeness: 0.95
|
|
max_error_rate: 0.05
|
|
check_referential_integrity: true
|
|
check_date_consistency: true
|
|
check_value_ranges: true
|
|
|
|
# Logging Configuration
|
|
logging:
|
|
level: INFO
|
|
file: logs/omop_pipeline.log
|
|
max_bytes: 10485760 # 10MB
|
|
backup_count: 5
|
|
format: "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
|
|
|
|
# Performance Configuration
|
|
performance:
|
|
enable_parallel_processing: true
|
|
monitor_memory: true
|
|
memory_threshold: 0.8 # 80% of available memory
|
|
circuit_breaker_threshold: 0.5 # 50% error rate
|
|
circuit_breaker_window: 100 # records
|
|
|
|
# Schema Configuration
|
|
schema:
|
|
omop_schema: omop
|
|
staging_schema: staging
|
|
audit_schema: audit
|
|
create_indexes: true
|
|
create_constraints: true
|