Initial commit

omop/config.yaml

@@ -0,0 +1,59 @@
+# OMOP Pipeline Configuration
+
+# Database Configuration
+database:
+  host: localhost
+  port: 5432
+  database: omop_cdm
+  user: dom
+  password: loli
+  pool_size: 10
+  max_overflow: 20
+  pool_timeout: 30
+  pool_recycle: 3600
+
+# ETL Configuration
+etl:
+  batch_size: 1000
+  num_workers: 8
+  max_retries: 3
+  retry_delay: 5  # seconds
+  checkpoint_interval: 10000  # records
+
+# Mapping Configuration
+mapping:
+  cache_size: 10000
+  use_custom_mappings: true
+  unmapped_concept_id: 0
+
+# Validation Configuration
+validation:
+  min_completeness: 0.95
+  max_error_rate: 0.05
+  check_referential_integrity: true
+  check_date_consistency: true
+  check_value_ranges: true
+
+# Logging Configuration
+logging:
+  level: INFO
+  file: logs/omop_pipeline.log
+  max_bytes: 10485760  # 10MB
+  backup_count: 5
+  format: "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+
+# Performance Configuration
+performance:
+  enable_parallel_processing: true
+  monitor_memory: true
+  memory_threshold: 0.8  # 80% of available memory
+  circuit_breaker_threshold: 0.5  # 50% error rate
+  circuit_breaker_window: 100  # records
+
+# Schema Configuration
+schema:
+  omop_schema: omop
+  staging_schema: staging
+  audit_schema: audit
+  create_indexes: true
+  create_constraints: true