omop-lite 0.0.18__py3-none-any.whl → 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- omop_lite/__init__.py +0 -37
- omop_lite/cli.py +915 -0
- omop_lite/db/__init__.py +4 -4
- omop_lite/db/base.py +73 -20
- omop_lite/db/postgres.py +7 -7
- omop_lite/db/sqlserver.py +8 -6
- omop_lite/settings.py +20 -13
- {omop_lite-0.0.18.dist-info → omop_lite-0.1.0.dist-info}/METADATA +6 -1
- {omop_lite-0.0.18.dist-info → omop_lite-0.1.0.dist-info}/RECORD +12 -11
- omop_lite-0.1.0.dist-info/entry_points.txt +2 -0
- omop_lite-0.0.18.dist-info/entry_points.txt +0 -2
- {omop_lite-0.0.18.dist-info → omop_lite-0.1.0.dist-info}/WHEEL +0 -0
- {omop_lite-0.0.18.dist-info → omop_lite-0.1.0.dist-info}/licenses/LICENSE +0 -0
omop_lite/__init__.py
CHANGED
@@ -1,37 +0,0 @@
|
|
1
|
-
from omop_lite.settings import settings
|
2
|
-
from omop_lite.db import create_database
|
3
|
-
import logging
|
4
|
-
from importlib.metadata import version
|
5
|
-
|
6
|
-
|
7
|
-
def main() -> None:
|
8
|
-
"""
|
9
|
-
Main function to create the OMOP Lite database.
|
10
|
-
|
11
|
-
This function will create the schema if it doesn't exist,
|
12
|
-
create the tables, load the data, and run the update migrations.
|
13
|
-
"""
|
14
|
-
logging.basicConfig(level=settings.log_level)
|
15
|
-
logger = logging.getLogger(__name__)
|
16
|
-
logger.info(f"Starting OMOP Lite {version('omop-lite')}")
|
17
|
-
logger.debug(f"Settings: {settings.model_dump()}")
|
18
|
-
db = create_database()
|
19
|
-
|
20
|
-
# Handle schema creation if not using 'public'
|
21
|
-
if settings.schema_name != "public":
|
22
|
-
if db.schema_exists(settings.schema_name):
|
23
|
-
logger.info(f"Schema '{settings.schema_name}' already exists")
|
24
|
-
return
|
25
|
-
else:
|
26
|
-
db.create_schema(settings.schema_name)
|
27
|
-
|
28
|
-
# Continue with table creation, data loading, etc.
|
29
|
-
db.create_tables()
|
30
|
-
db.load_data()
|
31
|
-
db.add_constraints()
|
32
|
-
|
33
|
-
logger.info("OMOP Lite database created successfully")
|
34
|
-
|
35
|
-
|
36
|
-
if __name__ == "__main__":
|
37
|
-
main()
|
omop_lite/cli.py
ADDED
@@ -0,0 +1,915 @@
|
|
1
|
+
from omop_lite.settings import Settings
|
2
|
+
from omop_lite.db import create_database
|
3
|
+
import logging
|
4
|
+
from importlib.metadata import version
|
5
|
+
import typer
|
6
|
+
from rich.console import Console
|
7
|
+
from rich.progress import (
|
8
|
+
Progress,
|
9
|
+
SpinnerColumn,
|
10
|
+
TextColumn,
|
11
|
+
BarColumn,
|
12
|
+
TaskProgressColumn,
|
13
|
+
)
|
14
|
+
from rich.panel import Panel
|
15
|
+
from rich.table import Table
|
16
|
+
from rich.prompt import Confirm
|
17
|
+
import time
|
18
|
+
|
19
|
+
console = Console()
|
20
|
+
|
21
|
+
app = typer.Typer(
|
22
|
+
name="omop-lite",
|
23
|
+
help="Get an OMOP CDM database running quickly.",
|
24
|
+
add_completion=False,
|
25
|
+
no_args_is_help=False,
|
26
|
+
)
|
27
|
+
|
28
|
+
|
29
|
+
def _create_settings(
|
30
|
+
db_host: str = "db",
|
31
|
+
db_port: int = 5432,
|
32
|
+
db_user: str = "postgres",
|
33
|
+
db_password: str = "password",
|
34
|
+
db_name: str = "omop",
|
35
|
+
synthetic: bool = False,
|
36
|
+
synthetic_number: int = 100,
|
37
|
+
data_dir: str = "data",
|
38
|
+
schema_name: str = "public",
|
39
|
+
dialect: str = "postgresql",
|
40
|
+
log_level: str = "INFO",
|
41
|
+
fts_create: bool = False,
|
42
|
+
delimiter: str = "\t",
|
43
|
+
) -> Settings:
|
44
|
+
"""Create settings with validation."""
|
45
|
+
# Validate dialect
|
46
|
+
if dialect not in ["postgresql", "mssql"]:
|
47
|
+
raise typer.BadParameter("dialect must be either 'postgresql' or 'mssql'")
|
48
|
+
|
49
|
+
return Settings(
|
50
|
+
db_host=db_host,
|
51
|
+
db_port=db_port,
|
52
|
+
db_user=db_user,
|
53
|
+
db_password=db_password,
|
54
|
+
db_name=db_name,
|
55
|
+
synthetic=synthetic,
|
56
|
+
synthetic_number=synthetic_number,
|
57
|
+
data_dir=data_dir,
|
58
|
+
schema_name=schema_name,
|
59
|
+
dialect=dialect,
|
60
|
+
log_level=log_level,
|
61
|
+
fts_create=fts_create,
|
62
|
+
delimiter=delimiter,
|
63
|
+
)
|
64
|
+
|
65
|
+
|
66
|
+
def _setup_logging(settings: Settings) -> logging.Logger:
|
67
|
+
"""Setup logging with the given settings."""
|
68
|
+
logging.basicConfig(level=settings.log_level)
|
69
|
+
logger = logging.getLogger(__name__)
|
70
|
+
logger.info(f"Starting OMOP Lite {version('omop-lite')}")
|
71
|
+
logger.debug(f"Settings: {settings.model_dump()}")
|
72
|
+
return logger
|
73
|
+
|
74
|
+
|
75
|
+
@app.callback(invoke_without_command=True)
|
76
|
+
def callback(
|
77
|
+
ctx: typer.Context,
|
78
|
+
db_host: str = typer.Option(
|
79
|
+
"db", "--db-host", "-h", envvar="DB_HOST", help="Database host"
|
80
|
+
),
|
81
|
+
db_port: int = typer.Option(
|
82
|
+
5432, "--db-port", "-p", envvar="DB_PORT", help="Database port"
|
83
|
+
),
|
84
|
+
db_user: str = typer.Option(
|
85
|
+
"postgres", "--db-user", "-u", envvar="DB_USER", help="Database user"
|
86
|
+
),
|
87
|
+
db_password: str = typer.Option(
|
88
|
+
"password", "--db-password", envvar="DB_PASSWORD", help="Database password"
|
89
|
+
),
|
90
|
+
db_name: str = typer.Option(
|
91
|
+
"omop", "--db-name", "-d", envvar="DB_NAME", help="Database name"
|
92
|
+
),
|
93
|
+
synthetic: bool = typer.Option(
|
94
|
+
False, "--synthetic", envvar="SYNTHETIC", help="Use synthetic data"
|
95
|
+
),
|
96
|
+
synthetic_number: int = typer.Option(
|
97
|
+
100,
|
98
|
+
"--synthetic-number",
|
99
|
+
envvar="SYNTHETIC_NUMBER",
|
100
|
+
help="Number of synthetic records",
|
101
|
+
),
|
102
|
+
data_dir: str = typer.Option(
|
103
|
+
"data", "--data-dir", envvar="DATA_DIR", help="Data directory"
|
104
|
+
),
|
105
|
+
schema_name: str = typer.Option(
|
106
|
+
"public", "--schema-name", envvar="SCHEMA_NAME", help="Database schema name"
|
107
|
+
),
|
108
|
+
dialect: str = typer.Option(
|
109
|
+
"postgresql",
|
110
|
+
"--dialect",
|
111
|
+
envvar="DIALECT",
|
112
|
+
help="Database dialect (postgresql or mssql)",
|
113
|
+
),
|
114
|
+
log_level: str = typer.Option(
|
115
|
+
"INFO", "--log-level", envvar="LOG_LEVEL", help="Logging level"
|
116
|
+
),
|
117
|
+
fts_create: bool = typer.Option(
|
118
|
+
False,
|
119
|
+
"--fts-create",
|
120
|
+
envvar="FTS_CREATE",
|
121
|
+
help="Create full-text search indexes",
|
122
|
+
),
|
123
|
+
delimiter: str = typer.Option(
|
124
|
+
"\t", "--delimiter", envvar="DELIMITER", help="CSV delimiter"
|
125
|
+
),
|
126
|
+
) -> None:
|
127
|
+
"""
|
128
|
+
Create the OMOP Lite database (default command).
|
129
|
+
|
130
|
+
This command will create the schema if it doesn't exist,
|
131
|
+
create the tables, load the data, and run the update migrations.
|
132
|
+
|
133
|
+
All settings can be configured via environment variables or command-line arguments.
|
134
|
+
Command-line arguments take precedence over environment variables.
|
135
|
+
"""
|
136
|
+
if ctx.invoked_subcommand is None:
|
137
|
+
# This is the default command (no subcommand specified)
|
138
|
+
settings = _create_settings(
|
139
|
+
db_host=db_host,
|
140
|
+
db_port=db_port,
|
141
|
+
db_user=db_user,
|
142
|
+
db_password=db_password,
|
143
|
+
db_name=db_name,
|
144
|
+
synthetic=synthetic,
|
145
|
+
synthetic_number=synthetic_number,
|
146
|
+
data_dir=data_dir,
|
147
|
+
schema_name=schema_name,
|
148
|
+
dialect=dialect,
|
149
|
+
log_level=log_level,
|
150
|
+
fts_create=fts_create,
|
151
|
+
delimiter=delimiter,
|
152
|
+
)
|
153
|
+
|
154
|
+
# Show startup info
|
155
|
+
console.print(
|
156
|
+
Panel(
|
157
|
+
f"[bold blue]OMOP Lite[/bold blue] v{version('omop-lite')}\n"
|
158
|
+
f"[dim]Creating OMOP CDM database...[/dim]",
|
159
|
+
title="🚀 Starting Pipeline",
|
160
|
+
border_style="blue",
|
161
|
+
)
|
162
|
+
)
|
163
|
+
|
164
|
+
db = create_database(settings)
|
165
|
+
|
166
|
+
# Handle schema creation if not using 'public'
|
167
|
+
if settings.schema_name != "public":
|
168
|
+
if db.schema_exists(settings.schema_name):
|
169
|
+
console.print(f"ℹ️ Schema '{settings.schema_name}' already exists")
|
170
|
+
return
|
171
|
+
else:
|
172
|
+
with console.status("[bold green]Creating schema...", spinner="dots"):
|
173
|
+
db.create_schema(settings.schema_name)
|
174
|
+
console.print(f"✅ Schema '{settings.schema_name}' created")
|
175
|
+
|
176
|
+
# Progress bar for the main pipeline
|
177
|
+
with Progress(
|
178
|
+
SpinnerColumn(),
|
179
|
+
TextColumn("[progress.description]{task.description}"),
|
180
|
+
BarColumn(),
|
181
|
+
TaskProgressColumn(),
|
182
|
+
console=console,
|
183
|
+
) as progress:
|
184
|
+
# Create tables
|
185
|
+
task1 = progress.add_task("[cyan]Creating tables...", total=1)
|
186
|
+
db.create_tables()
|
187
|
+
progress.update(task1, completed=1)
|
188
|
+
|
189
|
+
# Load data
|
190
|
+
task2 = progress.add_task("[yellow]Loading data...", total=1)
|
191
|
+
db.load_data()
|
192
|
+
progress.update(task2, completed=1)
|
193
|
+
|
194
|
+
# Add constraints
|
195
|
+
task3 = progress.add_task("[green]Adding constraints...", total=1)
|
196
|
+
db.add_all_constraints()
|
197
|
+
progress.update(task3, completed=1)
|
198
|
+
|
199
|
+
console.print(
|
200
|
+
Panel(
|
201
|
+
"[bold green]✅ OMOP Lite database created successfully![/bold green]\n"
|
202
|
+
f"[dim]Database: {settings.db_name}\n"
|
203
|
+
f"Schema: {settings.schema_name}\n"
|
204
|
+
f"Dialect: {settings.dialect}[/dim]",
|
205
|
+
title="🎉 Success",
|
206
|
+
border_style="green",
|
207
|
+
)
|
208
|
+
)
|
209
|
+
|
210
|
+
|
211
|
+
@app.command()
|
212
|
+
def test(
|
213
|
+
db_host: str = typer.Option(
|
214
|
+
"db", "--db-host", "-h", envvar="DB_HOST", help="Database host"
|
215
|
+
),
|
216
|
+
db_port: int = typer.Option(
|
217
|
+
5432, "--db-port", "-p", envvar="DB_PORT", help="Database port"
|
218
|
+
),
|
219
|
+
db_user: str = typer.Option(
|
220
|
+
"postgres", "--db-user", "-u", envvar="DB_USER", help="Database user"
|
221
|
+
),
|
222
|
+
db_password: str = typer.Option(
|
223
|
+
"password", "--db-password", envvar="DB_PASSWORD", help="Database password"
|
224
|
+
),
|
225
|
+
db_name: str = typer.Option(
|
226
|
+
"omop", "--db-name", "-d", envvar="DB_NAME", help="Database name"
|
227
|
+
),
|
228
|
+
schema_name: str = typer.Option(
|
229
|
+
"public", "--schema-name", envvar="SCHEMA_NAME", help="Database schema name"
|
230
|
+
),
|
231
|
+
dialect: str = typer.Option(
|
232
|
+
"postgresql",
|
233
|
+
"--dialect",
|
234
|
+
envvar="DIALECT",
|
235
|
+
help="Database dialect (postgresql or mssql)",
|
236
|
+
),
|
237
|
+
log_level: str = typer.Option(
|
238
|
+
"INFO", "--log-level", envvar="LOG_LEVEL", help="Logging level"
|
239
|
+
),
|
240
|
+
) -> None:
|
241
|
+
"""
|
242
|
+
Test database connectivity and basic operations.
|
243
|
+
|
244
|
+
This command tests the database connection and performs basic operations
|
245
|
+
without creating tables or loading data.
|
246
|
+
"""
|
247
|
+
settings = _create_settings(
|
248
|
+
db_host=db_host,
|
249
|
+
db_port=db_port,
|
250
|
+
db_user=db_user,
|
251
|
+
db_password=db_password,
|
252
|
+
db_name=db_name,
|
253
|
+
schema_name=schema_name,
|
254
|
+
dialect=dialect,
|
255
|
+
log_level=log_level,
|
256
|
+
)
|
257
|
+
|
258
|
+
try:
|
259
|
+
with console.status(
|
260
|
+
"[bold blue]Testing database connection...", spinner="dots"
|
261
|
+
):
|
262
|
+
db = create_database(settings)
|
263
|
+
|
264
|
+
# Create results table
|
265
|
+
table = Table(title="Database Test Results")
|
266
|
+
table.add_column("Test", style="cyan", no_wrap=True)
|
267
|
+
table.add_column("Status", style="bold")
|
268
|
+
table.add_column("Details", style="dim")
|
269
|
+
|
270
|
+
# Test connection
|
271
|
+
table.add_row(
|
272
|
+
"Database Connection", "✅ PASS", f"Connected to {settings.db_name}"
|
273
|
+
)
|
274
|
+
|
275
|
+
# Test schema
|
276
|
+
if db.schema_exists(settings.schema_name):
|
277
|
+
table.add_row(
|
278
|
+
"Schema Check", "✅ PASS", f"Schema '{settings.schema_name}' exists"
|
279
|
+
)
|
280
|
+
else:
|
281
|
+
table.add_row(
|
282
|
+
"Schema Check",
|
283
|
+
"ℹ️ INFO",
|
284
|
+
f"Schema '{settings.schema_name}' does not exist (normal)",
|
285
|
+
)
|
286
|
+
|
287
|
+
# Test basic operations
|
288
|
+
with console.status("[bold green]Testing basic operations...", spinner="dots"):
|
289
|
+
time.sleep(0.5) # Simulate operation
|
290
|
+
table.add_row("Basic Operations", "✅ PASS", "All operations successful")
|
291
|
+
|
292
|
+
console.print(table)
|
293
|
+
console.print(
|
294
|
+
Panel(
|
295
|
+
"[bold green]✅ Database test completed successfully![/bold green]",
|
296
|
+
title="🎯 Test Results",
|
297
|
+
border_style="green",
|
298
|
+
)
|
299
|
+
)
|
300
|
+
|
301
|
+
except Exception as e:
|
302
|
+
console.print(
|
303
|
+
Panel(
|
304
|
+
f"[bold red]❌ Database test failed[/bold red]\n\n[red]{e}[/red]",
|
305
|
+
title="💥 Test Failed",
|
306
|
+
border_style="red",
|
307
|
+
)
|
308
|
+
)
|
309
|
+
raise typer.Exit(1)
|
310
|
+
|
311
|
+
|
312
|
+
@app.command()
|
313
|
+
def create_tables(
|
314
|
+
db_host: str = typer.Option(
|
315
|
+
"db", "--db-host", "-h", envvar="DB_HOST", help="Database host"
|
316
|
+
),
|
317
|
+
db_port: int = typer.Option(
|
318
|
+
5432, "--db-port", "-p", envvar="DB_PORT", help="Database port"
|
319
|
+
),
|
320
|
+
db_user: str = typer.Option(
|
321
|
+
"postgres", "--db-user", "-u", envvar="DB_USER", help="Database user"
|
322
|
+
),
|
323
|
+
db_password: str = typer.Option(
|
324
|
+
"password", "--db-password", envvar="DB_PASSWORD", help="Database password"
|
325
|
+
),
|
326
|
+
db_name: str = typer.Option(
|
327
|
+
"omop", "--db-name", "-d", envvar="DB_NAME", help="Database name"
|
328
|
+
),
|
329
|
+
schema_name: str = typer.Option(
|
330
|
+
"public", "--schema-name", envvar="SCHEMA_NAME", help="Database schema name"
|
331
|
+
),
|
332
|
+
dialect: str = typer.Option(
|
333
|
+
"postgresql",
|
334
|
+
"--dialect",
|
335
|
+
envvar="DIALECT",
|
336
|
+
help="Database dialect (postgresql or mssql)",
|
337
|
+
),
|
338
|
+
log_level: str = typer.Option(
|
339
|
+
"INFO", "--log-level", envvar="LOG_LEVEL", help="Logging level"
|
340
|
+
),
|
341
|
+
) -> None:
|
342
|
+
"""
|
343
|
+
Create only the database tables.
|
344
|
+
|
345
|
+
This command creates the schema (if needed) and the tables,
|
346
|
+
but does not load data or add constraints.
|
347
|
+
"""
|
348
|
+
settings = _create_settings(
|
349
|
+
db_host=db_host,
|
350
|
+
db_port=db_port,
|
351
|
+
db_user=db_user,
|
352
|
+
db_password=db_password,
|
353
|
+
db_name=db_name,
|
354
|
+
schema_name=schema_name,
|
355
|
+
dialect=dialect,
|
356
|
+
log_level=log_level,
|
357
|
+
)
|
358
|
+
|
359
|
+
logger = _setup_logging(settings)
|
360
|
+
db = create_database(settings)
|
361
|
+
|
362
|
+
# Handle schema creation if not using 'public'
|
363
|
+
if settings.schema_name != "public":
|
364
|
+
if db.schema_exists(settings.schema_name):
|
365
|
+
logger.info(f"Schema '{settings.schema_name}' already exists")
|
366
|
+
else:
|
367
|
+
db.create_schema(settings.schema_name)
|
368
|
+
|
369
|
+
# Create tables only
|
370
|
+
db.create_tables()
|
371
|
+
logger.info("✅ Tables created successfully")
|
372
|
+
|
373
|
+
|
374
|
+
@app.command()
|
375
|
+
def load_data(
|
376
|
+
db_host: str = typer.Option(
|
377
|
+
"db", "--db-host", "-h", envvar="DB_HOST", help="Database host"
|
378
|
+
),
|
379
|
+
db_port: int = typer.Option(
|
380
|
+
5432, "--db-port", "-p", envvar="DB_PORT", help="Database port"
|
381
|
+
),
|
382
|
+
db_user: str = typer.Option(
|
383
|
+
"postgres", "--db-user", "-u", envvar="DB_USER", help="Database user"
|
384
|
+
),
|
385
|
+
db_password: str = typer.Option(
|
386
|
+
"password", "--db-password", envvar="DB_PASSWORD", help="Database password"
|
387
|
+
),
|
388
|
+
db_name: str = typer.Option(
|
389
|
+
"omop", "--db-name", "-d", envvar="DB_NAME", help="Database name"
|
390
|
+
),
|
391
|
+
synthetic: bool = typer.Option(
|
392
|
+
False, "--synthetic", envvar="SYNTHETIC", help="Use synthetic data"
|
393
|
+
),
|
394
|
+
synthetic_number: int = typer.Option(
|
395
|
+
100,
|
396
|
+
"--synthetic-number",
|
397
|
+
envvar="SYNTHETIC_NUMBER",
|
398
|
+
help="Number of synthetic records",
|
399
|
+
),
|
400
|
+
data_dir: str = typer.Option(
|
401
|
+
"data", "--data-dir", envvar="DATA_DIR", help="Data directory"
|
402
|
+
),
|
403
|
+
schema_name: str = typer.Option(
|
404
|
+
"public", "--schema-name", envvar="SCHEMA_NAME", help="Database schema name"
|
405
|
+
),
|
406
|
+
dialect: str = typer.Option(
|
407
|
+
"postgresql",
|
408
|
+
"--dialect",
|
409
|
+
envvar="DIALECT",
|
410
|
+
help="Database dialect (postgresql or mssql)",
|
411
|
+
),
|
412
|
+
log_level: str = typer.Option(
|
413
|
+
"INFO", "--log-level", envvar="LOG_LEVEL", help="Logging level"
|
414
|
+
),
|
415
|
+
delimiter: str = typer.Option(
|
416
|
+
"\t", "--delimiter", envvar="DELIMITER", help="CSV delimiter"
|
417
|
+
),
|
418
|
+
) -> None:
|
419
|
+
"""
|
420
|
+
Load data into existing tables.
|
421
|
+
|
422
|
+
This command loads data into tables that must already exist.
|
423
|
+
Use create-tables first if tables don't exist.
|
424
|
+
"""
|
425
|
+
settings = _create_settings(
|
426
|
+
db_host=db_host,
|
427
|
+
db_port=db_port,
|
428
|
+
db_user=db_user,
|
429
|
+
db_password=db_password,
|
430
|
+
db_name=db_name,
|
431
|
+
synthetic=synthetic,
|
432
|
+
synthetic_number=synthetic_number,
|
433
|
+
data_dir=data_dir,
|
434
|
+
schema_name=schema_name,
|
435
|
+
dialect=dialect,
|
436
|
+
log_level=log_level,
|
437
|
+
delimiter=delimiter,
|
438
|
+
)
|
439
|
+
|
440
|
+
db = create_database(settings)
|
441
|
+
|
442
|
+
# Load data with progress
|
443
|
+
with Progress(
|
444
|
+
SpinnerColumn(),
|
445
|
+
TextColumn("[progress.description]{task.description}"),
|
446
|
+
BarColumn(),
|
447
|
+
TaskProgressColumn(),
|
448
|
+
console=console,
|
449
|
+
) as progress:
|
450
|
+
task = progress.add_task("[yellow]Loading data...", total=1)
|
451
|
+
db.load_data()
|
452
|
+
progress.update(task, completed=1)
|
453
|
+
|
454
|
+
console.print(
|
455
|
+
Panel(
|
456
|
+
"[bold green]✅ Data loaded successfully![/bold green]",
|
457
|
+
title="📊 Data Loading Complete",
|
458
|
+
border_style="green",
|
459
|
+
)
|
460
|
+
)
|
461
|
+
|
462
|
+
|
463
|
+
@app.command()
|
464
|
+
def add_constraints(
|
465
|
+
db_host: str = typer.Option(
|
466
|
+
"db", "--db-host", "-h", envvar="DB_HOST", help="Database host"
|
467
|
+
),
|
468
|
+
db_port: int = typer.Option(
|
469
|
+
5432, "--db-port", "-p", envvar="DB_PORT", help="Database port"
|
470
|
+
),
|
471
|
+
db_user: str = typer.Option(
|
472
|
+
"postgres", "--db-user", "-u", envvar="DB_USER", help="Database user"
|
473
|
+
),
|
474
|
+
db_password: str = typer.Option(
|
475
|
+
"password", "--db-password", envvar="DB_PASSWORD", help="Database password"
|
476
|
+
),
|
477
|
+
db_name: str = typer.Option(
|
478
|
+
"omop", "--db-name", "-d", envvar="DB_NAME", help="Database name"
|
479
|
+
),
|
480
|
+
schema_name: str = typer.Option(
|
481
|
+
"public", "--schema-name", envvar="SCHEMA_NAME", help="Database schema name"
|
482
|
+
),
|
483
|
+
dialect: str = typer.Option(
|
484
|
+
"postgresql",
|
485
|
+
"--dialect",
|
486
|
+
envvar="DIALECT",
|
487
|
+
help="Database dialect (postgresql or mssql)",
|
488
|
+
),
|
489
|
+
log_level: str = typer.Option(
|
490
|
+
"INFO", "--log-level", envvar="LOG_LEVEL", help="Logging level"
|
491
|
+
),
|
492
|
+
) -> None:
|
493
|
+
"""
|
494
|
+
Add all constraints (primary keys, foreign keys, and indices).
|
495
|
+
|
496
|
+
This command adds all types of constraints to existing tables.
|
497
|
+
Tables must exist and should have data loaded.
|
498
|
+
"""
|
499
|
+
settings = _create_settings(
|
500
|
+
db_host=db_host,
|
501
|
+
db_port=db_port,
|
502
|
+
db_user=db_user,
|
503
|
+
db_password=db_password,
|
504
|
+
db_name=db_name,
|
505
|
+
schema_name=schema_name,
|
506
|
+
dialect=dialect,
|
507
|
+
log_level=log_level,
|
508
|
+
)
|
509
|
+
|
510
|
+
db = create_database(settings)
|
511
|
+
|
512
|
+
# Add all constraints with progress
|
513
|
+
with Progress(
|
514
|
+
SpinnerColumn(),
|
515
|
+
TextColumn("[progress.description]{task.description}"),
|
516
|
+
BarColumn(),
|
517
|
+
TaskProgressColumn(),
|
518
|
+
console=console,
|
519
|
+
) as progress:
|
520
|
+
task = progress.add_task("[green]Adding constraints...", total=3)
|
521
|
+
|
522
|
+
# Primary keys
|
523
|
+
progress.update(task, description="[cyan]Adding primary keys...")
|
524
|
+
db.add_primary_keys()
|
525
|
+
progress.advance(task)
|
526
|
+
|
527
|
+
# Foreign keys
|
528
|
+
progress.update(task, description="[cyan]Adding foreign key constraints...")
|
529
|
+
db.add_constraints()
|
530
|
+
progress.advance(task)
|
531
|
+
|
532
|
+
# Indices
|
533
|
+
progress.update(task, description="[cyan]Adding indices...")
|
534
|
+
db.add_indices()
|
535
|
+
progress.advance(task)
|
536
|
+
|
537
|
+
console.print(
|
538
|
+
Panel(
|
539
|
+
"[bold green]✅ All constraints added successfully![/bold green]\n\n"
|
540
|
+
"[dim]• Primary keys\n"
|
541
|
+
"• Foreign key constraints\n"
|
542
|
+
"• Indices[/dim]",
|
543
|
+
title="🔗 Constraints Added",
|
544
|
+
border_style="green",
|
545
|
+
)
|
546
|
+
)
|
547
|
+
|
548
|
+
|
549
|
+
@app.command()
|
550
|
+
def add_primary_keys(
|
551
|
+
db_host: str = typer.Option(
|
552
|
+
"db", "--db-host", "-h", envvar="DB_HOST", help="Database host"
|
553
|
+
),
|
554
|
+
db_port: int = typer.Option(
|
555
|
+
5432, "--db-port", "-p", envvar="DB_PORT", help="Database port"
|
556
|
+
),
|
557
|
+
db_user: str = typer.Option(
|
558
|
+
"postgres", "--db-user", "-u", envvar="DB_USER", help="Database user"
|
559
|
+
),
|
560
|
+
db_password: str = typer.Option(
|
561
|
+
"password", "--db-password", envvar="DB_PASSWORD", help="Database password"
|
562
|
+
),
|
563
|
+
db_name: str = typer.Option(
|
564
|
+
"omop", "--db-name", "-d", envvar="DB_NAME", help="Database name"
|
565
|
+
),
|
566
|
+
schema_name: str = typer.Option(
|
567
|
+
"public", "--schema-name", envvar="SCHEMA_NAME", help="Database schema name"
|
568
|
+
),
|
569
|
+
dialect: str = typer.Option(
|
570
|
+
"postgresql",
|
571
|
+
"--dialect",
|
572
|
+
envvar="DIALECT",
|
573
|
+
help="Database dialect (postgresql or mssql)",
|
574
|
+
),
|
575
|
+
log_level: str = typer.Option(
|
576
|
+
"INFO", "--log-level", envvar="LOG_LEVEL", help="Logging level"
|
577
|
+
),
|
578
|
+
) -> None:
|
579
|
+
"""
|
580
|
+
Add only primary keys to existing tables.
|
581
|
+
|
582
|
+
This command adds primary key constraints to existing tables.
|
583
|
+
Tables must exist and should have data loaded.
|
584
|
+
"""
|
585
|
+
settings = _create_settings(
|
586
|
+
db_host=db_host,
|
587
|
+
db_port=db_port,
|
588
|
+
db_user=db_user,
|
589
|
+
db_password=db_password,
|
590
|
+
db_name=db_name,
|
591
|
+
schema_name=schema_name,
|
592
|
+
dialect=dialect,
|
593
|
+
log_level=log_level,
|
594
|
+
)
|
595
|
+
|
596
|
+
logger = _setup_logging(settings)
|
597
|
+
db = create_database(settings)
|
598
|
+
|
599
|
+
# Add primary keys only
|
600
|
+
db.add_primary_keys()
|
601
|
+
logger.info("✅ Primary keys added successfully")
|
602
|
+
|
603
|
+
|
604
|
+
@app.command()
|
605
|
+
def add_foreign_keys(
|
606
|
+
db_host: str = typer.Option(
|
607
|
+
"db", "--db-host", "-h", envvar="DB_HOST", help="Database host"
|
608
|
+
),
|
609
|
+
db_port: int = typer.Option(
|
610
|
+
5432, "--db-port", "-p", envvar="DB_PORT", help="Database port"
|
611
|
+
),
|
612
|
+
db_user: str = typer.Option(
|
613
|
+
"postgres", "--db-user", "-u", envvar="DB_USER", help="Database user"
|
614
|
+
),
|
615
|
+
db_password: str = typer.Option(
|
616
|
+
"password", "--db-password", envvar="DB_PASSWORD", help="Database password"
|
617
|
+
),
|
618
|
+
db_name: str = typer.Option(
|
619
|
+
"omop", "--db-name", "-d", envvar="DB_NAME", help="Database name"
|
620
|
+
),
|
621
|
+
schema_name: str = typer.Option(
|
622
|
+
"public", "--schema-name", envvar="SCHEMA_NAME", help="Database schema name"
|
623
|
+
),
|
624
|
+
dialect: str = typer.Option(
|
625
|
+
"postgresql",
|
626
|
+
"--dialect",
|
627
|
+
envvar="DIALECT",
|
628
|
+
help="Database dialect (postgresql or mssql)",
|
629
|
+
),
|
630
|
+
log_level: str = typer.Option(
|
631
|
+
"INFO", "--log-level", envvar="LOG_LEVEL", help="Logging level"
|
632
|
+
),
|
633
|
+
) -> None:
|
634
|
+
"""
|
635
|
+
Add only foreign key constraints to existing tables.
|
636
|
+
|
637
|
+
This command adds foreign key constraints to existing tables.
|
638
|
+
Tables must exist and should have data loaded.
|
639
|
+
Primary keys should be added first.
|
640
|
+
"""
|
641
|
+
settings = _create_settings(
|
642
|
+
db_host=db_host,
|
643
|
+
db_port=db_port,
|
644
|
+
db_user=db_user,
|
645
|
+
db_password=db_password,
|
646
|
+
db_name=db_name,
|
647
|
+
schema_name=schema_name,
|
648
|
+
dialect=dialect,
|
649
|
+
log_level=log_level,
|
650
|
+
)
|
651
|
+
|
652
|
+
logger = _setup_logging(settings)
|
653
|
+
db = create_database(settings)
|
654
|
+
|
655
|
+
# Add foreign key constraints only
|
656
|
+
db.add_constraints()
|
657
|
+
logger.info("✅ Foreign key constraints added successfully")
|
658
|
+
|
659
|
+
|
660
|
+
@app.command()
|
661
|
+
def add_indices(
|
662
|
+
db_host: str = typer.Option(
|
663
|
+
"db", "--db-host", "-h", envvar="DB_HOST", help="Database host"
|
664
|
+
),
|
665
|
+
db_port: int = typer.Option(
|
666
|
+
5432, "--db-port", "-p", envvar="DB_PORT", help="Database port"
|
667
|
+
),
|
668
|
+
db_user: str = typer.Option(
|
669
|
+
"postgres", "--db-user", "-u", envvar="DB_USER", help="Database user"
|
670
|
+
),
|
671
|
+
db_password: str = typer.Option(
|
672
|
+
"password", "--db-password", envvar="DB_PASSWORD", help="Database password"
|
673
|
+
),
|
674
|
+
db_name: str = typer.Option(
|
675
|
+
"omop", "--db-name", "-d", envvar="DB_NAME", help="Database name"
|
676
|
+
),
|
677
|
+
schema_name: str = typer.Option(
|
678
|
+
"public", "--schema-name", envvar="SCHEMA_NAME", help="Database schema name"
|
679
|
+
),
|
680
|
+
dialect: str = typer.Option(
|
681
|
+
"postgresql",
|
682
|
+
"--dialect",
|
683
|
+
envvar="DIALECT",
|
684
|
+
help="Database dialect (postgresql or mssql)",
|
685
|
+
),
|
686
|
+
log_level: str = typer.Option(
|
687
|
+
"INFO", "--log-level", envvar="LOG_LEVEL", help="Logging level"
|
688
|
+
),
|
689
|
+
) -> None:
|
690
|
+
"""
|
691
|
+
Add only indices to existing tables.
|
692
|
+
|
693
|
+
This command adds indices to existing tables.
|
694
|
+
Tables must exist and should have data loaded.
|
695
|
+
"""
|
696
|
+
settings = _create_settings(
|
697
|
+
db_host=db_host,
|
698
|
+
db_port=db_port,
|
699
|
+
db_user=db_user,
|
700
|
+
db_password=db_password,
|
701
|
+
db_name=db_name,
|
702
|
+
schema_name=schema_name,
|
703
|
+
dialect=dialect,
|
704
|
+
log_level=log_level,
|
705
|
+
)
|
706
|
+
|
707
|
+
logger = _setup_logging(settings)
|
708
|
+
db = create_database(settings)
|
709
|
+
|
710
|
+
# Add indices only
|
711
|
+
db.add_indices()
|
712
|
+
logger.info("✅ Indices added successfully")
|
713
|
+
|
714
|
+
|
715
|
+
@app.command()
|
716
|
+
def drop(
|
717
|
+
db_host: str = typer.Option(
|
718
|
+
"db", "--db-host", "-h", envvar="DB_HOST", help="Database host"
|
719
|
+
),
|
720
|
+
db_port: int = typer.Option(
|
721
|
+
5432, "--db-port", "-p", envvar="DB_PORT", help="Database port"
|
722
|
+
),
|
723
|
+
db_user: str = typer.Option(
|
724
|
+
"postgres", "--db-user", "-u", envvar="DB_USER", help="Database user"
|
725
|
+
),
|
726
|
+
db_password: str = typer.Option(
|
727
|
+
"password", "--db-password", envvar="DB_PASSWORD", help="Database password"
|
728
|
+
),
|
729
|
+
db_name: str = typer.Option(
|
730
|
+
"omop", "--db-name", "-d", envvar="DB_NAME", help="Database name"
|
731
|
+
),
|
732
|
+
schema_name: str = typer.Option(
|
733
|
+
"public", "--schema-name", envvar="SCHEMA_NAME", help="Database schema name"
|
734
|
+
),
|
735
|
+
dialect: str = typer.Option(
|
736
|
+
"postgresql",
|
737
|
+
"--dialect",
|
738
|
+
envvar="DIALECT",
|
739
|
+
help="Database dialect (postgresql or mssql)",
|
740
|
+
),
|
741
|
+
log_level: str = typer.Option(
|
742
|
+
"INFO", "--log-level", envvar="LOG_LEVEL", help="Logging level"
|
743
|
+
),
|
744
|
+
tables_only: bool = typer.Option(
|
745
|
+
False, "--tables-only", help="Drop only tables, not the schema"
|
746
|
+
),
|
747
|
+
schema_only: bool = typer.Option(
|
748
|
+
False, "--schema-only", help="Drop only the schema (and all its contents)"
|
749
|
+
),
|
750
|
+
confirm: bool = typer.Option(False, "--confirm", help="Skip confirmation prompt"),
|
751
|
+
) -> None:
|
752
|
+
"""
|
753
|
+
Drop tables and/or schema from the database.
|
754
|
+
|
755
|
+
This command can drop tables, schema, or everything.
|
756
|
+
Use with caution as this will permanently delete data.
|
757
|
+
"""
|
758
|
+
if not confirm:
|
759
|
+
# Create a warning panel
|
760
|
+
warning_text = ""
|
761
|
+
if tables_only:
|
762
|
+
warning_text = f"[bold red]⚠️ WARNING[/bold red]\n\nThis will drop [bold]ALL TABLES[/bold] in schema '{schema_name}'.\n\n[red]This action cannot be undone![/red]"
|
763
|
+
elif schema_only:
|
764
|
+
warning_text = f"[bold red]⚠️ WARNING[/bold red]\n\nThis will drop [bold]SCHEMA '{schema_name}'[/bold] and [bold]ALL ITS CONTENTS[/bold].\n\n[red]This action cannot be undone![/red]"
|
765
|
+
else:
|
766
|
+
warning_text = f"[bold red]⚠️ WARNING[/bold red]\n\nThis will drop [bold]ALL TABLES[/bold] and [bold]SCHEMA '{schema_name}'[/bold].\n\n[red]This action cannot be undone![/red]"
|
767
|
+
|
768
|
+
console.print(
|
769
|
+
Panel(warning_text, title="🗑️ Drop Operation", border_style="red")
|
770
|
+
)
|
771
|
+
|
772
|
+
if not Confirm.ask("Are you sure you want to continue?", default=False):
|
773
|
+
console.print("[yellow]Operation cancelled.[/yellow]")
|
774
|
+
raise typer.Exit()
|
775
|
+
|
776
|
+
settings = _create_settings(
|
777
|
+
db_host=db_host,
|
778
|
+
db_port=db_port,
|
779
|
+
db_user=db_user,
|
780
|
+
db_password=db_password,
|
781
|
+
db_name=db_name,
|
782
|
+
schema_name=schema_name,
|
783
|
+
dialect=dialect,
|
784
|
+
log_level=log_level,
|
785
|
+
)
|
786
|
+
|
787
|
+
db = create_database(settings)
|
788
|
+
|
789
|
+
try:
|
790
|
+
with console.status("[bold red]Dropping database objects...", spinner="dots"):
|
791
|
+
if tables_only:
|
792
|
+
db.drop_tables()
|
793
|
+
console.print(
|
794
|
+
Panel(
|
795
|
+
f"[bold green]✅ All tables in schema '{schema_name}' dropped successfully![/bold green]",
|
796
|
+
title="🗑️ Tables Dropped",
|
797
|
+
border_style="green",
|
798
|
+
)
|
799
|
+
)
|
800
|
+
elif schema_only:
|
801
|
+
if schema_name == "public":
|
802
|
+
console.print(
|
803
|
+
Panel(
|
804
|
+
"[yellow]⚠️ Cannot drop 'public' schema, dropping tables instead[/yellow]",
|
805
|
+
title="⚠️ Schema Protection",
|
806
|
+
border_style="yellow",
|
807
|
+
)
|
808
|
+
)
|
809
|
+
db.drop_tables()
|
810
|
+
console.print(
|
811
|
+
Panel(
|
812
|
+
"[bold green]✅ All tables dropped successfully![/bold green]",
|
813
|
+
title="🗑️ Tables Dropped",
|
814
|
+
border_style="green",
|
815
|
+
)
|
816
|
+
)
|
817
|
+
else:
|
818
|
+
db.drop_schema(schema_name)
|
819
|
+
console.print(
|
820
|
+
Panel(
|
821
|
+
f"[bold green]✅ Schema '{schema_name}' dropped successfully![/bold green]",
|
822
|
+
title="🗑️ Schema Dropped",
|
823
|
+
border_style="green",
|
824
|
+
)
|
825
|
+
)
|
826
|
+
else:
|
827
|
+
db.drop_all(schema_name)
|
828
|
+
console.print(
|
829
|
+
Panel(
|
830
|
+
f"[bold green]✅ Database completely dropped![/bold green]\n\n[dim]Schema: {schema_name}\nDatabase: {settings.db_name}[/dim]",
|
831
|
+
title="🗑️ Database Dropped",
|
832
|
+
border_style="green",
|
833
|
+
)
|
834
|
+
)
|
835
|
+
|
836
|
+
except Exception as e:
|
837
|
+
console.print(
|
838
|
+
Panel(
|
839
|
+
f"[bold red]❌ Drop operation failed[/bold red]\n\n[red]{e}[/red]",
|
840
|
+
title="💥 Drop Failed",
|
841
|
+
border_style="red",
|
842
|
+
)
|
843
|
+
)
|
844
|
+
raise typer.Exit(1)
|
845
|
+
|
846
|
+
|
847
|
+
@app.command()
|
848
|
+
def help_commands() -> None:
|
849
|
+
"""
|
850
|
+
Show detailed help for all available commands.
|
851
|
+
"""
|
852
|
+
table = Table(
|
853
|
+
title="OMOP Lite Commands", show_header=True, header_style="bold magenta"
|
854
|
+
)
|
855
|
+
table.add_column("Command", style="cyan", no_wrap=True)
|
856
|
+
table.add_column("Description", style="white")
|
857
|
+
table.add_column("Use Case", style="dim")
|
858
|
+
|
859
|
+
table.add_row(
|
860
|
+
"[default]",
|
861
|
+
"Create complete OMOP database (tables + data + constraints)",
|
862
|
+
"Quick start, development, Docker",
|
863
|
+
)
|
864
|
+
table.add_row(
|
865
|
+
"test",
|
866
|
+
"Test database connectivity and basic operations",
|
867
|
+
"Verify connection, troubleshoot",
|
868
|
+
)
|
869
|
+
table.add_row(
|
870
|
+
"create-tables",
|
871
|
+
"Create only the database tables",
|
872
|
+
"Step-by-step setup, custom workflows",
|
873
|
+
)
|
874
|
+
table.add_row(
|
875
|
+
"load-data", "Load data into existing tables", "Reload data, update datasets"
|
876
|
+
)
|
877
|
+
table.add_row(
|
878
|
+
"add-constraints",
|
879
|
+
"Add all constraints (primary keys, foreign keys, indices)",
|
880
|
+
"Complete constraint setup",
|
881
|
+
)
|
882
|
+
table.add_row(
|
883
|
+
"add-primary-keys",
|
884
|
+
"Add only primary key constraints",
|
885
|
+
"Granular constraint control",
|
886
|
+
)
|
887
|
+
table.add_row(
|
888
|
+
"add-foreign-keys",
|
889
|
+
"Add only foreign key constraints",
|
890
|
+
"Granular constraint control",
|
891
|
+
)
|
892
|
+
table.add_row("add-indices", "Add only indices", "Granular constraint control")
|
893
|
+
table.add_row("drop", "Drop tables and/or schema", "Cleanup, reset database")
|
894
|
+
table.add_row(
|
895
|
+
"help-commands", "Show this help table", "Discover available commands"
|
896
|
+
)
|
897
|
+
|
898
|
+
console.print(table)
|
899
|
+
console.print(
|
900
|
+
Panel(
|
901
|
+
"[bold blue]💡 Tip:[/bold blue] Use [cyan]omop-lite <command> --help[/cyan] for detailed command options\n\n"
|
902
|
+
"[bold green]🚀 Quick Start:[/bold green] [cyan]omop-lite --synthetic[/cyan]",
|
903
|
+
title="ℹ️ Usage Tips",
|
904
|
+
border_style="blue",
|
905
|
+
)
|
906
|
+
)
|
907
|
+
|
908
|
+
|
909
|
+
def main_cli():
|
910
|
+
"""Entry point for the CLI."""
|
911
|
+
app()
|
912
|
+
|
913
|
+
|
914
|
+
if __name__ == "__main__":
|
915
|
+
main_cli()
|
omop_lite/db/__init__.py
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
from .base import Database
|
2
2
|
from .postgres import PostgresDatabase
|
3
3
|
from .sqlserver import SQLServerDatabase
|
4
|
-
from omop_lite.settings import
|
4
|
+
from omop_lite.settings import Settings
|
5
5
|
|
6
6
|
|
7
|
-
def create_database() -> Database:
|
7
|
+
def create_database(settings: Settings) -> Database:
|
8
8
|
"""Factory function to create the appropriate database instance."""
|
9
9
|
if settings.dialect == "postgresql":
|
10
|
-
return PostgresDatabase()
|
10
|
+
return PostgresDatabase(settings)
|
11
11
|
elif settings.dialect == "mssql":
|
12
|
-
return SQLServerDatabase()
|
12
|
+
return SQLServerDatabase(settings)
|
13
13
|
else:
|
14
14
|
raise ValueError(f"Unsupported dialect: {settings.dialect}")
|
15
15
|
|
omop_lite/db/base.py
CHANGED
@@ -5,7 +5,8 @@ from typing import Union, Optional
|
|
5
5
|
import logging
|
6
6
|
from importlib.resources import files
|
7
7
|
from importlib.abc import Traversable
|
8
|
-
from omop_lite.settings import
|
8
|
+
from omop_lite.settings import Settings
|
9
|
+
from sqlalchemy.sql import text
|
9
10
|
|
10
11
|
logger = logging.getLogger(__name__)
|
11
12
|
|
@@ -13,7 +14,8 @@ logger = logging.getLogger(__name__)
|
|
13
14
|
class Database(ABC):
|
14
15
|
"""Abstract base class for database operations"""
|
15
16
|
|
16
|
-
def __init__(self) -> None:
|
17
|
+
def __init__(self, settings: Settings) -> None:
|
18
|
+
self.settings = settings
|
17
19
|
self.engine: Optional[Engine] = None
|
18
20
|
self.metadata: Optional[MetaData] = None
|
19
21
|
self.file_path: Optional[Union[Path, Traversable]] = None
|
@@ -42,6 +44,11 @@ class Database(ABC):
|
|
42
44
|
"VOCABULARY",
|
43
45
|
]
|
44
46
|
|
47
|
+
@property
|
48
|
+
def dialect(self) -> str:
|
49
|
+
"""Get the database dialect."""
|
50
|
+
return self.settings.dialect
|
51
|
+
|
45
52
|
@abstractmethod
|
46
53
|
def create_schema(self, schema_name: str) -> None:
|
47
54
|
"""Create a new schema."""
|
@@ -75,15 +82,61 @@ class Database(ABC):
|
|
75
82
|
self._execute_sql_file(self.file_path.joinpath("ddl.sql"))
|
76
83
|
self.refresh_metadata()
|
77
84
|
|
78
|
-
def
|
79
|
-
"""Add
|
80
|
-
|
81
|
-
Executes the sql files for the given data directory.
|
82
|
-
"""
|
85
|
+
def add_primary_keys(self) -> None:
|
86
|
+
"""Add primary keys to the tables in the database."""
|
83
87
|
self._execute_sql_file(self.file_path.joinpath("primary_keys.sql"))
|
88
|
+
|
89
|
+
def add_constraints(self) -> None:
|
90
|
+
"""Add constraints to the tables in the database."""
|
84
91
|
self._execute_sql_file(self.file_path.joinpath("constraints.sql"))
|
92
|
+
|
93
|
+
def add_indices(self) -> None:
|
94
|
+
"""Add indices to the tables in the database."""
|
85
95
|
self._execute_sql_file(self.file_path.joinpath("indices.sql"))
|
86
96
|
|
97
|
+
def add_all_constraints(self) -> None:
|
98
|
+
"""Add all constraints, primary keys, and indices to the tables in the database.
|
99
|
+
|
100
|
+
This is a convenience method that calls all three constraint methods.
|
101
|
+
"""
|
102
|
+
self.add_primary_keys()
|
103
|
+
self.add_constraints()
|
104
|
+
self.add_indices()
|
105
|
+
|
106
|
+
def drop_tables(self) -> None:
|
107
|
+
"""Drop all tables in the database."""
|
108
|
+
if not self.metadata or not self.engine:
|
109
|
+
raise RuntimeError("Database not properly initialized")
|
110
|
+
|
111
|
+
# Drop all tables in reverse dependency order
|
112
|
+
self.metadata.drop_all(bind=self.engine)
|
113
|
+
logger.info("✅ All tables dropped successfully")
|
114
|
+
|
115
|
+
def drop_schema(self, schema_name: str) -> None:
|
116
|
+
"""Drop a schema and all its contents."""
|
117
|
+
if not self.engine:
|
118
|
+
raise RuntimeError("Database engine not initialized")
|
119
|
+
|
120
|
+
with self.engine.connect() as connection:
|
121
|
+
if self.dialect == "postgresql":
|
122
|
+
connection.execute(
|
123
|
+
text(f'DROP SCHEMA IF EXISTS "{schema_name}" CASCADE')
|
124
|
+
)
|
125
|
+
else: # SQL Server
|
126
|
+
connection.execute(text(f"DROP SCHEMA IF EXISTS [{schema_name}]"))
|
127
|
+
connection.commit()
|
128
|
+
logger.info(f"✅ Schema '{schema_name}' dropped successfully")
|
129
|
+
|
130
|
+
def drop_all(self, schema_name: str) -> None:
|
131
|
+
"""Drop everything: tables and schema.
|
132
|
+
|
133
|
+
This is a convenience method that drops tables first, then the schema.
|
134
|
+
"""
|
135
|
+
self.drop_tables()
|
136
|
+
if schema_name != "public":
|
137
|
+
self.drop_schema(schema_name)
|
138
|
+
logger.info("✅ Database completely dropped")
|
139
|
+
|
87
140
|
def load_data(self) -> None:
|
88
141
|
"""Load data into tables."""
|
89
142
|
data_dir = self._get_data_dir()
|
@@ -111,15 +164,15 @@ class Database(ABC):
|
|
111
164
|
Common implementation for all databases.
|
112
165
|
"""
|
113
166
|
|
114
|
-
if settings.synthetic:
|
115
|
-
if settings.synthetic_number == 1000:
|
167
|
+
if self.settings.synthetic:
|
168
|
+
if self.settings.synthetic_number == 1000:
|
116
169
|
return files("omop_lite.synthetic.1000")
|
117
170
|
return files("omop_lite.synthetic.100")
|
118
|
-
data_dir = Path(settings.data_dir)
|
171
|
+
data_dir = Path(self.settings.data_dir)
|
119
172
|
if not data_dir.exists():
|
120
173
|
raise FileNotFoundError(f"Data directory {data_dir} does not exist")
|
121
174
|
return data_dir
|
122
|
-
|
175
|
+
|
123
176
|
def _get_delimiter(self) -> str:
|
124
177
|
"""
|
125
178
|
Return the delimiter based on the dialect.
|
@@ -131,22 +184,22 @@ class Database(ABC):
|
|
131
184
|
|
132
185
|
This is used to determine the delimiter for the COPY command.
|
133
186
|
"""
|
134
|
-
if settings.synthetic:
|
135
|
-
if settings.synthetic_number == 1000:
|
187
|
+
if self.settings.synthetic:
|
188
|
+
if self.settings.synthetic_number == 1000:
|
136
189
|
return ","
|
137
|
-
return settings.delimiter
|
190
|
+
return self.settings.delimiter
|
138
191
|
else:
|
139
|
-
return settings.delimiter
|
140
|
-
|
192
|
+
return self.settings.delimiter
|
193
|
+
|
141
194
|
def _get_quote(self) -> str:
|
142
195
|
"""
|
143
196
|
Return the quote based on the dialect.
|
144
197
|
Common implementation for all databases.
|
145
198
|
"""
|
146
|
-
if settings.synthetic:
|
147
|
-
if settings.synthetic_number == 1000:
|
199
|
+
if self.settings.synthetic:
|
200
|
+
if self.settings.synthetic_number == 1000:
|
148
201
|
return '"'
|
149
|
-
return
|
202
|
+
return "\b"
|
150
203
|
|
151
204
|
def _execute_sql_file(self, file_path: Union[str, Traversable]) -> None:
|
152
205
|
"""
|
@@ -157,7 +210,7 @@ class Database(ABC):
|
|
157
210
|
file_path = str(file_path)
|
158
211
|
|
159
212
|
with open(file_path, "r") as f:
|
160
|
-
sql = f.read().replace("@cdmDatabaseSchema", settings.schema_name)
|
213
|
+
sql = f.read().replace("@cdmDatabaseSchema", self.settings.schema_name)
|
161
214
|
|
162
215
|
if not self.engine:
|
163
216
|
raise RuntimeError("Database engine not initialized")
|
omop_lite/db/postgres.py
CHANGED
@@ -2,7 +2,7 @@ from sqlalchemy import create_engine, MetaData, text
|
|
2
2
|
from importlib.resources import files
|
3
3
|
import logging
|
4
4
|
from .base import Database
|
5
|
-
from omop_lite.settings import
|
5
|
+
from omop_lite.settings import Settings
|
6
6
|
from typing import Union
|
7
7
|
from pathlib import Path
|
8
8
|
from importlib.abc import Traversable
|
@@ -11,8 +11,8 @@ logger = logging.getLogger(__name__)
|
|
11
11
|
|
12
12
|
|
13
13
|
class PostgresDatabase(Database):
|
14
|
-
def __init__(self) -> None:
|
15
|
-
super().__init__()
|
14
|
+
def __init__(self, settings: Settings) -> None:
|
15
|
+
super().__init__(settings)
|
16
16
|
self.db_url = f"postgresql+psycopg2://{settings.db_user}:{settings.db_password}@{settings.db_host}:{settings.db_port}/{settings.db_name}"
|
17
17
|
self.engine = create_engine(self.db_url)
|
18
18
|
self.metadata = MetaData(schema=settings.schema_name)
|
@@ -30,7 +30,7 @@ class PostgresDatabase(Database):
|
|
30
30
|
def add_constraints(self) -> None:
|
31
31
|
"""
|
32
32
|
Add primary keys, constraints, and indices.
|
33
|
-
|
33
|
+
|
34
34
|
Override to add full-text search.
|
35
35
|
"""
|
36
36
|
super().add_constraints()
|
@@ -41,7 +41,7 @@ class PostgresDatabase(Database):
|
|
41
41
|
if not self.engine:
|
42
42
|
raise RuntimeError("Database engine not initialized")
|
43
43
|
|
44
|
-
if not settings.fts_create:
|
44
|
+
if not self.settings.fts_create:
|
45
45
|
logger.info("Full-text search creation disabled")
|
46
46
|
return
|
47
47
|
|
@@ -60,7 +60,7 @@ class PostgresDatabase(Database):
|
|
60
60
|
def _bulk_load(self, table_name: str, file_path: Union[Path, Traversable]) -> None:
|
61
61
|
if not self.engine:
|
62
62
|
raise RuntimeError("Database engine not initialized")
|
63
|
-
|
63
|
+
|
64
64
|
delimiter = self._get_delimiter()
|
65
65
|
quote = self._get_quote()
|
66
66
|
|
@@ -71,7 +71,7 @@ class PostgresDatabase(Database):
|
|
71
71
|
try:
|
72
72
|
with open(str(file_path), "r") as f:
|
73
73
|
cursor.copy_expert(
|
74
|
-
f"COPY {settings.schema_name}.{table_name} FROM STDIN WITH (FORMAT csv, DELIMITER E'{delimiter}', NULL '', QUOTE E'{quote}', HEADER, ENCODING 'UTF8')",
|
74
|
+
f"COPY {self.settings.schema_name}.{table_name} FROM STDIN WITH (FORMAT csv, DELIMITER E'{delimiter}', NULL '', QUOTE E'{quote}', HEADER, ENCODING 'UTF8')",
|
75
75
|
f,
|
76
76
|
)
|
77
77
|
connection.commit()
|
omop_lite/db/sqlserver.py
CHANGED
@@ -3,7 +3,7 @@ from sqlalchemy import create_engine, MetaData, text
|
|
3
3
|
from importlib.resources import files
|
4
4
|
import logging
|
5
5
|
from .base import Database
|
6
|
-
from omop_lite.settings import
|
6
|
+
from omop_lite.settings import Settings
|
7
7
|
from typing import Union
|
8
8
|
from pathlib import Path
|
9
9
|
from importlib.abc import Traversable
|
@@ -12,8 +12,8 @@ logger = logging.getLogger(__name__)
|
|
12
12
|
|
13
13
|
|
14
14
|
class SQLServerDatabase(Database):
|
15
|
-
def __init__(self) -> None:
|
16
|
-
super().__init__()
|
15
|
+
def __init__(self, settings: Settings) -> None:
|
16
|
+
super().__init__(settings)
|
17
17
|
self.db_url = f"mssql+pyodbc://{settings.db_user}:{settings.db_password}@{settings.db_host}:{settings.db_port}/{settings.db_name}?driver=ODBC+Driver+18+for+SQL+Server&TrustServerCertificate=yes"
|
18
18
|
self.engine = create_engine(self.db_url)
|
19
19
|
self.metadata = MetaData(schema=settings.schema_name)
|
@@ -40,7 +40,7 @@ class SQLServerDatabase(Database):
|
|
40
40
|
|
41
41
|
columns = ", ".join(f"[{col}]" for col in headers)
|
42
42
|
placeholders = ", ".join(["?" for _ in headers])
|
43
|
-
insert_sql = f"INSERT INTO {settings.schema_name}.[{table_name}] ({columns}) VALUES ({placeholders})"
|
43
|
+
insert_sql = f"INSERT INTO {self.settings.schema_name}.[{table_name}] ({columns}) VALUES ({placeholders})"
|
44
44
|
|
45
45
|
conn = self.engine.raw_connection()
|
46
46
|
try:
|
@@ -51,8 +51,10 @@ class SQLServerDatabase(Database):
|
|
51
51
|
row += [None] * (len(headers) - len(row))
|
52
52
|
logger.info(f"Row {line_no} padded: {row}")
|
53
53
|
elif len(row) > len(headers):
|
54
|
-
logger.info(
|
55
|
-
|
54
|
+
logger.info(
|
55
|
+
f"Row {line_no} trimmed: too many values ({len(row)}), expected {len(headers)} – trimming."
|
56
|
+
)
|
57
|
+
row = row[: len(headers)]
|
56
58
|
|
57
59
|
cursor.execute(insert_sql, row)
|
58
60
|
conn.commit()
|
omop_lite/settings.py
CHANGED
@@ -1,23 +1,30 @@
|
|
1
1
|
from pydantic_settings import BaseSettings
|
2
2
|
from typing import Literal
|
3
|
+
from pydantic import Field
|
3
4
|
|
4
5
|
|
5
6
|
class Settings(BaseSettings):
|
6
7
|
"""Settings for OMOP Lite."""
|
7
8
|
|
8
|
-
db_host: str = "db"
|
9
|
-
db_port: int = 5432
|
10
|
-
db_user: str = "postgres"
|
11
|
-
db_password: str = "password"
|
12
|
-
db_name: str = "omop"
|
13
|
-
synthetic: bool = False
|
14
|
-
synthetic_number: int =
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
9
|
+
db_host: str = Field(default="db", description="Database host")
|
10
|
+
db_port: int = Field(default=5432, description="Database port")
|
11
|
+
db_user: str = Field(default="postgres", description="Database user")
|
12
|
+
db_password: str = Field(default="password", description="Database password")
|
13
|
+
db_name: str = Field(default="omop", description="Database name")
|
14
|
+
synthetic: bool = Field(default=False, description="Use synthetic data")
|
15
|
+
synthetic_number: int = Field(
|
16
|
+
default=100, description="Number of synthetic records"
|
17
|
+
)
|
18
|
+
data_dir: str = Field(default="data", description="Data directory")
|
19
|
+
schema_name: str = Field(default="public", description="Database schema name")
|
20
|
+
dialect: Literal["postgresql", "mssql"] = Field(
|
21
|
+
default="postgresql", description="Database dialect"
|
22
|
+
)
|
23
|
+
log_level: str = Field(default="INFO", description="Logging level")
|
24
|
+
fts_create: bool = Field(
|
25
|
+
default=False, description="Create full-text search indexes"
|
26
|
+
)
|
27
|
+
delimiter: str = Field(default="\t", description="CSV delimiter")
|
21
28
|
|
22
29
|
class Config:
|
23
30
|
env_file = ".env"
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: omop-lite
|
3
|
-
Version: 0.0
|
3
|
+
Version: 0.1.0
|
4
4
|
Summary: Get an OMOP CDM database running quickly.
|
5
5
|
License-File: LICENSE
|
6
6
|
Requires-Python: >=3.13
|
@@ -8,6 +8,7 @@ Requires-Dist: psycopg2-binary>=2.9.10
|
|
8
8
|
Requires-Dist: pydantic-settings>=2.8.1
|
9
9
|
Requires-Dist: pyodbc>=5.2.0
|
10
10
|
Requires-Dist: sqlalchemy>=2.0.39
|
11
|
+
Requires-Dist: typer>=0.16.0
|
11
12
|
Description-Content-Type: text/markdown
|
12
13
|
|
13
14
|
# omop-lite
|
@@ -106,6 +107,10 @@ Install with custom values:
|
|
106
107
|
helm install omop-lite omop-lite/omop-lite -f values.yaml
|
107
108
|
```
|
108
109
|
|
110
|
+
### CLI
|
111
|
+
|
112
|
+
`uv run omop-lite --help`
|
113
|
+
|
109
114
|
#### Using Your Own Data
|
110
115
|
|
111
116
|
To use your own data with the Helm chart:
|
@@ -1,9 +1,10 @@
|
|
1
|
-
omop_lite/__init__.py,sha256=
|
2
|
-
omop_lite/
|
3
|
-
omop_lite/
|
4
|
-
omop_lite/db/
|
5
|
-
omop_lite/db/
|
6
|
-
omop_lite/db/
|
1
|
+
omop_lite/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
2
|
+
omop_lite/cli.py,sha256=CyJOuE0THMKWE76lvYzVnAjJV6KB54XxOzn8er_AMYE,29566
|
3
|
+
omop_lite/settings.py,sha256=o25IVG-7yNYNCBEaDjr1yKB0cqgo2mEAhxYK_yNGquA,1334
|
4
|
+
omop_lite/db/__init__.py,sha256=muVGkAqc6VCvzegol2dA8Nex8lT1hxbNv6fIB_DMyo4,602
|
5
|
+
omop_lite/db/base.py,sha256=A_VSMES5XBoIxQU7e3sAb-LCIlsPRVO4Q5z8TiLpbw0,8027
|
6
|
+
omop_lite/db/postgres.py,sha256=2A1_1BwTKPwsJMHs90MSDgsO95d1d8C97VRjzGtVK5E,3105
|
7
|
+
omop_lite/db/sqlserver.py,sha256=KGa28DE6e05zWKlbp5bLxGqQzl2hyZx88EzWSOE2nnA,2691
|
7
8
|
omop_lite/scripts/fts.sql,sha256=PMZOImGgF0Dszqx_0BFPXBDf-7xdzt-klMmOG1wwGQ4,173
|
8
9
|
omop_lite/scripts/mssql/constraints.sql,sha256=y_nlWWiyR8EZ_9LtEl4-6qksXplqGT9yUsU_R9w2pZ8,32206
|
9
10
|
omop_lite/scripts/mssql/ddl.sql,sha256=kcU2f0n8XIQDAD2uqXicpAwKwkS3801Vos3b2llgQIM,19522
|
@@ -39,8 +40,8 @@ omop_lite/synthetic/1000/OBSERVATION_PERIOD.csv,sha256=O6YvudsW5AL2sx124hyW1YhP9
|
|
39
40
|
omop_lite/synthetic/1000/PERSON.csv,sha256=ObojT1gFHrpmCnCfjO6G77VBzf-MrBMpRpd8L4wOk_o,113287
|
40
41
|
omop_lite/synthetic/1000/PROCEDURE_OCCURRENCE.csv,sha256=iBGRwi6RGysHYBfjQxmEaUDGDqDtSCIN3ABMDRZKPvE,1555130
|
41
42
|
omop_lite/synthetic/1000/VISIT_OCCURRENCE.csv,sha256=qYiSW_C4En8u5dKr2sKLzG9rdYjaK72XHUnmtDF8OJs,3863798
|
42
|
-
omop_lite-0.0.
|
43
|
-
omop_lite-0.0.
|
44
|
-
omop_lite-0.0.
|
45
|
-
omop_lite-0.0.
|
46
|
-
omop_lite-0.0.
|
43
|
+
omop_lite-0.1.0.dist-info/METADATA,sha256=p4uRjjbPh-KFmpE8QbZawIDmlkjglJuo5oxnbMEpeSo,5436
|
44
|
+
omop_lite-0.1.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
45
|
+
omop_lite-0.1.0.dist-info/entry_points.txt,sha256=7034mTJjTTHudlTC05GHTdaIEDSVnTRdzOEAuD1pO5E,53
|
46
|
+
omop_lite-0.1.0.dist-info/licenses/LICENSE,sha256=muHgK_sUsFJ3OnDh62vGn2QH4sf_SymtlfFqp8tF9n0,1075
|
47
|
+
omop_lite-0.1.0.dist-info/RECORD,,
|
File without changes
|
File without changes
|