raxtik-etl 0.1.0.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- raxtik_etl/__init__.py +6 -0
- raxtik_etl/cli.py +298 -0
- raxtik_etl/config.py +30 -0
- raxtik_etl/distribution/__init__.py +13 -0
- raxtik_etl/distribution/huggingface.py +210 -0
- raxtik_etl/exporters/__init__.py +6 -0
- raxtik_etl/exporters/parquet_exporter.py +77 -0
- raxtik_etl/exporters/sqlite_exporter.py +137 -0
- raxtik_etl/longitudinal/__init__.py +11 -0
- raxtik_etl/longitudinal/merger.py +259 -0
- raxtik_etl/packaging/builder.py +98 -0
- raxtik_etl/reference/__init__.py +24 -0
- raxtik_etl/reference/codes.py +38 -0
- raxtik_etl/reference/dechallenge.py +54 -0
- raxtik_etl/reference/occupations.py +65 -0
- raxtik_etl/reference/outcomes.py +105 -0
- raxtik_etl/reference/rechallenge.py +54 -0
- raxtik_etl/reference/report_sources.py +87 -0
- raxtik_etl/reference/roles.py +99 -0
- raxtik_etl/schema/__init__.py +51 -0
- raxtik_etl/schema/canonical.py +264 -0
- raxtik_etl/schema/registry.py +854 -0
- raxtik_etl/schema/views.py +62 -0
- raxtik_etl/sdk.py +157 -0
- raxtik_etl/sources/__init__.py +17 -0
- raxtik_etl/sources/faers/__init__.py +13 -0
- raxtik_etl/sources/faers/fetcher.py +116 -0
- raxtik_etl/sources/faers/parser.py +108 -0
- raxtik_etl/sources/faers/resources.py +114 -0
- raxtik_etl/sources/faers/source.py +61 -0
- raxtik_etl/sources/protocol.py +49 -0
- raxtik_etl/sources/registry.py +61 -0
- raxtik_etl/transformations/__init__.py +5 -0
- raxtik_etl/transformations/metadata.py +437 -0
- raxtik_etl/transformations/normalization.py +62 -0
- raxtik_etl/transformations/runner.py +92 -0
- raxtik_etl/transformations/sql/01_deduplicate.sql +55 -0
- raxtik_etl/transformations/sql/02_normalize_drugs.sql +20 -0
- raxtik_etl/transformations/sql/03_aggregate_stats.sql +156 -0
- raxtik_etl-0.1.0.dev0.dist-info/METADATA +268 -0
- raxtik_etl-0.1.0.dev0.dist-info/RECORD +44 -0
- raxtik_etl-0.1.0.dev0.dist-info/WHEEL +4 -0
- raxtik_etl-0.1.0.dev0.dist-info/entry_points.txt +2 -0
- raxtik_etl-0.1.0.dev0.dist-info/licenses/LICENSE +661 -0
raxtik_etl/__init__.py
ADDED
raxtik_etl/cli.py
ADDED
|
@@ -0,0 +1,298 @@
|
|
|
1
|
+
"""Typer CLI interface for raxtik-etl pipeline commands."""
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from typing import List, Optional
|
|
5
|
+
|
|
6
|
+
import typer
|
|
7
|
+
from rich.console import Console
|
|
8
|
+
from rich.panel import Panel
|
|
9
|
+
|
|
10
|
+
from raxtik_etl.distribution.huggingface import DEFAULT_HF_REPO
|
|
11
|
+
from raxtik_etl.longitudinal import (
|
|
12
|
+
build_merged_longitudinal_package,
|
|
13
|
+
get_longitudinal_summary_report,
|
|
14
|
+
)
|
|
15
|
+
from raxtik_etl.packaging.builder import PACKAGE_OUTPUT_DIR, build_quarterly_package
|
|
16
|
+
from raxtik_etl.sdk import FAERS
|
|
17
|
+
|
|
18
|
+
app = typer.Typer(
|
|
19
|
+
name="raxtik-etl",
|
|
20
|
+
help="Enterprise-grade FAERS adverse event data platform & packaging CLI.",
|
|
21
|
+
add_completion=False,
|
|
22
|
+
)
|
|
23
|
+
console = Console()
|
|
24
|
+
CACHE_DIR = Path("data/raw/archives_cache")
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def parse_quarter_range(start_quarter: str, end_quarter: str) -> List[str]:
|
|
28
|
+
"""Generates an inclusive list of quarterly releases between start and end (e.g. '2004Q1' to '2006Q4')."""
|
|
29
|
+
start_clean = start_quarter.strip().upper()
|
|
30
|
+
end_clean = end_quarter.strip().upper()
|
|
31
|
+
|
|
32
|
+
start_y, start_q = int(start_clean[:4]), int(start_clean[-1])
|
|
33
|
+
end_y, end_q = int(end_clean[:4]), int(end_clean[-1])
|
|
34
|
+
|
|
35
|
+
quarters: List[str] = []
|
|
36
|
+
y, q = start_y, start_q
|
|
37
|
+
while (y < end_y) or (y == end_y and q <= end_q):
|
|
38
|
+
quarters.append(f"{y}Q{q}")
|
|
39
|
+
q += 1
|
|
40
|
+
if q > 4:
|
|
41
|
+
q = 1
|
|
42
|
+
y += 1
|
|
43
|
+
return quarters
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
@app.command("build-package")
|
|
47
|
+
def build_package_cmd(
|
|
48
|
+
source: str = typer.Option(
|
|
49
|
+
"faers", "--source", "-s", help="Source connector name (e.g. faers)"
|
|
50
|
+
),
|
|
51
|
+
release: str = typer.Option(
|
|
52
|
+
"2004Q1", "--release", "-r", help="Quarter release identifier (e.g. 2004Q1)"
|
|
53
|
+
),
|
|
54
|
+
zip_path: Optional[Path] = typer.Option(
|
|
55
|
+
None, "--zip-path", "-z", help="Path to source archive/zip file"
|
|
56
|
+
),
|
|
57
|
+
output_dir: Optional[Path] = typer.Option(
|
|
58
|
+
None, "--output-dir", "-o", help="Output package directory"
|
|
59
|
+
),
|
|
60
|
+
):
|
|
61
|
+
"""Build a standalone, self-describing Safety DuckDB and Parquet package."""
|
|
62
|
+
release_clean = release.upper()
|
|
63
|
+
archive_file = zip_path or (CACHE_DIR / f"{release_clean.lower()}_ascii.zip")
|
|
64
|
+
|
|
65
|
+
if not archive_file.exists():
|
|
66
|
+
console.print(f"[bold red]Source archive not found at {archive_file}.[/bold red]")
|
|
67
|
+
raise typer.Exit(code=1)
|
|
68
|
+
|
|
69
|
+
res = build_quarterly_package(
|
|
70
|
+
release=release_clean,
|
|
71
|
+
zip_path=archive_file,
|
|
72
|
+
source_name=source,
|
|
73
|
+
output_dir=output_dir,
|
|
74
|
+
)
|
|
75
|
+
console.print(
|
|
76
|
+
Panel(
|
|
77
|
+
f"Release: [bold]{res['release']}[/bold]\nSignature: {res['signature_id']}\nDuckDB: {res['duckdb_path']}\nParquet: {res['parquet_dir']}",
|
|
78
|
+
title="Package Generated Successfully",
|
|
79
|
+
style="green",
|
|
80
|
+
)
|
|
81
|
+
)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
@app.command("build-cohort")
|
|
85
|
+
def build_cohort_cmd(
|
|
86
|
+
start: str = typer.Option("2004Q1", "--start", help="Starting quarter release (e.g. 2004Q1)"),
|
|
87
|
+
end: str = typer.Option("2006Q4", "--end", help="Ending quarter release (e.g. 2006Q4)"),
|
|
88
|
+
source: str = typer.Option("faers", "--source", "-s", help="Source connector name"),
|
|
89
|
+
output_dir: Optional[Path] = typer.Option(
|
|
90
|
+
None, "--output-dir", "-o", help="Output directory for packages"
|
|
91
|
+
),
|
|
92
|
+
):
|
|
93
|
+
"""Batch build standalone packages for every quarter in the specified range."""
|
|
94
|
+
quarters = parse_quarter_range(start, end)
|
|
95
|
+
console.print(
|
|
96
|
+
f"[bold cyan]Building {len(quarters)} quarterly packages ({quarters[0]} to {quarters[-1]})...[/bold cyan]"
|
|
97
|
+
)
|
|
98
|
+
|
|
99
|
+
for q in quarters:
|
|
100
|
+
archive_file = CACHE_DIR / f"{q.lower()}_ascii.zip"
|
|
101
|
+
if not archive_file.exists():
|
|
102
|
+
console.print(f"[bold red]Archive not found for {q}: {archive_file}[/bold red]")
|
|
103
|
+
raise typer.Exit(code=1)
|
|
104
|
+
build_quarterly_package(
|
|
105
|
+
release=q,
|
|
106
|
+
zip_path=archive_file,
|
|
107
|
+
source_name=source,
|
|
108
|
+
output_dir=output_dir,
|
|
109
|
+
)
|
|
110
|
+
|
|
111
|
+
console.print(f"[bold green]✓ Successfully built all {len(quarters)} packages.[/bold green]")
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
@app.command("merge-cohort")
|
|
115
|
+
def merge_cohort_cmd(
|
|
116
|
+
start: str = typer.Option("2004Q1", "--start", help="Starting quarter release (e.g. 2004Q1)"),
|
|
117
|
+
end: str = typer.Option("2006Q4", "--end", help="Ending quarter release (e.g. 2006Q4)"),
|
|
118
|
+
package_dir: Optional[Path] = typer.Option(
|
|
119
|
+
None, "--package-dir", "-p", help="Directory containing quarterly packages"
|
|
120
|
+
),
|
|
121
|
+
output_name: Optional[str] = typer.Option(
|
|
122
|
+
None,
|
|
123
|
+
"--output-name",
|
|
124
|
+
"-n",
|
|
125
|
+
help="Merged output name (defaults to faers_{start}_{end}_merged)",
|
|
126
|
+
),
|
|
127
|
+
output_dir: Optional[Path] = typer.Option(
|
|
128
|
+
None, "--output-dir", "-o", help="Destination folder for merged package"
|
|
129
|
+
),
|
|
130
|
+
):
|
|
131
|
+
"""Merge quarterly packages into a consolidated longitudinal dataset with cross-quarter deduplication."""
|
|
132
|
+
quarters = parse_quarter_range(start, end)
|
|
133
|
+
resolved_output_name = output_name or f"faers_{start}_{end}_merged".lower()
|
|
134
|
+
res = build_merged_longitudinal_package(
|
|
135
|
+
releases=quarters,
|
|
136
|
+
package_dir=package_dir,
|
|
137
|
+
output_name=resolved_output_name,
|
|
138
|
+
output_dir=output_dir,
|
|
139
|
+
)
|
|
140
|
+
console.print(
|
|
141
|
+
Panel(
|
|
142
|
+
f"Merged Database: [bold]{res['output_duckdb']}[/bold]\n"
|
|
143
|
+
f"Merged Parquet: [bold]{res['output_parquet']}[/bold]\n"
|
|
144
|
+
f"Total Submissions: [cyan]{res['metrics'].get('total_raw_submissions')}[/cyan]\n"
|
|
145
|
+
f"Deduplicated Unique Cases: [green]{res['metrics'].get('total_unique_cases')}[/green]\n"
|
|
146
|
+
f"Cross-Quarter Cases: [yellow]{res['metrics'].get('cases_spanning_multiple_quarters')}[/yellow]\n"
|
|
147
|
+
f"Duplicates Removed: [magenta]{res['metrics'].get('deduplicated_duplicate_reports_removed')} ({res['metrics'].get('deduplication_reduction_pct')})[/magenta]",
|
|
148
|
+
title="Longitudinal Cohort Merged & Deduplicated",
|
|
149
|
+
style="green",
|
|
150
|
+
)
|
|
151
|
+
)
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
@app.command("inspect-package")
|
|
155
|
+
def inspect_package_cmd(
|
|
156
|
+
release: str = typer.Option(
|
|
157
|
+
"2004Q1", "--release", "-r", help="Quarter release identifier to inspect"
|
|
158
|
+
),
|
|
159
|
+
package_dir: Optional[Path] = typer.Option(
|
|
160
|
+
None, "--package-dir", "-p", help="Directory containing package files"
|
|
161
|
+
),
|
|
162
|
+
):
|
|
163
|
+
"""Inspect and display metadata and data quality report for a compiled package."""
|
|
164
|
+
try:
|
|
165
|
+
with FAERS(release=release, package_dir=package_dir) as faers:
|
|
166
|
+
faers.print_summary()
|
|
167
|
+
except Exception as e:
|
|
168
|
+
console.print(f"[bold red]Failed to inspect package:[/bold red] {e}")
|
|
169
|
+
raise typer.Exit(code=1)
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
@app.command("inspect-cohort")
|
|
173
|
+
def inspect_cohort_cmd(
|
|
174
|
+
cohort: Optional[str] = typer.Option(
|
|
175
|
+
None,
|
|
176
|
+
"--cohort",
|
|
177
|
+
"-c",
|
|
178
|
+
help="Cohort name or identifier (e.g. 'faers_2004_2006_merged' or '2004_2006')",
|
|
179
|
+
),
|
|
180
|
+
db_path: Optional[Path] = typer.Option(
|
|
181
|
+
None,
|
|
182
|
+
"--db-path",
|
|
183
|
+
"-d",
|
|
184
|
+
help="Direct path to merged DuckDB package file",
|
|
185
|
+
),
|
|
186
|
+
package_dir: Optional[Path] = typer.Option(
|
|
187
|
+
None,
|
|
188
|
+
"--package-dir",
|
|
189
|
+
"-p",
|
|
190
|
+
help="Directory containing packages (defaults to data/packages or RAXTIK_PACKAGE_DIR)",
|
|
191
|
+
),
|
|
192
|
+
):
|
|
193
|
+
"""Inspect and display metrics for a merged longitudinal cohort package."""
|
|
194
|
+
base_dir = package_dir or PACKAGE_OUTPUT_DIR
|
|
195
|
+
target_path: Optional[Path] = None
|
|
196
|
+
|
|
197
|
+
if db_path is not None:
|
|
198
|
+
target_path = db_path
|
|
199
|
+
elif cohort is not None:
|
|
200
|
+
filename = cohort if cohort.endswith(".duckdb") else f"{cohort}.duckdb"
|
|
201
|
+
if not filename.startswith("faers_") and not filename.startswith("/"):
|
|
202
|
+
filename = f"faers_{filename}"
|
|
203
|
+
candidate = base_dir / filename
|
|
204
|
+
if not candidate.exists() and not filename.endswith("_merged.duckdb"):
|
|
205
|
+
merged_candidate = base_dir / filename.replace(".duckdb", "_merged.duckdb")
|
|
206
|
+
if merged_candidate.exists():
|
|
207
|
+
candidate = merged_candidate
|
|
208
|
+
target_path = candidate
|
|
209
|
+
else:
|
|
210
|
+
# Default fallback: check for existing merged package
|
|
211
|
+
default_2004_2006 = base_dir / "faers_2004_2006_merged.duckdb"
|
|
212
|
+
if default_2004_2006.exists():
|
|
213
|
+
target_path = default_2004_2006
|
|
214
|
+
else:
|
|
215
|
+
merged_files = sorted(base_dir.glob("*merged*.duckdb"))
|
|
216
|
+
target_path = merged_files[0] if merged_files else default_2004_2006
|
|
217
|
+
|
|
218
|
+
if not target_path.exists():
|
|
219
|
+
console.print(f"[bold red]Database not found at {target_path}[/bold red]")
|
|
220
|
+
raise typer.Exit(code=1)
|
|
221
|
+
report_table = get_longitudinal_summary_report(target_path)
|
|
222
|
+
console.print(report_table)
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
@app.command("hf-download")
|
|
226
|
+
def hf_download_cmd(
|
|
227
|
+
release: str = typer.Option(
|
|
228
|
+
"2004Q1",
|
|
229
|
+
"--release",
|
|
230
|
+
"-r",
|
|
231
|
+
help="Release identifier (e.g. 2004Q1 or faers_2004_2006_merged)",
|
|
232
|
+
),
|
|
233
|
+
repo_id: str = typer.Option(
|
|
234
|
+
DEFAULT_HF_REPO,
|
|
235
|
+
"--repo-id",
|
|
236
|
+
help="Hugging Face dataset repository (defaults to RAXTIK_HF_REPO)",
|
|
237
|
+
),
|
|
238
|
+
dest_dir: Path = typer.Option(
|
|
239
|
+
Path("data/packages"), "--dest-dir", "-d", help="Destination folder"
|
|
240
|
+
),
|
|
241
|
+
force: bool = typer.Option(False, "--force", "-f", help="Overwrite existing local files"),
|
|
242
|
+
):
|
|
243
|
+
"""Download a compiled FAERS package from Hugging Face Hub (zero auth required)."""
|
|
244
|
+
from raxtik_etl.distribution.huggingface import download_package_from_hf
|
|
245
|
+
|
|
246
|
+
try:
|
|
247
|
+
download_package_from_hf(
|
|
248
|
+
release_or_name=release,
|
|
249
|
+
dest_dir=dest_dir,
|
|
250
|
+
repo_id=repo_id,
|
|
251
|
+
force=force,
|
|
252
|
+
)
|
|
253
|
+
except Exception as e:
|
|
254
|
+
console.print(f"[bold red]Download failed:[/bold red] {e}")
|
|
255
|
+
raise typer.Exit(code=1)
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
@app.command("hf-upload")
|
|
259
|
+
def hf_upload_cmd(
|
|
260
|
+
release: str = typer.Option(
|
|
261
|
+
"2004Q1",
|
|
262
|
+
"--release",
|
|
263
|
+
"-r",
|
|
264
|
+
help="Release identifier (e.g. 2004Q1 or faers_2004_2006_merged)",
|
|
265
|
+
),
|
|
266
|
+
repo_id: str = typer.Option(
|
|
267
|
+
DEFAULT_HF_REPO,
|
|
268
|
+
"--repo-id",
|
|
269
|
+
help="Hugging Face dataset repository (defaults to RAXTIK_HF_REPO)",
|
|
270
|
+
),
|
|
271
|
+
package_dir: Path = typer.Option(
|
|
272
|
+
Path("data/packages"), "--package-dir", "-p", help="Directory containing packages"
|
|
273
|
+
),
|
|
274
|
+
token: Optional[str] = typer.Option(
|
|
275
|
+
None, "--token", "-t", help="Hugging Face write access token (defaults to HF_TOKEN env var)"
|
|
276
|
+
),
|
|
277
|
+
parquet: bool = typer.Option(
|
|
278
|
+
True, "--parquet/--no-parquet", help="Whether to include Parquet shards"
|
|
279
|
+
),
|
|
280
|
+
):
|
|
281
|
+
"""Upload a compiled FAERS package to Hugging Face Hub (requires write token)."""
|
|
282
|
+
from raxtik_etl.distribution.huggingface import upload_package_to_hf
|
|
283
|
+
|
|
284
|
+
try:
|
|
285
|
+
upload_package_to_hf(
|
|
286
|
+
release_or_name=release,
|
|
287
|
+
package_dir=package_dir,
|
|
288
|
+
repo_id=repo_id,
|
|
289
|
+
token=token,
|
|
290
|
+
include_parquet=parquet,
|
|
291
|
+
)
|
|
292
|
+
except Exception as e:
|
|
293
|
+
console.print(f"[bold red]Upload failed:[/bold red] {e}")
|
|
294
|
+
raise typer.Exit(code=1)
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
if __name__ == "__main__":
|
|
298
|
+
app()
|
raxtik_etl/config.py
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
"""Global configuration and directory paths for raxtik-etl."""
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent
|
|
7
|
+
|
|
8
|
+
# Base data directories
|
|
9
|
+
DATA_DIR = Path(os.getenv("RAXTIK_DATA_DIR", PROJECT_ROOT / "data"))
|
|
10
|
+
RAW_DATA_DIR = DATA_DIR / "raw"
|
|
11
|
+
INTERMEDIATE_DATA_DIR = DATA_DIR / "intermediate"
|
|
12
|
+
OUTPUT_DATA_DIR = DATA_DIR / "output"
|
|
13
|
+
|
|
14
|
+
# DuckDB intermediate database
|
|
15
|
+
DEFAULT_DUCKDB_PATH = INTERMEDIATE_DATA_DIR / "faers.duckdb"
|
|
16
|
+
|
|
17
|
+
# Exporter output paths
|
|
18
|
+
DEFAULT_SQLITE_PATH = OUTPUT_DATA_DIR / "faers_seed.sqlite"
|
|
19
|
+
DEFAULT_PARQUET_DIR = OUTPUT_DATA_DIR / "parquet"
|
|
20
|
+
|
|
21
|
+
# FAERS configuration
|
|
22
|
+
FAERS_TABLE_TYPES = ["demo", "drug", "reac", "outc", "indi"]
|
|
23
|
+
FAERS_BASE_URL = "https://fis.fda.gov/content/Exports"
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def ensure_directories():
|
|
27
|
+
"""Ensure that standard data directories exist."""
|
|
28
|
+
RAW_DATA_DIR.mkdir(parents=True, exist_ok=True)
|
|
29
|
+
INTERMEDIATE_DATA_DIR.mkdir(parents=True, exist_ok=True)
|
|
30
|
+
OUTPUT_DATA_DIR.mkdir(parents=True, exist_ok=True)
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
"""Remote distribution subsystem for FAERS packages (Hugging Face Hub integration)."""
|
|
2
|
+
|
|
3
|
+
from raxtik_etl.distribution.huggingface import (
|
|
4
|
+
DEFAULT_HF_REPO,
|
|
5
|
+
download_package_from_hf,
|
|
6
|
+
upload_package_to_hf,
|
|
7
|
+
)
|
|
8
|
+
|
|
9
|
+
__all__ = [
|
|
10
|
+
"DEFAULT_HF_REPO",
|
|
11
|
+
"download_package_from_hf",
|
|
12
|
+
"upload_package_to_hf",
|
|
13
|
+
]
|
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
"""Hugging Face Hub distribution module for downloading and uploading compiled packages.
|
|
2
|
+
|
|
3
|
+
Designed with zero-auth public downloads:
|
|
4
|
+
- Downloads can use direct streaming HTTP via requests (no Hugging Face login required).
|
|
5
|
+
- Uploads require `huggingface_hub` (in dev dependencies) and a valid write token.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import os
|
|
11
|
+
import re
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
from typing import Optional
|
|
14
|
+
|
|
15
|
+
import requests
|
|
16
|
+
from rich.console import Console
|
|
17
|
+
from tqdm import tqdm
|
|
18
|
+
|
|
19
|
+
console = Console()
|
|
20
|
+
|
|
21
|
+
DEFAULT_HF_REPO = os.getenv("RAXTIK_HF_REPO", "raxtik-org/faers-safety-database")
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def download_package_from_hf(
|
|
25
|
+
release_or_name: str,
|
|
26
|
+
dest_dir: Path | str = "data/packages",
|
|
27
|
+
repo_id: str = DEFAULT_HF_REPO,
|
|
28
|
+
force: bool = False,
|
|
29
|
+
) -> Path:
|
|
30
|
+
"""
|
|
31
|
+
Downloads a compiled .duckdb safety package from Hugging Face Hub.
|
|
32
|
+
|
|
33
|
+
Publicly accessible without any authentication token or Hugging Face account.
|
|
34
|
+
|
|
35
|
+
Args:
|
|
36
|
+
release_or_name: e.g. '2004Q1' or 'faers_2004_2006_merged'
|
|
37
|
+
dest_dir: Destination folder (defaults to data/packages)
|
|
38
|
+
repo_id: Hugging Face dataset repository id
|
|
39
|
+
force: Whether to overwrite existing local file
|
|
40
|
+
|
|
41
|
+
Returns:
|
|
42
|
+
Path to downloaded DuckDB package.
|
|
43
|
+
"""
|
|
44
|
+
target_dir = Path(dest_dir)
|
|
45
|
+
target_dir.mkdir(parents=True, exist_ok=True)
|
|
46
|
+
|
|
47
|
+
rel_upper = release_or_name.strip()
|
|
48
|
+
if not rel_upper.startswith("faers_"):
|
|
49
|
+
filename = f"faers_{rel_upper.upper()}.duckdb"
|
|
50
|
+
else:
|
|
51
|
+
filename = f"{rel_upper}.duckdb"
|
|
52
|
+
|
|
53
|
+
match = re.search(r"(\d{4})Q[1-4]", rel_upper)
|
|
54
|
+
subfolder = match.group(1) if match else ("merged" if "merged" in rel_upper.lower() else "")
|
|
55
|
+
rel_path = f"{subfolder}/{filename}" if subfolder else filename
|
|
56
|
+
|
|
57
|
+
dest_file = target_dir / filename
|
|
58
|
+
if dest_file.exists() and not force:
|
|
59
|
+
console.print(f"[bold green]Package already present locally:[/bold green] {dest_file}")
|
|
60
|
+
return dest_file
|
|
61
|
+
|
|
62
|
+
# Try huggingface_hub if available
|
|
63
|
+
try:
|
|
64
|
+
from huggingface_hub import hf_hub_download # type: ignore
|
|
65
|
+
|
|
66
|
+
console.print(
|
|
67
|
+
f"[bold cyan]Downloading {filename} from Hugging Face Hub ({repo_id})...[/bold cyan]"
|
|
68
|
+
)
|
|
69
|
+
try:
|
|
70
|
+
downloaded_path = hf_hub_download(
|
|
71
|
+
repo_id=repo_id,
|
|
72
|
+
filename=rel_path,
|
|
73
|
+
repo_type="dataset",
|
|
74
|
+
local_dir=str(target_dir),
|
|
75
|
+
)
|
|
76
|
+
except Exception:
|
|
77
|
+
# Fall back to root path if not in year subfolder
|
|
78
|
+
downloaded_path = hf_hub_download(
|
|
79
|
+
repo_id=repo_id,
|
|
80
|
+
filename=filename,
|
|
81
|
+
repo_type="dataset",
|
|
82
|
+
local_dir=str(target_dir),
|
|
83
|
+
)
|
|
84
|
+
return Path(downloaded_path)
|
|
85
|
+
except ImportError:
|
|
86
|
+
# Fallback to direct HTTP download via requests (no huggingface_hub needed)
|
|
87
|
+
url = f"https://huggingface.co/datasets/{repo_id}/resolve/main/{rel_path}"
|
|
88
|
+
console.print(
|
|
89
|
+
f"[bold cyan]Downloading {filename} via public direct stream from {url}...[/bold cyan]"
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
response = requests.get(url, stream=True, timeout=120)
|
|
93
|
+
if response.status_code != 200:
|
|
94
|
+
# Try root URL fallback
|
|
95
|
+
fallback_url = f"https://huggingface.co/datasets/{repo_id}/resolve/main/{filename}"
|
|
96
|
+
response = requests.get(fallback_url, stream=True, timeout=120)
|
|
97
|
+
if response.status_code != 200:
|
|
98
|
+
raise RuntimeError(
|
|
99
|
+
f"Failed to download package '{filename}' from Hugging Face Hub at {url} (HTTP {response.status_code})."
|
|
100
|
+
)
|
|
101
|
+
|
|
102
|
+
total_size = int(response.headers.get("content-length", 0))
|
|
103
|
+
with (
|
|
104
|
+
open(dest_file, "wb") as f,
|
|
105
|
+
tqdm(
|
|
106
|
+
desc=filename,
|
|
107
|
+
total=total_size if total_size > 0 else None,
|
|
108
|
+
unit="iB",
|
|
109
|
+
unit_scale=True,
|
|
110
|
+
unit_divisor=1024,
|
|
111
|
+
) as bar,
|
|
112
|
+
):
|
|
113
|
+
for chunk in response.iter_content(chunk_size=1024 * 1024):
|
|
114
|
+
if chunk:
|
|
115
|
+
size = f.write(chunk)
|
|
116
|
+
bar.update(size)
|
|
117
|
+
|
|
118
|
+
console.print(f"[bold green]✓ Download complete:[/bold green] {dest_file}")
|
|
119
|
+
return dest_file
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def upload_package_to_hf(
|
|
123
|
+
release_or_name: str,
|
|
124
|
+
package_dir: Path | str = "data/packages",
|
|
125
|
+
repo_id: str = DEFAULT_HF_REPO,
|
|
126
|
+
token: Optional[str] = None,
|
|
127
|
+
include_parquet: bool = True,
|
|
128
|
+
) -> None:
|
|
129
|
+
"""
|
|
130
|
+
Uploads a compiled .duckdb package and optional Parquet shards to Hugging Face Hub.
|
|
131
|
+
|
|
132
|
+
Organizes releases cleanly into year subfolders (e.g. 2004/faers_2004Q1.duckdb, merged/...).
|
|
133
|
+
|
|
134
|
+
Requires `huggingface_hub` (available via `pip install 'raxtik-etl[dev]'`) and
|
|
135
|
+
a Hugging Face write token (via argument or HF_TOKEN env var).
|
|
136
|
+
|
|
137
|
+
Args:
|
|
138
|
+
release_or_name: e.g. '2004Q1' or 'faers_2004_2006_merged'
|
|
139
|
+
package_dir: Directory containing compiled packages
|
|
140
|
+
repo_id: Destination Hugging Face dataset repository id
|
|
141
|
+
token: Hugging Face user write access token
|
|
142
|
+
include_parquet: Whether to also upload the corresponding .parquet folder
|
|
143
|
+
"""
|
|
144
|
+
try:
|
|
145
|
+
from huggingface_hub import HfApi # type: ignore
|
|
146
|
+
except ImportError:
|
|
147
|
+
raise ImportError(
|
|
148
|
+
"Uploading requires 'huggingface_hub'. Please install it using: pip install 'raxtik-etl[dev]'"
|
|
149
|
+
)
|
|
150
|
+
|
|
151
|
+
auth_token = token or os.environ.get("HF_TOKEN") or os.environ.get("HUGGINGFACE_HUB_TOKEN")
|
|
152
|
+
if not auth_token:
|
|
153
|
+
raise ValueError(
|
|
154
|
+
"Hugging Face write token is required to upload packages. "
|
|
155
|
+
"Pass token=... or set the HF_TOKEN environment variable."
|
|
156
|
+
)
|
|
157
|
+
|
|
158
|
+
pkg_dir = Path(package_dir)
|
|
159
|
+
rel_clean = release_or_name.strip()
|
|
160
|
+
if not rel_clean.startswith("faers_"):
|
|
161
|
+
duckdb_name = f"faers_{rel_clean.upper()}.duckdb"
|
|
162
|
+
parquet_dir_name = f"faers_{rel_clean.upper()}.parquet"
|
|
163
|
+
else:
|
|
164
|
+
duckdb_name = f"{rel_clean}.duckdb"
|
|
165
|
+
parquet_dir_name = f"{rel_clean}.parquet"
|
|
166
|
+
|
|
167
|
+
# Route into year subfolder (e.g. 2004/...) or merged/...
|
|
168
|
+
match = re.search(r"(\d{4})Q[1-4]", rel_clean.upper())
|
|
169
|
+
if match:
|
|
170
|
+
subfolder = match.group(1)
|
|
171
|
+
elif "merged" in rel_clean.lower():
|
|
172
|
+
subfolder = "merged"
|
|
173
|
+
else:
|
|
174
|
+
subfolder = ""
|
|
175
|
+
|
|
176
|
+
path_in_repo_duckdb = f"{subfolder}/{duckdb_name}" if subfolder else duckdb_name
|
|
177
|
+
path_in_repo_parquet = f"{subfolder}/{parquet_dir_name}" if subfolder else parquet_dir_name
|
|
178
|
+
|
|
179
|
+
duckdb_file = pkg_dir / duckdb_name
|
|
180
|
+
parquet_dir = pkg_dir / parquet_dir_name
|
|
181
|
+
|
|
182
|
+
if not duckdb_file.exists():
|
|
183
|
+
raise FileNotFoundError(f"Package database not found at {duckdb_file}")
|
|
184
|
+
|
|
185
|
+
api = HfApi(token=auth_token)
|
|
186
|
+
|
|
187
|
+
# 1. Upload DuckDB file
|
|
188
|
+
console.print(
|
|
189
|
+
f"[bold cyan]Uploading {duckdb_name} -> {path_in_repo_duckdb} ({duckdb_file.stat().st_size / (1024 * 1024):.2f} MB)...[/bold cyan]"
|
|
190
|
+
)
|
|
191
|
+
api.upload_file(
|
|
192
|
+
path_or_fileobj=duckdb_file.as_posix(),
|
|
193
|
+
path_in_repo=path_in_repo_duckdb,
|
|
194
|
+
repo_id=repo_id,
|
|
195
|
+
repo_type="dataset",
|
|
196
|
+
)
|
|
197
|
+
console.print(f"[bold green]✓ Uploaded {path_in_repo_duckdb} successfully.[/bold green]")
|
|
198
|
+
|
|
199
|
+
# 2. Optionally upload Parquet shards
|
|
200
|
+
if include_parquet and parquet_dir.exists():
|
|
201
|
+
console.print(
|
|
202
|
+
f"[bold cyan]Uploading Parquet shards from {parquet_dir_name} -> {path_in_repo_parquet}...[/bold cyan]"
|
|
203
|
+
)
|
|
204
|
+
api.upload_folder(
|
|
205
|
+
folder_path=parquet_dir.as_posix(),
|
|
206
|
+
path_in_repo=path_in_repo_parquet,
|
|
207
|
+
repo_id=repo_id,
|
|
208
|
+
repo_type="dataset",
|
|
209
|
+
)
|
|
210
|
+
console.print(f"[bold green]✓ Uploaded {path_in_repo_parquet} successfully.[/bold green]")
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
"""Parquet Seed Exporter for analytical workloads."""
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from typing import Dict, List, Optional
|
|
5
|
+
|
|
6
|
+
import duckdb
|
|
7
|
+
from rich.console import Console
|
|
8
|
+
|
|
9
|
+
from raxtik_etl.config import DEFAULT_DUCKDB_PATH, DEFAULT_PARQUET_DIR
|
|
10
|
+
|
|
11
|
+
console = Console()
|
|
12
|
+
|
|
13
|
+
SEED_TABLES = [
|
|
14
|
+
"drug_reaction_stats",
|
|
15
|
+
"drug_demographics",
|
|
16
|
+
"drug_indications",
|
|
17
|
+
"drug_routes",
|
|
18
|
+
"drug_outcomes",
|
|
19
|
+
]
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def export_parquet(
|
|
23
|
+
db_path: Optional[Path | str] = None,
|
|
24
|
+
parquet_dir: Optional[Path | str] = None,
|
|
25
|
+
dataset_schema: str = "faers_raw",
|
|
26
|
+
tables: Optional[List[str]] = None,
|
|
27
|
+
verbose: bool = True,
|
|
28
|
+
) -> Path:
|
|
29
|
+
"""
|
|
30
|
+
Export transformed tables from DuckDB into compressed Parquet files.
|
|
31
|
+
|
|
32
|
+
Args:
|
|
33
|
+
db_path: Source DuckDB database path.
|
|
34
|
+
parquet_dir: Target directory for Parquet files.
|
|
35
|
+
dataset_schema: Schema where tables are located.
|
|
36
|
+
tables: List of tables to export (defaults to SEED_TABLES).
|
|
37
|
+
verbose: Log export progress.
|
|
38
|
+
|
|
39
|
+
Returns:
|
|
40
|
+
Path to output directory containing Parquet files.
|
|
41
|
+
"""
|
|
42
|
+
src_path = Path(db_path) if db_path else DEFAULT_DUCKDB_PATH
|
|
43
|
+
dest_dir = Path(parquet_dir) if parquet_dir else DEFAULT_PARQUET_DIR
|
|
44
|
+
|
|
45
|
+
dest_dir.mkdir(parents=True, exist_ok=True)
|
|
46
|
+
export_tables = tables or SEED_TABLES
|
|
47
|
+
|
|
48
|
+
duck_con = duckdb.connect(str(src_path), read_only=True)
|
|
49
|
+
|
|
50
|
+
exported: Dict[str, Path] = {}
|
|
51
|
+
for table_name in export_tables:
|
|
52
|
+
out_file = dest_dir / f"{table_name}.parquet"
|
|
53
|
+
success = False
|
|
54
|
+
for query_target in [f"{dataset_schema}.{table_name}", f"{table_name}"]:
|
|
55
|
+
try:
|
|
56
|
+
duck_con.execute(
|
|
57
|
+
f"COPY {query_target} TO '{out_file.as_posix()}' (FORMAT PARQUET, COMPRESSION ZSTD)"
|
|
58
|
+
)
|
|
59
|
+
exported[table_name] = out_file
|
|
60
|
+
success = True
|
|
61
|
+
if verbose:
|
|
62
|
+
console.print(f"[green]✓[/green] Exported [bold]{out_file.name}[/bold]")
|
|
63
|
+
break
|
|
64
|
+
except Exception:
|
|
65
|
+
continue
|
|
66
|
+
|
|
67
|
+
if not success and verbose:
|
|
68
|
+
console.print(f"[yellow]⚠[/yellow] Skipping {table_name}: not found in DuckDB.")
|
|
69
|
+
|
|
70
|
+
duck_con.close()
|
|
71
|
+
|
|
72
|
+
if verbose:
|
|
73
|
+
console.print(
|
|
74
|
+
f"[bold green]✓ Successfully generated Parquet seed files at:[/bold green] {dest_dir}"
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
return dest_dir
|