raxtik-etl 0.1.0.dev0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. raxtik_etl/__init__.py +6 -0
  2. raxtik_etl/cli.py +298 -0
  3. raxtik_etl/config.py +30 -0
  4. raxtik_etl/distribution/__init__.py +13 -0
  5. raxtik_etl/distribution/huggingface.py +210 -0
  6. raxtik_etl/exporters/__init__.py +6 -0
  7. raxtik_etl/exporters/parquet_exporter.py +77 -0
  8. raxtik_etl/exporters/sqlite_exporter.py +137 -0
  9. raxtik_etl/longitudinal/__init__.py +11 -0
  10. raxtik_etl/longitudinal/merger.py +259 -0
  11. raxtik_etl/packaging/builder.py +98 -0
  12. raxtik_etl/reference/__init__.py +24 -0
  13. raxtik_etl/reference/codes.py +38 -0
  14. raxtik_etl/reference/dechallenge.py +54 -0
  15. raxtik_etl/reference/occupations.py +65 -0
  16. raxtik_etl/reference/outcomes.py +105 -0
  17. raxtik_etl/reference/rechallenge.py +54 -0
  18. raxtik_etl/reference/report_sources.py +87 -0
  19. raxtik_etl/reference/roles.py +99 -0
  20. raxtik_etl/schema/__init__.py +51 -0
  21. raxtik_etl/schema/canonical.py +264 -0
  22. raxtik_etl/schema/registry.py +854 -0
  23. raxtik_etl/schema/views.py +62 -0
  24. raxtik_etl/sdk.py +157 -0
  25. raxtik_etl/sources/__init__.py +17 -0
  26. raxtik_etl/sources/faers/__init__.py +13 -0
  27. raxtik_etl/sources/faers/fetcher.py +116 -0
  28. raxtik_etl/sources/faers/parser.py +108 -0
  29. raxtik_etl/sources/faers/resources.py +114 -0
  30. raxtik_etl/sources/faers/source.py +61 -0
  31. raxtik_etl/sources/protocol.py +49 -0
  32. raxtik_etl/sources/registry.py +61 -0
  33. raxtik_etl/transformations/__init__.py +5 -0
  34. raxtik_etl/transformations/metadata.py +437 -0
  35. raxtik_etl/transformations/normalization.py +62 -0
  36. raxtik_etl/transformations/runner.py +92 -0
  37. raxtik_etl/transformations/sql/01_deduplicate.sql +55 -0
  38. raxtik_etl/transformations/sql/02_normalize_drugs.sql +20 -0
  39. raxtik_etl/transformations/sql/03_aggregate_stats.sql +156 -0
  40. raxtik_etl-0.1.0.dev0.dist-info/METADATA +268 -0
  41. raxtik_etl-0.1.0.dev0.dist-info/RECORD +44 -0
  42. raxtik_etl-0.1.0.dev0.dist-info/WHEEL +4 -0
  43. raxtik_etl-0.1.0.dev0.dist-info/entry_points.txt +2 -0
  44. raxtik_etl-0.1.0.dev0.dist-info/licenses/LICENSE +661 -0
raxtik_etl/__init__.py ADDED
@@ -0,0 +1,6 @@
1
+ """raxtik-etl: Versioned, reproducible FAERS data platform for high-performance analytics."""
2
+
3
+ from raxtik_etl.sdk import FAERS
4
+
5
+ __version__ = "0.1.0.dev0"
6
+ __all__ = ["FAERS", "__version__"]
raxtik_etl/cli.py ADDED
@@ -0,0 +1,298 @@
1
+ """Typer CLI interface for raxtik-etl pipeline commands."""
2
+
3
+ from pathlib import Path
4
+ from typing import List, Optional
5
+
6
+ import typer
7
+ from rich.console import Console
8
+ from rich.panel import Panel
9
+
10
+ from raxtik_etl.distribution.huggingface import DEFAULT_HF_REPO
11
+ from raxtik_etl.longitudinal import (
12
+ build_merged_longitudinal_package,
13
+ get_longitudinal_summary_report,
14
+ )
15
+ from raxtik_etl.packaging.builder import PACKAGE_OUTPUT_DIR, build_quarterly_package
16
+ from raxtik_etl.sdk import FAERS
17
+
18
+ app = typer.Typer(
19
+ name="raxtik-etl",
20
+ help="Enterprise-grade FAERS adverse event data platform & packaging CLI.",
21
+ add_completion=False,
22
+ )
23
+ console = Console()
24
+ CACHE_DIR = Path("data/raw/archives_cache")
25
+
26
+
27
+ def parse_quarter_range(start_quarter: str, end_quarter: str) -> List[str]:
28
+ """Generates an inclusive list of quarterly releases between start and end (e.g. '2004Q1' to '2006Q4')."""
29
+ start_clean = start_quarter.strip().upper()
30
+ end_clean = end_quarter.strip().upper()
31
+
32
+ start_y, start_q = int(start_clean[:4]), int(start_clean[-1])
33
+ end_y, end_q = int(end_clean[:4]), int(end_clean[-1])
34
+
35
+ quarters: List[str] = []
36
+ y, q = start_y, start_q
37
+ while (y < end_y) or (y == end_y and q <= end_q):
38
+ quarters.append(f"{y}Q{q}")
39
+ q += 1
40
+ if q > 4:
41
+ q = 1
42
+ y += 1
43
+ return quarters
44
+
45
+
46
+ @app.command("build-package")
47
+ def build_package_cmd(
48
+ source: str = typer.Option(
49
+ "faers", "--source", "-s", help="Source connector name (e.g. faers)"
50
+ ),
51
+ release: str = typer.Option(
52
+ "2004Q1", "--release", "-r", help="Quarter release identifier (e.g. 2004Q1)"
53
+ ),
54
+ zip_path: Optional[Path] = typer.Option(
55
+ None, "--zip-path", "-z", help="Path to source archive/zip file"
56
+ ),
57
+ output_dir: Optional[Path] = typer.Option(
58
+ None, "--output-dir", "-o", help="Output package directory"
59
+ ),
60
+ ):
61
+ """Build a standalone, self-describing Safety DuckDB and Parquet package."""
62
+ release_clean = release.upper()
63
+ archive_file = zip_path or (CACHE_DIR / f"{release_clean.lower()}_ascii.zip")
64
+
65
+ if not archive_file.exists():
66
+ console.print(f"[bold red]Source archive not found at {archive_file}.[/bold red]")
67
+ raise typer.Exit(code=1)
68
+
69
+ res = build_quarterly_package(
70
+ release=release_clean,
71
+ zip_path=archive_file,
72
+ source_name=source,
73
+ output_dir=output_dir,
74
+ )
75
+ console.print(
76
+ Panel(
77
+ f"Release: [bold]{res['release']}[/bold]\nSignature: {res['signature_id']}\nDuckDB: {res['duckdb_path']}\nParquet: {res['parquet_dir']}",
78
+ title="Package Generated Successfully",
79
+ style="green",
80
+ )
81
+ )
82
+
83
+
84
+ @app.command("build-cohort")
85
+ def build_cohort_cmd(
86
+ start: str = typer.Option("2004Q1", "--start", help="Starting quarter release (e.g. 2004Q1)"),
87
+ end: str = typer.Option("2006Q4", "--end", help="Ending quarter release (e.g. 2006Q4)"),
88
+ source: str = typer.Option("faers", "--source", "-s", help="Source connector name"),
89
+ output_dir: Optional[Path] = typer.Option(
90
+ None, "--output-dir", "-o", help="Output directory for packages"
91
+ ),
92
+ ):
93
+ """Batch build standalone packages for every quarter in the specified range."""
94
+ quarters = parse_quarter_range(start, end)
95
+ console.print(
96
+ f"[bold cyan]Building {len(quarters)} quarterly packages ({quarters[0]} to {quarters[-1]})...[/bold cyan]"
97
+ )
98
+
99
+ for q in quarters:
100
+ archive_file = CACHE_DIR / f"{q.lower()}_ascii.zip"
101
+ if not archive_file.exists():
102
+ console.print(f"[bold red]Archive not found for {q}: {archive_file}[/bold red]")
103
+ raise typer.Exit(code=1)
104
+ build_quarterly_package(
105
+ release=q,
106
+ zip_path=archive_file,
107
+ source_name=source,
108
+ output_dir=output_dir,
109
+ )
110
+
111
+ console.print(f"[bold green]✓ Successfully built all {len(quarters)} packages.[/bold green]")
112
+
113
+
114
+ @app.command("merge-cohort")
115
+ def merge_cohort_cmd(
116
+ start: str = typer.Option("2004Q1", "--start", help="Starting quarter release (e.g. 2004Q1)"),
117
+ end: str = typer.Option("2006Q4", "--end", help="Ending quarter release (e.g. 2006Q4)"),
118
+ package_dir: Optional[Path] = typer.Option(
119
+ None, "--package-dir", "-p", help="Directory containing quarterly packages"
120
+ ),
121
+ output_name: Optional[str] = typer.Option(
122
+ None,
123
+ "--output-name",
124
+ "-n",
125
+ help="Merged output name (defaults to faers_{start}_{end}_merged)",
126
+ ),
127
+ output_dir: Optional[Path] = typer.Option(
128
+ None, "--output-dir", "-o", help="Destination folder for merged package"
129
+ ),
130
+ ):
131
+ """Merge quarterly packages into a consolidated longitudinal dataset with cross-quarter deduplication."""
132
+ quarters = parse_quarter_range(start, end)
133
+ resolved_output_name = output_name or f"faers_{start}_{end}_merged".lower()
134
+ res = build_merged_longitudinal_package(
135
+ releases=quarters,
136
+ package_dir=package_dir,
137
+ output_name=resolved_output_name,
138
+ output_dir=output_dir,
139
+ )
140
+ console.print(
141
+ Panel(
142
+ f"Merged Database: [bold]{res['output_duckdb']}[/bold]\n"
143
+ f"Merged Parquet: [bold]{res['output_parquet']}[/bold]\n"
144
+ f"Total Submissions: [cyan]{res['metrics'].get('total_raw_submissions')}[/cyan]\n"
145
+ f"Deduplicated Unique Cases: [green]{res['metrics'].get('total_unique_cases')}[/green]\n"
146
+ f"Cross-Quarter Cases: [yellow]{res['metrics'].get('cases_spanning_multiple_quarters')}[/yellow]\n"
147
+ f"Duplicates Removed: [magenta]{res['metrics'].get('deduplicated_duplicate_reports_removed')} ({res['metrics'].get('deduplication_reduction_pct')})[/magenta]",
148
+ title="Longitudinal Cohort Merged & Deduplicated",
149
+ style="green",
150
+ )
151
+ )
152
+
153
+
154
+ @app.command("inspect-package")
155
+ def inspect_package_cmd(
156
+ release: str = typer.Option(
157
+ "2004Q1", "--release", "-r", help="Quarter release identifier to inspect"
158
+ ),
159
+ package_dir: Optional[Path] = typer.Option(
160
+ None, "--package-dir", "-p", help="Directory containing package files"
161
+ ),
162
+ ):
163
+ """Inspect and display metadata and data quality report for a compiled package."""
164
+ try:
165
+ with FAERS(release=release, package_dir=package_dir) as faers:
166
+ faers.print_summary()
167
+ except Exception as e:
168
+ console.print(f"[bold red]Failed to inspect package:[/bold red] {e}")
169
+ raise typer.Exit(code=1)
170
+
171
+
172
+ @app.command("inspect-cohort")
173
+ def inspect_cohort_cmd(
174
+ cohort: Optional[str] = typer.Option(
175
+ None,
176
+ "--cohort",
177
+ "-c",
178
+ help="Cohort name or identifier (e.g. 'faers_2004_2006_merged' or '2004_2006')",
179
+ ),
180
+ db_path: Optional[Path] = typer.Option(
181
+ None,
182
+ "--db-path",
183
+ "-d",
184
+ help="Direct path to merged DuckDB package file",
185
+ ),
186
+ package_dir: Optional[Path] = typer.Option(
187
+ None,
188
+ "--package-dir",
189
+ "-p",
190
+ help="Directory containing packages (defaults to data/packages or RAXTIK_PACKAGE_DIR)",
191
+ ),
192
+ ):
193
+ """Inspect and display metrics for a merged longitudinal cohort package."""
194
+ base_dir = package_dir or PACKAGE_OUTPUT_DIR
195
+ target_path: Optional[Path] = None
196
+
197
+ if db_path is not None:
198
+ target_path = db_path
199
+ elif cohort is not None:
200
+ filename = cohort if cohort.endswith(".duckdb") else f"{cohort}.duckdb"
201
+ if not filename.startswith("faers_") and not filename.startswith("/"):
202
+ filename = f"faers_{filename}"
203
+ candidate = base_dir / filename
204
+ if not candidate.exists() and not filename.endswith("_merged.duckdb"):
205
+ merged_candidate = base_dir / filename.replace(".duckdb", "_merged.duckdb")
206
+ if merged_candidate.exists():
207
+ candidate = merged_candidate
208
+ target_path = candidate
209
+ else:
210
+ # Default fallback: check for existing merged package
211
+ default_2004_2006 = base_dir / "faers_2004_2006_merged.duckdb"
212
+ if default_2004_2006.exists():
213
+ target_path = default_2004_2006
214
+ else:
215
+ merged_files = sorted(base_dir.glob("*merged*.duckdb"))
216
+ target_path = merged_files[0] if merged_files else default_2004_2006
217
+
218
+ if not target_path.exists():
219
+ console.print(f"[bold red]Database not found at {target_path}[/bold red]")
220
+ raise typer.Exit(code=1)
221
+ report_table = get_longitudinal_summary_report(target_path)
222
+ console.print(report_table)
223
+
224
+
225
+ @app.command("hf-download")
226
+ def hf_download_cmd(
227
+ release: str = typer.Option(
228
+ "2004Q1",
229
+ "--release",
230
+ "-r",
231
+ help="Release identifier (e.g. 2004Q1 or faers_2004_2006_merged)",
232
+ ),
233
+ repo_id: str = typer.Option(
234
+ DEFAULT_HF_REPO,
235
+ "--repo-id",
236
+ help="Hugging Face dataset repository (defaults to RAXTIK_HF_REPO)",
237
+ ),
238
+ dest_dir: Path = typer.Option(
239
+ Path("data/packages"), "--dest-dir", "-d", help="Destination folder"
240
+ ),
241
+ force: bool = typer.Option(False, "--force", "-f", help="Overwrite existing local files"),
242
+ ):
243
+ """Download a compiled FAERS package from Hugging Face Hub (zero auth required)."""
244
+ from raxtik_etl.distribution.huggingface import download_package_from_hf
245
+
246
+ try:
247
+ download_package_from_hf(
248
+ release_or_name=release,
249
+ dest_dir=dest_dir,
250
+ repo_id=repo_id,
251
+ force=force,
252
+ )
253
+ except Exception as e:
254
+ console.print(f"[bold red]Download failed:[/bold red] {e}")
255
+ raise typer.Exit(code=1)
256
+
257
+
258
+ @app.command("hf-upload")
259
+ def hf_upload_cmd(
260
+ release: str = typer.Option(
261
+ "2004Q1",
262
+ "--release",
263
+ "-r",
264
+ help="Release identifier (e.g. 2004Q1 or faers_2004_2006_merged)",
265
+ ),
266
+ repo_id: str = typer.Option(
267
+ DEFAULT_HF_REPO,
268
+ "--repo-id",
269
+ help="Hugging Face dataset repository (defaults to RAXTIK_HF_REPO)",
270
+ ),
271
+ package_dir: Path = typer.Option(
272
+ Path("data/packages"), "--package-dir", "-p", help="Directory containing packages"
273
+ ),
274
+ token: Optional[str] = typer.Option(
275
+ None, "--token", "-t", help="Hugging Face write access token (defaults to HF_TOKEN env var)"
276
+ ),
277
+ parquet: bool = typer.Option(
278
+ True, "--parquet/--no-parquet", help="Whether to include Parquet shards"
279
+ ),
280
+ ):
281
+ """Upload a compiled FAERS package to Hugging Face Hub (requires write token)."""
282
+ from raxtik_etl.distribution.huggingface import upload_package_to_hf
283
+
284
+ try:
285
+ upload_package_to_hf(
286
+ release_or_name=release,
287
+ package_dir=package_dir,
288
+ repo_id=repo_id,
289
+ token=token,
290
+ include_parquet=parquet,
291
+ )
292
+ except Exception as e:
293
+ console.print(f"[bold red]Upload failed:[/bold red] {e}")
294
+ raise typer.Exit(code=1)
295
+
296
+
297
+ if __name__ == "__main__":
298
+ app()
raxtik_etl/config.py ADDED
@@ -0,0 +1,30 @@
1
+ """Global configuration and directory paths for raxtik-etl."""
2
+
3
+ import os
4
+ from pathlib import Path
5
+
6
+ PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent
7
+
8
+ # Base data directories
9
+ DATA_DIR = Path(os.getenv("RAXTIK_DATA_DIR", PROJECT_ROOT / "data"))
10
+ RAW_DATA_DIR = DATA_DIR / "raw"
11
+ INTERMEDIATE_DATA_DIR = DATA_DIR / "intermediate"
12
+ OUTPUT_DATA_DIR = DATA_DIR / "output"
13
+
14
+ # DuckDB intermediate database
15
+ DEFAULT_DUCKDB_PATH = INTERMEDIATE_DATA_DIR / "faers.duckdb"
16
+
17
+ # Exporter output paths
18
+ DEFAULT_SQLITE_PATH = OUTPUT_DATA_DIR / "faers_seed.sqlite"
19
+ DEFAULT_PARQUET_DIR = OUTPUT_DATA_DIR / "parquet"
20
+
21
+ # FAERS configuration
22
+ FAERS_TABLE_TYPES = ["demo", "drug", "reac", "outc", "indi"]
23
+ FAERS_BASE_URL = "https://fis.fda.gov/content/Exports"
24
+
25
+
26
+ def ensure_directories():
27
+ """Ensure that standard data directories exist."""
28
+ RAW_DATA_DIR.mkdir(parents=True, exist_ok=True)
29
+ INTERMEDIATE_DATA_DIR.mkdir(parents=True, exist_ok=True)
30
+ OUTPUT_DATA_DIR.mkdir(parents=True, exist_ok=True)
@@ -0,0 +1,13 @@
1
+ """Remote distribution subsystem for FAERS packages (Hugging Face Hub integration)."""
2
+
3
+ from raxtik_etl.distribution.huggingface import (
4
+ DEFAULT_HF_REPO,
5
+ download_package_from_hf,
6
+ upload_package_to_hf,
7
+ )
8
+
9
+ __all__ = [
10
+ "DEFAULT_HF_REPO",
11
+ "download_package_from_hf",
12
+ "upload_package_to_hf",
13
+ ]
@@ -0,0 +1,210 @@
1
+ """Hugging Face Hub distribution module for downloading and uploading compiled packages.
2
+
3
+ Designed with zero-auth public downloads:
4
+ - Downloads can use direct streaming HTTP via requests (no Hugging Face login required).
5
+ - Uploads require `huggingface_hub` (in dev dependencies) and a valid write token.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import os
11
+ import re
12
+ from pathlib import Path
13
+ from typing import Optional
14
+
15
+ import requests
16
+ from rich.console import Console
17
+ from tqdm import tqdm
18
+
19
+ console = Console()
20
+
21
+ DEFAULT_HF_REPO = os.getenv("RAXTIK_HF_REPO", "raxtik-org/faers-safety-database")
22
+
23
+
24
+ def download_package_from_hf(
25
+ release_or_name: str,
26
+ dest_dir: Path | str = "data/packages",
27
+ repo_id: str = DEFAULT_HF_REPO,
28
+ force: bool = False,
29
+ ) -> Path:
30
+ """
31
+ Downloads a compiled .duckdb safety package from Hugging Face Hub.
32
+
33
+ Publicly accessible without any authentication token or Hugging Face account.
34
+
35
+ Args:
36
+ release_or_name: e.g. '2004Q1' or 'faers_2004_2006_merged'
37
+ dest_dir: Destination folder (defaults to data/packages)
38
+ repo_id: Hugging Face dataset repository id
39
+ force: Whether to overwrite existing local file
40
+
41
+ Returns:
42
+ Path to downloaded DuckDB package.
43
+ """
44
+ target_dir = Path(dest_dir)
45
+ target_dir.mkdir(parents=True, exist_ok=True)
46
+
47
+ rel_upper = release_or_name.strip()
48
+ if not rel_upper.startswith("faers_"):
49
+ filename = f"faers_{rel_upper.upper()}.duckdb"
50
+ else:
51
+ filename = f"{rel_upper}.duckdb"
52
+
53
+ match = re.search(r"(\d{4})Q[1-4]", rel_upper)
54
+ subfolder = match.group(1) if match else ("merged" if "merged" in rel_upper.lower() else "")
55
+ rel_path = f"{subfolder}/{filename}" if subfolder else filename
56
+
57
+ dest_file = target_dir / filename
58
+ if dest_file.exists() and not force:
59
+ console.print(f"[bold green]Package already present locally:[/bold green] {dest_file}")
60
+ return dest_file
61
+
62
+ # Try huggingface_hub if available
63
+ try:
64
+ from huggingface_hub import hf_hub_download # type: ignore
65
+
66
+ console.print(
67
+ f"[bold cyan]Downloading {filename} from Hugging Face Hub ({repo_id})...[/bold cyan]"
68
+ )
69
+ try:
70
+ downloaded_path = hf_hub_download(
71
+ repo_id=repo_id,
72
+ filename=rel_path,
73
+ repo_type="dataset",
74
+ local_dir=str(target_dir),
75
+ )
76
+ except Exception:
77
+ # Fall back to root path if not in year subfolder
78
+ downloaded_path = hf_hub_download(
79
+ repo_id=repo_id,
80
+ filename=filename,
81
+ repo_type="dataset",
82
+ local_dir=str(target_dir),
83
+ )
84
+ return Path(downloaded_path)
85
+ except ImportError:
86
+ # Fallback to direct HTTP download via requests (no huggingface_hub needed)
87
+ url = f"https://huggingface.co/datasets/{repo_id}/resolve/main/{rel_path}"
88
+ console.print(
89
+ f"[bold cyan]Downloading {filename} via public direct stream from {url}...[/bold cyan]"
90
+ )
91
+
92
+ response = requests.get(url, stream=True, timeout=120)
93
+ if response.status_code != 200:
94
+ # Try root URL fallback
95
+ fallback_url = f"https://huggingface.co/datasets/{repo_id}/resolve/main/{filename}"
96
+ response = requests.get(fallback_url, stream=True, timeout=120)
97
+ if response.status_code != 200:
98
+ raise RuntimeError(
99
+ f"Failed to download package '{filename}' from Hugging Face Hub at {url} (HTTP {response.status_code})."
100
+ )
101
+
102
+ total_size = int(response.headers.get("content-length", 0))
103
+ with (
104
+ open(dest_file, "wb") as f,
105
+ tqdm(
106
+ desc=filename,
107
+ total=total_size if total_size > 0 else None,
108
+ unit="iB",
109
+ unit_scale=True,
110
+ unit_divisor=1024,
111
+ ) as bar,
112
+ ):
113
+ for chunk in response.iter_content(chunk_size=1024 * 1024):
114
+ if chunk:
115
+ size = f.write(chunk)
116
+ bar.update(size)
117
+
118
+ console.print(f"[bold green]✓ Download complete:[/bold green] {dest_file}")
119
+ return dest_file
120
+
121
+
122
+ def upload_package_to_hf(
123
+ release_or_name: str,
124
+ package_dir: Path | str = "data/packages",
125
+ repo_id: str = DEFAULT_HF_REPO,
126
+ token: Optional[str] = None,
127
+ include_parquet: bool = True,
128
+ ) -> None:
129
+ """
130
+ Uploads a compiled .duckdb package and optional Parquet shards to Hugging Face Hub.
131
+
132
+ Organizes releases cleanly into year subfolders (e.g. 2004/faers_2004Q1.duckdb, merged/...).
133
+
134
+ Requires `huggingface_hub` (available via `pip install 'raxtik-etl[dev]'`) and
135
+ a Hugging Face write token (via argument or HF_TOKEN env var).
136
+
137
+ Args:
138
+ release_or_name: e.g. '2004Q1' or 'faers_2004_2006_merged'
139
+ package_dir: Directory containing compiled packages
140
+ repo_id: Destination Hugging Face dataset repository id
141
+ token: Hugging Face user write access token
142
+ include_parquet: Whether to also upload the corresponding .parquet folder
143
+ """
144
+ try:
145
+ from huggingface_hub import HfApi # type: ignore
146
+ except ImportError:
147
+ raise ImportError(
148
+ "Uploading requires 'huggingface_hub'. Please install it using: pip install 'raxtik-etl[dev]'"
149
+ )
150
+
151
+ auth_token = token or os.environ.get("HF_TOKEN") or os.environ.get("HUGGINGFACE_HUB_TOKEN")
152
+ if not auth_token:
153
+ raise ValueError(
154
+ "Hugging Face write token is required to upload packages. "
155
+ "Pass token=... or set the HF_TOKEN environment variable."
156
+ )
157
+
158
+ pkg_dir = Path(package_dir)
159
+ rel_clean = release_or_name.strip()
160
+ if not rel_clean.startswith("faers_"):
161
+ duckdb_name = f"faers_{rel_clean.upper()}.duckdb"
162
+ parquet_dir_name = f"faers_{rel_clean.upper()}.parquet"
163
+ else:
164
+ duckdb_name = f"{rel_clean}.duckdb"
165
+ parquet_dir_name = f"{rel_clean}.parquet"
166
+
167
+ # Route into year subfolder (e.g. 2004/...) or merged/...
168
+ match = re.search(r"(\d{4})Q[1-4]", rel_clean.upper())
169
+ if match:
170
+ subfolder = match.group(1)
171
+ elif "merged" in rel_clean.lower():
172
+ subfolder = "merged"
173
+ else:
174
+ subfolder = ""
175
+
176
+ path_in_repo_duckdb = f"{subfolder}/{duckdb_name}" if subfolder else duckdb_name
177
+ path_in_repo_parquet = f"{subfolder}/{parquet_dir_name}" if subfolder else parquet_dir_name
178
+
179
+ duckdb_file = pkg_dir / duckdb_name
180
+ parquet_dir = pkg_dir / parquet_dir_name
181
+
182
+ if not duckdb_file.exists():
183
+ raise FileNotFoundError(f"Package database not found at {duckdb_file}")
184
+
185
+ api = HfApi(token=auth_token)
186
+
187
+ # 1. Upload DuckDB file
188
+ console.print(
189
+ f"[bold cyan]Uploading {duckdb_name} -> {path_in_repo_duckdb} ({duckdb_file.stat().st_size / (1024 * 1024):.2f} MB)...[/bold cyan]"
190
+ )
191
+ api.upload_file(
192
+ path_or_fileobj=duckdb_file.as_posix(),
193
+ path_in_repo=path_in_repo_duckdb,
194
+ repo_id=repo_id,
195
+ repo_type="dataset",
196
+ )
197
+ console.print(f"[bold green]✓ Uploaded {path_in_repo_duckdb} successfully.[/bold green]")
198
+
199
+ # 2. Optionally upload Parquet shards
200
+ if include_parquet and parquet_dir.exists():
201
+ console.print(
202
+ f"[bold cyan]Uploading Parquet shards from {parquet_dir_name} -> {path_in_repo_parquet}...[/bold cyan]"
203
+ )
204
+ api.upload_folder(
205
+ folder_path=parquet_dir.as_posix(),
206
+ path_in_repo=path_in_repo_parquet,
207
+ repo_id=repo_id,
208
+ repo_type="dataset",
209
+ )
210
+ console.print(f"[bold green]✓ Uploaded {path_in_repo_parquet} successfully.[/bold green]")
@@ -0,0 +1,6 @@
1
+ """Seed exporters for SQLite and Parquet artifacts."""
2
+
3
+ from raxtik_etl.exporters.parquet_exporter import export_parquet
4
+ from raxtik_etl.exporters.sqlite_exporter import export_sqlite
5
+
6
+ __all__ = ["export_sqlite", "export_parquet"]
@@ -0,0 +1,77 @@
1
+ """Parquet Seed Exporter for analytical workloads."""
2
+
3
+ from pathlib import Path
4
+ from typing import Dict, List, Optional
5
+
6
+ import duckdb
7
+ from rich.console import Console
8
+
9
+ from raxtik_etl.config import DEFAULT_DUCKDB_PATH, DEFAULT_PARQUET_DIR
10
+
11
+ console = Console()
12
+
13
+ SEED_TABLES = [
14
+ "drug_reaction_stats",
15
+ "drug_demographics",
16
+ "drug_indications",
17
+ "drug_routes",
18
+ "drug_outcomes",
19
+ ]
20
+
21
+
22
+ def export_parquet(
23
+ db_path: Optional[Path | str] = None,
24
+ parquet_dir: Optional[Path | str] = None,
25
+ dataset_schema: str = "faers_raw",
26
+ tables: Optional[List[str]] = None,
27
+ verbose: bool = True,
28
+ ) -> Path:
29
+ """
30
+ Export transformed tables from DuckDB into compressed Parquet files.
31
+
32
+ Args:
33
+ db_path: Source DuckDB database path.
34
+ parquet_dir: Target directory for Parquet files.
35
+ dataset_schema: Schema where tables are located.
36
+ tables: List of tables to export (defaults to SEED_TABLES).
37
+ verbose: Log export progress.
38
+
39
+ Returns:
40
+ Path to output directory containing Parquet files.
41
+ """
42
+ src_path = Path(db_path) if db_path else DEFAULT_DUCKDB_PATH
43
+ dest_dir = Path(parquet_dir) if parquet_dir else DEFAULT_PARQUET_DIR
44
+
45
+ dest_dir.mkdir(parents=True, exist_ok=True)
46
+ export_tables = tables or SEED_TABLES
47
+
48
+ duck_con = duckdb.connect(str(src_path), read_only=True)
49
+
50
+ exported: Dict[str, Path] = {}
51
+ for table_name in export_tables:
52
+ out_file = dest_dir / f"{table_name}.parquet"
53
+ success = False
54
+ for query_target in [f"{dataset_schema}.{table_name}", f"{table_name}"]:
55
+ try:
56
+ duck_con.execute(
57
+ f"COPY {query_target} TO '{out_file.as_posix()}' (FORMAT PARQUET, COMPRESSION ZSTD)"
58
+ )
59
+ exported[table_name] = out_file
60
+ success = True
61
+ if verbose:
62
+ console.print(f"[green]✓[/green] Exported [bold]{out_file.name}[/bold]")
63
+ break
64
+ except Exception:
65
+ continue
66
+
67
+ if not success and verbose:
68
+ console.print(f"[yellow]⚠[/yellow] Skipping {table_name}: not found in DuckDB.")
69
+
70
+ duck_con.close()
71
+
72
+ if verbose:
73
+ console.print(
74
+ f"[bold green]✓ Successfully generated Parquet seed files at:[/bold green] {dest_dir}"
75
+ )
76
+
77
+ return dest_dir