omop-duck-db 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- omop_duck_db-0.2.0/.Rbuildignore +31 -0
- omop_duck_db-0.2.0/.Rprofile +1 -0
- omop_duck_db-0.2.0/.dockerignore +36 -0
- omop_duck_db-0.2.0/.github/workflows/ci.yml +105 -0
- omop_duck_db-0.2.0/.github/workflows/pkgdown.yml +58 -0
- omop_duck_db-0.2.0/.github/workflows/release.yml +65 -0
- omop_duck_db-0.2.0/.github/workflows/test-coverage.yml +88 -0
- omop_duck_db-0.2.0/.gitignore +48 -0
- omop_duck_db-0.2.0/DESCRIPTION +34 -0
- omop_duck_db-0.2.0/Dockerfile +76 -0
- omop_duck_db-0.2.0/LICENSE +2 -0
- omop_duck_db-0.2.0/LICENSE.md +21 -0
- omop_duck_db-0.2.0/NAMESPACE +17 -0
- omop_duck_db-0.2.0/NEWS.md +41 -0
- omop_duck_db-0.2.0/NOTES.md +25 -0
- omop_duck_db-0.2.0/PKG-INFO +256 -0
- omop_duck_db-0.2.0/R/build_schema.R +42 -0
- omop_duck_db-0.2.0/R/etl_pcornet.R +758 -0
- omop_duck_db-0.2.0/R/gdrive_sync.R +63 -0
- omop_duck_db-0.2.0/R/load_from_bigquery.R +77 -0
- omop_duck_db-0.2.0/R/load_vocabulary.R +171 -0
- omop_duck_db-0.2.0/R/mapping.R +170 -0
- omop_duck_db-0.2.0/R/omopduckdb-package.R +6 -0
- omop_duck_db-0.2.0/R/remapping.R +209 -0
- omop_duck_db-0.2.0/R/utils.R +97 -0
- omop_duck_db-0.2.0/README.Rmd +184 -0
- omop_duck_db-0.2.0/README.md +212 -0
- omop_duck_db-0.2.0/README.qmd +22 -0
- omop_duck_db-0.2.0/_pkgdown.yml +43 -0
- omop_duck_db-0.2.0/config/bigquery.yml.example +9 -0
- omop_duck_db-0.2.0/config/gdrive.yml.example +6 -0
- omop_duck_db-0.2.0/inst/extdata/5.4/duckdb/OMOPCDM_duckdb_5.4_constraints.sql +177 -0
- omop_duck_db-0.2.0/inst/extdata/5.4/duckdb/OMOPCDM_duckdb_5.4_ddl.sql +511 -0
- omop_duck_db-0.2.0/inst/extdata/5.4/duckdb/OMOPCDM_duckdb_5.4_indices.sql +98 -0
- omop_duck_db-0.2.0/inst/extdata/5.4/duckdb/OMOPCDM_duckdb_5.4_primary_keys.sql +29 -0
- omop_duck_db-0.2.0/inst/scripts/build_schema.R +14 -0
- omop_duck_db-0.2.0/inst/scripts/debug_output.txt +52 -0
- omop_duck_db-0.2.0/inst/scripts/debug_query.R +30 -0
- omop_duck_db-0.2.0/inst/scripts/etl_pcornet.R +18 -0
- omop_duck_db-0.2.0/inst/scripts/gdrive_sync.R +15 -0
- omop_duck_db-0.2.0/inst/scripts/load_from_bigquery.R +24 -0
- omop_duck_db-0.2.0/inst/scripts/load_vocabulary.R +19 -0
- omop_duck_db-0.2.0/inst/scripts/query.r +132 -0
- omop_duck_db-0.2.0/inst/sql/mapping_macros.sql +75 -0
- omop_duck_db-0.2.0/man/build_condition_era.Rd +14 -0
- omop_duck_db-0.2.0/man/build_drug_era.Rd +14 -0
- omop_duck_db-0.2.0/man/build_observation_period.Rd +14 -0
- omop_duck_db-0.2.0/man/build_schema.Rd +25 -0
- omop_duck_db-0.2.0/man/check_vocabulary_version.Rd +17 -0
- omop_duck_db-0.2.0/man/etl_pcornet.Rd +25 -0
- omop_duck_db-0.2.0/man/export_unmapped_codes.Rd +23 -0
- omop_duck_db-0.2.0/man/gdrive_download_latest.Rd +17 -0
- omop_duck_db-0.2.0/man/gdrive_upload_snapshot.Rd +17 -0
- omop_duck_db-0.2.0/man/import_source_to_concept_map.Rd +21 -0
- omop_duck_db-0.2.0/man/import_usagi_mappings.Rd +28 -0
- omop_duck_db-0.2.0/man/load_from_bigquery.Rd +51 -0
- omop_duck_db-0.2.0/man/load_vocabulary.Rd +41 -0
- omop_duck_db-0.2.0/man/omopduckdb-package.Rd +29 -0
- omop_duck_db-0.2.0/man/remap_all.Rd +21 -0
- omop_duck_db-0.2.0/man/remap_cdm_table.Rd +21 -0
- omop_duck_db-0.2.0/pyproject.toml +44 -0
- omop_duck_db-0.2.0/python/omop_etl/__init__.py +55 -0
- omop_duck_db-0.2.0/python/omop_etl/build_omop_cdm.py +905 -0
- omop_duck_db-0.2.0/python/omop_etl/mapping.py +171 -0
- omop_duck_db-0.2.0/python/omop_etl/remapping.py +208 -0
- omop_duck_db-0.2.0/python/omop_etl/vocabulary.py +180 -0
- omop_duck_db-0.2.0/python/requirements.txt +3 -0
- omop_duck_db-0.2.0/renv/.gitignore +7 -0
- omop_duck_db-0.2.0/renv/activate.R +1438 -0
- omop_duck_db-0.2.0/renv/settings.json +21 -0
- omop_duck_db-0.2.0/renv.lock +3185 -0
- omop_duck_db-0.2.0/scripts/release.R +59 -0
- omop_duck_db-0.2.0/scripts/release.py +160 -0
- omop_duck_db-0.2.0/tests/fixtures/pcornet_sample/demographic.csv +4 -0
- omop_duck_db-0.2.0/tests/fixtures/pcornet_sample/diagnosis.csv +5 -0
- omop_duck_db-0.2.0/tests/fixtures/pcornet_sample/encounter.csv +4 -0
- omop_duck_db-0.2.0/tests/fixtures/pcornet_sample/lab_result_cm.csv +4 -0
- omop_duck_db-0.2.0/tests/fixtures/pcornet_sample/prescribing.csv +4 -0
- omop_duck_db-0.2.0/tests/fixtures/pcornet_sample/procedures.csv +4 -0
- omop_duck_db-0.2.0/tests/fixtures/pcornet_sample/provider.csv +3 -0
- omop_duck_db-0.2.0/tests/test_etl_parity.py +122 -0
- omop_duck_db-0.2.0/tests/test_helpers.py +253 -0
- omop_duck_db-0.2.0/tests/test_schema.py +38 -0
- omop_duck_db-0.2.0/tests/testthat/test-etl-pcornet.R +41 -0
- omop_duck_db-0.2.0/tests/testthat/test-load-vocabulary.R +38 -0
- omop_duck_db-0.2.0/tests/testthat/test-mapping-remapping.R +154 -0
- omop_duck_db-0.2.0/tests/testthat/test-schema.R +16 -0
- omop_duck_db-0.2.0/tests/testthat/testdata/tiny_vocab/CONCEPT.csv +3 -0
- omop_duck_db-0.2.0/tests/testthat/testdata/tiny_vocab_bad/CONCEPT.csv +2 -0
- omop_duck_db-0.2.0/tests/testthat.R +4 -0
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
^renv$
|
|
2
|
+
^renv\.lock$
|
|
3
|
+
^\.git$
|
|
4
|
+
^\.github$
|
|
5
|
+
^\.gitignore$
|
|
6
|
+
^\.Rproj\.user$
|
|
7
|
+
^\.claude$
|
|
8
|
+
^tasks$
|
|
9
|
+
^computed_tasks$
|
|
10
|
+
^data$
|
|
11
|
+
^out$
|
|
12
|
+
^python$
|
|
13
|
+
^vocabulary_download_v5_.*$
|
|
14
|
+
(^|/)[^/]*\.duckdb$
|
|
15
|
+
^config$
|
|
16
|
+
^dist$
|
|
17
|
+
^build$
|
|
18
|
+
^.*\.egg-info$
|
|
19
|
+
^pyproject\.toml$
|
|
20
|
+
^requirements.*\.txt$
|
|
21
|
+
^\.venv$
|
|
22
|
+
^\.pytest_cache$
|
|
23
|
+
^__pycache__$
|
|
24
|
+
^\.Rprofile$
|
|
25
|
+
^README\.Rmd$
|
|
26
|
+
^LICENSE\.md$
|
|
27
|
+
^_pkgdown\.yml$
|
|
28
|
+
^docs$
|
|
29
|
+
.dockerignore
|
|
30
|
+
^Dockerfile$
|
|
31
|
+
^scripts$
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
source("renv/activate.R")
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
.git
|
|
2
|
+
.gitignore
|
|
3
|
+
.Rbuildignore
|
|
4
|
+
.Rproj.user
|
|
5
|
+
.vscode
|
|
6
|
+
NEWS.md
|
|
7
|
+
*.log
|
|
8
|
+
*.swp
|
|
9
|
+
*.swo
|
|
10
|
+
*~
|
|
11
|
+
.DS_Store
|
|
12
|
+
revdep/
|
|
13
|
+
pkgdown/
|
|
14
|
+
renv/library
|
|
15
|
+
renv/staging
|
|
16
|
+
|
|
17
|
+
# Project-specific: large generated artifacts and real credentials --
|
|
18
|
+
# never send these to the Docker build context.
|
|
19
|
+
*.duckdb
|
|
20
|
+
*.duckdb.wal
|
|
21
|
+
vocabulary_download_v5_*/
|
|
22
|
+
vocabulary_download_v5_*.zip
|
|
23
|
+
config/*.yml
|
|
24
|
+
!config/*.yml.example
|
|
25
|
+
data/
|
|
26
|
+
out/
|
|
27
|
+
dist/
|
|
28
|
+
build/
|
|
29
|
+
*.egg-info/
|
|
30
|
+
docs/
|
|
31
|
+
.venv/
|
|
32
|
+
__pycache__/
|
|
33
|
+
.pytest_cache/
|
|
34
|
+
tasks/
|
|
35
|
+
computed_tasks/
|
|
36
|
+
.claude/
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
workflow_dispatch:
|
|
8
|
+
|
|
9
|
+
jobs:
|
|
10
|
+
r-check:
|
|
11
|
+
runs-on: ${{ matrix.os }}
|
|
12
|
+
strategy:
|
|
13
|
+
fail-fast: false
|
|
14
|
+
matrix:
|
|
15
|
+
os: [ubuntu-latest, windows-latest]
|
|
16
|
+
env:
|
|
17
|
+
GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
|
|
18
|
+
# Avoid the renv-autoload trap we hit locally/in Docker: this job
|
|
19
|
+
# installs deps directly via r-lib/actions, not through renv::restore().
|
|
20
|
+
RENV_CONFIG_AUTOLOADER_ENABLED: false
|
|
21
|
+
steps:
|
|
22
|
+
- uses: actions/checkout@v4
|
|
23
|
+
|
|
24
|
+
- uses: r-lib/actions/setup-r@v2
|
|
25
|
+
with:
|
|
26
|
+
use-public-rspm: true
|
|
27
|
+
|
|
28
|
+
- uses: r-lib/actions/setup-r-dependencies@v2
|
|
29
|
+
with:
|
|
30
|
+
extra-packages: any::rcmdcheck
|
|
31
|
+
needs: check
|
|
32
|
+
|
|
33
|
+
- uses: r-lib/actions/check-r-package@v2
|
|
34
|
+
with:
|
|
35
|
+
error-on: '"warning"'
|
|
36
|
+
upload-snapshots: true
|
|
37
|
+
|
|
38
|
+
python-test:
|
|
39
|
+
runs-on: ${{ matrix.os }}
|
|
40
|
+
strategy:
|
|
41
|
+
fail-fast: false
|
|
42
|
+
matrix:
|
|
43
|
+
os: [ubuntu-latest, windows-latest]
|
|
44
|
+
python-version: ["3.10", "3.11", "3.12"]
|
|
45
|
+
env:
|
|
46
|
+
RENV_CONFIG_AUTOLOADER_ENABLED: false
|
|
47
|
+
steps:
|
|
48
|
+
- uses: actions/checkout@v4
|
|
49
|
+
|
|
50
|
+
- uses: actions/setup-python@v5
|
|
51
|
+
with:
|
|
52
|
+
python-version: ${{ matrix.python-version }}
|
|
53
|
+
|
|
54
|
+
# R is needed too: tests/test_etl_parity.py shells out to Rscript to
|
|
55
|
+
# run the R ETL and diff it against the Python one.
|
|
56
|
+
- uses: r-lib/actions/setup-r@v2
|
|
57
|
+
with:
|
|
58
|
+
use-public-rspm: true
|
|
59
|
+
|
|
60
|
+
- name: Install R package + its hard dependencies
|
|
61
|
+
run: Rscript -e "install.packages(c('DBI','duckdb','stringr','yaml')); install.packages('.', repos = NULL, type = 'source')"
|
|
62
|
+
|
|
63
|
+
- name: Install Python package (+ dev extras)
|
|
64
|
+
run: pip install -e ".[dev]"
|
|
65
|
+
|
|
66
|
+
- name: Run pytest
|
|
67
|
+
run: pytest tests/ -v
|
|
68
|
+
|
|
69
|
+
python-only:
|
|
70
|
+
# Proves the Python package works standalone with no R installed at all --
|
|
71
|
+
# the real target for this: GPU/deep-learning machines running just the
|
|
72
|
+
# ETL/inference side, no R toolchain. tests/test_etl_parity.py must SKIP
|
|
73
|
+
# here (via its shutil.which("Rscript") is None guard), not fail; if it
|
|
74
|
+
# fails instead, that guard has regressed.
|
|
75
|
+
runs-on: ubuntu-latest
|
|
76
|
+
steps:
|
|
77
|
+
- uses: actions/checkout@v4
|
|
78
|
+
|
|
79
|
+
- uses: actions/setup-python@v5
|
|
80
|
+
with:
|
|
81
|
+
python-version: "3.12"
|
|
82
|
+
|
|
83
|
+
- name: Install Python package (+ dev extras) -- no R anywhere in this job
|
|
84
|
+
run: pip install -e ".[dev]"
|
|
85
|
+
|
|
86
|
+
- name: Run pytest (parity test should skip, not fail)
|
|
87
|
+
run: pytest tests/ -v
|
|
88
|
+
|
|
89
|
+
docker-build:
|
|
90
|
+
# Docker isn't available the same way on windows-latest/macos-latest
|
|
91
|
+
# runners; this validates the containerized path on Linux only.
|
|
92
|
+
runs-on: ubuntu-latest
|
|
93
|
+
steps:
|
|
94
|
+
- uses: actions/checkout@v4
|
|
95
|
+
|
|
96
|
+
- uses: docker/setup-buildx-action@v3
|
|
97
|
+
|
|
98
|
+
- name: Build image (runs both test suites at build time)
|
|
99
|
+
uses: docker/build-push-action@v6
|
|
100
|
+
with:
|
|
101
|
+
context: .
|
|
102
|
+
push: false
|
|
103
|
+
tags: omopduckdb:ci
|
|
104
|
+
cache-from: type=gha
|
|
105
|
+
cache-to: type=gha,mode=max
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
name: pkgdown
|
|
2
|
+
|
|
3
|
+
# Deploys the pkgdown reference site to GitHub Pages on every push to main.
|
|
4
|
+
#
|
|
5
|
+
# One-time manual setup required before this will work:
|
|
6
|
+
# Repo Settings -> Pages -> Build and deployment -> Source: "GitHub Actions"
|
|
7
|
+
# Without that, the deploy job below will fail even if the site builds fine.
|
|
8
|
+
|
|
9
|
+
on:
|
|
10
|
+
push:
|
|
11
|
+
branches: [main]
|
|
12
|
+
workflow_dispatch:
|
|
13
|
+
|
|
14
|
+
permissions:
|
|
15
|
+
contents: read
|
|
16
|
+
pages: write
|
|
17
|
+
id-token: write
|
|
18
|
+
|
|
19
|
+
concurrency:
|
|
20
|
+
group: pkgdown-pages
|
|
21
|
+
cancel-in-progress: true
|
|
22
|
+
|
|
23
|
+
jobs:
|
|
24
|
+
build:
|
|
25
|
+
runs-on: ubuntu-latest
|
|
26
|
+
env:
|
|
27
|
+
RENV_CONFIG_AUTOLOADER_ENABLED: false
|
|
28
|
+
steps:
|
|
29
|
+
- uses: actions/checkout@v4
|
|
30
|
+
|
|
31
|
+
- uses: r-lib/actions/setup-r@v2
|
|
32
|
+
with:
|
|
33
|
+
use-public-rspm: true
|
|
34
|
+
|
|
35
|
+
- uses: r-lib/actions/setup-pandoc@v2
|
|
36
|
+
|
|
37
|
+
- uses: r-lib/actions/setup-r-dependencies@v2
|
|
38
|
+
with:
|
|
39
|
+
extra-packages: any::pkgdown, local::.
|
|
40
|
+
needs: website
|
|
41
|
+
|
|
42
|
+
- name: Build site
|
|
43
|
+
run: pkgdown::build_site_github_pages(new_process = FALSE, install = FALSE)
|
|
44
|
+
shell: Rscript {0}
|
|
45
|
+
|
|
46
|
+
- uses: actions/upload-pages-artifact@v3
|
|
47
|
+
with:
|
|
48
|
+
path: docs
|
|
49
|
+
|
|
50
|
+
deploy:
|
|
51
|
+
needs: build
|
|
52
|
+
runs-on: ubuntu-latest
|
|
53
|
+
environment:
|
|
54
|
+
name: github-pages
|
|
55
|
+
url: ${{ steps.deployment.outputs.page_url }}
|
|
56
|
+
steps:
|
|
57
|
+
- id: deployment
|
|
58
|
+
uses: actions/deploy-pages@v4
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
name: Release (Python -> PyPI)
|
|
2
|
+
|
|
3
|
+
# Builds and checks the Python distribution on every version tag push, then
|
|
4
|
+
# publishes to PyPI -- but only after a human approves it. Nothing here can
|
|
5
|
+
# publish without that manual click; and even then, it needs one-time setup
|
|
6
|
+
# on both PyPI and GitHub before it can succeed at all (see below).
|
|
7
|
+
#
|
|
8
|
+
# One-time setup required before publish-pypi can work:
|
|
9
|
+
# 1. Create the PyPI project (or use "pending publisher" for one that
|
|
10
|
+
# doesn't exist yet: https://pypi.org/manage/account/publishing/).
|
|
11
|
+
# 2. Register this repo + workflow file as a Trusted Publisher on PyPI --
|
|
12
|
+
# no token/secret needed, PyPI verifies GitHub's OIDC token instead.
|
|
13
|
+
# 3. Repo Settings -> Environments -> New environment named
|
|
14
|
+
# "pypi-release" -> add yourself as a required reviewer. That's the
|
|
15
|
+
# approval gate: the publish job pauses here until manually approved,
|
|
16
|
+
# every single time, regardless of how the tag push happened.
|
|
17
|
+
#
|
|
18
|
+
# Until all three exist, build-check still runs and is useful on its own
|
|
19
|
+
# (confirms the tag actually produces a valid, installable distribution);
|
|
20
|
+
# publish-pypi will simply fail or sit waiting for an environment that
|
|
21
|
+
# doesn't exist yet, and can be safely ignored until you're ready.
|
|
22
|
+
|
|
23
|
+
on:
|
|
24
|
+
push:
|
|
25
|
+
tags:
|
|
26
|
+
- "v*"
|
|
27
|
+
workflow_dispatch:
|
|
28
|
+
|
|
29
|
+
jobs:
|
|
30
|
+
build-check:
|
|
31
|
+
runs-on: ubuntu-latest
|
|
32
|
+
steps:
|
|
33
|
+
- uses: actions/checkout@v4
|
|
34
|
+
|
|
35
|
+
- uses: actions/setup-python@v5
|
|
36
|
+
with:
|
|
37
|
+
python-version: "3.12"
|
|
38
|
+
|
|
39
|
+
- run: pip install build twine
|
|
40
|
+
|
|
41
|
+
- run: python -m build
|
|
42
|
+
|
|
43
|
+
- run: twine check dist/*
|
|
44
|
+
|
|
45
|
+
- uses: actions/upload-artifact@v4
|
|
46
|
+
with:
|
|
47
|
+
name: python-dist
|
|
48
|
+
path: dist/
|
|
49
|
+
|
|
50
|
+
publish-pypi:
|
|
51
|
+
needs: build-check
|
|
52
|
+
runs-on: ubuntu-latest
|
|
53
|
+
# This environment is the manual-approval gate -- see setup notes above.
|
|
54
|
+
environment:
|
|
55
|
+
name: pypi-release
|
|
56
|
+
url: https://pypi.org/p/omop-duck-db
|
|
57
|
+
permissions:
|
|
58
|
+
id-token: write # Trusted Publishing (OIDC) -- no PYPI_API_TOKEN secret needed
|
|
59
|
+
steps:
|
|
60
|
+
- uses: actions/download-artifact@v4
|
|
61
|
+
with:
|
|
62
|
+
name: python-dist
|
|
63
|
+
path: dist/
|
|
64
|
+
|
|
65
|
+
- uses: pypa/gh-action-pypi-publish@release/v1
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
name: Test Coverage
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
workflow_dispatch:
|
|
8
|
+
|
|
9
|
+
permissions:
|
|
10
|
+
contents: read
|
|
11
|
+
|
|
12
|
+
jobs:
|
|
13
|
+
r-coverage:
|
|
14
|
+
runs-on: ubuntu-latest
|
|
15
|
+
env:
|
|
16
|
+
GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
|
|
17
|
+
RENV_CONFIG_AUTOLOADER_ENABLED: false
|
|
18
|
+
steps:
|
|
19
|
+
- uses: actions/checkout@v4
|
|
20
|
+
|
|
21
|
+
- uses: r-lib/actions/setup-r@v2
|
|
22
|
+
with:
|
|
23
|
+
use-public-rspm: true
|
|
24
|
+
|
|
25
|
+
- uses: r-lib/actions/setup-r-dependencies@v2
|
|
26
|
+
with:
|
|
27
|
+
extra-packages: any::covr, any::xml2, local::.
|
|
28
|
+
needs: coverage
|
|
29
|
+
|
|
30
|
+
- name: Calculate R Test Coverage
|
|
31
|
+
run: |
|
|
32
|
+
cov <- covr::package_coverage()
|
|
33
|
+
covr::to_cobertura(cov, filename = "r_coverage.xml")
|
|
34
|
+
shell: Rscript {0}
|
|
35
|
+
|
|
36
|
+
- name: Upload R Coverage Report
|
|
37
|
+
uses: actions/upload-artifact@v4
|
|
38
|
+
with:
|
|
39
|
+
name: r-coverage-report
|
|
40
|
+
path: r_coverage.xml
|
|
41
|
+
if-no-files-found: ignore
|
|
42
|
+
|
|
43
|
+
- name: Upload R Coverage to Codecov
|
|
44
|
+
uses: codecov/codecov-action@v4
|
|
45
|
+
with:
|
|
46
|
+
token: ${{ secrets.CODECOV_TOKEN }}
|
|
47
|
+
files: ./r_coverage.xml
|
|
48
|
+
flags: r
|
|
49
|
+
fail_ci_if_error: false
|
|
50
|
+
|
|
51
|
+
python-coverage:
|
|
52
|
+
runs-on: ubuntu-latest
|
|
53
|
+
env:
|
|
54
|
+
RENV_CONFIG_AUTOLOADER_ENABLED: false
|
|
55
|
+
steps:
|
|
56
|
+
- uses: actions/checkout@v4
|
|
57
|
+
|
|
58
|
+
- uses: actions/setup-python@v5
|
|
59
|
+
with:
|
|
60
|
+
python-version: "3.12"
|
|
61
|
+
|
|
62
|
+
- uses: r-lib/actions/setup-r@v2
|
|
63
|
+
with:
|
|
64
|
+
use-public-rspm: true
|
|
65
|
+
|
|
66
|
+
- name: Install R package (required by parity tests)
|
|
67
|
+
run: Rscript -e "install.packages(c('DBI','duckdb','stringr','yaml')); install.packages('.', repos = NULL, type = 'source')"
|
|
68
|
+
|
|
69
|
+
- name: Install Python package with dev dependencies
|
|
70
|
+
run: pip install -e ".[dev]"
|
|
71
|
+
|
|
72
|
+
- name: Run pytest with coverage
|
|
73
|
+
run: pytest tests/ --cov=python/omop_etl --cov-report=xml:py_coverage.xml --cov-report=term
|
|
74
|
+
|
|
75
|
+
- name: Upload Python Coverage Report
|
|
76
|
+
uses: actions/upload-artifact@v4
|
|
77
|
+
with:
|
|
78
|
+
name: python-coverage-report
|
|
79
|
+
path: py_coverage.xml
|
|
80
|
+
if-no-files-found: ignore
|
|
81
|
+
|
|
82
|
+
- name: Upload Python Coverage to Codecov
|
|
83
|
+
uses: codecov/codecov-action@v4
|
|
84
|
+
with:
|
|
85
|
+
token: ${{ secrets.CODECOV_TOKEN }}
|
|
86
|
+
files: ./py_coverage.xml
|
|
87
|
+
flags: python
|
|
88
|
+
fail_ci_if_error: false
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
# Large generated/downloaded artifacts (synced via Google Drive instead, see R/gdrive_sync.R)
|
|
2
|
+
*.duckdb
|
|
3
|
+
*.duckdb.wal
|
|
4
|
+
vocabulary_download_v5_*/
|
|
5
|
+
vocabulary_download_v5_*.zip
|
|
6
|
+
out/
|
|
7
|
+
|
|
8
|
+
# Claude Code / ComputeGate session scratch (not part of the project)
|
|
9
|
+
tasks/
|
|
10
|
+
computed_tasks/
|
|
11
|
+
|
|
12
|
+
# R
|
|
13
|
+
.Rhistory
|
|
14
|
+
.Rdata
|
|
15
|
+
.Rproj.user/
|
|
16
|
+
renv/library/
|
|
17
|
+
renv/local/
|
|
18
|
+
renv/staging/
|
|
19
|
+
README.knit.md
|
|
20
|
+
*_files/
|
|
21
|
+
docs/
|
|
22
|
+
Meta/
|
|
23
|
+
doc/
|
|
24
|
+
/*.tar.gz
|
|
25
|
+
/*.Rcheck/
|
|
26
|
+
|
|
27
|
+
# Python packaging build artifacts
|
|
28
|
+
dist/
|
|
29
|
+
build/
|
|
30
|
+
*.egg-info/
|
|
31
|
+
.pytest_cache/
|
|
32
|
+
|
|
33
|
+
# Python
|
|
34
|
+
__pycache__/
|
|
35
|
+
*.pyc
|
|
36
|
+
.venv/
|
|
37
|
+
venv/
|
|
38
|
+
|
|
39
|
+
# Credentials / tokens - never commit
|
|
40
|
+
.env
|
|
41
|
+
*token*.json
|
|
42
|
+
*credentials*.json
|
|
43
|
+
config/*.yml
|
|
44
|
+
!config/*.yml.example
|
|
45
|
+
|
|
46
|
+
# Raw site data, manifests, and row samples (may contain PHI-adjacent
|
|
47
|
+
# patient-level data) - never commit anything under here
|
|
48
|
+
secrets/
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
Package: omopduckdb
|
|
2
|
+
Title: Build and Populate an OMOP CDM v5.4 Database in DuckDB
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Authors@R: person("J Kyle", "Armstrong", email = "j.kyle.armstrong@gmail.com", role = c("aut", "cre"), comment = c(ORCID = "0000-0002-4269-0121"))
|
|
5
|
+
Description: Tools for building an OMOP Common Data Model (CDM) v5.4 schema
|
|
6
|
+
in DuckDB, loading the OHDSI Athena vocabulary (from a local download or
|
|
7
|
+
live from BigQuery), and mapping PCORnet-format source data into the CDM
|
|
8
|
+
via parallel R and Python ETL implementations that share the same
|
|
9
|
+
concept-mapping SQL. Also includes a Google Drive sync helper for the
|
|
10
|
+
large generated database/vocabulary files.
|
|
11
|
+
License: GPL-3 | MIT + file LICENSE
|
|
12
|
+
URL: https://github.com/jkylearmstrong/omop-duck-db, https://jkylearmstrong.github.io/omop-duck-db
|
|
13
|
+
BugReports: https://github.com/jkylearmstrong/omop-duck-db/issues
|
|
14
|
+
Encoding: UTF-8
|
|
15
|
+
Roxygen: list(markdown = TRUE)
|
|
16
|
+
Depends:
|
|
17
|
+
R (>= 4.1)
|
|
18
|
+
Imports:
|
|
19
|
+
DBI,
|
|
20
|
+
duckdb,
|
|
21
|
+
stringr,
|
|
22
|
+
yaml
|
|
23
|
+
Suggests:
|
|
24
|
+
googledrive,
|
|
25
|
+
bigrquery,
|
|
26
|
+
CommonDataModel,
|
|
27
|
+
covr,
|
|
28
|
+
xml2,
|
|
29
|
+
testthat (>= 3.0.0),
|
|
30
|
+
usethis (>= 3.2.1)
|
|
31
|
+
Config/testthat/edition: 3
|
|
32
|
+
Config/roxygen2/version: 8.0.0
|
|
33
|
+
Remotes:
|
|
34
|
+
jkylearmstrong/usethis
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
# syntax=docker/dockerfile:1.6
|
|
2
|
+
|
|
3
|
+
# Multi-stage build for R package: omopduckdb
|
|
4
|
+
# (also installs and tests the Python package, omop-duck-db, since this repo
|
|
5
|
+
# ships both -- usethis::use_dockerfile() only knows about the R side)
|
|
6
|
+
#
|
|
7
|
+
# NOTE on the repos date: usethis::use_dockerfile() date-pins the Posit
|
|
8
|
+
# Package Manager snapshot to Sys.Date() at generation time. On this machine
|
|
9
|
+
# that produced a *future* date (2026-07-31) that Posit's snapshot service
|
|
10
|
+
# won't have data for yet in the real world -- swapped for the rolling
|
|
11
|
+
# `/latest` alias instead. Worth fixing upstream in the PR: pinning to
|
|
12
|
+
# Sys.Date() is only safe if that's guaranteed to be <= the real current date.
|
|
13
|
+
|
|
14
|
+
# Stage 1: builder
|
|
15
|
+
FROM rocker/r-ver:4.4.2 AS builder
|
|
16
|
+
|
|
17
|
+
# The repo's .Rprofile activates renv on load; renv's own project library is
|
|
18
|
+
# irrelevant here (this image installs straight into /rlib instead), and
|
|
19
|
+
# without this it silently swaps .libPaths() to an empty renv library,
|
|
20
|
+
# hiding everything just installed. Same fix needed locally on Windows.
|
|
21
|
+
ENV RENV_CONFIG_AUTOLOADER_ENABLED=FALSE
|
|
22
|
+
ENV R_LIBS=/rlib
|
|
23
|
+
WORKDIR /build
|
|
24
|
+
|
|
25
|
+
# Install system dependencies (git from the generator; python3 added so this
|
|
26
|
+
# stage can also build the Python package)
|
|
27
|
+
RUN apt-get update && apt-get install -y --no-install-recommends \
|
|
28
|
+
git python3 python3-pip python3-venv libicu-dev \
|
|
29
|
+
&& rm -rf /var/lib/apt/lists/*
|
|
30
|
+
|
|
31
|
+
# Install R packages into a dedicated library
|
|
32
|
+
RUN mkdir -p /rlib && Rscript -e 'install.packages(c("bigrquery", "CommonDataModel", "DBI", "duckdb", "googledrive", "quarto", "stringr", "testthat", "usethis", "yaml"), repos = "https://packagemanager.posit.co/cran/__linux__/noble/latest", lib = "/rlib")'
|
|
33
|
+
|
|
34
|
+
# Build + test both packages in the builder stage, so a failing test fails
|
|
35
|
+
# the image build, not just a later `docker run`.
|
|
36
|
+
COPY . /build
|
|
37
|
+
RUN R_LIBS=/rlib R CMD INSTALL --no-multiarch --with-keep.source --install-tests /build --library=/rlib
|
|
38
|
+
RUN cd /build/tests && R_LIBS=/rlib Rscript testthat.R
|
|
39
|
+
|
|
40
|
+
RUN python3 -m venv /pyvenv \
|
|
41
|
+
&& /pyvenv/bin/pip install --no-cache-dir "/build[dev]" \
|
|
42
|
+
&& R_LIBS=/rlib PATH="/pyvenv/bin:$PATH" /pyvenv/bin/python -m pytest /build/tests -v
|
|
43
|
+
|
|
44
|
+
# Stage 2: final runtime
|
|
45
|
+
FROM rocker/r-ver:4.4.2
|
|
46
|
+
|
|
47
|
+
LABEL org.opencontainers.image.title="omopduckdb"
|
|
48
|
+
|
|
49
|
+
# Create non-root user (skip if already exists)
|
|
50
|
+
RUN id -u ruser >/dev/null 2>&1 || useradd -m -u 1000 ruser
|
|
51
|
+
|
|
52
|
+
# Install runtime system deps
|
|
53
|
+
RUN apt-get update && apt-get install -y --no-install-recommends \
|
|
54
|
+
git python3 python3-venv libicu-dev \
|
|
55
|
+
&& rm -rf /var/lib/apt/lists/*
|
|
56
|
+
|
|
57
|
+
WORKDIR /workspace
|
|
58
|
+
|
|
59
|
+
# Copy R packages and the built Python venv from builder
|
|
60
|
+
COPY --from=builder /rlib /rlib
|
|
61
|
+
COPY --from=builder /pyvenv /pyvenv
|
|
62
|
+
ENV RENV_CONFIG_AUTOLOADER_ENABLED=FALSE
|
|
63
|
+
ENV R_LIBS=/rlib
|
|
64
|
+
ENV PATH="/pyvenv/bin:$PATH"
|
|
65
|
+
|
|
66
|
+
# Copy project source
|
|
67
|
+
COPY --chown=ruser:ruser . .
|
|
68
|
+
|
|
69
|
+
# Ensure working directory is owned by non-root user
|
|
70
|
+
RUN chown -R ruser:ruser /workspace
|
|
71
|
+
|
|
72
|
+
# Switch to non-root user
|
|
73
|
+
USER ruser
|
|
74
|
+
|
|
75
|
+
# Default: R interactive
|
|
76
|
+
CMD ["R", "--no-save"]
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 J Kyle Armstrong
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
|
13
|
+
all copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# Generated by roxygen2: do not edit by hand
|
|
2
|
+
|
|
3
|
+
export(build_condition_era)
|
|
4
|
+
export(build_drug_era)
|
|
5
|
+
export(build_observation_period)
|
|
6
|
+
export(build_schema)
|
|
7
|
+
export(check_vocabulary_version)
|
|
8
|
+
export(etl_pcornet)
|
|
9
|
+
export(export_unmapped_codes)
|
|
10
|
+
export(gdrive_download_latest)
|
|
11
|
+
export(gdrive_upload_snapshot)
|
|
12
|
+
export(import_source_to_concept_map)
|
|
13
|
+
export(import_usagi_mappings)
|
|
14
|
+
export(load_from_bigquery)
|
|
15
|
+
export(load_vocabulary)
|
|
16
|
+
export(remap_all)
|
|
17
|
+
export(remap_cdm_table)
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
# omopduckdb 0.2.0
|
|
2
|
+
|
|
3
|
+
### Architecture & Production Hardening
|
|
4
|
+
- **Vectorized Set-Based Mapping**: Replaced correlated scalar subqueries in concept resolution macros with vectorized hash joins across `concept`, `concept_relationship` (`Maps to`), and `source_to_concept_map`, dramatically improving query execution speed on large extracts.
|
|
5
|
+
- **Deterministic Surrogate Primary Keys**: Eliminated hash collision risks and non-deterministic IDs by migrating all CDM occurrence tables to deterministic `ROW_NUMBER() OVER (...) + MAX(id)` sequence generation.
|
|
6
|
+
- **Date & Datetime Macro Parsing**: Added flexible multi-format date and datetime parsing macros (`parse_omop_date` and `parse_omop_datetime`) supporting `%Y-%m-%d`, `%Y%m%d`, `%d%b%Y`, and `%m/%d/%Y`.
|
|
7
|
+
- **Source Column Resilience**: Implemented automated source view preparation (`prepare_source_view`) handling case-insensitive column names, de-identified ID aliases (`ssid` to `PATID`), and projecting `NULL` for missing optional columns without failing execution.
|
|
8
|
+
- **Mandatory OHDSI Synthesizers**: Added automated generation for required OMOP CDM v5.4 tables: `observation_period` (computed from patient event boundaries), `drug_era` (30-day persistence window collapse), `condition_era` (30-day persistence window collapse), and `cdm_source` (`cdm_version_concept_id = 756265`).
|
|
9
|
+
- **Modular Death Table Handling**: Ingestion of `death.csv` is now optional and skips gracefully with an informative message if omitted, while deduplicating patient death records when provided.
|
|
10
|
+
|
|
11
|
+
### Vocabulary Management & Remapping Helpers
|
|
12
|
+
- **CPT4 Null Concept Name Sanitization**: Athena CPT4 concepts with null names are automatically replaced with `'CPT4 ' || concept_code` upon ingestion to prevent downstream failures.
|
|
13
|
+
- **Vocabulary Version Inspector**: Added `check_vocabulary_version()` in both R and Python to query vocabulary metadata and verify Athena release dates.
|
|
14
|
+
- **Usagi & Custom Mapping Tools**: Added `import_source_to_concept_map()` and `import_usagi_mappings()` to easily load custom crosswalks directly into `source_to_concept_map`, and `export_unmapped_codes()` to audit unmapped source codes into CSV format ready for OHDSI Usagi review.
|
|
15
|
+
- **In-Place Retroactive Remapping**: Added `remap_cdm_table()` and `remap_all()` supporting dry-run previews and transactional in-place re-mapping of existing CDM tables when vocabularies are updated, with automatic era synthesizer regeneration.
|
|
16
|
+
|
|
17
|
+
### Dual-Language Parity
|
|
18
|
+
- Verified 100% byte-for-byte parity across R and Python ETL implementations for all 12 CDM tables in `tests/test_etl_parity.py`.
|
|
19
|
+
|
|
20
|
+
# omopduckdb 0.1.1
|
|
21
|
+
|
|
22
|
+
- Added unified cross-language test coverage tracking (`covr` for R, `pytest-cov` for Python) and a dedicated `.github/workflows/test-coverage.yml` CI workflow.
|
|
23
|
+
- Created consolidated developer release scripts (`scripts/release.py` and `scripts/release.R`) for automated dual-package version bumping, Quarto rendering, test suite execution, wheel building, and git tagging.
|
|
24
|
+
- Enhanced Docker container configuration for multi-stage testing and deployment.
|
|
25
|
+
|
|
26
|
+
# omopduckdb 0.1.0
|
|
27
|
+
|
|
28
|
+
Initial packaged release.
|
|
29
|
+
|
|
30
|
+
- Build an OMOP CDM v5.4 schema in DuckDB from a shared, checked-in DDL.
|
|
31
|
+
- Load and refresh the OHDSI Athena vocabulary, transactionally per table.
|
|
32
|
+
- Map PCORnet-format source data into the CDM via parallel R and Python ETL
|
|
33
|
+
implementations that share the same concept-mapping SQL, verified to
|
|
34
|
+
produce identical output against a shared test fixture.
|
|
35
|
+
- Load OMOP-shaped tables directly from BigQuery (vocab refresh or demo data),
|
|
36
|
+
as an alternative to a local vocabulary download.
|
|
37
|
+
- Sync the generated database and vocabulary archive through Google Drive
|
|
38
|
+
instead of committing large binaries to git.
|
|
39
|
+
- Packaged as both an installable R package (`DESCRIPTION`/`NAMESPACE`,
|
|
40
|
+
`testthat` suite, `pkgdown` site) and a Python package (`pyproject.toml`,
|
|
41
|
+
`pytest` suite).
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# Notes
|
|
2
|
+
|
|
3
|
+
Internal working notes on where this package came from and where it's headed. See `README.Rmd`/`README.md` for the technical package overview.
|
|
4
|
+
|
|
5
|
+
## Where it came from
|
|
6
|
+
|
|
7
|
+
`omop-duck-db` (R package `omopduckdb`) builds and populates an OMOP CDM v5.4 database in DuckDB, loads the OHDSI Athena vocabulary, and maps PCORnet-format source data into the CDM — with parallel R and Python ETL implementations sharing the same schema and concept-mapping SQL, so either language gets the same result. It's deliberately the public/generic piece of a bigger initiative: no institution names, no site specifics, no protected data, ever — real-world use against production data happens entirely outside this repo, against your own infrastructure (see `kyle_out_zip/omop_bridge/` for that private, site-aware layer).
|
|
8
|
+
|
|
9
|
+
## Key decisions (and why)
|
|
10
|
+
|
|
11
|
+
- **License: dual `GPL-3 | MIT`**, changed from plain MIT on 2026-08-05 to match the rest of the public track (`TempleCBE`, `pslongSim`, `ML-PScore`, `quarto_temple_brand`). Still local-only, not pushed.
|
|
12
|
+
- **Schema is generated, never hand-declared.** `inst/extdata/5.4/duckdb/*.sql` (from OHDSI's `CommonDataModel` R package) is the single source of truth for both ETL paths — this is why `kyle_out_zip`'s bridging script calls this package's real entrypoint instead of writing new mapping SQL; a hand-rolled alternative (`kyle_out_zip/omop_duckdb_gpu_server_integration_plan.md`) diverged from this and would have produced a broken database (wrong ID scheme, wrong glob paths, invented columns).
|
|
13
|
+
|
|
14
|
+
## Current status (2026-08-05)
|
|
15
|
+
|
|
16
|
+
Verified directly against source this session, not assumed:
|
|
17
|
+
- The Python ETL (`build_omop_cdm.py`) has **no `VITAL` → `MEASUREMENT` mapping** yet — only `demographic`, `provider`, `encounter`, `diagnosis`, `procedures`, `lab_result_cm`, `prescribing` are handled.
|
|
18
|
+
- `load_vocabulary()` is **R-only** — no Python equivalent exists yet.
|
|
19
|
+
- PCORnet source file/column expectations were cross-checked against `tests/fixtures/pcornet_sample/` and confirmed to match what `kyle_out_zip`'s real per-site extracts actually look like (modulo one `ssid`→`PATID` header rename).
|
|
20
|
+
|
|
21
|
+
## Where it's heading
|
|
22
|
+
|
|
23
|
+
- Close the two gaps above (`VITAL` mapping, Python-side vocabulary loader) — needed before `kyle_out_zip`'s eDerri pipeline can capture vitals data or run entirely in Python without an R fallback.
|
|
24
|
+
- Commit + push the license change.
|
|
25
|
+
- Once `kyle_out_zip/omop_bridge/` runs successfully against real site data, OHDSI's DataQualityDashboard is the next milestone (tracked in `tasks/Big_Picture.txt`), still not wired up here.
|