omop-duck-db 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. omop_duck_db-0.2.0/.Rbuildignore +31 -0
  2. omop_duck_db-0.2.0/.Rprofile +1 -0
  3. omop_duck_db-0.2.0/.dockerignore +36 -0
  4. omop_duck_db-0.2.0/.github/workflows/ci.yml +105 -0
  5. omop_duck_db-0.2.0/.github/workflows/pkgdown.yml +58 -0
  6. omop_duck_db-0.2.0/.github/workflows/release.yml +65 -0
  7. omop_duck_db-0.2.0/.github/workflows/test-coverage.yml +88 -0
  8. omop_duck_db-0.2.0/.gitignore +48 -0
  9. omop_duck_db-0.2.0/DESCRIPTION +34 -0
  10. omop_duck_db-0.2.0/Dockerfile +76 -0
  11. omop_duck_db-0.2.0/LICENSE +2 -0
  12. omop_duck_db-0.2.0/LICENSE.md +21 -0
  13. omop_duck_db-0.2.0/NAMESPACE +17 -0
  14. omop_duck_db-0.2.0/NEWS.md +41 -0
  15. omop_duck_db-0.2.0/NOTES.md +25 -0
  16. omop_duck_db-0.2.0/PKG-INFO +256 -0
  17. omop_duck_db-0.2.0/R/build_schema.R +42 -0
  18. omop_duck_db-0.2.0/R/etl_pcornet.R +758 -0
  19. omop_duck_db-0.2.0/R/gdrive_sync.R +63 -0
  20. omop_duck_db-0.2.0/R/load_from_bigquery.R +77 -0
  21. omop_duck_db-0.2.0/R/load_vocabulary.R +171 -0
  22. omop_duck_db-0.2.0/R/mapping.R +170 -0
  23. omop_duck_db-0.2.0/R/omopduckdb-package.R +6 -0
  24. omop_duck_db-0.2.0/R/remapping.R +209 -0
  25. omop_duck_db-0.2.0/R/utils.R +97 -0
  26. omop_duck_db-0.2.0/README.Rmd +184 -0
  27. omop_duck_db-0.2.0/README.md +212 -0
  28. omop_duck_db-0.2.0/README.qmd +22 -0
  29. omop_duck_db-0.2.0/_pkgdown.yml +43 -0
  30. omop_duck_db-0.2.0/config/bigquery.yml.example +9 -0
  31. omop_duck_db-0.2.0/config/gdrive.yml.example +6 -0
  32. omop_duck_db-0.2.0/inst/extdata/5.4/duckdb/OMOPCDM_duckdb_5.4_constraints.sql +177 -0
  33. omop_duck_db-0.2.0/inst/extdata/5.4/duckdb/OMOPCDM_duckdb_5.4_ddl.sql +511 -0
  34. omop_duck_db-0.2.0/inst/extdata/5.4/duckdb/OMOPCDM_duckdb_5.4_indices.sql +98 -0
  35. omop_duck_db-0.2.0/inst/extdata/5.4/duckdb/OMOPCDM_duckdb_5.4_primary_keys.sql +29 -0
  36. omop_duck_db-0.2.0/inst/scripts/build_schema.R +14 -0
  37. omop_duck_db-0.2.0/inst/scripts/debug_output.txt +52 -0
  38. omop_duck_db-0.2.0/inst/scripts/debug_query.R +30 -0
  39. omop_duck_db-0.2.0/inst/scripts/etl_pcornet.R +18 -0
  40. omop_duck_db-0.2.0/inst/scripts/gdrive_sync.R +15 -0
  41. omop_duck_db-0.2.0/inst/scripts/load_from_bigquery.R +24 -0
  42. omop_duck_db-0.2.0/inst/scripts/load_vocabulary.R +19 -0
  43. omop_duck_db-0.2.0/inst/scripts/query.r +132 -0
  44. omop_duck_db-0.2.0/inst/sql/mapping_macros.sql +75 -0
  45. omop_duck_db-0.2.0/man/build_condition_era.Rd +14 -0
  46. omop_duck_db-0.2.0/man/build_drug_era.Rd +14 -0
  47. omop_duck_db-0.2.0/man/build_observation_period.Rd +14 -0
  48. omop_duck_db-0.2.0/man/build_schema.Rd +25 -0
  49. omop_duck_db-0.2.0/man/check_vocabulary_version.Rd +17 -0
  50. omop_duck_db-0.2.0/man/etl_pcornet.Rd +25 -0
  51. omop_duck_db-0.2.0/man/export_unmapped_codes.Rd +23 -0
  52. omop_duck_db-0.2.0/man/gdrive_download_latest.Rd +17 -0
  53. omop_duck_db-0.2.0/man/gdrive_upload_snapshot.Rd +17 -0
  54. omop_duck_db-0.2.0/man/import_source_to_concept_map.Rd +21 -0
  55. omop_duck_db-0.2.0/man/import_usagi_mappings.Rd +28 -0
  56. omop_duck_db-0.2.0/man/load_from_bigquery.Rd +51 -0
  57. omop_duck_db-0.2.0/man/load_vocabulary.Rd +41 -0
  58. omop_duck_db-0.2.0/man/omopduckdb-package.Rd +29 -0
  59. omop_duck_db-0.2.0/man/remap_all.Rd +21 -0
  60. omop_duck_db-0.2.0/man/remap_cdm_table.Rd +21 -0
  61. omop_duck_db-0.2.0/pyproject.toml +44 -0
  62. omop_duck_db-0.2.0/python/omop_etl/__init__.py +55 -0
  63. omop_duck_db-0.2.0/python/omop_etl/build_omop_cdm.py +905 -0
  64. omop_duck_db-0.2.0/python/omop_etl/mapping.py +171 -0
  65. omop_duck_db-0.2.0/python/omop_etl/remapping.py +208 -0
  66. omop_duck_db-0.2.0/python/omop_etl/vocabulary.py +180 -0
  67. omop_duck_db-0.2.0/python/requirements.txt +3 -0
  68. omop_duck_db-0.2.0/renv/.gitignore +7 -0
  69. omop_duck_db-0.2.0/renv/activate.R +1438 -0
  70. omop_duck_db-0.2.0/renv/settings.json +21 -0
  71. omop_duck_db-0.2.0/renv.lock +3185 -0
  72. omop_duck_db-0.2.0/scripts/release.R +59 -0
  73. omop_duck_db-0.2.0/scripts/release.py +160 -0
  74. omop_duck_db-0.2.0/tests/fixtures/pcornet_sample/demographic.csv +4 -0
  75. omop_duck_db-0.2.0/tests/fixtures/pcornet_sample/diagnosis.csv +5 -0
  76. omop_duck_db-0.2.0/tests/fixtures/pcornet_sample/encounter.csv +4 -0
  77. omop_duck_db-0.2.0/tests/fixtures/pcornet_sample/lab_result_cm.csv +4 -0
  78. omop_duck_db-0.2.0/tests/fixtures/pcornet_sample/prescribing.csv +4 -0
  79. omop_duck_db-0.2.0/tests/fixtures/pcornet_sample/procedures.csv +4 -0
  80. omop_duck_db-0.2.0/tests/fixtures/pcornet_sample/provider.csv +3 -0
  81. omop_duck_db-0.2.0/tests/test_etl_parity.py +122 -0
  82. omop_duck_db-0.2.0/tests/test_helpers.py +253 -0
  83. omop_duck_db-0.2.0/tests/test_schema.py +38 -0
  84. omop_duck_db-0.2.0/tests/testthat/test-etl-pcornet.R +41 -0
  85. omop_duck_db-0.2.0/tests/testthat/test-load-vocabulary.R +38 -0
  86. omop_duck_db-0.2.0/tests/testthat/test-mapping-remapping.R +154 -0
  87. omop_duck_db-0.2.0/tests/testthat/test-schema.R +16 -0
  88. omop_duck_db-0.2.0/tests/testthat/testdata/tiny_vocab/CONCEPT.csv +3 -0
  89. omop_duck_db-0.2.0/tests/testthat/testdata/tiny_vocab_bad/CONCEPT.csv +2 -0
  90. omop_duck_db-0.2.0/tests/testthat.R +4 -0
@@ -0,0 +1,31 @@
1
+ ^renv$
2
+ ^renv\.lock$
3
+ ^\.git$
4
+ ^\.github$
5
+ ^\.gitignore$
6
+ ^\.Rproj\.user$
7
+ ^\.claude$
8
+ ^tasks$
9
+ ^computed_tasks$
10
+ ^data$
11
+ ^out$
12
+ ^python$
13
+ ^vocabulary_download_v5_.*$
14
+ (^|/)[^/]*\.duckdb$
15
+ ^config$
16
+ ^dist$
17
+ ^build$
18
+ ^.*\.egg-info$
19
+ ^pyproject\.toml$
20
+ ^requirements.*\.txt$
21
+ ^\.venv$
22
+ ^\.pytest_cache$
23
+ ^__pycache__$
24
+ ^\.Rprofile$
25
+ ^README\.Rmd$
26
+ ^LICENSE\.md$
27
+ ^_pkgdown\.yml$
28
+ ^docs$
29
+ .dockerignore
30
+ ^Dockerfile$
31
+ ^scripts$
@@ -0,0 +1 @@
1
+ source("renv/activate.R")
@@ -0,0 +1,36 @@
1
+ .git
2
+ .gitignore
3
+ .Rbuildignore
4
+ .Rproj.user
5
+ .vscode
6
+ NEWS.md
7
+ *.log
8
+ *.swp
9
+ *.swo
10
+ *~
11
+ .DS_Store
12
+ revdep/
13
+ pkgdown/
14
+ renv/library
15
+ renv/staging
16
+
17
+ # Project-specific: large generated artifacts and real credentials --
18
+ # never send these to the Docker build context.
19
+ *.duckdb
20
+ *.duckdb.wal
21
+ vocabulary_download_v5_*/
22
+ vocabulary_download_v5_*.zip
23
+ config/*.yml
24
+ !config/*.yml.example
25
+ data/
26
+ out/
27
+ dist/
28
+ build/
29
+ *.egg-info/
30
+ docs/
31
+ .venv/
32
+ __pycache__/
33
+ .pytest_cache/
34
+ tasks/
35
+ computed_tasks/
36
+ .claude/
@@ -0,0 +1,105 @@
1
+ name: CI
2
+
3
+ on:
4
+ push:
5
+ branches: [main]
6
+ pull_request:
7
+ workflow_dispatch:
8
+
9
+ jobs:
10
+ r-check:
11
+ runs-on: ${{ matrix.os }}
12
+ strategy:
13
+ fail-fast: false
14
+ matrix:
15
+ os: [ubuntu-latest, windows-latest]
16
+ env:
17
+ GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
18
+ # Avoid the renv-autoload trap we hit locally/in Docker: this job
19
+ # installs deps directly via r-lib/actions, not through renv::restore().
20
+ RENV_CONFIG_AUTOLOADER_ENABLED: false
21
+ steps:
22
+ - uses: actions/checkout@v4
23
+
24
+ - uses: r-lib/actions/setup-r@v2
25
+ with:
26
+ use-public-rspm: true
27
+
28
+ - uses: r-lib/actions/setup-r-dependencies@v2
29
+ with:
30
+ extra-packages: any::rcmdcheck
31
+ needs: check
32
+
33
+ - uses: r-lib/actions/check-r-package@v2
34
+ with:
35
+ error-on: '"warning"'
36
+ upload-snapshots: true
37
+
38
+ python-test:
39
+ runs-on: ${{ matrix.os }}
40
+ strategy:
41
+ fail-fast: false
42
+ matrix:
43
+ os: [ubuntu-latest, windows-latest]
44
+ python-version: ["3.10", "3.11", "3.12"]
45
+ env:
46
+ RENV_CONFIG_AUTOLOADER_ENABLED: false
47
+ steps:
48
+ - uses: actions/checkout@v4
49
+
50
+ - uses: actions/setup-python@v5
51
+ with:
52
+ python-version: ${{ matrix.python-version }}
53
+
54
+ # R is needed too: tests/test_etl_parity.py shells out to Rscript to
55
+ # run the R ETL and diff it against the Python one.
56
+ - uses: r-lib/actions/setup-r@v2
57
+ with:
58
+ use-public-rspm: true
59
+
60
+ - name: Install R package + its hard dependencies
61
+ run: Rscript -e "install.packages(c('DBI','duckdb','stringr','yaml')); install.packages('.', repos = NULL, type = 'source')"
62
+
63
+ - name: Install Python package (+ dev extras)
64
+ run: pip install -e ".[dev]"
65
+
66
+ - name: Run pytest
67
+ run: pytest tests/ -v
68
+
69
+ python-only:
70
+ # Proves the Python package works standalone with no R installed at all --
71
+ # the real target for this: GPU/deep-learning machines running just the
72
+ # ETL/inference side, no R toolchain. tests/test_etl_parity.py must SKIP
73
+ # here (via its shutil.which("Rscript") is None guard), not fail; if it
74
+ # fails instead, that guard has regressed.
75
+ runs-on: ubuntu-latest
76
+ steps:
77
+ - uses: actions/checkout@v4
78
+
79
+ - uses: actions/setup-python@v5
80
+ with:
81
+ python-version: "3.12"
82
+
83
+ - name: Install Python package (+ dev extras) -- no R anywhere in this job
84
+ run: pip install -e ".[dev]"
85
+
86
+ - name: Run pytest (parity test should skip, not fail)
87
+ run: pytest tests/ -v
88
+
89
+ docker-build:
90
+ # Docker isn't available the same way on windows-latest/macos-latest
91
+ # runners; this validates the containerized path on Linux only.
92
+ runs-on: ubuntu-latest
93
+ steps:
94
+ - uses: actions/checkout@v4
95
+
96
+ - uses: docker/setup-buildx-action@v3
97
+
98
+ - name: Build image (runs both test suites at build time)
99
+ uses: docker/build-push-action@v6
100
+ with:
101
+ context: .
102
+ push: false
103
+ tags: omopduckdb:ci
104
+ cache-from: type=gha
105
+ cache-to: type=gha,mode=max
@@ -0,0 +1,58 @@
1
+ name: pkgdown
2
+
3
+ # Deploys the pkgdown reference site to GitHub Pages on every push to main.
4
+ #
5
+ # One-time manual setup required before this will work:
6
+ # Repo Settings -> Pages -> Build and deployment -> Source: "GitHub Actions"
7
+ # Without that, the deploy job below will fail even if the site builds fine.
8
+
9
+ on:
10
+ push:
11
+ branches: [main]
12
+ workflow_dispatch:
13
+
14
+ permissions:
15
+ contents: read
16
+ pages: write
17
+ id-token: write
18
+
19
+ concurrency:
20
+ group: pkgdown-pages
21
+ cancel-in-progress: true
22
+
23
+ jobs:
24
+ build:
25
+ runs-on: ubuntu-latest
26
+ env:
27
+ RENV_CONFIG_AUTOLOADER_ENABLED: false
28
+ steps:
29
+ - uses: actions/checkout@v4
30
+
31
+ - uses: r-lib/actions/setup-r@v2
32
+ with:
33
+ use-public-rspm: true
34
+
35
+ - uses: r-lib/actions/setup-pandoc@v2
36
+
37
+ - uses: r-lib/actions/setup-r-dependencies@v2
38
+ with:
39
+ extra-packages: any::pkgdown, local::.
40
+ needs: website
41
+
42
+ - name: Build site
43
+ run: pkgdown::build_site_github_pages(new_process = FALSE, install = FALSE)
44
+ shell: Rscript {0}
45
+
46
+ - uses: actions/upload-pages-artifact@v3
47
+ with:
48
+ path: docs
49
+
50
+ deploy:
51
+ needs: build
52
+ runs-on: ubuntu-latest
53
+ environment:
54
+ name: github-pages
55
+ url: ${{ steps.deployment.outputs.page_url }}
56
+ steps:
57
+ - id: deployment
58
+ uses: actions/deploy-pages@v4
@@ -0,0 +1,65 @@
1
+ name: Release (Python -> PyPI)
2
+
3
+ # Builds and checks the Python distribution on every version tag push, then
4
+ # publishes to PyPI -- but only after a human approves it. Nothing here can
5
+ # publish without that manual click; and even then, it needs one-time setup
6
+ # on both PyPI and GitHub before it can succeed at all (see below).
7
+ #
8
+ # One-time setup required before publish-pypi can work:
9
+ # 1. Create the PyPI project (or use "pending publisher" for one that
10
+ # doesn't exist yet: https://pypi.org/manage/account/publishing/).
11
+ # 2. Register this repo + workflow file as a Trusted Publisher on PyPI --
12
+ # no token/secret needed, PyPI verifies GitHub's OIDC token instead.
13
+ # 3. Repo Settings -> Environments -> New environment named
14
+ # "pypi-release" -> add yourself as a required reviewer. That's the
15
+ # approval gate: the publish job pauses here until manually approved,
16
+ # every single time, regardless of how the tag push happened.
17
+ #
18
+ # Until all three exist, build-check still runs and is useful on its own
19
+ # (confirms the tag actually produces a valid, installable distribution);
20
+ # publish-pypi will simply fail or sit waiting for an environment that
21
+ # doesn't exist yet, and can be safely ignored until you're ready.
22
+
23
+ on:
24
+ push:
25
+ tags:
26
+ - "v*"
27
+ workflow_dispatch:
28
+
29
+ jobs:
30
+ build-check:
31
+ runs-on: ubuntu-latest
32
+ steps:
33
+ - uses: actions/checkout@v4
34
+
35
+ - uses: actions/setup-python@v5
36
+ with:
37
+ python-version: "3.12"
38
+
39
+ - run: pip install build twine
40
+
41
+ - run: python -m build
42
+
43
+ - run: twine check dist/*
44
+
45
+ - uses: actions/upload-artifact@v4
46
+ with:
47
+ name: python-dist
48
+ path: dist/
49
+
50
+ publish-pypi:
51
+ needs: build-check
52
+ runs-on: ubuntu-latest
53
+ # This environment is the manual-approval gate -- see setup notes above.
54
+ environment:
55
+ name: pypi-release
56
+ url: https://pypi.org/p/omop-duck-db
57
+ permissions:
58
+ id-token: write # Trusted Publishing (OIDC) -- no PYPI_API_TOKEN secret needed
59
+ steps:
60
+ - uses: actions/download-artifact@v4
61
+ with:
62
+ name: python-dist
63
+ path: dist/
64
+
65
+ - uses: pypa/gh-action-pypi-publish@release/v1
@@ -0,0 +1,88 @@
1
+ name: Test Coverage
2
+
3
+ on:
4
+ push:
5
+ branches: [main]
6
+ pull_request:
7
+ workflow_dispatch:
8
+
9
+ permissions:
10
+ contents: read
11
+
12
+ jobs:
13
+ r-coverage:
14
+ runs-on: ubuntu-latest
15
+ env:
16
+ GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
17
+ RENV_CONFIG_AUTOLOADER_ENABLED: false
18
+ steps:
19
+ - uses: actions/checkout@v4
20
+
21
+ - uses: r-lib/actions/setup-r@v2
22
+ with:
23
+ use-public-rspm: true
24
+
25
+ - uses: r-lib/actions/setup-r-dependencies@v2
26
+ with:
27
+ extra-packages: any::covr, any::xml2, local::.
28
+ needs: coverage
29
+
30
+ - name: Calculate R Test Coverage
31
+ run: |
32
+ cov <- covr::package_coverage()
33
+ covr::to_cobertura(cov, filename = "r_coverage.xml")
34
+ shell: Rscript {0}
35
+
36
+ - name: Upload R Coverage Report
37
+ uses: actions/upload-artifact@v4
38
+ with:
39
+ name: r-coverage-report
40
+ path: r_coverage.xml
41
+ if-no-files-found: ignore
42
+
43
+ - name: Upload R Coverage to Codecov
44
+ uses: codecov/codecov-action@v4
45
+ with:
46
+ token: ${{ secrets.CODECOV_TOKEN }}
47
+ files: ./r_coverage.xml
48
+ flags: r
49
+ fail_ci_if_error: false
50
+
51
+ python-coverage:
52
+ runs-on: ubuntu-latest
53
+ env:
54
+ RENV_CONFIG_AUTOLOADER_ENABLED: false
55
+ steps:
56
+ - uses: actions/checkout@v4
57
+
58
+ - uses: actions/setup-python@v5
59
+ with:
60
+ python-version: "3.12"
61
+
62
+ - uses: r-lib/actions/setup-r@v2
63
+ with:
64
+ use-public-rspm: true
65
+
66
+ - name: Install R package (required by parity tests)
67
+ run: Rscript -e "install.packages(c('DBI','duckdb','stringr','yaml')); install.packages('.', repos = NULL, type = 'source')"
68
+
69
+ - name: Install Python package with dev dependencies
70
+ run: pip install -e ".[dev]"
71
+
72
+ - name: Run pytest with coverage
73
+ run: pytest tests/ --cov=python/omop_etl --cov-report=xml:py_coverage.xml --cov-report=term
74
+
75
+ - name: Upload Python Coverage Report
76
+ uses: actions/upload-artifact@v4
77
+ with:
78
+ name: python-coverage-report
79
+ path: py_coverage.xml
80
+ if-no-files-found: ignore
81
+
82
+ - name: Upload Python Coverage to Codecov
83
+ uses: codecov/codecov-action@v4
84
+ with:
85
+ token: ${{ secrets.CODECOV_TOKEN }}
86
+ files: ./py_coverage.xml
87
+ flags: python
88
+ fail_ci_if_error: false
@@ -0,0 +1,48 @@
1
+ # Large generated/downloaded artifacts (synced via Google Drive instead, see R/gdrive_sync.R)
2
+ *.duckdb
3
+ *.duckdb.wal
4
+ vocabulary_download_v5_*/
5
+ vocabulary_download_v5_*.zip
6
+ out/
7
+
8
+ # Claude Code / ComputeGate session scratch (not part of the project)
9
+ tasks/
10
+ computed_tasks/
11
+
12
+ # R
13
+ .Rhistory
14
+ .Rdata
15
+ .Rproj.user/
16
+ renv/library/
17
+ renv/local/
18
+ renv/staging/
19
+ README.knit.md
20
+ *_files/
21
+ docs/
22
+ Meta/
23
+ doc/
24
+ /*.tar.gz
25
+ /*.Rcheck/
26
+
27
+ # Python packaging build artifacts
28
+ dist/
29
+ build/
30
+ *.egg-info/
31
+ .pytest_cache/
32
+
33
+ # Python
34
+ __pycache__/
35
+ *.pyc
36
+ .venv/
37
+ venv/
38
+
39
+ # Credentials / tokens - never commit
40
+ .env
41
+ *token*.json
42
+ *credentials*.json
43
+ config/*.yml
44
+ !config/*.yml.example
45
+
46
+ # Raw site data, manifests, and row samples (may contain PHI-adjacent
47
+ # patient-level data) - never commit anything under here
48
+ secrets/
@@ -0,0 +1,34 @@
1
+ Package: omopduckdb
2
+ Title: Build and Populate an OMOP CDM v5.4 Database in DuckDB
3
+ Version: 0.2.0
4
+ Authors@R: person("J Kyle", "Armstrong", email = "j.kyle.armstrong@gmail.com", role = c("aut", "cre"), comment = c(ORCID = "0000-0002-4269-0121"))
5
+ Description: Tools for building an OMOP Common Data Model (CDM) v5.4 schema
6
+ in DuckDB, loading the OHDSI Athena vocabulary (from a local download or
7
+ live from BigQuery), and mapping PCORnet-format source data into the CDM
8
+ via parallel R and Python ETL implementations that share the same
9
+ concept-mapping SQL. Also includes a Google Drive sync helper for the
10
+ large generated database/vocabulary files.
11
+ License: GPL-3 | MIT + file LICENSE
12
+ URL: https://github.com/jkylearmstrong/omop-duck-db, https://jkylearmstrong.github.io/omop-duck-db
13
+ BugReports: https://github.com/jkylearmstrong/omop-duck-db/issues
14
+ Encoding: UTF-8
15
+ Roxygen: list(markdown = TRUE)
16
+ Depends:
17
+ R (>= 4.1)
18
+ Imports:
19
+ DBI,
20
+ duckdb,
21
+ stringr,
22
+ yaml
23
+ Suggests:
24
+ googledrive,
25
+ bigrquery,
26
+ CommonDataModel,
27
+ covr,
28
+ xml2,
29
+ testthat (>= 3.0.0),
30
+ usethis (>= 3.2.1)
31
+ Config/testthat/edition: 3
32
+ Config/roxygen2/version: 8.0.0
33
+ Remotes:
34
+ jkylearmstrong/usethis
@@ -0,0 +1,76 @@
1
+ # syntax=docker/dockerfile:1.6
2
+
3
+ # Multi-stage build for R package: omopduckdb
4
+ # (also installs and tests the Python package, omop-duck-db, since this repo
5
+ # ships both -- usethis::use_dockerfile() only knows about the R side)
6
+ #
7
+ # NOTE on the repos date: usethis::use_dockerfile() date-pins the Posit
8
+ # Package Manager snapshot to Sys.Date() at generation time. On this machine
9
+ # that produced a *future* date (2026-07-31) that Posit's snapshot service
10
+ # won't have data for yet in the real world -- swapped for the rolling
11
+ # `/latest` alias instead. Worth fixing upstream in the PR: pinning to
12
+ # Sys.Date() is only safe if that's guaranteed to be <= the real current date.
13
+
14
+ # Stage 1: builder
15
+ FROM rocker/r-ver:4.4.2 AS builder
16
+
17
+ # The repo's .Rprofile activates renv on load; renv's own project library is
18
+ # irrelevant here (this image installs straight into /rlib instead), and
19
+ # without this it silently swaps .libPaths() to an empty renv library,
20
+ # hiding everything just installed. Same fix needed locally on Windows.
21
+ ENV RENV_CONFIG_AUTOLOADER_ENABLED=FALSE
22
+ ENV R_LIBS=/rlib
23
+ WORKDIR /build
24
+
25
+ # Install system dependencies (git from the generator; python3 added so this
26
+ # stage can also build the Python package)
27
+ RUN apt-get update && apt-get install -y --no-install-recommends \
28
+ git python3 python3-pip python3-venv libicu-dev \
29
+ && rm -rf /var/lib/apt/lists/*
30
+
31
+ # Install R packages into a dedicated library
32
+ RUN mkdir -p /rlib && Rscript -e 'install.packages(c("bigrquery", "CommonDataModel", "DBI", "duckdb", "googledrive", "quarto", "stringr", "testthat", "usethis", "yaml"), repos = "https://packagemanager.posit.co/cran/__linux__/noble/latest", lib = "/rlib")'
33
+
34
+ # Build + test both packages in the builder stage, so a failing test fails
35
+ # the image build, not just a later `docker run`.
36
+ COPY . /build
37
+ RUN R_LIBS=/rlib R CMD INSTALL --no-multiarch --with-keep.source --install-tests /build --library=/rlib
38
+ RUN cd /build/tests && R_LIBS=/rlib Rscript testthat.R
39
+
40
+ RUN python3 -m venv /pyvenv \
41
+ && /pyvenv/bin/pip install --no-cache-dir "/build[dev]" \
42
+ && R_LIBS=/rlib PATH="/pyvenv/bin:$PATH" /pyvenv/bin/python -m pytest /build/tests -v
43
+
44
+ # Stage 2: final runtime
45
+ FROM rocker/r-ver:4.4.2
46
+
47
+ LABEL org.opencontainers.image.title="omopduckdb"
48
+
49
+ # Create non-root user (skip if already exists)
50
+ RUN id -u ruser >/dev/null 2>&1 || useradd -m -u 1000 ruser
51
+
52
+ # Install runtime system deps
53
+ RUN apt-get update && apt-get install -y --no-install-recommends \
54
+ git python3 python3-venv libicu-dev \
55
+ && rm -rf /var/lib/apt/lists/*
56
+
57
+ WORKDIR /workspace
58
+
59
+ # Copy R packages and the built Python venv from builder
60
+ COPY --from=builder /rlib /rlib
61
+ COPY --from=builder /pyvenv /pyvenv
62
+ ENV RENV_CONFIG_AUTOLOADER_ENABLED=FALSE
63
+ ENV R_LIBS=/rlib
64
+ ENV PATH="/pyvenv/bin:$PATH"
65
+
66
+ # Copy project source
67
+ COPY --chown=ruser:ruser . .
68
+
69
+ # Ensure working directory is owned by non-root user
70
+ RUN chown -R ruser:ruser /workspace
71
+
72
+ # Switch to non-root user
73
+ USER ruser
74
+
75
+ # Default: R interactive
76
+ CMD ["R", "--no-save"]
@@ -0,0 +1,2 @@
1
+ YEAR: 2026
2
+ COPYRIGHT HOLDER: J Kyle Armstrong
@@ -0,0 +1,21 @@
1
+ # MIT License
2
+
3
+ Copyright (c) 2026 J Kyle Armstrong
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in
13
+ all copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,17 @@
1
+ # Generated by roxygen2: do not edit by hand
2
+
3
+ export(build_condition_era)
4
+ export(build_drug_era)
5
+ export(build_observation_period)
6
+ export(build_schema)
7
+ export(check_vocabulary_version)
8
+ export(etl_pcornet)
9
+ export(export_unmapped_codes)
10
+ export(gdrive_download_latest)
11
+ export(gdrive_upload_snapshot)
12
+ export(import_source_to_concept_map)
13
+ export(import_usagi_mappings)
14
+ export(load_from_bigquery)
15
+ export(load_vocabulary)
16
+ export(remap_all)
17
+ export(remap_cdm_table)
@@ -0,0 +1,41 @@
1
+ # omopduckdb 0.2.0
2
+
3
+ ### Architecture & Production Hardening
4
+ - **Vectorized Set-Based Mapping**: Replaced correlated scalar subqueries in concept resolution macros with vectorized hash joins across `concept`, `concept_relationship` (`Maps to`), and `source_to_concept_map`, dramatically improving query execution speed on large extracts.
5
+ - **Deterministic Surrogate Primary Keys**: Eliminated hash collision risks and non-deterministic IDs by migrating all CDM occurrence tables to deterministic `ROW_NUMBER() OVER (...) + MAX(id)` sequence generation.
6
+ - **Date & Datetime Macro Parsing**: Added flexible multi-format date and datetime parsing macros (`parse_omop_date` and `parse_omop_datetime`) supporting `%Y-%m-%d`, `%Y%m%d`, `%d%b%Y`, and `%m/%d/%Y`.
7
+ - **Source Column Resilience**: Implemented automated source view preparation (`prepare_source_view`) handling case-insensitive column names, de-identified ID aliases (`ssid` to `PATID`), and projecting `NULL` for missing optional columns without failing execution.
8
+ - **Mandatory OHDSI Synthesizers**: Added automated generation for required OMOP CDM v5.4 tables: `observation_period` (computed from patient event boundaries), `drug_era` (30-day persistence window collapse), `condition_era` (30-day persistence window collapse), and `cdm_source` (`cdm_version_concept_id = 756265`).
9
+ - **Modular Death Table Handling**: Ingestion of `death.csv` is now optional and skips gracefully with an informative message if omitted, while deduplicating patient death records when provided.
10
+
11
+ ### Vocabulary Management & Remapping Helpers
12
+ - **CPT4 Null Concept Name Sanitization**: Athena CPT4 concepts with null names are automatically replaced with `'CPT4 ' || concept_code` upon ingestion to prevent downstream failures.
13
+ - **Vocabulary Version Inspector**: Added `check_vocabulary_version()` in both R and Python to query vocabulary metadata and verify Athena release dates.
14
+ - **Usagi & Custom Mapping Tools**: Added `import_source_to_concept_map()` and `import_usagi_mappings()` to easily load custom crosswalks directly into `source_to_concept_map`, and `export_unmapped_codes()` to audit unmapped source codes into CSV format ready for OHDSI Usagi review.
15
+ - **In-Place Retroactive Remapping**: Added `remap_cdm_table()` and `remap_all()` supporting dry-run previews and transactional in-place re-mapping of existing CDM tables when vocabularies are updated, with automatic era synthesizer regeneration.
16
+
17
+ ### Dual-Language Parity
18
+ - Verified 100% byte-for-byte parity across R and Python ETL implementations for all 12 CDM tables in `tests/test_etl_parity.py`.
19
+
20
+ # omopduckdb 0.1.1
21
+
22
+ - Added unified cross-language test coverage tracking (`covr` for R, `pytest-cov` for Python) and a dedicated `.github/workflows/test-coverage.yml` CI workflow.
23
+ - Created consolidated developer release scripts (`scripts/release.py` and `scripts/release.R`) for automated dual-package version bumping, Quarto rendering, test suite execution, wheel building, and git tagging.
24
+ - Enhanced Docker container configuration for multi-stage testing and deployment.
25
+
26
+ # omopduckdb 0.1.0
27
+
28
+ Initial packaged release.
29
+
30
+ - Build an OMOP CDM v5.4 schema in DuckDB from a shared, checked-in DDL.
31
+ - Load and refresh the OHDSI Athena vocabulary, transactionally per table.
32
+ - Map PCORnet-format source data into the CDM via parallel R and Python ETL
33
+ implementations that share the same concept-mapping SQL, verified to
34
+ produce identical output against a shared test fixture.
35
+ - Load OMOP-shaped tables directly from BigQuery (vocab refresh or demo data),
36
+ as an alternative to a local vocabulary download.
37
+ - Sync the generated database and vocabulary archive through Google Drive
38
+ instead of committing large binaries to git.
39
+ - Packaged as both an installable R package (`DESCRIPTION`/`NAMESPACE`,
40
+ `testthat` suite, `pkgdown` site) and a Python package (`pyproject.toml`,
41
+ `pytest` suite).
@@ -0,0 +1,25 @@
1
+ # Notes
2
+
3
+ Internal working notes on where this package came from and where it's headed. See `README.Rmd`/`README.md` for the technical package overview.
4
+
5
+ ## Where it came from
6
+
7
+ `omop-duck-db` (R package `omopduckdb`) builds and populates an OMOP CDM v5.4 database in DuckDB, loads the OHDSI Athena vocabulary, and maps PCORnet-format source data into the CDM — with parallel R and Python ETL implementations sharing the same schema and concept-mapping SQL, so either language gets the same result. It's deliberately the public/generic piece of a bigger initiative: no institution names, no site specifics, no protected data, ever — real-world use against production data happens entirely outside this repo, against your own infrastructure (see `kyle_out_zip/omop_bridge/` for that private, site-aware layer).
8
+
9
+ ## Key decisions (and why)
10
+
11
+ - **License: dual `GPL-3 | MIT`**, changed from plain MIT on 2026-08-05 to match the rest of the public track (`TempleCBE`, `pslongSim`, `ML-PScore`, `quarto_temple_brand`). Still local-only, not pushed.
12
+ - **Schema is generated, never hand-declared.** `inst/extdata/5.4/duckdb/*.sql` (from OHDSI's `CommonDataModel` R package) is the single source of truth for both ETL paths — this is why `kyle_out_zip`'s bridging script calls this package's real entrypoint instead of writing new mapping SQL; a hand-rolled alternative (`kyle_out_zip/omop_duckdb_gpu_server_integration_plan.md`) diverged from this and would have produced a broken database (wrong ID scheme, wrong glob paths, invented columns).
13
+
14
+ ## Current status (2026-08-05)
15
+
16
+ Verified directly against source this session, not assumed:
17
+ - The Python ETL (`build_omop_cdm.py`) has **no `VITAL` → `MEASUREMENT` mapping** yet — only `demographic`, `provider`, `encounter`, `diagnosis`, `procedures`, `lab_result_cm`, `prescribing` are handled.
18
+ - `load_vocabulary()` is **R-only** — no Python equivalent exists yet.
19
+ - PCORnet source file/column expectations were cross-checked against `tests/fixtures/pcornet_sample/` and confirmed to match what `kyle_out_zip`'s real per-site extracts actually look like (modulo one `ssid`→`PATID` header rename).
20
+
21
+ ## Where it's heading
22
+
23
+ - Close the two gaps above (`VITAL` mapping, Python-side vocabulary loader) — needed before `kyle_out_zip`'s eDerri pipeline can capture vitals data or run entirely in Python without an R fallback.
24
+ - Commit + push the license change.
25
+ - Once `kyle_out_zip/omop_bridge/` runs successfully against real site data, OHDSI's DataQualityDashboard is the next milestone (tracked in `tasks/Big_Picture.txt`), still not wired up here.