INSPIRE-MEDS 0.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- inspire_meds-0.0.1/.github/workflows/code-quality-main.yaml +31 -0
- inspire_meds-0.0.1/.github/workflows/code-quality-pr.yaml +45 -0
- inspire_meds-0.0.1/.github/workflows/python-build.yaml +99 -0
- inspire_meds-0.0.1/.github/workflows/tests.yaml +48 -0
- inspire_meds-0.0.1/.gitignore +162 -0
- inspire_meds-0.0.1/.pre-commit-config.yaml +137 -0
- inspire_meds-0.0.1/CONTRIBUTING.md +175 -0
- inspire_meds-0.0.1/LICENSE +21 -0
- inspire_meds-0.0.1/PKG-INFO +83 -0
- inspire_meds-0.0.1/README.md +45 -0
- inspire_meds-0.0.1/pyproject.toml +42 -0
- inspire_meds-0.0.1/setup.cfg +4 -0
- inspire_meds-0.0.1/src/INSPIRE_MEDS/__init__.py +38 -0
- inspire_meds-0.0.1/src/INSPIRE_MEDS/__main__.py +90 -0
- inspire_meds-0.0.1/src/INSPIRE_MEDS/commands.py +79 -0
- inspire_meds-0.0.1/src/INSPIRE_MEDS/configs/ETL.yaml +45 -0
- inspire_meds-0.0.1/src/INSPIRE_MEDS/configs/__init__.py +0 -0
- inspire_meds-0.0.1/src/INSPIRE_MEDS/configs/event_configs.yaml +456 -0
- inspire_meds-0.0.1/src/INSPIRE_MEDS/configs/main.yaml +25 -0
- inspire_meds-0.0.1/src/INSPIRE_MEDS/configs/pre_MEDS.yaml +16 -0
- inspire_meds-0.0.1/src/INSPIRE_MEDS/configs/runner.yaml +7 -0
- inspire_meds-0.0.1/src/INSPIRE_MEDS/configs/table_preprocessors.yaml +197 -0
- inspire_meds-0.0.1/src/INSPIRE_MEDS/dataset.yaml +13 -0
- inspire_meds-0.0.1/src/INSPIRE_MEDS/download.py +301 -0
- inspire_meds-0.0.1/src/INSPIRE_MEDS/pre_MEDS.py +258 -0
- inspire_meds-0.0.1/src/INSPIRE_MEDS.egg-info/PKG-INFO +83 -0
- inspire_meds-0.0.1/src/INSPIRE_MEDS.egg-info/SOURCES.txt +32 -0
- inspire_meds-0.0.1/src/INSPIRE_MEDS.egg-info/dependency_links.txt +1 -0
- inspire_meds-0.0.1/src/INSPIRE_MEDS.egg-info/entry_points.txt +2 -0
- inspire_meds-0.0.1/src/INSPIRE_MEDS.egg-info/requires.txt +29 -0
- inspire_meds-0.0.1/src/INSPIRE_MEDS.egg-info/top_level.txt +1 -0
- inspire_meds-0.0.1/static/pypi_trusted_publisher_example.png +0 -0
- inspire_meds-0.0.1/tests/e2e_demo_test.py +56 -0
- inspire_meds-0.0.1/tests/operations_synthetic.csv +1 -0
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
# Same as `code-quality-pr.yaml` but triggered on commit to main branch
|
|
2
|
+
# and runs on all files (instead of only the changed ones)
|
|
3
|
+
|
|
4
|
+
name: Code Quality Main
|
|
5
|
+
|
|
6
|
+
on:
|
|
7
|
+
push:
|
|
8
|
+
branches: [main]
|
|
9
|
+
|
|
10
|
+
jobs:
|
|
11
|
+
code-quality:
|
|
12
|
+
runs-on: ubuntu-latest
|
|
13
|
+
|
|
14
|
+
strategy:
|
|
15
|
+
matrix:
|
|
16
|
+
python-version: ["3.12"]
|
|
17
|
+
|
|
18
|
+
steps:
|
|
19
|
+
- uses: actions/checkout@v4
|
|
20
|
+
|
|
21
|
+
- name: Set up Python ${{ matrix.python-version }}
|
|
22
|
+
uses: actions/setup-python@v5
|
|
23
|
+
with:
|
|
24
|
+
python-version: ${{ matrix.python-version }}
|
|
25
|
+
|
|
26
|
+
- name: Install packages
|
|
27
|
+
run: |
|
|
28
|
+
pip install .[dev]
|
|
29
|
+
|
|
30
|
+
- name: Run pre-commits
|
|
31
|
+
uses: pre-commit/action@v3.0.1
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# This workflow finds which files were changed, prints them,
|
|
2
|
+
# and runs `pre-commit` on those files.
|
|
3
|
+
|
|
4
|
+
# Inspired by the sktime library:
|
|
5
|
+
# https://github.com/alan-turing-institute/sktime/blob/main/.github/workflows/test.yml
|
|
6
|
+
|
|
7
|
+
name: Code Quality PR
|
|
8
|
+
|
|
9
|
+
on:
|
|
10
|
+
pull_request:
|
|
11
|
+
branches: [main, "release/*", "dev"]
|
|
12
|
+
|
|
13
|
+
jobs:
|
|
14
|
+
code-quality:
|
|
15
|
+
runs-on: ubuntu-latest
|
|
16
|
+
|
|
17
|
+
strategy:
|
|
18
|
+
matrix:
|
|
19
|
+
python-version: ["3.12"]
|
|
20
|
+
|
|
21
|
+
steps:
|
|
22
|
+
- uses: actions/checkout@v4
|
|
23
|
+
|
|
24
|
+
- name: Set up Python ${{ matrix.python-version }}
|
|
25
|
+
uses: actions/setup-python@v5
|
|
26
|
+
with:
|
|
27
|
+
python-version: ${{ matrix.python-version }}
|
|
28
|
+
|
|
29
|
+
- name: Install packages
|
|
30
|
+
run: |
|
|
31
|
+
pip install .[dev]
|
|
32
|
+
|
|
33
|
+
- name: Find modified files
|
|
34
|
+
id: file_changes
|
|
35
|
+
uses: trilom/file-changes-action@v1.2.4
|
|
36
|
+
with:
|
|
37
|
+
output: " "
|
|
38
|
+
|
|
39
|
+
- name: List modified files
|
|
40
|
+
run: echo '${{ steps.file_changes.outputs.files}}'
|
|
41
|
+
|
|
42
|
+
- name: Run pre-commits
|
|
43
|
+
uses: pre-commit/action@v3.0.1
|
|
44
|
+
with:
|
|
45
|
+
extra_args: --files ${{ steps.file_changes.outputs.files}}
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
name: Publish Python 🐍 distribution 📦 to PyPI and TestPyPI
|
|
2
|
+
|
|
3
|
+
on: push
|
|
4
|
+
|
|
5
|
+
jobs:
|
|
6
|
+
build:
|
|
7
|
+
name: Build distribution 📦
|
|
8
|
+
runs-on: ubuntu-latest
|
|
9
|
+
|
|
10
|
+
strategy:
|
|
11
|
+
matrix:
|
|
12
|
+
python-version: ["3.12"]
|
|
13
|
+
|
|
14
|
+
steps:
|
|
15
|
+
- uses: actions/checkout@v4
|
|
16
|
+
- name: Set up Python ${{ matrix.python-version }}
|
|
17
|
+
uses: actions/setup-python@v5
|
|
18
|
+
with:
|
|
19
|
+
python-version: ${{ matrix.python-version }}
|
|
20
|
+
- name: Install pypa/build
|
|
21
|
+
run: >-
|
|
22
|
+
python3 -m
|
|
23
|
+
pip install
|
|
24
|
+
build
|
|
25
|
+
--user
|
|
26
|
+
- name: Build a binary wheel and a source tarball
|
|
27
|
+
run: python3 -m build
|
|
28
|
+
- name: Store the distribution packages
|
|
29
|
+
uses: actions/upload-artifact@v4
|
|
30
|
+
with:
|
|
31
|
+
name: python-package-distributions
|
|
32
|
+
path: dist/
|
|
33
|
+
|
|
34
|
+
publish-to-pypi:
|
|
35
|
+
name: >-
|
|
36
|
+
Publish Python 🐍 distribution 📦 to PyPI
|
|
37
|
+
if: startsWith(github.ref, 'refs/tags/') # only publish to PyPI on tag pushes
|
|
38
|
+
needs:
|
|
39
|
+
- build
|
|
40
|
+
runs-on: ubuntu-latest
|
|
41
|
+
environment:
|
|
42
|
+
name: pypi
|
|
43
|
+
url: https://pypi.org/p/MEDS-transforms # Replace <package-name> with your PyPI project name
|
|
44
|
+
permissions:
|
|
45
|
+
id-token: write # IMPORTANT: mandatory for trusted publishing
|
|
46
|
+
|
|
47
|
+
steps:
|
|
48
|
+
- name: Download all the dists
|
|
49
|
+
uses: actions/download-artifact@v4
|
|
50
|
+
with:
|
|
51
|
+
name: python-package-distributions
|
|
52
|
+
path: dist/
|
|
53
|
+
|
|
54
|
+
- name: Publish distribution 📦 to PyPI
|
|
55
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
56
|
+
|
|
57
|
+
github-release:
|
|
58
|
+
name: >-
|
|
59
|
+
Sign the Python 🐍 distribution 📦 with Sigstore
|
|
60
|
+
and upload them to GitHub Release
|
|
61
|
+
needs:
|
|
62
|
+
- publish-to-pypi
|
|
63
|
+
runs-on: ubuntu-latest
|
|
64
|
+
|
|
65
|
+
permissions:
|
|
66
|
+
contents: write # IMPORTANT: mandatory for making GitHub Releases
|
|
67
|
+
id-token: write # IMPORTANT: mandatory for sigstore
|
|
68
|
+
|
|
69
|
+
steps:
|
|
70
|
+
- name: Download all the dists
|
|
71
|
+
uses: actions/download-artifact@v4
|
|
72
|
+
with:
|
|
73
|
+
name: python-package-distributions
|
|
74
|
+
path: dist/
|
|
75
|
+
|
|
76
|
+
- name: Sign the dists with Sigstore
|
|
77
|
+
uses: sigstore/gh-action-sigstore-python@v3.0.0
|
|
78
|
+
with:
|
|
79
|
+
inputs: >-
|
|
80
|
+
./dist/*.tar.gz
|
|
81
|
+
./dist/*.whl
|
|
82
|
+
- name: Create GitHub Release
|
|
83
|
+
env:
|
|
84
|
+
GITHUB_TOKEN: ${{ github.token }}
|
|
85
|
+
run: >-
|
|
86
|
+
gh release create
|
|
87
|
+
'${{ github.ref_name }}'
|
|
88
|
+
--repo '${{ github.repository }}'
|
|
89
|
+
--notes ""
|
|
90
|
+
- name: Upload artifact signatures to GitHub Release
|
|
91
|
+
env:
|
|
92
|
+
GITHUB_TOKEN: ${{ github.token }}
|
|
93
|
+
# Upload to GitHub Release using the `gh` CLI.
|
|
94
|
+
# `dist/` contains the built packages, and the
|
|
95
|
+
# sigstore-produced signatures and certificates.
|
|
96
|
+
run: >-
|
|
97
|
+
gh release upload
|
|
98
|
+
'${{ github.ref_name }}' dist/**
|
|
99
|
+
--repo '${{ github.repository }}'
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
name: Tests
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
branches: [main, "release/*", "dev"]
|
|
8
|
+
|
|
9
|
+
jobs:
|
|
10
|
+
run_tests_ubuntu:
|
|
11
|
+
runs-on: ubuntu-latest
|
|
12
|
+
|
|
13
|
+
strategy:
|
|
14
|
+
matrix:
|
|
15
|
+
python-version: ["3.11", "3.12"]
|
|
16
|
+
fail-fast: false
|
|
17
|
+
|
|
18
|
+
timeout-minutes: 30
|
|
19
|
+
|
|
20
|
+
steps:
|
|
21
|
+
- name: Checkout
|
|
22
|
+
uses: actions/checkout@v4
|
|
23
|
+
|
|
24
|
+
- name: Set up Python ${{ matrix.python-version }}
|
|
25
|
+
uses: actions/setup-python@v5
|
|
26
|
+
with:
|
|
27
|
+
python-version: ${{ matrix.python-version }}
|
|
28
|
+
|
|
29
|
+
- name: Install packages
|
|
30
|
+
run: |
|
|
31
|
+
pip install -e .[tests]
|
|
32
|
+
|
|
33
|
+
#----------------------------------------------
|
|
34
|
+
# run test suite
|
|
35
|
+
#----------------------------------------------
|
|
36
|
+
- name: Run tests
|
|
37
|
+
run: |
|
|
38
|
+
pytest src/ tests/ -v --doctest-modules --cov=src --junitxml=junit.xml -s
|
|
39
|
+
|
|
40
|
+
- name: Upload coverage to Codecov
|
|
41
|
+
uses: codecov/codecov-action@v4.0.1
|
|
42
|
+
with:
|
|
43
|
+
token: ${{ secrets.CODECOV_TOKEN }}
|
|
44
|
+
- name: Upload test results to Codecov
|
|
45
|
+
if: ${{ !cancelled() }}
|
|
46
|
+
uses: codecov/test-results-action@v1
|
|
47
|
+
with:
|
|
48
|
+
token: ${{ secrets.CODECOV_TOKEN }}
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
# Byte-compiled / optimized / DLL files
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*$py.class
|
|
5
|
+
|
|
6
|
+
# C extensions
|
|
7
|
+
*.so
|
|
8
|
+
|
|
9
|
+
# Distribution / packaging
|
|
10
|
+
.Python
|
|
11
|
+
build/
|
|
12
|
+
develop-eggs/
|
|
13
|
+
dist/
|
|
14
|
+
downloads/
|
|
15
|
+
eggs/
|
|
16
|
+
.eggs/
|
|
17
|
+
lib/
|
|
18
|
+
lib64/
|
|
19
|
+
parts/
|
|
20
|
+
sdist/
|
|
21
|
+
var/
|
|
22
|
+
wheels/
|
|
23
|
+
share/python-wheels/
|
|
24
|
+
*.egg-info/
|
|
25
|
+
.installed.cfg
|
|
26
|
+
*.egg
|
|
27
|
+
MANIFEST
|
|
28
|
+
|
|
29
|
+
# PyInstaller
|
|
30
|
+
# Usually these files are written by a python script from a template
|
|
31
|
+
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
|
32
|
+
*.manifest
|
|
33
|
+
*.spec
|
|
34
|
+
|
|
35
|
+
# Installer logs
|
|
36
|
+
pip-log.txt
|
|
37
|
+
pip-delete-this-directory.txt
|
|
38
|
+
|
|
39
|
+
# Unit test / coverage reports
|
|
40
|
+
htmlcov/
|
|
41
|
+
.tox/
|
|
42
|
+
.nox/
|
|
43
|
+
.coverage
|
|
44
|
+
.coverage.*
|
|
45
|
+
.cache
|
|
46
|
+
nosetests.xml
|
|
47
|
+
coverage.xml
|
|
48
|
+
*.cover
|
|
49
|
+
*.py,cover
|
|
50
|
+
.hypothesis/
|
|
51
|
+
.pytest_cache/
|
|
52
|
+
cover/
|
|
53
|
+
|
|
54
|
+
# Translations
|
|
55
|
+
*.mo
|
|
56
|
+
*.pot
|
|
57
|
+
|
|
58
|
+
# Django stuff:
|
|
59
|
+
*.log
|
|
60
|
+
local_settings.py
|
|
61
|
+
db.sqlite3
|
|
62
|
+
db.sqlite3-journal
|
|
63
|
+
|
|
64
|
+
# Flask stuff:
|
|
65
|
+
instance/
|
|
66
|
+
.webassets-cache
|
|
67
|
+
|
|
68
|
+
# Scrapy stuff:
|
|
69
|
+
.scrapy
|
|
70
|
+
|
|
71
|
+
# Sphinx documentation
|
|
72
|
+
docs/_build/
|
|
73
|
+
|
|
74
|
+
# PyBuilder
|
|
75
|
+
.pybuilder/
|
|
76
|
+
target/
|
|
77
|
+
|
|
78
|
+
# Jupyter Notebook
|
|
79
|
+
.ipynb_checkpoints
|
|
80
|
+
|
|
81
|
+
# IPython
|
|
82
|
+
profile_default/
|
|
83
|
+
ipython_config.py
|
|
84
|
+
|
|
85
|
+
# pyenv
|
|
86
|
+
# For a library or package, you might want to ignore these files since the code is
|
|
87
|
+
# intended to run in multiple environments; otherwise, check them in:
|
|
88
|
+
# .python-version
|
|
89
|
+
|
|
90
|
+
# pipenv
|
|
91
|
+
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
|
92
|
+
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
|
93
|
+
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
|
94
|
+
# install all needed dependencies.
|
|
95
|
+
#Pipfile.lock
|
|
96
|
+
|
|
97
|
+
# poetry
|
|
98
|
+
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
|
|
99
|
+
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
|
100
|
+
# commonly ignored for libraries.
|
|
101
|
+
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
|
|
102
|
+
#poetry.lock
|
|
103
|
+
|
|
104
|
+
# pdm
|
|
105
|
+
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
|
|
106
|
+
#pdm.lock
|
|
107
|
+
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
|
|
108
|
+
# in version control.
|
|
109
|
+
# https://pdm.fming.dev/latest/usage/project/#working-with-version-control
|
|
110
|
+
.pdm.toml
|
|
111
|
+
.pdm-python
|
|
112
|
+
.pdm-build/
|
|
113
|
+
|
|
114
|
+
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
|
115
|
+
__pypackages__/
|
|
116
|
+
|
|
117
|
+
# Celery stuff
|
|
118
|
+
celerybeat-schedule
|
|
119
|
+
celerybeat.pid
|
|
120
|
+
|
|
121
|
+
# SageMath parsed files
|
|
122
|
+
*.sage.py
|
|
123
|
+
|
|
124
|
+
# Environments
|
|
125
|
+
.env
|
|
126
|
+
.venv
|
|
127
|
+
env/
|
|
128
|
+
venv/
|
|
129
|
+
ENV/
|
|
130
|
+
env.bak/
|
|
131
|
+
venv.bak/
|
|
132
|
+
|
|
133
|
+
# Spyder project settings
|
|
134
|
+
.spyderproject
|
|
135
|
+
.spyproject
|
|
136
|
+
|
|
137
|
+
# Rope project settings
|
|
138
|
+
.ropeproject
|
|
139
|
+
|
|
140
|
+
# mkdocs documentation
|
|
141
|
+
/site
|
|
142
|
+
|
|
143
|
+
# mypy
|
|
144
|
+
.mypy_cache/
|
|
145
|
+
.dmypy.json
|
|
146
|
+
dmypy.json
|
|
147
|
+
|
|
148
|
+
# Pyre type checker
|
|
149
|
+
.pyre/
|
|
150
|
+
|
|
151
|
+
# pytype static type analyzer
|
|
152
|
+
.pytype/
|
|
153
|
+
|
|
154
|
+
# Cython debug symbols
|
|
155
|
+
cython_debug/
|
|
156
|
+
|
|
157
|
+
# PyCharm
|
|
158
|
+
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
|
|
159
|
+
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
|
160
|
+
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
|
161
|
+
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
|
162
|
+
#.idea/
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
default_language_version:
|
|
2
|
+
python: python3.12
|
|
3
|
+
|
|
4
|
+
#exclude:
|
|
5
|
+
|
|
6
|
+
repos:
|
|
7
|
+
- repo: https://github.com/pre-commit/pre-commit-hooks
|
|
8
|
+
rev: v5.0.0
|
|
9
|
+
hooks:
|
|
10
|
+
# list of supported hooks: https://pre-commit.com/hooks.html
|
|
11
|
+
- id: trailing-whitespace
|
|
12
|
+
- id: end-of-file-fixer
|
|
13
|
+
- id: check-docstring-first
|
|
14
|
+
- id: check-yaml
|
|
15
|
+
- id: debug-statements
|
|
16
|
+
- id: detect-private-key
|
|
17
|
+
- id: check-executables-have-shebangs
|
|
18
|
+
- id: check-toml
|
|
19
|
+
- id: check-case-conflict
|
|
20
|
+
- id: check-added-large-files
|
|
21
|
+
args: [--maxkb, "800"]
|
|
22
|
+
|
|
23
|
+
# python code formatting
|
|
24
|
+
- repo: https://github.com/psf/black
|
|
25
|
+
rev: 24.10.0
|
|
26
|
+
hooks:
|
|
27
|
+
- id: black
|
|
28
|
+
args: [--line-length, "110"]
|
|
29
|
+
|
|
30
|
+
# python import sorting
|
|
31
|
+
- repo: https://github.com/PyCQA/isort
|
|
32
|
+
rev: 5.13.2
|
|
33
|
+
hooks:
|
|
34
|
+
- id: isort
|
|
35
|
+
args: ["--profile", "black", "--filter-files", "-o", "wandb"]
|
|
36
|
+
|
|
37
|
+
- repo: https://github.com/PyCQA/autoflake
|
|
38
|
+
rev: v2.3.1
|
|
39
|
+
hooks:
|
|
40
|
+
- id: autoflake
|
|
41
|
+
args: [--in-place, --remove-all-unused-imports]
|
|
42
|
+
|
|
43
|
+
# python upgrading syntax to newer version
|
|
44
|
+
- repo: https://github.com/asottile/pyupgrade
|
|
45
|
+
rev: v3.19.0
|
|
46
|
+
hooks:
|
|
47
|
+
- id: pyupgrade
|
|
48
|
+
args: [--py311-plus]
|
|
49
|
+
|
|
50
|
+
# python docstring formatting
|
|
51
|
+
# - repo: https://github.com/myint/docformatter
|
|
52
|
+
# rev: v1.7.5
|
|
53
|
+
# hooks:
|
|
54
|
+
# - id: docformatter
|
|
55
|
+
# name: docformatter
|
|
56
|
+
# entry: docformatter
|
|
57
|
+
# language: python
|
|
58
|
+
# args: [--in-place, --wrap-summaries=110, --wrap-descriptions=110]
|
|
59
|
+
# files: \.py$
|
|
60
|
+
|
|
61
|
+
# python check (PEP8), programming errors and code complexity
|
|
62
|
+
- repo: https://github.com/PyCQA/flake8
|
|
63
|
+
rev: 7.1.1
|
|
64
|
+
hooks:
|
|
65
|
+
- id: flake8
|
|
66
|
+
args:
|
|
67
|
+
[
|
|
68
|
+
"--max-complexity=10",
|
|
69
|
+
"--extend-ignore",
|
|
70
|
+
"E402,E701,E251,E226,E302,W504,E704,E402,E401,C901,E203",
|
|
71
|
+
"--max-line-length=110",
|
|
72
|
+
"--exclude",
|
|
73
|
+
"logs/*,data/*",
|
|
74
|
+
"--per-file-ignores",
|
|
75
|
+
"__init__.py:F401",
|
|
76
|
+
]
|
|
77
|
+
|
|
78
|
+
# yaml formatting
|
|
79
|
+
- repo: https://github.com/pre-commit/mirrors-prettier
|
|
80
|
+
rev: v4.0.0-alpha.8
|
|
81
|
+
hooks:
|
|
82
|
+
- id: prettier
|
|
83
|
+
types: [yaml]
|
|
84
|
+
exclude: "environment.yaml"
|
|
85
|
+
|
|
86
|
+
# shell scripts linter
|
|
87
|
+
- repo: https://github.com/shellcheck-py/shellcheck-py
|
|
88
|
+
rev: v0.10.0.1
|
|
89
|
+
hooks:
|
|
90
|
+
- id: shellcheck
|
|
91
|
+
|
|
92
|
+
# md formatting
|
|
93
|
+
- repo: https://github.com/executablebooks/mdformat
|
|
94
|
+
rev: 0.7.18
|
|
95
|
+
hooks:
|
|
96
|
+
- id: mdformat
|
|
97
|
+
args: ["--number"]
|
|
98
|
+
additional_dependencies:
|
|
99
|
+
- mdformat-gfm
|
|
100
|
+
- mdformat-tables
|
|
101
|
+
- mdformat_frontmatter
|
|
102
|
+
- mdformat-black
|
|
103
|
+
- mdformat-config
|
|
104
|
+
- mdformat-shfmt
|
|
105
|
+
- mdformat-mkdocs
|
|
106
|
+
- mdformat-toc
|
|
107
|
+
- mdformat-admon
|
|
108
|
+
|
|
109
|
+
# word spelling linter
|
|
110
|
+
- repo: https://github.com/codespell-project/codespell
|
|
111
|
+
rev: v2.3.0
|
|
112
|
+
hooks:
|
|
113
|
+
- id: codespell
|
|
114
|
+
args:
|
|
115
|
+
- --skip=logs/**,data/**,*.ipynb,*.bib,env.yml,env_cpu.yml,*.svg,poetry.lock
|
|
116
|
+
- --ignore-words-list=ehr,crate
|
|
117
|
+
|
|
118
|
+
# jupyter notebook cell output clearing
|
|
119
|
+
- repo: https://github.com/kynan/nbstripout
|
|
120
|
+
rev: 0.7.1
|
|
121
|
+
hooks:
|
|
122
|
+
- id: nbstripout
|
|
123
|
+
|
|
124
|
+
# jupyter notebook linting
|
|
125
|
+
- repo: https://github.com/nbQA-dev/nbQA
|
|
126
|
+
rev: 1.8.7
|
|
127
|
+
hooks:
|
|
128
|
+
- id: nbqa-black
|
|
129
|
+
args: ["--line-length=110"]
|
|
130
|
+
- id: nbqa-isort
|
|
131
|
+
args: ["--profile=black"]
|
|
132
|
+
- id: nbqa-flake8
|
|
133
|
+
args:
|
|
134
|
+
[
|
|
135
|
+
"--extend-ignore=E203,E402,E501,F401,F841",
|
|
136
|
+
"--exclude=logs/*,data/*",
|
|
137
|
+
]
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
A template repository for a MEDS-Transforms powered extraction pipeline for a custom dataset. Once you have
|
|
2
|
+
customized the repository to your dataset (see instructions below), you will be able to run your extraction
|
|
3
|
+
pipeline with a few simple command-line commands, such as:
|
|
4
|
+
|
|
5
|
+
```bash
|
|
6
|
+
pip install PACKAGE_NAME # you can do this locally or via PyPI
|
|
7
|
+
# Download your data or set download credentials
|
|
8
|
+
COMMAND_NAME root_output_dir=$ROOT_OUTPUT_DIR
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
See the [MIMIC-IV MEDS Extraction ETL](https://github.com/rvandewater/MIMIC_IV_MEDS) for an end to end example!
|
|
12
|
+
|
|
13
|
+
## How to use this repository
|
|
14
|
+
|
|
15
|
+
1. Initialize a new repository using this template repository.
|
|
16
|
+
2. Rename the directory after `src/` to the name of your package in python-friendly format (e.g.,
|
|
17
|
+
`MIMIC_IV_MEDS`).
|
|
18
|
+
3. Customize the following code points:
|
|
19
|
+
- [`pyproject.toml`](#pyprojecttoml)
|
|
20
|
+
- [`src/.../__init__.py`](#srcinitpy)
|
|
21
|
+
- [`src/.../dataset.yaml`](#srcdatasetyaml)
|
|
22
|
+
- [`src/.../pre_MEDS.py`](#srcpre_medspy)
|
|
23
|
+
- [`src/.../event_configs.yaml`](#srcevent_configsyaml)
|
|
24
|
+
- [`README.md`](#readmemd)
|
|
25
|
+
- [`tests/e2e_demo_test.py`](#testse2e_demo_testpy)
|
|
26
|
+
4. Customize the following external services:
|
|
27
|
+
- CodeCov
|
|
28
|
+
- PyPI
|
|
29
|
+
|
|
30
|
+
### Code Points:
|
|
31
|
+
|
|
32
|
+
#### `pyproject.toml`
|
|
33
|
+
|
|
34
|
+
In the `pyproject.toml` file, you will need to update the following fields:
|
|
35
|
+
|
|
36
|
+
1. Under `[project]`:
|
|
37
|
+
- `name = "ETL-MEDS"`: Update `ETL-MEDS` to the name of your package (e.g., `MIMIC-IV-MEDS`)
|
|
38
|
+
- `authors = [...]`: Update the author information to your name and email.
|
|
39
|
+
- `description = "..."`: Update the description to a brief description of your dataset.
|
|
40
|
+
- `dependencies = [...]`: Update the dependencies to include the necessary packages for your ETL pipeline
|
|
41
|
+
(if any additional packages are needed).
|
|
42
|
+
2. Under `[project.scripts]`
|
|
43
|
+
- `MEDS_extract-sample_dataset = "INSPIRE_MEDS.__main__:main"`: Update `MEDS_extract-sample_dataset` to the
|
|
44
|
+
name of your command-line pipeline (e.g., `MIMIC-IV_extract`) and update `INSPIRE_MEDS` to the name of your
|
|
45
|
+
package that you would import in python (e.g., `MIMIC_IV_MEDS`). This will be the same as the directory
|
|
46
|
+
name between `src` and your actual code.
|
|
47
|
+
3. Under `[project.urls]`
|
|
48
|
+
- `Homepage = "..."` Update the homepage to the URL of your GitHub repository.
|
|
49
|
+
- `Issues = "..."` Update the issues URL to the URL of your GitHub repository issues page.
|
|
50
|
+
|
|
51
|
+
#### `src/.../__init__.py`
|
|
52
|
+
|
|
53
|
+
In this file, you simply need to update the `__package_name__ = "INSPIRE_MEDS"` line to refer not to `INSPIRE_MEDS`
|
|
54
|
+
but to your new package import name (e.g., `MIMIC_IV_MEDS`)
|
|
55
|
+
|
|
56
|
+
#### `src/.../dataset.yaml`
|
|
57
|
+
|
|
58
|
+
In this file, you can add details about the dataset you are working with. This will be used to record metadata
|
|
59
|
+
about the dataset and to provide links from which the dataset can be downloaded. You'll need to modify:
|
|
60
|
+
|
|
61
|
+
1. `dataset_name`: The name of the dataset.
|
|
62
|
+
2. `raw_dataset_version`: The version this version of your pipeline is designed to work with.
|
|
63
|
+
3. `urls`: This block contains the URLs from which the dataset can be downloaded. This field requires
|
|
64
|
+
additional commentary, explored below.
|
|
65
|
+
|
|
66
|
+
##### URLs:
|
|
67
|
+
|
|
68
|
+
This field is an object and contains three sub-keys:
|
|
69
|
+
|
|
70
|
+
1. `dataset`: The URLs for the full dataset.
|
|
71
|
+
2. `demo`: The URLs for a smaller, open, demo version of the dataset.
|
|
72
|
+
3. `common`: The URLs for shared metadata files or other shared resources.
|
|
73
|
+
|
|
74
|
+
Each of these sub-keys should be a list of either strings (plain URLs) or dictionaries containing the URL (in
|
|
75
|
+
the key `url`) and username and password authentication information (in the keys `username` and `password`).
|
|
76
|
+
Note that we _strongly_ recommend that you _do not_ include your username and password in the raw file.
|
|
77
|
+
Instead, leverage the OmegaConf resolvers to reference external environment variables or other secure methods
|
|
78
|
+
of storing this information. In the example in this repository, we include one URL with the following
|
|
79
|
+
configuration:
|
|
80
|
+
|
|
81
|
+
```yaml
|
|
82
|
+
- url: EXAMPLE_CONTROLLED_URL
|
|
83
|
+
username: ${oc.env:DATASET_DOWNLOAD_USERNAME}
|
|
84
|
+
password: ${oc.env:DATASET_DOWNLOAD_PASSWORD}
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
which would resolve to fill in the `username` and `password` from the environment variables
|
|
88
|
+
`DATASET_DOWNLOAD_USERNAME` and `DATASET_DOWNLOAD_PASSWORD`, respectively.
|
|
89
|
+
|
|
90
|
+
###### What if my dataset doesn't support direct download/this style of downloading?
|
|
91
|
+
|
|
92
|
+
That's no problem! You can simply turn off downloading entirely by setting `do_download=False` in the
|
|
93
|
+
`configs/main.yaml` or on the command line when you run the pipeline and ensure that your data files are
|
|
94
|
+
manually downloaded and placed in the appropriate directory (the `raw_input_dir` in the `configs/main.yaml`).
|
|
95
|
+
If there is a technical issue with downloading the data through the format supported so far, you can also file
|
|
96
|
+
a [GitHub Issue](https://github.com/rvandewater/INSPIRE_MEDS_Template/issues) outlining your issue and we can
|
|
97
|
+
attempt to expand the supported libraries to cover your use case!
|
|
98
|
+
|
|
99
|
+
#### `pre_MEDS.py`
|
|
100
|
+
|
|
101
|
+
This script should be generally modified to include any "pre-MEDS" steps that are necessary to prepare the
|
|
102
|
+
dataset for MEDS-Transforms based extraction. Critically, these steps often include:
|
|
103
|
+
|
|
104
|
+
1. De-compressing files or otherwise preparing the raw data for extraction at a technical level.
|
|
105
|
+
2. Joining tables together so that all relevant rows include the unifying `subject_id`.
|
|
106
|
+
3. Converting any offsets into timestamps.
|
|
107
|
+
4. Any other modifications of interest.
|
|
108
|
+
|
|
109
|
+
See [MEDS-Transforms](TODO) for more documentation on the appropriate construction of the pre-MEDS script.
|
|
110
|
+
|
|
111
|
+
#### `event_configs.yaml`
|
|
112
|
+
|
|
113
|
+
This file is the configuration file for mapping the rows in your various raw data tables to MEDS events via
|
|
114
|
+
the MEDS-Transforms pipeline. See [MEDS-Transforms](TODO) for more documentation on the format and usage of
|
|
115
|
+
this file.
|
|
116
|
+
|
|
117
|
+
#### `README.md`
|
|
118
|
+
|
|
119
|
+
Insert badges like below:
|
|
120
|
+
|
|
121
|
+
```markdown
|
|
122
|
+
[](https://pypi.org/project/PACKAGE_NAME/)
|
|
123
|
+
[](https://REPO_NAME.readthedocs.io/en/stable/?badge=stable)
|
|
124
|
+
[](https://codecov.io/gh/rvandewater/REPO_NAME)
|
|
125
|
+
[](https://github.com/rvandewater/REPO_NAME/actions/workflows/tests.yml)
|
|
126
|
+
[](https://github.com/rvandewater/REPO_NAME/actions/workflows/code-quality-main.yaml)
|
|
127
|
+

|
|
128
|
+
[](https://github.com/rvandewater/REPO_NAME#license)
|
|
129
|
+
[](https://github.com/rvandewater/REPO_NAME/pulls)
|
|
130
|
+
[](https://github.com/rvandewater/REPO_NAME/graphs/contributors)
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
#### `tests/e2e_demo_test.py`
|
|
134
|
+
|
|
135
|
+
If your dataset does not have an open demo version, you can remove this file, as there is no way to set up
|
|
136
|
+
automated testing of the end-to-end pipeline in a safe manner without a demo dataset.
|
|
137
|
+
|
|
138
|
+
If you do have a demo dataset, ensure that it is included in your `dataset.yaml` file and update the
|
|
139
|
+
`e2e_demo_test.py` file as follows:
|
|
140
|
+
|
|
141
|
+
1. Update the command in the `command_parts` variable to match the command you set in your `pyproject.toml`
|
|
142
|
+
file for the executable for your pipeline (e.g., `MIMIC-IV_extract`).
|
|
143
|
+
2. Remove the `pytest.mark.skip` decorator from the test function so that it runs successfully!
|
|
144
|
+
|
|
145
|
+
The test file (and the internal doctests, which can help unittest your `pre-MEDS` file) can then be run via
|
|
146
|
+
`pytest --doctest-modules -s` from the root directory to ensure correctness of your pipeline. These tests will
|
|
147
|
+
also be run on pull requests or pushes to the `main` branch of your repository via GitHub Actions, and test
|
|
148
|
+
code coverage will be tracked via CodeCov.
|
|
149
|
+
|
|
150
|
+
### External Services
|
|
151
|
+
|
|
152
|
+
#### CodeCov
|
|
153
|
+
|
|
154
|
+
1. Go to [CodeCov](https://codecov.io/) and add make an account or log-in as needed.
|
|
155
|
+
2. Follow the instructions to configure your new repository with CodeCov.
|
|
156
|
+
3. Copy the badge markdown from CodeCov and paste it into the `README.md` file. To find the badge markdown
|
|
157
|
+
link, go to your repository in CodeCov, click on the "Configuration" tab, click on the "Badges and
|
|
158
|
+
Graphs" option, then copy the markdown link from the top section and paste it in the corresponding line
|
|
159
|
+
of the README, in place of the default link included above.
|
|
160
|
+
4. It will now track the test coverage of your ETL, including running the full pipeline against the linked
|
|
161
|
+
demo data you provide in `dataset.yaml`.
|
|
162
|
+
|
|
163
|
+
#### PyPI
|
|
164
|
+
|
|
165
|
+
1. Go to [PyPI](https://pypi.org/) and add make an account or log-in as needed.
|
|
166
|
+
2. Go to your account settings and go to the "Publishing" settings.
|
|
167
|
+
3. Set up a new "Trusted Publisher" for your GitHub Repository (e.g., see the image below). Ensure your
|
|
168
|
+
package name matches in the trusted publisher and in your `pyproject.toml` file!
|
|
169
|
+
4. Now, if, on the local command line, you run `git tag 0.0.1`, then `git push origin 0.0.1`, it will push a
|
|
170
|
+
new, tagged version of your code as of the local commit when you ran the command both to a new GitHub
|
|
171
|
+
Release and to PyPI. This will allow you to install your package via `pip install PACKAGE_NAME` and to manage
|
|
172
|
+
versions effectively!
|
|
173
|
+
|
|
174
|
+
Example trusted publisher set-up:
|
|
175
|
+

|