cehrbert-data 0.0.1__tar.gz

cehrbert_data-0.0.1/.github/workflows/tests.yml

@@ -0,0 +1,39 @@
+# This workflow will install Python dependencies, run tests and lint with a single version of Python
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
+
+name: Tests
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python 3.10.0
+        uses: actions/setup-python@v3
+        with:
+          python-version: "3.10"
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install flake8 pytest
+          pip install -e .
+      - name: Lint with flake8
+        run: |
+          # stop the build if there are Python syntax errors or undefined names
+          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+          flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+      - name: Test with pytest
+        run: |
+          PYTHONPATH=./: pytest

cehrbert_data-0.0.1/.gitignore

@@ -0,0 +1,6 @@
+.idea
+build/
+.eggs/
+*.egg-info/
+*__pycache__/
+*venv*

cehrbert_data-0.0.1/.pre-commit-config.yaml

@@ -0,0 +1,83 @@
+# For documentation on pre-commit usage, see https://pre-commit.com/
+# This file should be updated quarterly by a developer running `pre-commit autoupdate`
+# with changes added and committed.
+# This will run all defined formatters prior to adding a commit.
+default_language_version:
+  python: python3  # or python3.10 to set a specific default version
+
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.6.0
+    hooks:
+      - id: check-yaml
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+
+  - repo: https://github.com/DanielNoord/pydocstringformatter
+    rev: 'v0.7.3'
+    hooks:
+      - id: pydocstringformatter
+
+  - repo: https://github.com/PyCQA/autoflake
+    rev: v2.2.0
+    hooks:
+      - id: autoflake
+
+  - repo: https://github.com/psf/black
+    rev: '24.1.1'
+    hooks:
+      - id: black
+        # It is recommended to specify the latest version of Python
+        # supported by your project here, or alternatively use
+        # pre-commit's default_language_version, see
+        # https://pre-commit.com/#top_level-default_language_version
+        # Pre-commit hook info from: https://black.readthedocs.io/en/stable/integrations/source_version_control.html
+        # Editor integration here:  https://black.readthedocs.io/en/stable/integrations/editors.html
+
+  - repo: https://github.com/adamchainz/blacken-docs
+    rev: "v1.12.1"  # replace with latest tag on GitHub
+    hooks:
+      - id: blacken-docs
+        additional_dependencies:
+          - black>=22.12.0
+
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: 'v4.5.0'
+    hooks:
+      - id: trailing-whitespace
+        exclude: .git/COMMIT_EDITMSG
+      - id: end-of-file-fixer
+        exclude: .git/COMMIT_EDITMSG
+      - id: detect-private-key
+      - id: debug-statements
+      - id: check-json
+      - id: pretty-format-json
+      - id: check-yaml
+      - id: name-tests-test
+      - id: requirements-txt-fixer
+
+  - repo: https://github.com/pre-commit/pygrep-hooks
+    rev: 'v1.10.0'
+    hooks:
+      - id: python-no-eval
+      - id: python-no-log-warn
+      - id: python-use-type-annotations
+
+  - repo: https://github.com/Lucas-C/pre-commit-hooks
+    rev: v1.5.4
+    hooks:
+      - id: remove-crlf
+      - id: remove-tabs  # defaults to: 4
+        exclude: .git/COMMIT_EDITMSG
+
+  - repo: https://github.com/PyCQA/isort.git
+    rev: 5.13.2
+    hooks:
+      - id: isort
+        args: [ "--profile", "black" ]
+
+  - repo: https://github.com/PyCQA/bandit
+    rev: '1.7.7'
+    hooks:
+      - id: bandit
+        args: ["--skip", "B101,B106,B107,B301,B311,B105,B608,B403"]

cehrbert_data-0.0.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Department of Biomedical Informatics
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

cehrbert_data-0.0.1/PKG-INFO

@@ -0,0 +1,116 @@
+Metadata-Version: 2.1
+Name: cehrbert_data
+Version: 0.0.1
+Summary: The Spark ETL tools for generating the CEHR-BERT and CEHR-GPT pre-training and finetuning data
+Author-email: Chao Pang <chaopang229@gmail.com>, Xinzhuo Jiang <xj2193@cumc.columbia.edu>, Krishna Kalluri <kk3326@cumc.columbia.edu>, Nishanth Parameshwar Pavinkurve <np2689@cumc.columbia.edu>, Karthik Natarajan <kn2174@cumc.columbia.edu>
+License: MIT License
+Project-URL: Homepage, https://github.com/knatarajan-lab/cehrbert_data
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.10.0
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy==1.24.3
+Requires-Dist: packaging==23.2
+Requires-Dist: pandas==2.2.0
+Requires-Dist: pyspark==3.1.2
+Provides-Extra: dev
+Requires-Dist: pre-commit; extra == "dev"
+Requires-Dist: pytest; extra == "dev"
+Requires-Dist: pytest-cov; extra == "dev"
+Requires-Dist: pytest-subtests; extra == "dev"
+Requires-Dist: rootutils; extra == "dev"
+Requires-Dist: hypothesis; extra == "dev"
+Requires-Dist: black; extra == "dev"
+
+# cehrbert_data
+
+cehrbert_data is the ETL tool that generates the pretraining and finetuning datasets for CEHRbERT, which is a large language model developed for the structured EHR data, the work has been published
+at https://proceedings.mlr.press/v158/pang21a.html.
+
+## Patient Representation
+For each patient, all medical codes were aggregated and constructed into a sequence chronologically.
+In order to incorporate temporal information, we inserted an artificial time token (ATT) between two neighboring visits
+based on their time interval.
+The following logic was used for creating ATTs based on the following time intervals between visits, if less than 28
+days, ATTs take on the form of $W_n$ where n represents the week number ranging from 0-3 (e.g. $W_1$); 2) if between 28
+days and 365 days, ATTs are in the form of **$M_n$** where n represents the month number ranging from 1-11 e.g $M_{11}$;
+
+3) beyond 365 days then a **LT** (Long Term) token is inserted. In addition, we added two more special tokens — **VS**
+   and **VE** to represent the start and the end of a visit to explicitly define the visit segment, where all the
+   concepts
+   associated with the visit are subsumed by **VS** and **VE**.
+
+!["patient_representation"](https://raw.githubusercontent.com/cumc-dbmi/cehr-bert/main/images/tokenization_att_generation.png)
+
+## Pre-requisite
+The project is built in python 3.10, and project dependency needs to be installed
+
+Create a new Python virtual environment
+
+```console
+python3.10 -m venv .venv;
+source .venv/bin/activate;
+```
+
+Build the project
+
+```console
+pip install -e .
+```
+
+Download [jtds-1.3.1.jar](jtds-1.3.1.jar) into the spark jars folder in the python environment
+```console
+cp jtds-1.3.1.jar .venv/lib/python3.10/site-packages/pyspark/jars/
+```
+## Instructions for Use
+
+### 1. Download OMOP tables as parquet files
+
+We created a spark app to download OMOP tables from SQL Server as parquet files. You need adjust the properties
+in `db_properties.ini` to match with your database setup.
+
+```console
+PYTHONPATH=./: spark-submit tools/download_omop_tables.py -c db_properties.ini -tc person visit_occurrence condition_occurrence procedure_occurrence drug_exposure measurement observation_period concept concept_relationship concept_ancestor -o ~/Documents/omop_test/
+```
+
+We have prepared a synthea dataset with 1M patients for you to test, you could download it
+at [omop_synthea.tar.gz](https://drive.google.com/file/d/1k7-cZACaDNw8A1JRI37mfMAhEErxKaQJ/view?usp=share_link)
+
+```console
+tar -xvf omop_synthea.tar ~/Document/omop_test/
+```
+
+### 2. Generate training data for CEHR-BERT
+We order the patient events in chronological order and put all data points in a sequence. We insert artificial tokens
+VS (visit start) and VE (visit end) to the start and the end of the visit. In addition, we insert artificial time
+tokens (ATT) between visits to indicate the time interval between visits. This approach allows us to apply BERT to
+structured EHR as-is.
+The sequence can be seen conceptually as [VS] [V1] [VE] [ATT] [VS] [V2] [VE], where [V1] and [V2] represent a list of
+concepts associated with those visits.
+
+```console
+PYTHONPATH=./: spark-submit spark_apps/generate_training_data.py -i ~/Documents/omop_test/ -o ~/Documents/omop_test/cehr-bert -tc condition_occurrence procedure_occurrence drug_exposure -d 1985-01-01 --is_new_patient_representation -iv
+```
+### 3. Generate hf readmission prediction task
+If you don't have your own OMOP instance, we have provided a sample of patient sequence data generated using Synthea
+at `sample/hf_readmissioon` in the repo
+
+```console
+PYTHONPATH=./:$PYTHONPATH spark-submit spark_apps/prediction_cohorts/hf_readmission.py -c hf_readmission -i ~/Documents/omop_test/ -o ~/Documents/omop_test/cehr-bert -dl 1985-01-01 -du 2020-12-31 -l 18 -u 100 -ow 360 -ps 0 -pw 30 --is_new_patient_representation
+```
+
+## Contact us
+If you have any questions, feel free to contact us at CEHR-BERT@lists.cumc.columbia.edu
+
+## Citation
+Please acknowledge the following work in papers
+
+Chao Pang, Xinzhuo Jiang, Krishna S. Kalluri, Matthew Spotnitz, RuiJun Chen, Adler
+Perotte, and Karthik Natarajan. "Cehr-bert: Incorporating temporal information from
+structured ehr data to improve prediction tasks." In Proceedings of Machine Learning for
+Health, volume 158 of Proceedings of Machine Learning Research, pages 239–260. PMLR,
+04 Dec 2021.

cehrbert_data-0.0.1/README.md

@@ -0,0 +1,88 @@
+# cehrbert_data
+
+cehrbert_data is the ETL tool that generates the pretraining and finetuning datasets for CEHRbERT, which is a large language model developed for the structured EHR data, the work has been published
+at https://proceedings.mlr.press/v158/pang21a.html.
+
+## Patient Representation
+For each patient, all medical codes were aggregated and constructed into a sequence chronologically.
+In order to incorporate temporal information, we inserted an artificial time token (ATT) between two neighboring visits
+based on their time interval.
+The following logic was used for creating ATTs based on the following time intervals between visits, if less than 28
+days, ATTs take on the form of $W_n$ where n represents the week number ranging from 0-3 (e.g. $W_1$); 2) if between 28
+days and 365 days, ATTs are in the form of **$M_n$** where n represents the month number ranging from 1-11 e.g $M_{11}$;
+
+3) beyond 365 days then a **LT** (Long Term) token is inserted. In addition, we added two more special tokens — **VS**
+   and **VE** to represent the start and the end of a visit to explicitly define the visit segment, where all the
+   concepts
+   associated with the visit are subsumed by **VS** and **VE**.
+
+!["patient_representation"](https://raw.githubusercontent.com/cumc-dbmi/cehr-bert/main/images/tokenization_att_generation.png)
+
+## Pre-requisite
+The project is built in python 3.10, and project dependency needs to be installed
+
+Create a new Python virtual environment
+
+```console
+python3.10 -m venv .venv;
+source .venv/bin/activate;
+```
+
+Build the project
+
+```console
+pip install -e .
+```
+
+Download [jtds-1.3.1.jar](jtds-1.3.1.jar) into the spark jars folder in the python environment
+```console
+cp jtds-1.3.1.jar .venv/lib/python3.10/site-packages/pyspark/jars/
+```
+## Instructions for Use
+
+### 1. Download OMOP tables as parquet files
+
+We created a spark app to download OMOP tables from SQL Server as parquet files. You need adjust the properties
+in `db_properties.ini` to match with your database setup.
+
+```console
+PYTHONPATH=./: spark-submit tools/download_omop_tables.py -c db_properties.ini -tc person visit_occurrence condition_occurrence procedure_occurrence drug_exposure measurement observation_period concept concept_relationship concept_ancestor -o ~/Documents/omop_test/
+```
+
+We have prepared a synthea dataset with 1M patients for you to test, you could download it
+at [omop_synthea.tar.gz](https://drive.google.com/file/d/1k7-cZACaDNw8A1JRI37mfMAhEErxKaQJ/view?usp=share_link)
+
+```console
+tar -xvf omop_synthea.tar ~/Document/omop_test/
+```
+
+### 2. Generate training data for CEHR-BERT
+We order the patient events in chronological order and put all data points in a sequence. We insert artificial tokens
+VS (visit start) and VE (visit end) to the start and the end of the visit. In addition, we insert artificial time
+tokens (ATT) between visits to indicate the time interval between visits. This approach allows us to apply BERT to
+structured EHR as-is.
+The sequence can be seen conceptually as [VS] [V1] [VE] [ATT] [VS] [V2] [VE], where [V1] and [V2] represent a list of
+concepts associated with those visits.
+
+```console
+PYTHONPATH=./: spark-submit spark_apps/generate_training_data.py -i ~/Documents/omop_test/ -o ~/Documents/omop_test/cehr-bert -tc condition_occurrence procedure_occurrence drug_exposure -d 1985-01-01 --is_new_patient_representation -iv
+```
+### 3. Generate hf readmission prediction task
+If you don't have your own OMOP instance, we have provided a sample of patient sequence data generated using Synthea
+at `sample/hf_readmissioon` in the repo
+
+```console
+PYTHONPATH=./:$PYTHONPATH spark-submit spark_apps/prediction_cohorts/hf_readmission.py -c hf_readmission -i ~/Documents/omop_test/ -o ~/Documents/omop_test/cehr-bert -dl 1985-01-01 -du 2020-12-31 -l 18 -u 100 -ow 360 -ps 0 -pw 30 --is_new_patient_representation
+```
+
+## Contact us
+If you have any questions, feel free to contact us at CEHR-BERT@lists.cumc.columbia.edu
+
+## Citation
+Please acknowledge the following work in papers
+
+Chao Pang, Xinzhuo Jiang, Krishna S. Kalluri, Matthew Spotnitz, RuiJun Chen, Adler
+Perotte, and Karthik Natarajan. "Cehr-bert: Incorporating temporal information from
+structured ehr data to improve prediction tasks." In Proceedings of Machine Learning for
+Health, volume 158 of Proceedings of Machine Learning Research, pages 239–260. PMLR,
+04 Dec 2021.

cehrbert_data-0.0.1/pyproject.toml

@@ -0,0 +1,54 @@
+[build-system]
+requires = ["setuptools", "wheel", "setuptools_scm"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "cehrbert_data"
+dynamic = ["version"]
+authors = [
+    { name = "Chao Pang", email = "chaopang229@gmail.com" },
+    { name = "Xinzhuo Jiang", email = "xj2193@cumc.columbia.edu" },
+    { name = "Krishna Kalluri", email = "kk3326@cumc.columbia.edu" },
+    { name = "Nishanth Parameshwar Pavinkurve", email = "np2689@cumc.columbia.edu" },
+    { name = "Karthik Natarajan", email = "kn2174@cumc.columbia.edu" }
+]
+description = "The Spark ETL tools for generating the CEHR-BERT and CEHR-GPT pre-training and finetuning data"
+readme = "README.md"
+license = { text = "MIT License" }
+requires-python = ">=3.10.0"
+
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3"
+]
+
+dependencies = [
+    "numpy==1.24.3",
+    "packaging==23.2",
+    "pandas==2.2.0",
+    "pyspark==3.1.2"
+]
+
+[tool.setuptools_scm]
+
+[project.urls]
+Homepage = "https://github.com/knatarajan-lab/cehrbert_data"
+
+[project.optional-dependencies]
+dev = [
+    "pre-commit", "pytest", "pytest-cov", "pytest-subtests", "rootutils", "hypothesis", "black"
+]
+
+[tool.isort]
+multi_line_output = 3
+include_trailing_comma = true
+force_grid_wrap = 0
+use_parentheses = true
+ensure_newline_before_comments = true
+line_length = 120
+
+[tool.black]
+line_length = 120

cehrbert_data-0.0.1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+