cumulus-library-rxnorm 1.0__tar.gz

cumulus_library_rxnorm-1.0/PKG-INFO

@@ -0,0 +1,68 @@
+Metadata-Version: 2.4
+Name: cumulus-library-rxnorm
+Version: 1.0
+Summary: A RxNorm study for the Cumulus project
+Keywords: FHIR,SQL,Health Informatics
+Requires-Python: >= 3.11
+Description-Content-Type: text/markdown
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Dist: cumulus-library >= 6, <7.0
+Requires-Dist: ruff == 0.15.20 ; extra == "dev"
+Requires-Dist: pre-commit ; extra == "dev"
+Requires-Dist: pytest ; extra == "test"
+Requires-Dist: responses ; extra == "test"
+Project-URL: Documentation, https://docs.smarthealthit.org/cumulus/
+Project-URL: Home, https://smarthealthit.org/cumulus
+Project-URL: Source, https://github.com/smart-on-fhir/cumulus-library-rxnorm
+Provides-Extra: dev
+Provides-Extra: test
+
+# Cumulus Library UMLS
+
+An installation of the Unified Medical Language System® RxNorm dataset. Part of the [SMART on FHIR Cumulus Project](https://smarthealthit.org/cumulus)
+
+For more information, [browse the documentation](https://docs.smarthealthit.org/cumulus/library).
+## Usage
+
+In order to use RxNorm, you'll need to get an API key for access from the National Library of Medicine, which you can sign up for [here](https://uts.nlm.nih.gov/uts/signup-login).
+
+You can then install this module by running `pip install cumulus-library-rxnorm`.
+
+This will add an `rxnorm` target to `cumulus-library`. You'll need to pass your
+API key via the `--umls-key` CLI flag, or set the `UMLS_API_KEY` environment variable
+to the key you received from NIH.
+
+We download a file from NIH, extract it, create parquet files for Athena, and then upload it.
+We try to preserve some of those artifacts along the way to make rebuilds faster. If you need 
+to force recreation from scratch, the `--force-upload` CLI flag will handle this.
+
+Note: This study is explicitly namespaced in its own schema, `rxnorm`. Make sure your
+database is not using this schema for another use. Do not create tables inside this
+schema by another means.
+
+## Licensing details
+
+The `cumulus-library-rxnorm` study is provided as a convenience to install the
+UMLS RxNorm dataset, but is not shipped with said dataset. It will
+require an API key to download the data from NIH directly.
+
+As a reminder, the 
+[RxNorm terms of service](https://www.nlm.nih.gov/research/umls/rxnorm/docs/termsofservice.html)
+provides several restrictions on this usage of this data (including distributing
+the dataset). When you sign up for a UMLS key, you are assuming responsibility
+for complying with these terms, or an alternate licensing agreement with the
+owners of the RxNorm data if you are provided with one.
+
+
+## NIH Requried Disclaimer
+
+This product uses publicly available data courtesy of the U.S. National Library of Medicine (NLM), National Institutes of Health, Department of Health and Human Services; NLM is not responsible for the product and does not endorse or recommend this or any other product.
+
+## Citations
+
+Bodenreider O. The Unified Medical Language System (UMLS): integrating biomedical terminology. Nucleic Acids Res. 2004 Jan 1;32(Database issue):D267-70. doi: 10.1093/nar/gkh061. PubMed PMID: 14681409; PubMed Central PMCID: PMC308795.
+
+Nelson SJ, Zeng K, Kilbourne J, Powell T, Moore R. Normalized names for clinical drugs: RxNorm at 6 years. J Am Med Inform Assoc. 2011 Jul-Aug;18(4)441-8. doi: 10.1136/amiajnl-2011-000116. Epub 2011 Apr 21. PubMed PMID: 21515544; PubMed Central PMCID: PMC3128404.

cumulus_library_rxnorm-1.0/README.md

@@ -0,0 +1,46 @@
+# Cumulus Library UMLS
+
+An installation of the Unified Medical Language System® RxNorm dataset. Part of the [SMART on FHIR Cumulus Project](https://smarthealthit.org/cumulus)
+
+For more information, [browse the documentation](https://docs.smarthealthit.org/cumulus/library).
+## Usage
+
+In order to use RxNorm, you'll need to get an API key for access from the National Library of Medicine, which you can sign up for [here](https://uts.nlm.nih.gov/uts/signup-login).
+
+You can then install this module by running `pip install cumulus-library-rxnorm`.
+
+This will add an `rxnorm` target to `cumulus-library`. You'll need to pass your
+API key via the `--umls-key` CLI flag, or set the `UMLS_API_KEY` environment variable
+to the key you received from NIH.
+
+We download a file from NIH, extract it, create parquet files for Athena, and then upload it.
+We try to preserve some of those artifacts along the way to make rebuilds faster. If you need 
+to force recreation from scratch, the `--force-upload` CLI flag will handle this.
+
+Note: This study is explicitly namespaced in its own schema, `rxnorm`. Make sure your
+database is not using this schema for another use. Do not create tables inside this
+schema by another means.
+
+## Licensing details
+
+The `cumulus-library-rxnorm` study is provided as a convenience to install the
+UMLS RxNorm dataset, but is not shipped with said dataset. It will
+require an API key to download the data from NIH directly.
+
+As a reminder, the 
+[RxNorm terms of service](https://www.nlm.nih.gov/research/umls/rxnorm/docs/termsofservice.html)
+provides several restrictions on this usage of this data (including distributing
+the dataset). When you sign up for a UMLS key, you are assuming responsibility
+for complying with these terms, or an alternate licensing agreement with the
+owners of the RxNorm data if you are provided with one.
+
+
+## NIH Requried Disclaimer
+
+This product uses publicly available data courtesy of the U.S. National Library of Medicine (NLM), National Institutes of Health, Department of Health and Human Services; NLM is not responsible for the product and does not endorse or recommend this or any other product.
+
+## Citations
+
+Bodenreider O. The Unified Medical Language System (UMLS): integrating biomedical terminology. Nucleic Acids Res. 2004 Jan 1;32(Database issue):D267-70. doi: 10.1093/nar/gkh061. PubMed PMID: 14681409; PubMed Central PMCID: PMC308795.
+
+Nelson SJ, Zeng K, Kilbourne J, Powell T, Moore R. Normalized names for clinical drugs: RxNorm at 6 years. J Am Med Inform Assoc. 2011 Jul-Aug;18(4)441-8. doi: 10.1136/amiajnl-2011-000116. Epub 2011 Apr 21. PubMed PMID: 21515544; PubMed Central PMCID: PMC3128404.

cumulus_library_rxnorm-1.0/cumulus_library_rxnorm/manifest.toml

@@ -0,0 +1,9 @@
+study_prefix = "rxnorm"
+
+[[stages.default]]
+files = [
+    "rxnorm_builder.py"
+]
+
+[advanced_options]
+dedicated_schema="rxnorm"

cumulus_library_rxnorm-1.0/cumulus_library_rxnorm/rxnorm_builder.py

@@ -0,0 +1,209 @@
+import pathlib
+
+import pandas
+from cumulus_library import BaseTableBuilder, base_utils, study_manifest
+from cumulus_library.apis import umls
+from cumulus_library.template_sql import base_templates
+
+
+class RxNormBuilder(BaseTableBuilder):
+    def rmtree(self, root: pathlib.Path):
+        """Deletes a dir and all files underneath
+
+        :param root: the location at the base of the path you want to remove
+
+        TODO: replace with native pathlib.walk when upgrading to python 3.12
+        """
+
+        # just in case, if we get passed a file (like if there's an error and a
+        # zip file exists in the download dir)
+        if not root.is_dir():
+            root.unlink()
+        else:
+            for p in root.iterdir():
+                if p.is_dir():
+                    self.rmtree(p)
+                else:
+                    p.unlink()
+            root.rmdir()
+
+    def get_rxnorm_data(
+        self,
+        download_path: pathlib.Path,
+        parquet_path: pathlib.Path,
+        force_upload: bool,
+        umls_key: str,
+    ) -> (list, bool, str):
+        """Fetches and extracts data from the UMLS API
+
+        :param download_path: the location to read from
+        :param parquet_path: the location output is written; only used for deletion
+            if a new dataset is downloaded
+        :param force_upload: if True, will download from UMLS regardless of data on disk
+        :param umls_key: the UMLS API key to use to auth requests
+        :returns:
+            - filtered_files - a list of files to process (excluding language tables)
+            - download_required - if True, a new UMLS release needed to be retrieved
+            - release_version - the name of the folder data was extracted to
+        """
+        api = umls.UmlsApi(api_key=umls_key)
+        metadata = api.get_latest_umls_file_release(
+            target="rxnorm-full-monthly-release"
+        )
+        download_required = False
+
+        if not (download_path / metadata["releaseVersion"]).exists():
+            print("New RxNorm release available, downloading & updating...")
+            download_required = True
+            for version in download_path.iterdir():
+                self.rmtree(version)
+            for version in (parquet_path).iterdir():
+                self.rmtree(version)
+        if download_required or force_upload:
+            api.download_umls_files(
+                target="rxnorm-full-monthly-release", path=download_path, unzip=False
+            )
+            base_utils.unzip_file(
+                download_path / metadata["fileName"],
+                download_path / metadata["releaseVersion"],
+            )
+            (download_path / metadata["fileName"]).unlink()
+        files = list(
+            download_path.glob(f"./{metadata['releaseVersion']}/scripts/oracle/*.ctl")
+        )
+        return files, download_required, metadata["releaseVersion"]
+
+    def sql_type_to_df_parquet_type(self, text: str) -> str:
+        """Converts types extract from the MySQL .ctl definition to parquet types
+
+        :param text: the type to convert
+        :returns: the parquet type
+        """
+        text = text.split("(")[0].strip(",").replace(" external", "")
+        match text:
+            case "char":
+                return "string", "String"
+            case "integer":
+                return "Int64", "Integer"
+            case "float":
+                return "float", "Float"
+            case _:
+                raise Exception(f"'{text}' missing a type converter")
+
+    def parse_ctl_file(self, contents: list[str]) -> (str, dict):
+        """Extracts table and type definitions from a *.ctl file
+
+        :param contents: an array of strings, expected from a file.readlines call()
+        :returns:
+            - datasource - the name of the datasource for population
+            - table -a dict describing the table
+        """
+        datasource = None
+        table = {"headers": [], "dtype": {}, "parquet_types": []}
+        is_col_def_section = False
+        for line in contents:
+            if line is None:
+                continue
+            if line.startswith("infile"):
+                datasource = line.split(" ")[1].rstrip().replace("'", "")
+            elif line == "trailing nullcols(\n":
+                is_col_def_section = True
+                continue
+            elif line.startswith("("):
+                is_col_def_section = True
+                line = line[1:]
+            elif line.strip(" ").startswith(")"):
+                is_col_def_section = False
+            if is_col_def_section:
+                if line is not None:
+                    line = line.strip().split("\t")
+                    # Some files in RxNorm ctls are formatted differently
+                    # (i.e. space deliniated, extra whitespace), so we
+                    # do some sanity checking here.
+                    if len(line) == 1:
+                        if len(line[0]) == 0:
+                            continue
+                        else:
+                            line = line[0].split()
+                    df_type, parquet_type = self.sql_type_to_df_parquet_type(line[1])
+                    table["headers"].append(line[0])
+                    table["dtype"][line[0]] = df_type
+                    table["parquet_types"].append(parquet_type)
+        return datasource, table
+
+    def create_parquet(
+        self,
+        rrf_path: pathlib.Path,
+        parquet_path: pathlib.Path,
+        table: dict[list],
+        config: base_utils.StudyConfig,
+    ):
+        """Creates a parquet file from a .rrf metathesaurus file
+
+        :param rrf_path: the location of the .rrf files
+        :param parquet_path: the location to write output parquet to
+        :param table: a table definition created by parse_ctl_files
+        :param force_upload: if true, upload to a remote source regardless of what
+            already exists there
+        """
+        if not config.force_upload:
+            if (parquet_path / f"{rrf_path.stem}/{rrf_path.stem}.parquet").exists():
+                return
+            else:
+                (parquet_path / rrf_path.stem).mkdir(parents=True, exist_ok=True)
+        df = pandas.read_csv(
+            rrf_path,
+            delimiter="|",
+            names=table["headers"],
+            dtype=table["dtype"],
+            index_col=False,
+        )
+        df.to_parquet(parquet_path / f"{rrf_path.stem}/{rrf_path.stem}.parquet")
+
+    def prepare_queries(
+        self,
+        config: base_utils.StudyConfig,
+        manifest: study_manifest.StudyManifest,
+        *args,
+        **kwargs,
+    ):
+        download_path = pathlib.Path(__file__).resolve().parent / "downloads"
+        download_path.mkdir(exist_ok=True, parents=True)
+        parquet_path = pathlib.Path(__file__).resolve().parent / "generated_parquet"
+        parquet_path.mkdir(exist_ok=True, parents=True)
+        files, new_version, folder = self.get_rxnorm_data(
+            download_path, parquet_path, config.force_upload, config.umls_key
+        )
+        parquet_path = parquet_path / folder
+        parquet_path.mkdir(exist_ok=True, parents=True)
+        schema = base_utils.get_schema(config, manifest)
+        with base_utils.get_progress_bar() as progress:
+            task = progress.add_task(
+                None,
+                total=len(files),
+            )
+            for file in files:
+                with open(file) as f:
+                    datasource, table = self.parse_ctl_file(f.readlines())
+                    progress.update(task, description=f"Compressing {datasource}...")
+                    rrf_path = download_path / f"./{folder}/rrf/{datasource}"
+                    self.create_parquet(rrf_path, parquet_path, table, config)
+                    progress.update(task, description=f"Uploading {datasource}...")
+                    remote_path = config.db.upload_file(
+                        file=parquet_path / f"{file.stem}/{file.stem}.parquet",
+                        study="rxnorm",
+                        topic=file.stem,
+                        remote_filename=f"{file.stem}.parquet",
+                        force_upload=config.force_upload or new_version,
+                    )
+                    self.queries.append(
+                        base_templates.get_ctas_from_parquet_query(
+                            schema_name=schema,
+                            table_name=f"rxnorm__{file.stem}",
+                            local_location=parquet_path / f"{file.stem}",
+                            remote_location=remote_path,
+                            table_cols=table["headers"],
+                            remote_table_cols_types=table["parquet_types"],
+                        )
+                    )
+                    progress.advance(task)

cumulus_library_rxnorm-1.0/pyproject.toml

@@ -0,0 +1,65 @@
+[project]
+version = "1.0"
+name = "cumulus-library-rxnorm"
+requires-python = ">= 3.11"
+dependencies = [
+    "cumulus-library >= 6, <7.0",
+]
+description = "A RxNorm study for the Cumulus project"
+readme = "README.md"
+license = { text="Apache License 2.0" }
+keywords = ["FHIR", "SQL", "Health Informatics"]
+classifiers = [
+    "License :: OSI Approved :: Apache Software License",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+]
+[project.optional-dependencies]
+dev = [
+    "ruff == 0.15.20",
+    "pre-commit",
+]
+test = [
+    "pytest",
+    "responses"
+]
+
+[project.urls]
+Home = "https://smarthealthit.org/cumulus"
+Documentation = "https://docs.smarthealthit.org/cumulus/"
+Source = "https://github.com/smart-on-fhir/cumulus-library-rxnorm"
+
+[build-system]
+requires = ["flit_core >=3.4,<4"]
+build-backend = "flit_core.buildapi"
+
+[tool.flit.sdist]
+include = [".sqlfluff"]
+
+[tool.pytest.ini_options]
+minversion = "6.0"
+testpaths = [
+    "tests",
+]
+
+[tool.ruff]
+target-version = "py310"
+
+[tool.ruff.lint]
+select = [
+    "A",  # prevent using keywords that clobber python builtins
+    "B",  # bugbear: security warnings
+    "E",  # pycodestyle
+    "F",  # pyflakes
+    "I",  # isort
+    "ISC",  # implicit string concatenation
+    "PLE",  # pylint errors
+    "RUF",  # the ruff developer's own rules
+    "UP",  # alert you when better syntax is available in your python version
+]
+ignore = [
+# Recommended ingore from `ruff format` due to in-project conflicts with check.
+# It's expected that this will be fixed in the coming months.
+    "ISC001"
+]