carrot-transform 0.3.1__tar.gz → 0.3.3__tar.gz

carrot_transform-0.3.3/PKG-INFO

@@ -0,0 +1,48 @@
+Metadata-Version: 2.3
+Name: carrot_transform
+Version: 0.3.3
+Summary: 
+Author: anwarfg
+Author-email: 913028+anwarfg@users.noreply.github.com
+Requires-Python: >=3.10,<4.0
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Dist: click (>=8.1.7,<9.0.0)
+Requires-Dist: jinja2 (>=3.1.4,<4.0.0)
+Requires-Dist: pandas (>=2.2.3,<3.0.0)
+Description-Content-Type: text/markdown
+
+<p align="center">
+  <a href="https://carrot.ac.uk/" target="_blank">
+  <picture>
+    <source media="(prefers-color-scheme: dark)" srcset="/images/logo-dark.png">
+    <img alt="Carrot Logo" src="/images/logo-primary.png" width="280"/>
+  </picture>
+  </a>
+</p>
+<div align="center">
+  <strong>
+  <h2>Streamlined Data Mapping to OMOP</h2>
+  <a href="https://carrot.ac.uk/">Carrot Tranform</a> executes the conversion of the data to the OMOP CDM.<br />
+  </strong>
+</div>
+
+TODO:
+
+- Document carrot-transform
+- Add more comments in-code
+- Handle capture of ddl and json config via the command-line as optional args
+
+Reduction in complexity over the original CaRROT-CDM version for the Transform part of _ETL_ - In practice _Extract_ is always
+performed by Data Partners, _Load_ by database bulk-load software.
+
+Statistics
+
+External libraries imported (approximate)
+
+carrot-cdm 61
+carrot-transform 12
+

carrot_transform-0.3.3/README.md

@@ -0,0 +1,30 @@
+<p align="center">
+  <a href="https://carrot.ac.uk/" target="_blank">
+  <picture>
+    <source media="(prefers-color-scheme: dark)" srcset="/images/logo-dark.png">
+    <img alt="Carrot Logo" src="/images/logo-primary.png" width="280"/>
+  </picture>
+  </a>
+</p>
+<div align="center">
+  <strong>
+  <h2>Streamlined Data Mapping to OMOP</h2>
+  <a href="https://carrot.ac.uk/">Carrot Tranform</a> executes the conversion of the data to the OMOP CDM.<br />
+  </strong>
+</div>
+
+TODO:
+
+- Document carrot-transform
+- Add more comments in-code
+- Handle capture of ddl and json config via the command-line as optional args
+
+Reduction in complexity over the original CaRROT-CDM version for the Transform part of _ETL_ - In practice _Extract_ is always
+performed by Data Partners, _Load_ by database bulk-load software.
+
+Statistics
+
+External libraries imported (approximate)
+
+carrot-cdm 61
+carrot-transform 12

{carrot_transform-0.3.1 → carrot_transform-0.3.3}/carrottransform/_version.py

@@ -1,2 +1,2 @@
 # TODO - pick this up automatically when building
-__version__ = '0.3.1'
+__version__ = '0.3.2'

{carrot_transform-0.3.1 → carrot_transform-0.3.3}/carrottransform/cli/subcommands/run.py

@@ -27,8 +27,14 @@ def run():
 @click.option("--person-file",
               required=True,
               help="File containing person_ids in the first column")
+@click.option("--omop-ddl-file",
+              required=False,
+              help="File containing OHDSI ddl statements for OMOP tables")
+@click.option("--omop-config-file",
+              required=False,
+              help="File containing additional / override json config for omop outputs")
 @click.option("--omop-version",
-              required=True,
+              required=False,
               help="Quoted string containing opmop version - eg '5.3'")
 @click.option("--saved-person-id-file",
               default=None,
@@ -49,7 +55,10 @@ def run():
 @click.argument("input-dir",
                 required=False,
                 nargs=-1)
-def mapstream(rules_file, output_dir, write_mode, person_file, omop_version, saved_person_id_file, use_input_person_ids, last_used_ids_file, log_file_threshold, input_dir):
+def mapstream(rules_file, output_dir, write_mode, 
+              person_file, omop_ddl_file, omop_config_file, 
+              omop_version, saved_person_id_file, use_input_person_ids, 
+              last_used_ids_file, log_file_threshold, input_dir):
     """
     Map to output using input streams
     """
@@ -59,9 +68,10 @@ def mapstream(rules_file, output_dir, write_mode, person_file, omop_version, sav
     # - check main directories for existence
     # - handle saved persion ids
     # - initialise metrics 
-    omop_config_file = str(importlib.resources.files('carrottransform')) + '/' + 'config/omop.json'
-    omop_ddl_file_name = "OMOPCDM_postgresql_" + omop_version + "_ddl.sql"
-    omop_ddl_file = str(importlib.resources.files('carrottransform')) + '/' + 'config/' + omop_ddl_file_name
+    if (omop_ddl_file == None) and (omop_config_file == None) and (omop_version != None):
+      omop_config_file = str(importlib.resources.files('carrottransform')) + '/' + 'config/omop.json'
+      omop_ddl_file_name = "OMOPCDM_postgresql_" + omop_version + "_ddl.sql"
+      omop_ddl_file = str(importlib.resources.files('carrottransform')) + '/' + 'config/' + omop_ddl_file_name
 
     if os.path.isdir(input_dir[0]) == False:
         print("Not a directory, input dir {0}".format(input_dir[0]))
@@ -78,13 +88,12 @@ def mapstream(rules_file, output_dir, write_mode, person_file, omop_version, sav
    
     starttime = time.time()
     omopcdm = tools.omopcdm.OmopCDM(omop_ddl_file, omop_config_file)
-    #print(omopcdm.dump_ddl())
     mappingrules = tools.mappingrules.MappingRules(rules_file, omopcdm)
     metrics = tools.metrics.Metrics(mappingrules.get_dataset_name(), log_file_threshold)
     nowtime = time.time()
 
     print("--------------------------------------------------------------------------------")
-    print("Loaded mapping rules from: {0} after {1:.5f} secs".format(rules_file, (nowtime - starttime)))
+    print("Loaded mapping rules from: {0} in {1:.5f} secs".format(rules_file, (nowtime - starttime)))
     output_files = mappingrules.get_all_outfile_names()
     record_numbers = {}
     for output_file in output_files:
@@ -132,12 +141,10 @@ def mapstream(rules_file, output_dir, write_mode, person_file, omop_version, sav
         if infile not in rules_input_files:
             msg = "ERROR: no mapping rules found for existing input file - {0}".format(infile)
             print(msg)
-            metrics.add_log_data(msg)
     for infile in rules_input_files:
         if infile not in existing_input_files:
             msg = "ERROR: no data for mapped input file - {0}".format(infile)
             print(msg)
-            metrics.add_log_data(msg)
 
     # set up overall counts
     rejidcounts = {}
@@ -243,26 +250,21 @@ def mapstream(rules_file, output_dir, write_mode, person_file, omop_version, sav
         print("INPUT file data : {0}: input count {1}, time since start {2:.5} secs".format(srcfilename, str(rcount), (nowtime - starttime)))
         for outtablename, count in outcounts.items():
             print("TARGET: {0}: output count {1}".format(outtablename, str(count)))
+    # END main processing loop
 
     print("--------------------------------------------------------------------------------")
     data_summary = metrics.get_mapstream_summary()
-    log_report = metrics.get_log_data()
     try:
         dsfh = open(output_dir + "/summary_mapstream.tsv", mode="w")
         dsfh.write(data_summary)
         dsfh.close()
-        logfh = open(output_dir + "/error_report.txt", mode="w")
-        logfh.write(log_report)
-        logfh.close()
     except IOError as e:
         print("I/O error({0}): {1}".format(e.errno, e.strerror))
         print("Unable to write file")
 
+    # END mapstream
     nowtime = time.time()
     print("Elapsed time = {0:.5f} secs".format(nowtime - starttime))
-    #profiler.disable()
-    #stats = pstats.Stats(profiler).sort_stats('ncalls')
-    #stats.print_stats()
 
 def get_target_records(tgtfilename, tgtcolmap, rulesmap, srcfield, srcdata, srccolmap, srcfilename, omopcdm, metrics):
     """

{carrot_transform-0.3.1 → carrot_transform-0.3.3}/carrottransform/config/omop.json

@@ -26,6 +26,10 @@
     "visit_occurrence": {
 	    "visit_start_datetime": "visit_start_date", 
 	    "visit_end_datetime": "visit_end_date"
+    },
+    "device_exposure": {
+	    "device_exposure_start_datetime": "device_exposure_start_date", 
+	    "device_exposure_end_datetime": "device_exposure_end_date"
     }
   },
   "date_field_components": {
@@ -46,6 +50,7 @@
     "person": "person_id",
     "procedure_occurrence": "person_id",
     "specimen": "person_id",
+    "device_exposure": "person_id",
     "visit_occurrence": "person_id"
   },
   "auto_number_field": {
@@ -56,6 +61,7 @@
     "observation": "observation_id",
     "procedure_occurrence": "procedure_occurrence_id",
     "specimen": "specimen_id",
+    "device_exposure": "device_exposure_id",
     "visit_occurrence": "visit_occurrence_id"
   }
 }

carrot_transform-0.3.3/carrottransform/tools/file_helpers.py

@@ -0,0 +1,15 @@
+import os
+import sys
+import json
+
+# Function inherited from the "old" CaRROT-CDM (modfied to exit on error)
+    
+def load_json(f_in):
+    try:
+        data = json.load(open(f_in))
+    except Exception as err:
+        print ("{0} not found. Or cannot parse as json".format(f_in))
+        sys.exit()
+
+    return data
+

{carrot_transform-0.3.1 → carrot_transform-0.3.3}/carrottransform/tools/metrics.py

@@ -9,7 +9,6 @@ class Metrics():
         """
         self.datasummary={}
         self.allcounts={}
-        self.log_data=""
         self.dataset_name=dataset_name
         self.log_threshold = log_threshold
 
@@ -128,10 +127,3 @@ class Metrics():
               summary_str += self.dataset_name + "\t" + source + "\t" + fieldname + "\t" + tablename + "\t" + concept_id + "\t" + additional +"\t" + input_count + "\t" + invalid_person_ids + "\t" + invalid_date_fields + "\t" + invalid_source_fields + "\t" + output_count + "\n"
 
         return summary_str
-
-    def add_log_data(self, msg):
-        self.log_data += msg + "\n"
-
-    def get_log_data(self):
-        return self.log_data
-    

{carrot_transform-0.3.1 → carrot_transform-0.3.3}/carrottransform/tools/omopcdm.py

@@ -29,8 +29,8 @@ class OmopCDM:
     def load_ddl(self, omopddl):
         try:
             fp = open(omopddl, "r") 
-        except IOError as e:
-            print("I/O error for ddl file ({0}): {1}".format(e.errno, e.strerror))
+        except Exception as err:
+            print("OMOP ddl file ({0}) not found".format(omopddl))
             sys.exit()
         
         return(self.process_ddl(fp))

carrot_transform-0.3.3/pyproject.toml

@@ -0,0 +1,20 @@
+[tool.poetry]
+name = "carrot_transform"
+version = "0.3.3"
+description = ""
+authors = ["anwarfg <913028+anwarfg@users.noreply.github.com>"]
+readme = "README.md"
+packages = [
+    { include = "carrottransform" }
+]
+
+[tool.poetry.dependencies]
+python = "^3.10"
+click = "^8.1.7"
+pandas = "^2.2.3"
+jinja2 = "^3.1.4"
+
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

carrot_transform-0.3.1/.github/workflows/pypi.publish.yml

@@ -1,55 +0,0 @@
-# This workflow will upload a Python Package using Twine when a release is created
-# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries
-
-name: Upload Python Package
-
-# on:
-#   release:
-#     types: [published]
-on: push
-
-permissions:
-  contents: read
-
-jobs:
-  build:
-    name: Build distribution
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v4
-    - name: Set up Python
-      uses: actions/setup-python@v3
-      with:
-        python-version: '3.x'
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install build
-    - name: Build package
-      run: python -m build
-    - name: Store the distribution packages
-      uses: actions/upload-artifact@v4
-      with:
-        name: python-package-distributions
-        path: dist/
-
-  publish-to-pypi:
-    name: Publish Python distribution to PyPI
-    needs:
-    - build
-    runs-on: ubuntu-latest
-    environment:
-      name: pypi
-      url: https://pypi.org/p/carrot-transform
-    permissions:
-      id-token: write
-
-    steps:
-    - name: Download all the dists
-      uses: actions/download-artifact@v4
-      with:
-        name: python-package-distributions
-        path: dist/
-    - name: Publish distribution 📦 to PyPI
-      uses: pypa/gh-action-pypi-publish@release/v1

carrot_transform-0.3.1/.gitignore

@@ -1,12 +0,0 @@
-data
-.ipynb_checkpoints
-*_pycache_*
-*.whl
-*.spec
-*egg-info
-dist
-.DS_store
-build
-*.env
-temp
-.python-version

carrot_transform-0.3.1/MANIFEST.in

@@ -1,2 +0,0 @@
-include carrottransform/config/*.sql
-include carrottransform/config/*.json

carrot_transform-0.3.1/PKG-INFO

@@ -1,28 +0,0 @@
-Metadata-Version: 2.1
-Name: carrot-transform
-Version: 0.3.1
-Summary: Carrot simple transformer, input rules and data csv's, output OMOP
-Author-email: PD Appleby <pdappleby@gmail.com>
-Classifier: Programming Language :: Python :: 3
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Operating System :: OS Independent
-Requires-Python: >=3.9
-Description-Content-Type: text/markdown
-License-File: LICENSE
-
-# carrot-transform
-
-TODO: 
-* Document carrot-transform 
-* Add more comments in-code
-* Handle capture of ddl and json config via the command-line as optional args
-
-Reduction in complexity over the original CaRROT-CDM version for the Transform part of *ETL* - In practice *Extract* is always 
-performed by Data Partners, *Load* by database bulk-load software.
-
-Statistics
-
-External libraries imported (approximate)
-
-carrot-cdm      61
-carrot-transform 12

carrot_transform-0.3.1/README.md

@@ -1,16 +0,0 @@
-# carrot-transform
-
-TODO: 
-* Document carrot-transform 
-* Add more comments in-code
-* Handle capture of ddl and json config via the command-line as optional args
-
-Reduction in complexity over the original CaRROT-CDM version for the Transform part of *ETL* - In practice *Extract* is always 
-performed by Data Partners, *Load* by database bulk-load software.
-
-Statistics
-
-External libraries imported (approximate)
-
-carrot-cdm      61
-carrot-transform 12

carrot_transform-0.3.1/carrot_transform.egg-info/PKG-INFO

@@ -1,28 +0,0 @@
-Metadata-Version: 2.1
-Name: carrot-transform
-Version: 0.3.1
-Summary: Carrot simple transformer, input rules and data csv's, output OMOP
-Author-email: PD Appleby <pdappleby@gmail.com>
-Classifier: Programming Language :: Python :: 3
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Operating System :: OS Independent
-Requires-Python: >=3.9
-Description-Content-Type: text/markdown
-License-File: LICENSE
-
-# carrot-transform
-
-TODO: 
-* Document carrot-transform 
-* Add more comments in-code
-* Handle capture of ddl and json config via the command-line as optional args
-
-Reduction in complexity over the original CaRROT-CDM version for the Transform part of *ETL* - In practice *Extract* is always 
-performed by Data Partners, *Load* by database bulk-load software.
-
-Statistics
-
-External libraries imported (approximate)
-
-carrot-cdm      61
-carrot-transform 12

carrot_transform-0.3.1/carrot_transform.egg-info/SOURCES.txt

@@ -1,25 +0,0 @@
-.gitignore
-LICENSE
-MANIFEST.in
-README.md
-carrot_transform.py
-pyproject.toml
-.github/workflows/pypi.publish.yml
-carrot_transform.egg-info/PKG-INFO
-carrot_transform.egg-info/SOURCES.txt
-carrot_transform.egg-info/dependency_links.txt
-carrot_transform.egg-info/entry_points.txt
-carrot_transform.egg-info/top_level.txt
-carrottransform/__init__.py
-carrottransform/_version.py
-carrottransform/cli/__init__.py
-carrottransform/cli/command.py
-carrottransform/cli/subcommands/__init__.py
-carrottransform/cli/subcommands/run.py
-carrottransform/config/OMOPCDM_postgresql_5.3_ddl.sql
-carrottransform/config/omop.json
-carrottransform/tools/__init__.py
-carrottransform/tools/file_helpers.py
-carrottransform/tools/mappingrules.py
-carrottransform/tools/metrics.py
-carrottransform/tools/omopcdm.py

carrot_transform-0.3.1/carrot_transform.egg-info/dependency_links.txt

@@ -1 +0,0 @@
-

carrot_transform-0.3.1/carrot_transform.egg-info/entry_points.txt

@@ -1,2 +0,0 @@
-[console_scripts]
-carrot-transform = carrottransform.cli.command:transform

carrot_transform-0.3.1/carrot_transform.egg-info/top_level.txt

@@ -1 +0,0 @@
-carrottransform

carrot_transform-0.3.1/carrot_transform.py

@@ -1,4 +0,0 @@
-# Provides an entry point for the built executable
-from carrottransform.cli.command import transform
-if __name__ == '__main__':
-  transform()

carrot_transform-0.3.1/carrottransform/tools/file_helpers.py

@@ -1,16 +0,0 @@
-import os
-import json
-
-# Function inherited from the "old" CaRROT-CDM
-    
-def load_json(f_in):
-  if os.path.exists(f_in):
-    data = json.load(open(f_in))
-  else:
-    try:
-      data = json.loads(f_in)
-    except Exception as err:
-      raise FileNotFoundError(f"{f_in} not found. Or cannot parse as json")
-
-  return data
-

carrot_transform-0.3.1/pyproject.toml

@@ -1,20 +0,0 @@
-[build-system]
-requires = ["setuptools", "setuptools-scm"]
-build-backend = "setuptools.build_meta"
-
-[project]
-name = "carrot-transform"
-version = "0.3.1"
-authors = [
-  { name="PD Appleby", email="pdappleby@gmail.com" },
-]
-description = "Carrot simple transformer, input rules and data csv's, output OMOP"
-readme = "README.md"
-requires-python = ">=3.9"
-classifiers = [
-    "Programming Language :: Python :: 3",
-    "License :: OSI Approved :: MIT License",
-    "Operating System :: OS Independent",
-]
-[project.scripts]
-carrot-transform="carrottransform.cli.command:transform"

carrot_transform-0.3.1/setup.cfg

@@ -1,4 +0,0 @@
-[egg_info]
-tag_build = 
-tag_date = 0
-