carrot-transform 0.3.2__tar.gz → 0.3.4__tar.gz

carrot_transform-0.3.4/PKG-INFO

@@ -0,0 +1,106 @@
+Metadata-Version: 2.3
+Name: carrot_transform
+Version: 0.3.4
+Summary: 
+Author: anwarfg
+Author-email: 913028+anwarfg@users.noreply.github.com
+Requires-Python: >=3.10,<4.0
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Dist: click (>=8.1.7,<9.0.0)
+Requires-Dist: jinja2 (>=3.1.4,<4.0.0)
+Requires-Dist: pandas (>=2.2.3,<3.0.0)
+Requires-Dist: pytest (>=8.3.4,<9.0.0)
+Description-Content-Type: text/markdown
+
+<p align="center">
+  <a href="https://carrot.ac.uk/" target="_blank">
+  <picture>
+    <source media="(prefers-color-scheme: dark)" srcset="/images/logo-dark.png">
+    <img alt="Carrot Logo" src="/images/logo-primary.png" width="280"/>
+  </picture>
+  </a>
+</p>
+
+<p align="center">
+
+<a href="https://github.com/Health-Informatics-UoN/carrot-transform/releases">
+  <img src="https://img.shields.io/github/v/release/Health-Informatics-UoN/carrot-transform" alt="Release">
+</a>
+<a href="https://opensource.org/license/mit">
+  <img src="https://img.shields.io/badge/License-MIT-yellow.svg" alt="License">
+</a>
+</p>
+
+
+<div align="center">
+  <strong>
+  <h2>Streamlined Data Transformation to OMOP</h2><br />
+<a href="https://carrot.ac.uk/">Carrot Transform</a> automates data transformation processes and facilitates the standardisation of datasets to the OMOP vocabulary, simplifying the integration of diverse data sources.
+  <br />
+  </strong>
+</div>
+
+<p align="center">
+  <br />
+  <a href="https://carrot.ac.uk/transform" rel="dofollow"><strong>Explore the docs »</strong></a>
+  <br />
+<br />  
+
+<a href="https://carrot.ac.uk/">Carrot Mapper</a> is a webapp which allows the user to use the metadata (as output by [WhiteRabbit](https://github.com/OHDSI/WhiteRabbit)) from a dataset to produce mapping rules to the OMOP standard, in the JSON format. These can be ingested by [Carrot Transform](https://carrot.ac.uk/transform/quickstart) to perform the mapping of the contents of the dataset to OMOP.
+
+Carrot Transform transforms input data into tab separated variable files of standard OMOP tables, with  concepts mapped according to the provided rules (generated from Carrot Mapper).
+
+## Quick Start for Developers
+
+To have the project up and running, please follow the [Quick Start Guide](https://carrot.ac.uk/transform/quickstart).
+
+## Release Procedure 
+To release a new version of `carrot-transform` follow these steps: 
+
+### 1. Prepare the repository
+  - First ensure that repository is clean and all required changes have been merged. 
+  - Pull the latest changes from `main` with `git pull origin main`. 
+
+### 2. Create a release branch 
+
+- Now create a new feature branch name `release/v<NEW-VERSION>` (e.g. `release/v0.2.0`). 
+
+### 3. Update the version number 
+- Use poetry to bump the version. For example, for a minor version update invoke: 
+```bash
+poetry version minor 
+```
+- Commit and push the changes (to the release feature branch):
+```bash 
+NEW_VERSION=$(poetry version -s)
+git add pyproject.toml
+git commit -m "Bump version to $NEW_VERSION"
+git push --set-upstream origin release/v$NEW_VERSION
+```
+
+### 4. Create pull request 
+- Open a pull request from `release/v$NEW_VERSION` to `main` and await approval.
+### 5. Merge and tag 
+- After approval merge the the feature branch to `main`. 
+- Checkout to `main`, pull updates, and create a tag corresponding to the new version number. 
+```bash 
+git checkout main
+git pull origin main
+git tag -a "$NEW_VERSION" -m "Release $NEW_VERSION"
+git push origin "$NEW_VERSION"
+```
+
+### 6. Create a release
+- We must now link the tag to a release in the GitHub repository. To do this from the command line first install GitHub command line tools `gh` and then invoke: 
+```bash 
+gh release create "$TAG" --title "$TAG" --notes "Release for $VERSION"
+```
+
+- Alternatively, follow the instructions in the [GitHub documentation](https://docs.github.com/en/repositories/releasing-projects-on-github/managing-releases-in-a-repository) to manually create a release. 
+## License
+
+This repository's source code is available under the [MIT license](LICENSE).

carrot_transform-0.3.4/README.md

@@ -0,0 +1,88 @@
+<p align="center">
+  <a href="https://carrot.ac.uk/" target="_blank">
+  <picture>
+    <source media="(prefers-color-scheme: dark)" srcset="/images/logo-dark.png">
+    <img alt="Carrot Logo" src="/images/logo-primary.png" width="280"/>
+  </picture>
+  </a>
+</p>
+
+<p align="center">
+
+<a href="https://github.com/Health-Informatics-UoN/carrot-transform/releases">
+  <img src="https://img.shields.io/github/v/release/Health-Informatics-UoN/carrot-transform" alt="Release">
+</a>
+<a href="https://opensource.org/license/mit">
+  <img src="https://img.shields.io/badge/License-MIT-yellow.svg" alt="License">
+</a>
+</p>
+
+
+<div align="center">
+  <strong>
+  <h2>Streamlined Data Transformation to OMOP</h2><br />
+<a href="https://carrot.ac.uk/">Carrot Transform</a> automates data transformation processes and facilitates the standardisation of datasets to the OMOP vocabulary, simplifying the integration of diverse data sources.
+  <br />
+  </strong>
+</div>
+
+<p align="center">
+  <br />
+  <a href="https://carrot.ac.uk/transform" rel="dofollow"><strong>Explore the docs »</strong></a>
+  <br />
+<br />  
+
+<a href="https://carrot.ac.uk/">Carrot Mapper</a> is a webapp which allows the user to use the metadata (as output by [WhiteRabbit](https://github.com/OHDSI/WhiteRabbit)) from a dataset to produce mapping rules to the OMOP standard, in the JSON format. These can be ingested by [Carrot Transform](https://carrot.ac.uk/transform/quickstart) to perform the mapping of the contents of the dataset to OMOP.
+
+Carrot Transform transforms input data into tab separated variable files of standard OMOP tables, with  concepts mapped according to the provided rules (generated from Carrot Mapper).
+
+## Quick Start for Developers
+
+To have the project up and running, please follow the [Quick Start Guide](https://carrot.ac.uk/transform/quickstart).
+
+## Release Procedure 
+To release a new version of `carrot-transform` follow these steps: 
+
+### 1. Prepare the repository
+  - First ensure that repository is clean and all required changes have been merged. 
+  - Pull the latest changes from `main` with `git pull origin main`. 
+
+### 2. Create a release branch 
+
+- Now create a new feature branch name `release/v<NEW-VERSION>` (e.g. `release/v0.2.0`). 
+
+### 3. Update the version number 
+- Use poetry to bump the version. For example, for a minor version update invoke: 
+```bash
+poetry version minor 
+```
+- Commit and push the changes (to the release feature branch):
+```bash 
+NEW_VERSION=$(poetry version -s)
+git add pyproject.toml
+git commit -m "Bump version to $NEW_VERSION"
+git push --set-upstream origin release/v$NEW_VERSION
+```
+
+### 4. Create pull request 
+- Open a pull request from `release/v$NEW_VERSION` to `main` and await approval.
+### 5. Merge and tag 
+- After approval merge the the feature branch to `main`. 
+- Checkout to `main`, pull updates, and create a tag corresponding to the new version number. 
+```bash 
+git checkout main
+git pull origin main
+git tag -a "$NEW_VERSION" -m "Release $NEW_VERSION"
+git push origin "$NEW_VERSION"
+```
+
+### 6. Create a release
+- We must now link the tag to a release in the GitHub repository. To do this from the command line first install GitHub command line tools `gh` and then invoke: 
+```bash 
+gh release create "$TAG" --title "$TAG" --notes "Release for $VERSION"
+```
+
+- Alternatively, follow the instructions in the [GitHub documentation](https://docs.github.com/en/repositories/releasing-projects-on-github/managing-releases-in-a-repository) to manually create a release. 
+## License
+
+This repository's source code is available under the [MIT license](LICENSE).

carrot_transform-0.3.4/carrottransform/_version.py

@@ -0,0 +1,6 @@
+from importlib.metadata import version
+
+try:
+    __version__ = version("carrot_transform")  # Defined in the pyproject.toml 
+except Exception:
+    __version__ = "unknown" 

{carrot_transform-0.3.2 → carrot_transform-0.3.4}/carrottransform/cli/subcommands/run.py

@@ -8,6 +8,9 @@ import json
 import importlib.resources
 import carrottransform
 import carrottransform.tools as tools
+from carrottransform.tools.omopcdm import OmopCDM
+from typing import Iterator, IO
+
 
 @click.group(help="Commands for mapping data to the OMOP CommonDataModel (CDM).")
 def run():
@@ -35,7 +38,7 @@ def run():
               help="File containing additional / override json config for omop outputs")
 @click.option("--omop-version",
               required=False,
-              help="Quoted string containing opmop version - eg '5.3'")
+              help="Quoted string containing omop version - eg '5.3'")
 @click.option("--saved-person-id-file",
               default=None,
               required=False,
@@ -66,28 +69,26 @@ def mapstream(rules_file, output_dir, write_mode,
     # - check for values in optional arguments
     # - read in configuration files
     # - check main directories for existence
-    # - handle saved persion ids
-    # - initialise metrics 
-    if (omop_ddl_file == None) and (omop_config_file == None) and (omop_version != None):
-      omop_config_file = str(importlib.resources.files('carrottransform')) + '/' + 'config/omop.json'
-      omop_ddl_file_name = "OMOPCDM_postgresql_" + omop_version + "_ddl.sql"
-      omop_ddl_file = str(importlib.resources.files('carrottransform')) + '/' + 'config/' + omop_ddl_file_name
-
-    if os.path.isdir(input_dir[0]) == False:
-        print("Not a directory, input dir {0}".format(input_dir[0]))
-        sys.exit(1)
-
-    if os.path.isdir(output_dir) == False:
-        print("Not a directory, output dir {0}".format(output_dir))
-        sys.exit(1)
-
-    if saved_person_id_file == None:
-        saved_person_id_file = output_dir + "/" + "person_ids.tsv"
-        if os.path.exists(saved_person_id_file):
-            os.remove(saved_person_id_file)
+    # - handle saved person ids
+    # - initialise metrics
+    print(rules_file, output_dir, write_mode,
+              person_file, omop_ddl_file, omop_config_file,
+              omop_version, saved_person_id_file, use_input_person_ids,
+              last_used_ids_file, log_file_threshold, input_dir)
+
+    ## set omop filenames
+    omop_config_file, omop_ddl_file = set_omop_filenames(omop_ddl_file, omop_config_file, omop_version)
+    ## check directories are valid
+    check_dir_isvalid(input_dir)
+    check_dir_isvalid(output_dir)
+
+    saved_person_id_file = set_saved_person_id_file(saved_person_id_file, output_dir)
    
     starttime = time.time()
+    ## create OmopCDM object, which contains attributes and methods for the omop data tables.
     omopcdm = tools.omopcdm.OmopCDM(omop_ddl_file, omop_config_file)
+
+    ## mapping rules determine the ouput files? which input files and fields in the source data, AND the mappings to omop concepts
     mappingrules = tools.mappingrules.MappingRules(rules_file, omopcdm)
     metrics = tools.metrics.Metrics(mappingrules.get_dataset_name(), log_file_threshold)
     nowtime = time.time()
@@ -95,36 +96,41 @@ def mapstream(rules_file, output_dir, write_mode,
     print("--------------------------------------------------------------------------------")
     print("Loaded mapping rules from: {0} in {1:.5f} secs".format(rules_file, (nowtime - starttime)))
     output_files = mappingrules.get_all_outfile_names()
+
+    ## set record number
+    ## will keep track of the current record number in each file, e.g., measurement_id, observation_id.
     record_numbers = {}
     for output_file in output_files:
         record_numbers[output_file] = 1
+    if last_used_ids_file != None:
+        if os.path.isfile(last_used_ids_file):
+            record_numbers = load_last_used_ids(last_used_ids_file, record_numbers)
 
     fhd = {}
     tgtcolmaps = {}
 
+
+
     try:
-        # Saved-person-file existence test, reload if found, return last used integer
-        if os.path.isfile(saved_person_id_file):
-            person_lookup, last_used_integer = load_saved_person_ids(saved_person_id_file)
-        else:
-            person_lookup = {}
-            last_used_integer = 1
-        if last_used_ids_file != None:
-            if os.path.isfile(last_used_ids_file):
-                record_numbers = load_last_used_ids(last_used_ids_file, record_numbers)
- 
-        person_lookup, rejected_person_count = load_person_ids(person_file, person_lookup, mappingrules, use_input_person_ids, last_used_integer)
-        fhpout = open(saved_person_id_file, mode="w")
-        fhpout.write("SOURCE_SUBJECT\tTARGET_SUBJECT\n")
-        for person_id, person_assigned_id in person_lookup.items():
-            fhpout.write("{0}\t{1}\n".format(str(person_id), str(person_assigned_id)))
-        fhpout.close()
-        # Initialise output files, output a header for each 
+        ## get all person_ids from file and either renumber with an int or take directly, and add to a dict
+        person_lookup, rejected_person_count = load_person_ids(saved_person_id_file, person_file, mappingrules, use_input_person_ids)
+        ## open person_ids output file
+        with open(saved_person_id_file, mode="w") as fhpout:
+            ## write the header to the file
+            fhpout.write("SOURCE_SUBJECT\tTARGET_SUBJECT\n")
+            ##iterate through the ids and write them to the file.
+            for person_id, person_assigned_id in person_lookup.items():
+                fhpout.write("{0}\t{1}\n".format(str(person_id), str(person_assigned_id)))
+
+        ## Initialise output files (adding them to a dict), output a header for each
+        ## these aren't being closed deliberately
         for tgtfile in output_files:
             fhd[tgtfile] = open(output_dir + "/" + tgtfile + ".tsv", mode=write_mode)
             if write_mode == 'w':
                 outhdr = omopcdm.get_omop_column_list(tgtfile)
                 fhd[tgtfile].write("\t".join(outhdr) + "\n")
+            ## maps all omop columns for each file into a dict containing the column name and the index
+            ## so tgtcolmaps is a dict of dicts.
             tgtcolmaps[tgtfile] = omopcdm.get_omop_column_map(tgtfile)
 
     except IOError as e:
@@ -133,43 +139,35 @@ def mapstream(rules_file, output_dir, write_mode,
 
     print("person_id stats: total loaded {0}, reject count {1}".format(len(person_lookup), rejected_person_count))
 
-    # Compare files found in the input_dir with those expected based on mapping rules
+    ## Compare files found in the input_dir with those expected based on mapping rules
     existing_input_files = fnmatch.filter(os.listdir(input_dir[0]), '*.csv')
     rules_input_files = mappingrules.get_all_infile_names()
-    # Log mismatches but continue
-    for infile in existing_input_files:
-        if infile not in rules_input_files:
-            msg = "ERROR: no mapping rules found for existing input file - {0}".format(infile)
-            print(msg)
-    for infile in rules_input_files:
-        if infile not in existing_input_files:
-            msg = "ERROR: no data for mapped input file - {0}".format(infile)
-            print(msg)
 
-    # set up overall counts
+    ## Log mismatches but continue
+    check_files_in_rules_exist(rules_input_files, existing_input_files)
+
+    ## set up overall counts
     rejidcounts = {}
     rejdatecounts = {}
     print(rules_input_files)
 
-    # set up per-input counts
+    ## set up per-input counts
     for srcfilename in rules_input_files:
         rejidcounts[srcfilename] = 0
         rejdatecounts[srcfilename] = 0
 
-    # main processing loop, for each input file
+    ## main processing loop, for each input file
     for srcfilename in rules_input_files:
         outcounts = {}
         rejcounts = {}
         rcount = 0
 
-        try:
-            fh = open(input_dir[0] + "/" + srcfilename, mode="r", encoding="utf-8-sig")
-            csvr = csv.reader(fh)
-        except IOError as e:
-            print("Unable to open: {0}".format(input_dir[0] + "/" + srcfilename))
-            print("I/O error({0}): {1}".format(e.errno, e.strerror))
+        fh, csvr = open_file(input_dir[0], srcfilename)
+        if fh is None:
             continue
 
+
+        ## create dict for input file, giving the data and output file
         tgtfiles, src_to_tgt = mappingrules.parse_rules_src_to_tgt(srcfilename)
         infile_datetime_source, infile_person_id_source = mappingrules.get_infile_date_person_id(srcfilename)
         for tgtfile in tgtfiles:
@@ -185,12 +183,13 @@ def mapstream(rules_file, output_dir, write_mode,
         datetime_col = inputcolmap[infile_datetime_source]
         print("--------------------------------------------------------------------------------")
         print("Processing input: {0}".format(srcfilename))
-  
+
         # for each input record
         for indata in csvr:
             key = srcfilename + "~all~all~all~"
             metrics.increment_key_count(key, "input_count")
             rcount += 1
+            # if there is a date, parse it - read it is a string and convert to YYYY-MM-DD
             strdate = indata[datetime_col].split(" ")[0]
             fulldate = parse_date(strdate)
             if fulldate != None:
@@ -214,30 +213,15 @@ def mapstream(rules_file, output_dir, write_mode,
                         for outrecord in outrecords:
                             if auto_num_col != None:
                                 outrecord[tgtcolmap[auto_num_col]] = str(record_numbers[tgtfile])
+                                ### most of the rest of this section is actually to do with metrics
                                 record_numbers[tgtfile] += 1
                             if (outrecord[tgtcolmap[pers_id_col]]) in person_lookup:
                                 outrecord[tgtcolmap[pers_id_col]] = person_lookup[outrecord[tgtcolmap[pers_id_col]]]
                                 outcounts[tgtfile] += 1
-                                key = srcfilename + "~all~all~all~"
-                                metrics.increment_key_count(key, "output_count")
-                                key = "all~all~" + tgtfile + "~all~"
-                                metrics.increment_key_count(key, "output_count")
-                                key = srcfilename + "~all~" + tgtfile + "~all~"
-                                metrics.increment_key_count(key, "output_count")
-                                if tgtfile == "person":
-                                    key = srcfilename + "~all~" + tgtfile + "~" + outrecord[1] +"~"
-                                    metrics.increment_key_count(key, "output_count")
-                                    key = srcfilename + "~" + datacol +"~" + tgtfile + "~" + outrecord[1] + "~" + outrecord[2]
-                                    metrics.increment_key_count(key, "output_count")
-                                else:
-                                    key = srcfilename + "~" + datacol +"~" + tgtfile + "~" + outrecord[2] + "~"
-                                    metrics.increment_key_count(key, "output_count")
-                                    key = srcfilename + "~all~" + tgtfile + "~" + outrecord[2] + "~"
-                                    metrics.increment_key_count(key, "output_count")
-                                    key = "all~all~" + tgtfile + "~" + outrecord[2] + "~"
-                                    metrics.increment_key_count(key, "output_count")
-                                    key = "all~all~all~" + outrecord[2] + "~"
-                                    metrics.increment_key_count(key, "output_count")
+
+                                increment_key_counts(srcfilename, metrics, tgtfile, datacol, outrecord)
+
+                                # write the line to the file
                                 fhd[tgtfile].write("\t".join(outrecord) + "\n")
                             else:
                                 key = srcfilename + "~all~" + tgtfile + "~all~"
@@ -266,7 +250,39 @@ def mapstream(rules_file, output_dir, write_mode,
     nowtime = time.time()
     print("Elapsed time = {0:.5f} secs".format(nowtime - starttime))
 
-def get_target_records(tgtfilename, tgtcolmap, rulesmap, srcfield, srcdata, srccolmap, srcfilename, omopcdm, metrics):
+def increment_key_counts(srcfilename: str, metrics: tools.metrics.Metrics, tgtfile: str, datacol: str, outrecord: list[str]) -> None:
+    key = srcfilename + "~all~all~all~"
+    metrics.increment_key_count(key, "output_count")
+
+    key = "all~all~" + tgtfile + "~all~"
+    metrics.increment_key_count(key, "output_count")
+
+    key = srcfilename + "~all~" + tgtfile + "~all~"
+    metrics.increment_key_count(key, "output_count")
+
+    if tgtfile == "person":
+        key = srcfilename + "~all~" + tgtfile + "~" + outrecord[1] + "~"
+        metrics.increment_key_count(key, "output_count")
+
+        key = srcfilename + "~" + datacol + "~" + tgtfile + "~" + outrecord[1] + "~" + outrecord[2]
+        metrics.increment_key_count(key, "output_count")
+    else:
+        key = srcfilename + "~" + datacol + "~" + tgtfile + "~" + outrecord[2] + "~"
+        metrics.increment_key_count(key, "output_count")
+
+        key = srcfilename + "~all~" + tgtfile + "~" + outrecord[2] + "~"
+        metrics.increment_key_count(key, "output_count")
+
+        key = "all~all~" + tgtfile + "~" + outrecord[2] + "~"
+        metrics.increment_key_count(key, "output_count")
+
+        key = "all~all~all~" + outrecord[2] + "~"
+        metrics.increment_key_count(key, "output_count")
+    return
+
+
+def get_target_records(tgtfilename: str, tgtcolmap: dict[str, dict[str, int]], rulesmap: dict[str, list[dict[str, list[str]]]], srcfield: str, srcdata: list[str], srccolmap: dict[str, int], srcfilename: str, omopcdm: OmopCDM, metrics: tools.metrics.Metrics) -> \
+tuple[bool, list[str], tools.metrics.Metrics]:
     """
     build all target records for a given input field
     """
@@ -279,6 +295,7 @@ def get_target_records(tgtfilename, tgtcolmap, rulesmap, srcfield, srcdata, srcc
     srckey = srcfilename + "~" + srcfield + "~" + tgtfilename
     summarykey = srcfilename + "~" + srcfield + "~" + tgtfilename + "~all~"
     if valid_value(str(srcdata[srccolmap[srcfield]])):
+        ## check if either or both of the srckey and summarykey are in the rules
         srcfullkey = srcfilename + "~" + srcfield + "~" + str(srcdata[srccolmap[srcfield]]) + "~" + tgtfilename
         dictkeys = []
         if srcfullkey in rulesmap:
@@ -291,6 +308,7 @@ def get_target_records(tgtfilename, tgtcolmap, rulesmap, srcfield, srcdata, srcc
             for dictkey in dictkeys:
                 for out_data_elem in rulesmap[dictkey]:
                     valid_data_elem = True
+                    ## create empty list to store the data. Populate numerical data elements with 0 instead of empty string.
                     tgtarray = ['']*len(tgtcolmap)
                     for req_integer in notnull_numeric_fields:
                         tgtarray[tgtcolmap[req_integer]] = "0"
@@ -302,6 +320,7 @@ def get_target_records(tgtfilename, tgtcolmap, rulesmap, srcfield, srcdata, srcc
                             else:
                                 tgtarray[tgtcolmap[output_col_data]] = srcdata[srccolmap[infield]]
                             if output_col_data in date_component_data:
+                                ## parse the date and store it in the proper format
                                 strdate = srcdata[srccolmap[infield]].split(" ")[0]
                                 dt = get_datetime_value(strdate)
                                 if dt != None:
@@ -453,7 +472,9 @@ def load_saved_person_ids(person_file):
     fh.close()
     return person_ids, last_int
 
-def load_person_ids(person_file, person_ids, mappingrules, use_input_person_ids, person_number=1, delim=","):
+def load_person_ids(saved_person_id_file, person_file, mappingrules, use_input_person_ids, delim=","):
+    person_ids, person_number = get_person_lookup(saved_person_id_file)
+
     fh = open(person_file, mode="r", encoding="utf-8-sig")
     csvr = csv.reader(fh, delimiter=delim)
     person_columns = {}
@@ -468,23 +489,25 @@ def load_person_ids(person_file, person_ids, mappingrules, use_input_person_ids,
         person_columns[col] = person_col_in_hdr_number
         person_col_in_hdr_number += 1
 
+## check the mapping rules for person to find where to get the person data) i.e., which column in the person file contains dob, sex
     birth_datetime_source, person_id_source = mappingrules.get_person_source_field_info("person")
     print("Load Person Data {0}, {1}".format(birth_datetime_source, person_id_source))
+    ## get the column index of the PersonID from the input file
     person_col = person_columns[person_id_source]
 
     for persondata in csvr:
-        if not valid_value(persondata[person_columns[person_id_source]]):
+        if not valid_value(persondata[person_columns[person_id_source]]): #just checking that the id is not an empty string
             reject_count += 1
             continue
         if not valid_date_value(persondata[person_columns[birth_datetime_source]]):
             reject_count += 1
             continue
-        if persondata[person_col] not in person_ids:
+        if persondata[person_col] not in person_ids: #if not already in person_ids dict, add it
             if use_input_person_ids == "N":
-                person_ids[persondata[person_col]] = str(person_number)
+                person_ids[persondata[person_col]] = str(person_number) #create a new integer person_id
                 person_number += 1
             else:
-                person_ids[persondata[person_col]] = str(persondata[person_col])
+                person_ids[persondata[person_col]] = str(persondata[person_col]) #use existing person_id
     fh.close()
 
     return person_ids, reject_count
@@ -493,4 +516,62 @@ def load_person_ids(person_file, person_ids, mappingrules, use_input_person_ids,
 def py():
     pass
 
+def check_dir_isvalid(directory: str | tuple[str, ...]) -> None:
+    ## check output dir is valid
+    if type(directory) is tuple:
+        directory = directory[0]
+
+    if not os.path.isdir(directory):
+        print("Not a directory, dir {0}".format(directory))
+        sys.exit(1)
+
+def set_saved_person_id_file(saved_person_id_file: str, output_dir: str) -> str:
+## check if there is a saved person id file set in options - if not, check if the file exists and remove it
+    if saved_person_id_file is None:
+        saved_person_id_file = output_dir + "/" + "person_ids.tsv"
+        if os.path.exists(saved_person_id_file):
+            os.remove(saved_person_id_file)
+    return saved_person_id_file
+
+
+def check_files_in_rules_exist(rules_input_files: list[str], existing_input_files: list[str]) -> None:
+    for infile in existing_input_files:
+        if infile not in rules_input_files:
+            msg = "WARNING: no mapping rules found for existing input file - {0}".format(infile)
+            print(msg)
+    for infile in rules_input_files:
+        if infile not in existing_input_files:
+            msg = "WARNING: no data for mapped input file - {0}".format(infile)
+            print(msg)
+
+def open_file(directory: str, filename: str) -> tuple[IO[str], Iterator[list[str]]] | None:
+#def open_file(directory: str, filename: str):
+    try:
+        fh = open(directory + "/" + filename, mode="r", encoding="utf-8-sig")
+        csvr = csv.reader(fh)
+        return fh, csvr
+    except IOError as e:
+        print("Unable to open: {0}".format(directory + "/" + filename))
+        print("I/O error({0}): {1}".format(e.errno, e.strerror))
+        return None
+
+def set_omop_filenames(omop_ddl_file: str, omop_config_file: str, omop_version: str) -> tuple[str, str]:
+    if (omop_ddl_file is None) and (omop_config_file is None) and (omop_version is not None):
+        omop_config_file = str(importlib.resources.files('carrottransform')) + '/' + 'config/omop.json'
+        omop_ddl_file_name = "OMOPCDM_postgresql_" + omop_version + "_ddl.sql"
+        omop_ddl_file = str(importlib.resources.files('carrottransform')) + '/' + 'config/' + omop_ddl_file_name
+    return omop_config_file, omop_ddl_file
+
+def get_person_lookup(saved_person_id_file: str) -> tuple[dict[str, str], int]:
+    # Saved-person-file existence test, reload if found, return last used integer
+    if os.path.isfile(saved_person_id_file):
+        person_lookup, last_used_integer = load_saved_person_ids(saved_person_id_file)
+    else:
+        person_lookup = {}
+        last_used_integer = 1
+    return person_lookup, last_used_integer
+
 run.add_command(mapstream,"mapstream")
+
+if __name__== '__main__':
+    mapstream()

{carrot_transform-0.3.2 → carrot_transform-0.3.4}/carrottransform/config/omop.json

@@ -26,6 +26,10 @@
     "visit_occurrence": {
 	    "visit_start_datetime": "visit_start_date", 
 	    "visit_end_datetime": "visit_end_date"
+    },
+    "device_exposure": {
+	    "device_exposure_start_datetime": "device_exposure_start_date", 
+	    "device_exposure_end_datetime": "device_exposure_end_date"
     }
   },
   "date_field_components": {
@@ -46,6 +50,7 @@
     "person": "person_id",
     "procedure_occurrence": "person_id",
     "specimen": "person_id",
+    "device_exposure": "person_id",
     "visit_occurrence": "person_id"
   },
   "auto_number_field": {
@@ -56,6 +61,7 @@
     "observation": "observation_id",
     "procedure_occurrence": "procedure_occurrence_id",
     "specimen": "specimen_id",
+    "device_exposure": "device_exposure_id",
     "visit_occurrence": "visit_occurrence_id"
   }
 }