carrot-transform 0.3.5__py3-none-any.whl → 0.4.0__py3-none-any.whl

carrottransform/tools/metrics.py

@@ -1,17 +1,18 @@
-
-import logging
-logger = logging.getLogger(__name__)
-
 from dataclasses import dataclass, field
 from typing import Dict, List
 
+from carrottransform.tools.logger import logger_setup
+
+logger = logger_setup()
+
+
 @dataclass
 class DataKey:
     source: str
-    fieldname:str
-    tablename:str
-    concept_id:str
-    additional:str
+    fieldname: str
+    tablename: str
+    concept_id: str
+    additional: str
 
     def __str__(self) -> str:
         """
@@ -19,11 +20,21 @@ class DataKey:
         This is here in case that representation is needed somewhere
         """
         return f"{self.source}~{self.fieldname}~{self.tablename}~{self.concept_id}~{self.additional}"
+
     def __hash__(self) -> int:
         """
         The DataKey is used as a key for a dictionary of key counts
         """
-        return hash((self.source, self.fieldname, self.tablename, self.concept_id, self.additional))
+        return hash(
+            (
+                self.source,
+                self.fieldname,
+                self.tablename,
+                self.concept_id,
+                self.additional,
+            )
+        )
+
 
 @dataclass
 class CountData:
@@ -33,13 +44,15 @@ class CountData:
         if count_type not in self.counts:
             self.counts[count_type] = 0
         self.counts[count_type] += 1
-    
-    def get_count(self, count_type: str, default: int=0):
+
+    def get_count(self, count_type: str, default: int = 0):
         return self.counts.get(count_type, default)
 
+
 @dataclass
 class MapstreamSummaryRow:
     """Represents a single row in the mapstream summary"""
+
     dataset_name: str
     source: str
     fieldname: str
@@ -51,57 +64,62 @@ class MapstreamSummaryRow:
     invalid_date_fields: int = 0
     invalid_source_fields: int = 0
     output_count: int = 0
-    
+
     def to_tsv_row(self) -> str:
         """Convert the row to a tab-separated string"""
-        row_list = [str(col) for col in [
-            self.dataset_name,
-            self.source,
-            self.fieldname,
-            self.tablename,
-            self.concept_id,
-            self.additional,
-            self.input_count,
-            self.invalid_person_ids,
-            self.invalid_date_fields,
-            self.invalid_source_fields,
-            self.output_count
-            ]]
+        row_list = [
+            str(col)
+            for col in [
+                self.dataset_name,
+                self.source,
+                self.fieldname,
+                self.tablename,
+                self.concept_id,
+                self.additional,
+                self.input_count,
+                self.invalid_person_ids,
+                self.invalid_date_fields,
+                self.invalid_source_fields,
+                self.output_count,
+            ]
+        ]
         # If python gets updated, you can move the row_str expression into the f-string
-        row_str = '\t'.join(row_list)
+        row_str = "\t".join(row_list)
         return f"{row_str}\n"
-    
+
     @classmethod
     def get_header(cls) -> str:
         """Return the TSV header row"""
         header = [
-                "dsname",
-                "source",
-                "source_field",
-                "target",
-                "concept_id",
-                "additional",
-                "incount",
-                "invalid_persid",
-                "invalid_date",
-                "invalid_source",
-                "outcount"
-                ]
-        header_str = '\t'.join(header)
+            "dsname",
+            "source",
+            "source_field",
+            "target",
+            "concept_id",
+            "additional",
+            "incount",
+            "invalid_persid",
+            "invalid_date",
+            "invalid_source",
+            "outcount",
+        ]
+        header_str = "\t".join(header)
         return f"{header_str}\n"
 
-class Metrics():
+
+class Metrics:
     """
     Capture metrics for output to a summary tsv file, record counts at multiple levels
     The main principle is to increment counts associated with datakeys (dkey) at different levels
     """
+
     def __init__(self, dataset_name, log_threshold=0):
         """
         self.datasummary holds all the saved counts
         """
-        self.datasummary={}
-        self.allcounts={}
-        self.dataset_name=dataset_name
+        self.datasummary = {}
+        self.allcounts = {}
+        self.dataset_name = dataset_name
         self.log_threshold = log_threshold
 
     def get_new_mapstream_counts(self):
@@ -135,8 +153,18 @@ class Metrics():
                     prfx = "NA"
                     if "source_files" in increment:
                         if fieldname in increment["source_files"]:
-                            prfx = self.get_prefix(increment["source_files"][fieldname]["table"])
-                            dkey = prfx + "." + desttablename + "." + name + "." + fieldname
+                            prfx = self.get_prefix(
+                                increment["source_files"][fieldname]["table"]
+                            )
+                            dkey = (
+                                prfx
+                                + "."
+                                + desttablename
+                                + "."
+                                + name
+                                + "."
+                                + fieldname
+                            )
                             self.add_counts_to_summary(dkey, dataitem[fieldname])
 
     def get_prefix(self, fname):
@@ -150,7 +178,9 @@ class Metrics():
                 self.datasummary[dkey][counttype] = 0
             self.datasummary[dkey][counttype] += int(count_block[counttype])
 
-    def increment_key_count(self, source, fieldname, tablename, concept_id, additional, count_type):
+    def increment_key_count(
+        self, source, fieldname, tablename, concept_id, additional, count_type
+    ):
         dkey = DataKey(source, fieldname, tablename, concept_id, additional)
 
         if dkey not in self.datasummary:
@@ -159,87 +189,111 @@ class Metrics():
         self.datasummary[dkey].increment(count_type)
 
     def increment_with_datacol(
-            self,
-            source_path: str,
-            target_file: str,
-            datacol: str,
-            out_record: List[str],
-            ) -> None:
-        #Are the parameters for DataKeys hierarchical?
-        #If so, a nested structure where a Source contains n Fields etc. and each has a method to sum its children would be better
-        #But I don't know if that's the desired behaviour
-
-        #A lot of these increment the same thing, so I have defined `increment_this`
+        self,
+        source_path: str,
+        target_file: str,
+        datacol: str,
+        out_record: List[str],
+    ) -> None:
+        # Are the parameters for DataKeys hierarchical?
+        # If so, a nested structure where a Source contains n Fields etc. and each has a method to sum its children would be better
+        # But I don't know if that's the desired behaviour
+
+        # A lot of these increment the same thing, so I have defined `increment_this`
         def increment_this(
-                fieldname: str,
-                concept_id: str,
-                additional = "",
-                ) -> None:
+            fieldname: str,
+            concept_id: str,
+            additional="",
+        ) -> None:
             self.increment_key_count(
-                    source=source_path,
-                    fieldname=fieldname,
-                    tablename=target_file,
-                    concept_id=concept_id,
-                    additional=additional,
-                    count_type="output_count"
-                    )
-        self.increment_key_count(
                 source=source_path,
-                fieldname="all",
-                tablename="all",
-                concept_id="all",
-                additional="",
-                count_type="output_count"
-                )
+                fieldname=fieldname,
+                tablename=target_file,
+                concept_id=concept_id,
+                additional=additional,
+                count_type="output_count",
+            )
 
         self.increment_key_count(
-                source="all",
-                fieldname="all",
-                tablename=target_file,
-                concept_id="all",
-                additional="",
-                count_type="output_count"
-                )
+            source=source_path,
+            fieldname="all",
+            tablename="all",
+            concept_id="all",
+            additional="",
+            count_type="output_count",
+        )
+
+        self.increment_key_count(
+            source="all",
+            fieldname="all",
+            tablename=target_file,
+            concept_id="all",
+            additional="",
+            count_type="output_count",
+        )
         increment_this(fieldname="all", concept_id="all")
-       
+
         if target_file == "person":
             increment_this(fieldname="all", concept_id=out_record[1])
-            increment_this(fieldname="all", concept_id=out_record[1], additional=out_record[2])
+            increment_this(
+                fieldname="all", concept_id=out_record[1], additional=out_record[2]
+            )
         else:
             increment_this(fieldname=datacol, concept_id=out_record[2])
             increment_this(fieldname="all", concept_id=out_record[2])
             self.increment_key_count(
-                    source="all",
-                    fieldname="all",
-                    tablename=target_file,
-                    concept_id=out_record[2],
-                    additional="",
-                    count_type="output_count"
-                    )
+                source="all",
+                fieldname="all",
+                tablename=target_file,
+                concept_id=out_record[2],
+                additional="",
+                count_type="output_count",
+            )
             self.increment_key_count(
-                    source="all",
-                    fieldname="all",
-                    tablename="all",
-                    concept_id=out_record[2],
-                    additional="",
-                    count_type="output_count"
-                    )
-
+                source="all",
+                fieldname="all",
+                tablename="all",
+                concept_id=out_record[2],
+                additional="",
+                count_type="output_count",
+            )
 
     def get_summary(self):
         summary_str = "source\ttablename\tname\tcolumn name\tbefore\tafter content check\tpct reject content check\tafter date format check\tpct reject date format\n"
 
         for dkey in self.datasummary:
             logger.debug(dkey)
-            source, tablename, name, colname = dkey.split('.')
+            source, tablename, name, colname = dkey.split(".")
             before_count = int(self.datasummary[dkey]["before"])
             after_count = int(self.datasummary[dkey]["after"])
             after_pct = (float)(before_count - after_count) * 100 / before_count
-            summary_str += source + "\t" + tablename + "\t" + name + "\t" + colname + "\t" + str(before_count) + "\t" + str(after_count) + "\t" + "{0:.3f}".format(after_pct) + "\t"
+            summary_str += (
+                source
+                + "\t"
+                + tablename
+                + "\t"
+                + name
+                + "\t"
+                + colname
+                + "\t"
+                + str(before_count)
+                + "\t"
+                + str(after_count)
+                + "\t"
+                + "{0:.3f}".format(after_pct)
+                + "\t"
+            )
             if "after_formatting" in self.datasummary[dkey]:
                 after_format_count = int(self.datasummary[dkey]["after_formatting"])
-                after_format_pct = (float)(after_count - after_format_count) * 100 / after_count
-                summary_str += str(after_format_count) + "\t" + "{0:.3f}".format(after_format_pct) + "\n"
+                after_format_pct = (
+                    (float)(after_count - after_format_count) * 100 / after_count
+                )
+                summary_str += (
+                    str(after_format_count)
+                    + "\t"
+                    + "{0:.3f}".format(after_format_pct)
+                    + "\n"
+                )
             else:
                 summary_str += "NA\tNA\n"
 
@@ -269,24 +323,23 @@ class Metrics():
                 invalid_person_ids=count_data.get_count("invalid_person_ids"),
                 invalid_date_fields=count_data.get_count("invalid_date_fields"),
                 invalid_source_fields=count_data.get_count("invalid_source_fields"),
-                output_count=count_data.get_count("output_count")
+                output_count=count_data.get_count("output_count"),
             )
 
             if row.output_count >= self.log_threshold:
                 rows.append(row)
         return rows
 
-
     def get_mapstream_summary(self) -> str:
         """
         Makes a TSV string of the mapstream summary
         """
         summary_rows = self.get_mapstream_summary_rows()
         result = MapstreamSummaryRow.get_header()
-        
+
         for row in summary_rows:
             result += row.to_tsv_row()
-            
+
         return result
 
     def get_mapstream_summary_dict(self) -> Dict:
@@ -297,5 +350,5 @@ class Metrics():
         return {
             "dataset": self.dataset_name,
             "threshold": self.log_threshold,
-            "rows": [vars(row) for row in rows]
+            "rows": [vars(row) for row in rows],
         }

carrottransform/tools/omopcdm.py

@@ -1,18 +1,19 @@
 import carrottransform.tools as tools
 import json
-import logging
+from carrottransform.tools.logger import logger_setup
 import re
 import sys
 
 from pathlib import Path
 
-logger = logging.getLogger(__name__)
+logger = logger_setup()
+
 
 class OmopCDM:
     """
     Load and parse OMOP DDL data, to make an in-memory json CDM
     Merge in manual additions (currently necessary to identify, person id, date / time fields and autonumber fields)
-    Define a series of "get" functions to allow CDM component discovery 
+    Define a series of "get" functions to allow CDM component discovery
     """
 
     def __init__(self, omopddl, omopcfg):
@@ -33,15 +34,14 @@ class OmopCDM:
         self.person_id_field = self.get_columns("person_id_field")
         self.auto_number_field = self.get_columns("auto_number_field")
 
-
     def load_ddl(self, omopddl: Path):
         try:
-            fp = omopddl.open("r") 
-        except Exception as err:
+            fp = omopddl.open("r")
+        except Exception:
             logger.exception("OMOP ddl file ({0}) not found".format(omopddl))
             sys.exit()
-        
-        return(self.process_ddl(fp))
+
+        return self.process_ddl(fp)
 
     def process_ddl(self, fp):
         """
@@ -56,13 +56,13 @@ class OmopCDM:
         output_dict["date_fields"] = {}
 
         ## matching for version number - matches '--postgres', any number of chars and some digits of the form X.Y, plus an end of string or end of line
-        ver_rgx = re.compile(r'^--postgresql.*(\d+\.\d+)$')
+        ver_rgx = re.compile(r"^--postgresql.*(\d+\.\d+)$")
         ## matching for table name - matches 'CREATE TABLE @', some letters (upper and lower case), '.' and some more letters (lower case)
-        start_rgx = re.compile(r'^CREATE\s*TABLE\s*(\@?[a-zA-Z]+\.)?([a-zA-Z_]+)')
+        start_rgx = re.compile(r"^CREATE\s*TABLE\s*(\@?[a-zA-Z]+\.)?([a-zA-Z_]+)")
         ## matches some whitespace, lower case letters(or underscores), whitespace, letters (upper/lower and underscores)
-        datatype_rgx = re.compile(r'^\s*([a-z_]+)\s+([a-zA-Z_]+)')
+        datatype_rgx = re.compile(r"^\s*([a-z_]+)\s+([a-zA-Z_]+)")
         ## matching for end of file - matches close bracket, semi colon, end of file or line
-        end_rgx = re.compile(r'.*[)];$')
+        end_rgx = re.compile(r".*[)];$")
         vermatched = False
         processing_table_data = False
         tabname = ""
@@ -70,21 +70,22 @@ class OmopCDM:
         for line in fp:
             line = line.strip()
             # check for line with version, if present
-            if vermatched == False:
+            if not vermatched:
                 vmatch = ver_rgx.search(line)
-                if vmatch != None:
+                if vmatch is not None:
                     version_string = vmatch.group(1)
                     output_dict["omop_version"] = version_string
                     vermatched = True
+
             # check for start of table definition
-            if processing_table_data == False:
+            if not processing_table_data:
                 smatch = start_rgx.search(line)
-                if smatch != None:
+                if smatch is not None:
                     processing_table_data = True
                     tabname = smatch.group(2).lower()
             else:
                 idtmatch = datatype_rgx.search(line)
-                if idtmatch != None:
+                if idtmatch is not None:
                     fname = idtmatch.group(1)
                     ftype = idtmatch.group(2)
 
@@ -99,12 +100,16 @@ class OmopCDM:
                         output_dict["datetime_fields"][tabname] = []
                     if tabname not in output_dict["date_fields"]:
                         output_dict["date_fields"][tabname] = []
-  
+
                     # Add in required column / field data
                     output_dict["all_columns"][tabname].append(fname)
                     if ftype.lower() in self.numeric_types:
                         output_dict["numeric_fields"][tabname].append(fname)
-                    if ftype.lower() in self.numeric_types and "NOT" in line and "NULL" in line:
+                    if (
+                        ftype.lower() in self.numeric_types
+                        and "NOT" in line
+                        and "NULL" in line
+                    ):
                         output_dict["notnull_numeric_fields"][tabname].append(fname)
                     if ftype.lower() in self.datetime_types:
                         output_dict["datetime_fields"][tabname].append(fname)
@@ -112,19 +117,19 @@ class OmopCDM:
                         output_dict["date_fields"][tabname].append(fname)
 
             ematch = end_rgx.search(line)
-            if ematch != None:
+            if ematch is not None:
                 processing_table_data = False
-  
-        return(output_dict)
-    
+
+        return output_dict
+
     def dump_ddl(self):
-        return(json.dumps(self.omop_json, indent=2))
+        return json.dumps(self.omop_json, indent=2)
 
     def merge_json(self, omopjson, omopcfg):
         tmp_json = tools.load_json(omopcfg)
         for key, data in tmp_json.items():
             omopjson[key] = data
-        return(omopjson)
+        return omopjson
 
     def get_columns(self, colkey):
         if colkey in self.omop_json:
@@ -157,43 +162,43 @@ class OmopCDM:
         return True
 
     def get_omop_numeric_fields(self, tablename):
-        if self.numeric_fields != None:
+        if self.numeric_fields is not None:
             if tablename in self.numeric_fields:
                 return self.numeric_fields[tablename]
         return []
 
     def get_omop_notnull_numeric_fields(self, tablename):
-        if self.notnull_numeric_fields != None:
+        if self.notnull_numeric_fields is not None:
             if tablename in self.notnull_numeric_fields:
                 return self.notnull_numeric_fields[tablename]
         return []
 
     def get_omop_datetime_linked_fields(self, tablename):
-        if self.datetime_linked_fields != None:
+        if self.datetime_linked_fields is not None:
             if tablename in self.datetime_linked_fields:
                 return self.datetime_linked_fields[tablename]
         return {}
 
     def get_omop_date_field_components(self, tablename):
-        if self.date_field_components != None:
+        if self.date_field_components is not None:
             if tablename in self.date_field_components:
                 return self.date_field_components[tablename]
         return {}
 
     def get_omop_datetime_fields(self, tablename):
-        if self.datetime_fields != None:
+        if self.datetime_fields is not None:
             if tablename in self.datetime_fields:
                 return self.datetime_fields[tablename]
         return []
 
     def get_omop_person_id_field(self, tablename):
-        if self.person_id_field != None:
+        if self.person_id_field is not None:
             if tablename in self.person_id_field:
                 return self.person_id_field[tablename]
         return None
 
     def get_omop_auto_number_field(self, tablename):
-        if self.auto_number_field != None:
+        if self.auto_number_field is not None:
             if tablename in self.auto_number_field:
                 return self.auto_number_field[tablename]
         return None