fhir-sheets 1.3.0__tar.gz → 2.0.0__tar.gz

{fhir_sheets-1.3.0 → fhir_sheets-2.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fhir-sheets
-Version: 1.3.0
+Version: 2.0.0
 Summary: FhirSheets is a command-line tool that reads an Excel file in FHIR cohort format and generates FHIR bundle JSON files from it. Each row in the template Excel file is used to create an individual JSON file, outputting them to a specified folder.
 License-File: LICENSE
 Author: Michael Riley

{fhir_sheets-1.3.0 → fhir_sheets-2.0.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "fhir-sheets"
-version = "1.3.0"
+version = "2.0.0"
 description = "FhirSheets is a command-line tool that reads an Excel file in FHIR cohort format and generates FHIR bundle JSON files from it. Each row in the template Excel file is used to create an individual JSON file, outputting them to a specified folder."
 authors = ["Michael Riley <Michael.Riley@gtri.gatech.edu>"]
 packages = [{include = "fhir_sheets", from = "src"}]

{fhir_sheets-1.3.0 → fhir_sheets-2.0.0}/src/fhir_sheets/cli/main.py

@@ -31,7 +31,7 @@ def main(input_file, output_folder):
     resource_definition_entities, resource_link_entities, cohort_data = read_input.read_xlsx_and_process(input_file)
     pprint(cohort_data)
     #For each index of patients
-    for i in range(0,cohort_data.num_entries):
+    for i in range(0,cohort_data.get_num_patients()):
         # Construct the file path for each JSON file
         file_path = output_folder_path / f"{i}.json"
         #Create a bundle

{fhir_sheets-1.3.0 → fhir_sheets-2.0.0}/src/fhir_sheets/core/conversion.py

@@ -3,7 +3,7 @@ import uuid
 from jsonpath_ng.jsonpath import Fields, Slice, Where
 from jsonpath_ng.ext import parse as parse_ext
 
-from .model.cohort_data_entity import CohortData, FieldEntry
+from .model.cohort_data_entity import CohortData, CohortData
 from .model.resource_definition_entity import ResourceDefinition
 from .model.resource_link_entity import ResourceLink
 from . import fhir_formatting
@@ -62,17 +62,27 @@ def initialize_resource(resource_definition):
 # Creates a fhir-json structure from a resource definition entity and the patient_data_sheet
 def create_fhir_resource(resource_definition: ResourceDefinition, cohort_data: CohortData, index = 0):
     resource_dict = initialize_resource(resource_definition)
-    #Get field entries for this entitiy
-    try:
-        all_field_entries = cohort_data.entities[resource_definition.entity_name].fields
-    except KeyError:
+    #Get field entries for this entity
+    header_entries_for_resourcename = [
+        headerEntry
+        for headerEntry in cohort_data.headers
+        if headerEntry.entityName == resource_definition.entity_name
+    ]
+    dataelements_for_resourcename = {
+        key: value
+        for key, value in cohort_data.patients[index].entries.items()
+        if value['entity_name'] == resource_definition.entity_name
+    }
+    if len(dataelements_for_resourcename.keys()) == 0:
         print(f"WARNING: Patient index {index} - Create Fhir Resource Error - {resource_definition.entity_name} - No columns for entity '{resource_definition.entity_name}' found for resource in 'PatientData' sheet")
         return resource_dict
+        all_field_entries = cohort_data.entities[resource_definition.entity_name].fields
     #For each field within the entity
-    for field_entry_key, field_entry in all_field_entries.items():
-        #Create a jsonpath from each provided json path and value for this resource
-        if field_entry.values and len(field_entry.values) > index:
-            create_structure_from_jsonpath(resource_dict, field_entry.jsonpath, resource_definition, field_entry, field_entry.value_type, field_entry.values[index])
+    for fieldname, dataelement in dataelements_for_resourcename.items():
+        header_element = next((header for header in header_entries_for_resourcename if header.fieldName == fieldname), None)
+        if header_element is None:
+            print(f"WARNING: Field Name {fieldname} - No Header Entry found.")
+        create_structure_from_jsonpath(resource_dict, header_element.jsonpath, resource_definition, header_element.value_type, dataelement['value'])
     return resource_dict
 
 #Create a resource_link for default references in the cases where only 1 resourceType of the source and destination exist
@@ -204,7 +214,7 @@ def add_resource_to_transaction_bundle(root_bundle, fhir_resource):
 # resource_definition: resource description model from import
 # entity_definition: specific field entry information for this function
 # value: Actual value to assign
-def create_structure_from_jsonpath(root_struct: Dict, json_path: str, resource_definition: ResourceDefinition, field_entry: FieldEntry, dataType: str, value: Any):
+def create_structure_from_jsonpath(root_struct: Dict, json_path: str, resource_definition: ResourceDefinition,  dataType: str, value: Any):
     #Get all dot notation components as seperate 
     if dataType is not None and dataType.strip().lower() == 'string':
         value = str(value)
@@ -214,10 +224,10 @@ def create_structure_from_jsonpath(root_struct: Dict, json_path: str, resource_d
         return root_struct
     #Start of top-level function which calls the enclosed recursive function
     parts = json_path.split('.')
-    return build_structure(root_struct, json_path, resource_definition, field_entry, parts, value, [])
+    return build_structure(root_struct, json_path, resource_definition, dataType, parts, value, [])
 
 # main recursive function to drill into the json structure, assign paths, and create structure where needed
-def build_structure(current_struct, json_path, resource_definition, entity_definition, parts, value, previous_parts):
+def build_structure(current_struct: Dict, json_path: str, resource_definition: ResourceDefinition, dataType: str, parts: List[str], value: Any, previous_parts: List[str]):
     if len(parts) == 0:
         return current_struct
     #Grab current part
@@ -225,11 +235,11 @@ def build_structure(current_struct, json_path, resource_definition, entity_defin
     #SPECIAL HANDLING CLAUSE
     matching_handler = next((handler for handler in special_values.custom_handlers if (json_path.startswith(handler) or json_path == handler)), None)
     if matching_handler is not None:
-        return special_values.custom_handlers[matching_handler].assign_value(json_path, resource_definition, entity_definition, current_struct, parts[-1], value)
+        return special_values.custom_handlers[matching_handler].assign_value(json_path, resource_definition,  current_struct, parts[-1], value)
     #Ignore dollar sign ($) and drill farther down
     if part == '$' or part == resource_definition.resource_type.strip():
         #Ignore the dollar sign and the resourcetype
-        return build_structure_recurse(current_struct, json_path, resource_definition, entity_definition, parts, value, previous_parts, part)
+        return build_structure_recurse(current_struct, json_path, resource_definition, dataType, parts, value, previous_parts, part)
     
     # If parts length is one then this is the final key to access and pair
     if len(parts) == 1:
@@ -252,7 +262,7 @@ def build_structure(current_struct, json_path, resource_definition, entity_defin
                 if part + 1 > len(current_struct):
                     current_struct.extend({} for x in range (part + 1 - len(current_struct)))
         #Actual assigning to the path
-        fhir_formatting.assign_value(current_struct, part, value, entity_definition.value_type)
+        fhir_formatting.assign_value(current_struct, part, value, dataType)
         return current_struct
     
     # If there is a simple qualifier with '['and ']'
@@ -274,7 +284,7 @@ def build_structure(current_struct, json_path, resource_definition, entity_defin
             if qualifier_as_number + 1 > len(current_struct):
                 current_struct.extend({} for x in range (qualifier_as_number + 1 - len(current_struct)))
             inner_struct = current_struct[qualifier_as_number]
-            inner_struct = build_structure_recurse(inner_struct, json_path, resource_definition, entity_definition, parts, value, previous_parts, part)
+            inner_struct = build_structure_recurse(inner_struct, json_path, resource_definition, dataType, parts, value, previous_parts, part)
             current_struct[qualifier_as_number] = inner_struct
             return current_struct
         # Create the key part in the structure
@@ -296,7 +306,7 @@ def build_structure(current_struct, json_path, resource_definition, entity_defin
                 inner_struct = {qualifier_key: qualifier_value}
                 current_struct[key_part].append(inner_struct)
             #Recurse into that innerstructure where the qualifier matched to continue the part traversal
-            inner_struct = build_structure_recurse(inner_struct, json_path, resource_definition, entity_definition, parts, value, previous_parts, part)
+            inner_struct = build_structure_recurse(inner_struct, json_path, resource_definition, dataType, parts, value, previous_parts, part)
             return current_struct
         #If there's no qualifier condition, but an index aka '[0]', '[1]' etc, then it's a simple accessor
         elif qualifier.isdigit():
@@ -306,19 +316,19 @@ def build_structure(current_struct, json_path, resource_definition, entity_defin
             if qualifier_as_number > len(current_struct):
                 current_struct[key_part].extend({} for x in range (qualifier_as_number - len(current_struct)))
             inner_struct = current_struct[key_part][qualifier_as_number]
-            inner_struct = build_structure_recurse(inner_struct, json_path, resource_definition, entity_definition, parts, value, previous_parts, part)
+            inner_struct = build_structure_recurse(inner_struct, json_path, resource_definition, dataType, parts, value, previous_parts, part)
             current_struct[key_part][qualifier_as_number] = inner_struct
             return current_struct
     #None qualifier accessor
     else:
         if(part not in current_struct):
             current_struct[part] = {}
-        inner_struct = build_structure_recurse(current_struct[part], json_path, resource_definition, entity_definition, parts, value, previous_parts, part)
+        inner_struct = build_structure_recurse(current_struct[part], json_path, resource_definition, dataType, parts, value, previous_parts, part)
         current_struct[part] = inner_struct
         return current_struct
     
 #Helper function to quickly recurse and return the next level of structure. Used by main recursive function
-def build_structure_recurse(current_struct, json_path, resource_definition, entity_definition, parts, value, previous_parts, part):
+def build_structure_recurse(current_struct, json_path, resource_definition, dataType, parts, value, previous_parts, part):
     previous_parts.append(part)
-    return_struct = build_structure(current_struct, json_path, resource_definition, entity_definition, parts[1:], value, previous_parts)
+    return_struct = build_structure(current_struct, json_path, resource_definition, dataType, parts[1:], value, previous_parts)
     return return_struct

fhir_sheets-2.0.0/src/fhir_sheets/core/model/cohort_data_entity.py

@@ -0,0 +1,33 @@
+import pprint
+from typing import Dict, Any, List, Optional, Tuple
+
+class HeaderEntry:
+    def __init__(self, data: Dict[str, Any]):
+        self.entityName: Optional[str] = data.get('entityName')
+        self.fieldName: Optional[str] = data.get('fieldName')
+        self.jsonpath: Optional[str] = data.get('jsonpath')
+        self.value_type: Optional[str] = data.get('valueType')
+        self.valuesets: Optional[str] = data.get('valuesets')
+        
+    def __repr__(self) -> str:
+        return (f"\nHeaderEntry(entityName='{self.entityName}', \n\tfieldName='{self.fieldName}', \n\tjsonpath='{self.jsonpath}',\n\tvalue_type='{self.value_type}', "
+                f"\n\tvaluesets='{self.valuesets}')")
+    
+class PatientEntry:
+
+    def __init__(self, entries:Dict[str,str]):
+        self.entries = entries
+
+    def __repr__(self) -> str:
+        return (f"PatientEntry(\n\t'{self.entries}')")
+    
+class CohortData:
+    def __init__(self, headers: List[Dict[str,Any]], patients: List[Dict[str,str]]):
+        self.headers = [HeaderEntry(header_data) for header_data in headers]
+        self.patients = [PatientEntry(patient_data) for patient_data in patients]
+
+    def __repr__(self) -> str:
+        return (f"CohortData(\n\t-----\n\theaders='{self.headers}',\n\t-----\n\tpatients='{self.patients}')")
+    
+    def get_num_patients(self):
+        return len(self.patients)

{fhir_sheets-1.3.0 → fhir_sheets-2.0.0}/src/fhir_sheets/core/read_input.py

@@ -1,6 +1,6 @@
 import openpyxl
 
-from .model.cohort_data_entity import CohortData, EntityData, FieldEntry
+from .model.cohort_data_entity import CohortData, CohortData
 
 from .model.resource_definition_entity import ResourceDefinition
 from .model.resource_link_entity import ResourceLink
@@ -21,7 +21,7 @@ def read_xlsx_and_process(file_path):
 
     if 'PatientData' in workbook.sheetnames:
         sheet = workbook['PatientData']
-        cohort_data = process_sheet_patient_data(sheet, resource_definition_entities)
+        cohort_data = process_sheet_patient_data_revised(sheet, resource_definition_entities)
     
     return resource_definition_entities, resource_link_entities, cohort_data
 
@@ -104,4 +104,44 @@ def process_sheet_patient_data(sheet, resource_definition_entities):
                 # Append the actual data values to the 'values' array
                 cohort_data.entities[entity_name].fields[field_name].values.append(value)
     cohort_data.num_entries = num_entries
+    return cohort_data
+
+# Function to process the "PatientData" sheet for the Revised CohortData
+def process_sheet_patient_data_revised(sheet, resource_definition_entities):
+    headers = []
+    patients = []
+    # Initialize the dictionary to store the processed data
+    # Process the Header Entries from the first 6 rows (Entity To Query, JsonPath, etc.) and the data from the rest.
+    for col in sheet.iter_cols(min_row=1, min_col=3, values_only=True):  # Start from 3rd column
+        if all(entry is None for entry in col):
+            continue
+        entity_name = col[0]  # The entity name comes from the first row (Entity To Query)
+        field_name = col[5]  #The "Data Element" comes from the fifth row
+        if (entity_name is None or entity_name == "") and (field_name is not None and field_name != ""):
+            print(f"WARNING: - Reading Patient Data Issue - {field_name} - 'Entity To Query' cell missing for column labelled '{field_name}', please provide entity name from the ResourceDefinitions tab.")
+
+        if entity_name not in [entry.entity_name for entry in resource_definition_entities]:
+            print(f"WARNING: - Reading Patient Data Issue - {field_name} - 'Entity To Query' cell has entity named '{entity_name}', however, the ResourceDefinition tab has no matching resource. Please provide a corresponding entry in the ResourceDefinition tab.")
+
+        # Create a header entry
+        header_data = {
+            "fieldName": field_name,
+            "entityName": entity_name,
+            "jsonpath": col[1],  # JsonPath from the second row
+            "valueType": col[2], # Value Type from the third row
+            "valuesets": col[3] # Value Set from the fourth row
+        }
+        headers.append(header_data)
+        # Create a data entry
+        values = col[6:] # The values come from the 6th row and below
+        values = tuple(item for item in values if item is not None)
+        #Expand the patient dictionary set if needed
+        if len(values) > len(patients):
+            needed_count = len(values) - len(patients)
+            patients.extend([{}] * needed_count)
+        for patient_dict, value in zip(patients, values):
+            patient_dict[field_name] = {}
+            patient_dict[field_name]["entity_name"] = entity_name
+            patient_dict[field_name]["value"] = value
+    cohort_data = CohortData(headers=headers, patients=patients)
     return cohort_data

{fhir_sheets-1.3.0 → fhir_sheets-2.0.0}/src/fhir_sheets/core/special_values.py

@@ -7,7 +7,7 @@ from abc import ABC, abstractmethod
 class AbstractCustomValueHandler(ABC):
     
     @abstractmethod
-    def assign_value(self, json_path, resource_definition, entity_definition, final_struct, key, value):
+    def assign_value(self, json_path, resource_definition, final_struct, key, value):
         pass
     
 class PatientRaceExtensionValueHandler(AbstractCustomValueHandler):
@@ -67,7 +67,7 @@ class PatientRaceExtensionValueHandler(AbstractCustomValueHandler):
       "url" : "http://hl7.org/fhir/us/core/StructureDefinition/us-core-race"
     }
     #Create an ombcategory and detailed section of race extension
-    def assign_value(self, json_path, resource_definition, entity_definition, final_struct, key, value):
+    def assign_value(self, json_path, resource_definition, final_struct, key, value):
         #Retrieve the race extension if it exists; make it if it does not.
         if 'extension' not in final_struct:
             final_struct['extension'] = []
@@ -128,7 +128,7 @@ class PatientEthnicityExtensionValueHandler(AbstractCustomValueHandler):
       "url" : "http://hl7.org/fhir/us/core/StructureDefinition/us-core-ethnicity"
     }
     #Create an ombcategory and detailed section of ethnicitiy extension
-    def assign_value(self, json_path, resource_definition, entity_definition, final_struct, key, value):
+    def assign_value(self, json_path, resource_definition, final_struct, key, value):
         #Retrieve the ethncitiy extension if it exists; make it if it does not.
         if 'extension' not in final_struct:
             final_struct['extension'] = []
@@ -154,7 +154,7 @@ class PatientBirthSexExtensionValueHandler(AbstractCustomValueHandler):
       "valueCode" : "$value"
     }
     #Assigna birthsex extension
-    def assign_value(self, json_path, resource_definition, entity_definition, final_struct, key, value):
+    def assign_value(self, json_path, resource_definition, final_struct, key, value):
         #Retrieve the birthsex extension if it exists; make it if it does not.
         if 'extension' not in final_struct:
             final_struct['extension'] = []
@@ -182,7 +182,7 @@ class PatientMRNIdentifierValueHandler(AbstractCustomValueHandler):
       "value" : "$value"
     }
     #Assign a MRN identifier
-    def assign_value(self, json_path, resource_definition, entity_definition, final_struct, key, value):
+    def assign_value(self, json_path, resource_definition, final_struct, key, value):
         #Retrieve the MRN identifier if it exists; make it if it does not.
         target_identifier = self.patient_mrn_block
         new_identifier = True
@@ -215,7 +215,7 @@ class PatientSSNIdentifierValueHandler(AbstractCustomValueHandler):
       "value" : "$value"
     }
     #Assign a MRN identifier
-    def assign_value(self, json_path, resource_definition, entity_definition, final_struct, key, value):
+    def assign_value(self, json_path, resource_definition, final_struct, key, value):
         #Retrieve the MRN identifier if it exists; make it if it does not.
         target_identifier = self.patient_mrn_block
         new_identifier = True
@@ -238,7 +238,7 @@ class OrganizationIdentiferNPIValueHandler(AbstractCustomValueHandler):
       "value" : "$value"
     }
     #Assigna birthsex extension
-    def assign_value(self, json_path, resource_definition, entity_definition, final_struct, key, value):
+    def assign_value(self, json_path, resource_definition, final_struct, key, value):
         #Retrieve the birthsex extension if it exists; make it if it does not.
         if 'identifier' not in final_struct:
             final_struct['identifier'] = []
@@ -255,7 +255,7 @@ class OrganizationIdentiferCLIAValueHandler(AbstractCustomValueHandler):
       "value" : "$value"
     }
     #Assign a birthsex extension
-    def assign_value(self, json_path, resource_definition, entity_definition, final_struct, key, value):
+    def assign_value(self, json_path, resource_definition, final_struct, key, value):
         #Retrieve the birthsex extension if it exists; make it if it does not.
         if 'identifier' not in final_struct:
             final_struct['identifier'] = []
@@ -272,7 +272,7 @@ class PractitionerIdentiferNPIValueHandler(AbstractCustomValueHandler):
       "value" : "$value"
     }
     #Assigna birthsex extension
-    def assign_value(self, json_path, resource_definition, entity_definition, final_struct, key, value):
+    def assign_value(self, json_path, resource_definition, final_struct, key, value):
         #Retrieve the birthsex extension if it exists; make it if it does not.
         if 'identifier' not in final_struct:
             final_struct['identifier'] = []
@@ -309,7 +309,7 @@ class ObservationComponentHandler(AbstractCustomValueHandler):
       }
     }
     #Find the appropriate component for the observaiton; then call build_structure again to continue the drill down
-    def assign_value(self, json_path, resource_definition, entity_definition, final_struct, key, value):
+    def assign_value(self, json_path, resource_definition, final_struct, key, value):
         #Check to make sure the component part exists
         if 'component' not in final_struct:
           final_struct['component'] = []
@@ -330,7 +330,7 @@ class ObservationComponentHandler(AbstractCustomValueHandler):
           if target_component is self.pulse_oximetry_oxygen_concentration:
             components.append(target_component)
         #Recurse back down into 
-        return conversion.build_structure(target_component, '.'.join(parts[2:]), resource_definition, entity_definition, parts[2:], value, parts[:2])
+        return conversion.build_structure(target_component, '.'.join(parts[2:]), resource_definition, parts[2:], value, parts[:2])
         pass
 
 def utilFindExtensionWithURL(extension_block, url):

fhir_sheets-1.3.0/src/fhir_sheets/core/model/cohort_data_entity.py

@@ -1,97 +0,0 @@
-import pprint
-from typing import Dict, Any, List, Optional
-
-class FieldEntry:
-    def __init__(self, data: Dict[str, Any]):
-        self.jsonpath: Optional[str] = data.get('jsonpath')
-        self.value_type: Optional[str] = data.get('valueType')
-        self.valuesets: Optional[str] = data.get('valuesets')
-        self.values: Optional[List[str]] = data.get('values')
-        
-    def __repr__(self) -> str:
-        return (f"FieldEntry(\n\tjsonpath='{self.jsonpath}',\n\tvalue_type='{self.value_type}', "
-                f"\n\tvaluesets='{self.valuesets}', \n\tvalues={self.values})")
-
-class EntityData:
-    def __init__(self, data: Dict[str, Dict[str, Any]]):
-        """
-        Initializes the EntityData object. Accepts either a dictionary of raw data
-        or a list of FieldEntry objects.
-
-        Args:
-            data (Dict[str, Dict[str, Any]]): A dictionary mapping names to raw field data.
-            entries (List[FieldEntry]): A list of pre-created FieldEntry objects.
-        """
-        self.fields: Dict[str, FieldEntry] = {}
-        for name, field_data in data.items():
-            self.fields[name] = FieldEntry(field_data)
-
-    def __repr__(self) -> str:
-        return f"EntityData(fields=\n{pprint.pformat(self.fields, indent=4)})"
-    
-    def insert(self, name: str, entry: FieldEntry):
-        """
-        Inserts a new FieldEntry into the collection.
-        
-        Args:
-            name (str): The referential name for the field.
-            entry (FieldEntry): The FieldEntry object to insert.
-        """
-        self.fields[name] = entry
-
-    def remove(self, name: str) -> bool:
-        """
-        Removes a FieldEntry by its referential name.
-        
-        Args:
-            name (str): The referential name of the field to remove.
-        
-        Returns:
-            bool: True if the field was removed, False otherwise.
-        """
-        if name in self.fields:
-            del self.fields[name]
-            return True
-        return False
-    
-class CohortData:
-    def __init__(self, data: Dict[str, EntityData] = None):
-        """
-        Initializes the CohortData object.
-
-        Args:
-            data (Dict[str, EntityData]): A dictionary where keys are entity names
-                                          and values are EntityData objects.
-        """
-        self.entities: Dict[str, EntityData] = {}
-        self.num_entries = 0
-        if data:
-            self.entities.update(data)
-            
-    def __repr__(self) -> str:
-        return f"CohortData(entities={self.entities})"
-    
-    def insert_entity(self, name: str, entity_data: EntityData):
-        """
-        Inserts a new EntityData object into the cohort.
-
-        Args:
-            name (str): The name of the entity (e.g., 'PrimaryPatient').
-            entity_data (EntityData): The EntityData object to insert.
-        """
-        self.entities[name] = entity_data
-        
-    def remove_entity(self, name: str) -> bool:
-        """
-        Removes an EntityData object by its name.
-
-        Args:
-            name (str): The name of the entity to remove.
-
-        Returns:
-            bool: True if the entity was removed, False otherwise.
-        """
-        if name in self.entities:
-            del self.entities[name]
-            return True
-        return False