medicafe 0.240809.0__py3-none-any.whl → 0.241015.0__py3-none-any.whl

MediBot/MediBot_Preprocessor_lib.py

@@ -1,20 +1,18 @@
+#MediBot_Preprocessor_lib.py
 from collections import OrderedDict, defaultdict
-from datetime import datetime
-import os
-import csv
-import sys
+from datetime import datetime, timedelta
+import os, csv, sys
 
-# Add parent directory of the project to the Python path
-project_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
-sys.path.append(project_dir)
+# Add the parent directory of the project to the Python path
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
 
+# Attempt to import necessary modules, falling back if they are not found
 try:
     import MediLink_ConfigLoader
     import MediLink_DataMgmt
 except ImportError:
-    from MediLink import MediLink_ConfigLoader
-    from MediLink import MediLink_DataMgmt
-    
+    from MediLink import MediLink_ConfigLoader, MediLink_DataMgmt
+
 try:
     from MediBot_UI import app_control
     from MediBot_docx_decoder import parse_docx
@@ -32,26 +30,18 @@ class InitializationError(Exception):
 def initialize(config):
     global AHK_EXECUTABLE, CSV_FILE_PATH, field_mapping, page_end_markers
     
-    try:
-        AHK_EXECUTABLE = config.get('AHK_EXECUTABLE', "")
-    except AttributeError:
-        raise InitializationError("Error: 'AHK_EXECUTABLE' not found in config.")
-    
-    try:
-        CSV_FILE_PATH = config.get('CSV_FILE_PATH', "")
-    except AttributeError:
-        raise InitializationError("Error: 'CSV_FILE_PATH' not found in config.")
-    
-    try:
-        field_mapping = OrderedDict(config.get('field_mapping', {}))
-    except AttributeError:
-        raise InitializationError("Error: 'field_mapping' not found in config.")
+    required_keys = {
+        'AHK_EXECUTABLE': "",
+        'CSV_FILE_PATH': "",
+        'field_mapping': {},
+        'page_end_markers': []
+    }
     
-    try:
-        page_end_markers = config.get('page_end_markers', [])
-    except AttributeError:
-        raise InitializationError("Error: 'page_end_markers' not found in config.")
-
+    for key, default in required_keys.items():
+        try:
+            globals()[key] = config.get(key, default) if key != 'field_mapping' else OrderedDict(config.get(key, default))
+        except AttributeError:
+            raise InitializationError("Error: '{}' not found in config.".format(key))
 
 def open_csv_for_editing(csv_file_path):
     try:
@@ -103,117 +93,237 @@ def add_columns(csv_data, column_headers):
 
 # Extracting the list to a variable for future refactoring:
 def filter_rows(csv_data):
-    # TODO This should go to the crosswalk.
-    excluded_insurance = ['AETNA', 'AETNA MEDICARE', 'HUMANA MED HMO']
-    csv_data[:] = [row for row in csv_data if row.get('Patient ID', '').strip()]
-    csv_data[:] = [row for row in csv_data if row.get('Primary Insurance', '').strip() not in excluded_insurance]
+    # TODO: This should be handled in the crosswalk.
+    excluded_insurance = {'AETNA', 'AETNA MEDICARE', 'HUMANA MED HMO'}
+    csv_data[:] = [row for row in csv_data if row.get('Patient ID') and row.get('Primary Insurance') not in excluded_insurance]
 
 def convert_surgery_date(csv_data):
     for row in csv_data:
-        try:
-            row['Surgery Date'] = datetime.strptime(row.get('Surgery Date', ''), '%m/%d/%Y')
-        except ValueError:
-            row['Surgery Date'] = datetime.min  # Assign a minimum datetime value for sorting purposes
+        surgery_date_str = row.get('Surgery Date', '')
+        row['Surgery Date'] = (datetime.strptime(surgery_date_str, '%m/%d/%Y') 
+                               if surgery_date_str else datetime.min)  # Assign a minimum datetime value if empty
 
 def sort_and_deduplicate(csv_data):
-    # TODO we need to figure out a new logic here for doing second-eye charges. I don't know what the flow should be yet.
-    csv_data.sort(key=lambda x: (x['Surgery Date'], x.get('Patient Last', '').strip()))
+    # Create a dictionary to hold unique patients based on Patient ID
     unique_patients = {}
+    
+    # Iterate through the CSV data and populate the unique_patients dictionary
     for row in csv_data:
         patient_id = row.get('Patient ID')
-        if patient_id not in unique_patients or row['Surgery Date'] < unique_patients[patient_id]['Surgery Date']:
+        if patient_id not in unique_patients:
             unique_patients[patient_id] = row
-    csv_data[:] = list(unique_patients.values())
-    # TODO Sorting, now that we're going to have the Surgery Schedules available, should (or shouldn't?? 
-    # maybe we should build in the option as liek a 'setting' in the config) be ordered as the patients show up on the schedule.
-    # If we don't have that surgery schedule yet for some reason, we should default to the current ordering strategy.
-    csv_data.sort(key=lambda x: (x['Surgery Date'], x.get('Patient Last', '').strip()))
+        else:
+            # If the patient ID already exists, compare surgery dates
+            existing_row = unique_patients[patient_id]
+            if row['Surgery Date'] < existing_row['Surgery Date']:
+                unique_patients[patient_id] = row
+
+    # Convert the unique_patients dictionary back to a list and sort it
+    csv_data[:] = sorted(unique_patients.values(), key=lambda x: (x['Surgery Date'], x.get('Patient Last', '').strip())) # TODO Does this need to be sorted twice? once before and once after?
+    
+    # TODO: Consider adding an option in the config to sort based on Surgery Schedules when available.
+    # If no schedule is available, the current sorting strategy will be used.
 
 def combine_fields(csv_data):
     for row in csv_data:
-        row['Surgery Date'] = row['Surgery Date'].strftime('%m/%d/%Y')
+        # Safely handle the 'Surgery Date' conversion
+        surgery_date = row.get('Surgery Date')
+        row['Surgery Date'] = surgery_date.strftime('%m/%d/%Y') if surgery_date else ''
+        
         first_name = row.get('Patient First', '').strip()
         middle_name = row.get('Patient Middle', '').strip()
-        if len(middle_name) > 1:
-            middle_name = middle_name[0]  # Take only the first character
+        middle_name = middle_name[0] if len(middle_name) > 1 else ''  # Take only the first character or empty
         last_name = row.get('Patient Last', '').strip()
-        row['Patient Name'] = "{}, {} {}".format(last_name, first_name, middle_name).strip()
+        row['Patient Name'] = ', '.join(filter(None, [last_name, first_name, middle_name]))  # Join non-empty parts
+        
         address1 = row.get('Patient Address1', '').strip()
         address2 = row.get('Patient Address2', '').strip()
-        row['Patient Street'] = "{} {}".format(address1, address2).strip()
+        row['Patient Street'] = ' '.join(filter(None, [address1, address2]))  # Join non-empty addresses
 
 def apply_replacements(csv_data, crosswalk):
     replacements = crosswalk.get('csv_replacements', {})
     for row in csv_data:
         for old_value, new_value in replacements.items():
-            if row.get('Patient SSN', '') == old_value:
-                row['Patient SSN'] = new_value
-            elif row.get('Primary Insurance', '') == old_value:
-                row['Primary Insurance'] = new_value
-            elif row.get('Ins1 Payer ID') == old_value:
-                row['Ins1 Payer ID'] = new_value
-
-def update_insurance_ids(csv_data, crosswalk):
+            for key in ['Patient SSN', 'Primary Insurance', 'Ins1 Payer ID']:
+                if row.get(key) == old_value:
+                    row[key] = new_value
+                    break  # Exit the loop once a replacement is made
+
+import difflib
+from collections import defaultdict
+
+def find_best_medisoft_id(insurance_name, medisoft_ids, medisoft_to_mains_names):
+    """
+    Finds the best matching Medisoft ID for a given insurance name using fuzzy matching.
+
+    Parameters:
+    - insurance_name (str): The insurance name from the CSV row.
+    - medisoft_ids (list): List of Medisoft IDs associated with the Payer ID.
+    - medisoft_to_mains_names (dict): Mapping from Medisoft ID to list of MAINS names.
+
+    Returns:
+    - int or None: The best matching Medisoft ID or None if no match is found.
+    """
+    best_match_ratio = 0
+    best_medisoft_id = None
+
+    for medisoft_id in medisoft_ids:
+        mains_names = medisoft_to_mains_names.get(medisoft_id, [])
+        for mains_name in mains_names:
+            # Preprocess names by extracting non-numeric characters and converting to uppercase
+            processed_mains = ''.join(filter(lambda x: not x.isdigit(), mains_name)).upper()
+            processed_insurance = ''.join(filter(lambda x: not x.isdigit(), insurance_name)).upper()
+
+            # Log the processed names before computing the match ratio
+            MediLink_ConfigLoader.log("Processing Medisoft ID '{}': Comparing processed insurance '{}' with processed mains '{}'.".format(medisoft_id, processed_insurance, processed_mains), level="DEBUG")
+
+            # Compute the similarity ratio
+            match_ratio = difflib.SequenceMatcher(None, processed_insurance, processed_mains).ratio()
+
+            # Log the match ratio
+            MediLink_ConfigLoader.log("Match ratio for Medisoft ID '{}': {:.2f}".format(medisoft_id, match_ratio), level="DEBUG")
+
+            if match_ratio > best_match_ratio:
+                best_match_ratio = match_ratio
+                best_medisoft_id = medisoft_id
+                # Log the current best match
+                MediLink_ConfigLoader.log("New best match found: Medisoft ID '{}' with match ratio {:.2f}".format(best_medisoft_id, best_match_ratio), level="DEBUG")
+
+    # Log the final best match ratio and ID
+    MediLink_ConfigLoader.log("Final best match ratio: {:.2f} for Medisoft ID '{}'".format(best_match_ratio, best_medisoft_id), level="DEBUG")
+
+    # No threshold applied, return the best match found
+    return best_medisoft_id
+
+def NEW_update_insurance_ids(csv_data, config, crosswalk):
+    """
+    Updates the 'Ins1 Insurance ID' field in each row of csv_data based on the crosswalk and MAINS data.
+
+    Parameters:
+    - csv_data (list of dict): The CSV data where each row is represented as a dictionary.
+    - config (dict): Configuration object containing necessary paths and parameters.
+    - crosswalk (dict): Crosswalk data containing mappings between Payer IDs and Medisoft IDs.
+
+    Returns:
+    - None: The function modifies the csv_data in place.
+    """
+    processed_payer_ids = set()  # Track processed Payer IDs
+    MediLink_ConfigLoader.log("Starting update of insurance IDs.", level="INFO")
+
+    # Load MAINS data to get mapping from Medisoft ID to MAINS names
+    insurance_to_id = load_insurance_data_from_mains(config)  # Assuming it returns a dict mapping insurance names to IDs
+    MediLink_ConfigLoader.log("Loaded MAINS data for insurance to ID mapping.", level="DEBUG")
+    
+    # Invert the mapping to get Medisoft ID to MAINS names
+    medisoft_to_mains_names = defaultdict(list)
+    for insurance_name, medisoft_id in insurance_to_id.items():
+        medisoft_to_mains_names[medisoft_id].append(insurance_name)
+
     for row in csv_data:
         ins1_payer_id = row.get('Ins1 Payer ID', '').strip()
-        # MediLink_ConfigLoader.log("Ins1 Payer ID '{}' associated with Patient ID {}.".format(ins1_payer_id, row.get('Patient ID', "None")))
+        MediLink_ConfigLoader.log("Processing row with Ins1 Payer ID: '{}'.".format(ins1_payer_id), level="DEBUG")
+        
         if ins1_payer_id:
-            if ins1_payer_id in crosswalk.get('payer_id', {}):
-                medisoft_ids = crosswalk['payer_id'][ins1_payer_id].get('medisoft_id', [])
-                if medisoft_ids:
-                    medisoft_ids = [int(id) for id in medisoft_ids]
-                    # TODO Try to match OpenPM's Insurance Name to get a better match. 
-                    # Potential approach:
-                    # 1. Retrieve the insurance name from the current row
-                    # insurance_name = row.get('Primary Insurnace', '').strip()
-                    # 2. Check if the insurance name exists in the subset of MAINS names associated with 
-                    # crosswalk medisoft ID values for the given payer ID.
-                    # 3. If an approximate match is found above a certain confidence, use the corresponding medisoft_id.
-                    # else: 4. If no match is found, default to the first medisoft_id
-                    #     row['Ins1 Insurance ID'] = medisoft_ids[0]
-                    
-                    row['Ins1 Insurance ID'] = medisoft_ids[0] 
-                    # MediLink_ConfigLoader.log("Ins1 Insurance ID '{}' used for Payer ID {} in crosswalk.".format(row.get('Ins1 Insurance ID', ''), ins1_payer_id))
-            else:
-                MediLink_ConfigLoader.log("Ins1 Payer ID '{}' not found in the crosswalk.".format(ins1_payer_id))
-                # Create a placeholder entry in the crosswalk, need to consider the medisoft_medicare_id handling later.
-                if 'payer_id' not in crosswalk:
-                    crosswalk['payer_id'] = {}
-                crosswalk['payer_id'][ins1_payer_id] = {
-                    'medisoft_id': [],
-                    'medisoft_medicare_id': [],
-                    'endpoint': 'OPTUMEDI' # Default probably should be a flag for the crosswalk update function to deal with. BUG HARDCODE THERE ARE 3 of these defaults
-                } 
-
-def update_procedure_codes(csv_data):
-    # Define the Medisoft shorthand to diagnostic codes dictionary
-    # TODO The reverse of this will be in the crosswalk. We'll need to reverse it here for lookup.
-    medisoft_to_diagnosis = {
-        "25811": "H25.811",
-        "25812": "H25.812",
-        "2512": "H25.12",
-        "2511": "H25.11",
-        "529XA": "T85.29XA",
-        "4301": "H43.01",
-        "4302": "H43.02",
-        "011X2": "H40.11X2",
-        "051X3": "H40.51X3",
-        "5398A": "T85.398A"
-    }
+            # Mark this Payer ID as processed
+            if ins1_payer_id not in processed_payer_ids:
+                processed_payer_ids.add(ins1_payer_id)  # Add to set
+                MediLink_ConfigLoader.log("Marked Payer ID '{}' as processed.".format(ins1_payer_id), level="DEBUG")
+                
+                # Retrieve Medisoft IDs for the current Payer ID
+                medisoft_ids = crosswalk.get('payer_id', {}).get(ins1_payer_id, {}).get('medisoft_id', [])
+                MediLink_ConfigLoader.log("Retrieved Medisoft IDs for Payer ID '{}': {}".format(ins1_payer_id, medisoft_ids), level="DEBUG")
+
+        if not medisoft_ids:
+            MediLink_ConfigLoader.log("No Medisoft IDs available for Payer ID '{}', creating placeholder entry.".format(ins1_payer_id), level="WARNING")
+            # Create a placeholder entry in the crosswalk
+            if 'payer_id' not in crosswalk:
+                crosswalk['payer_id'] = {}
+            crosswalk['payer_id'][ins1_payer_id] = {
+                'medisoft_id': [],  # Placeholder for future Medisoft IDs
+                'medisoft_medicare_id': [],  # Placeholder for future Medicare IDs
+                'endpoint': None  # Placeholder for future endpoint
+            }
+            continue  # Skip further processing for this Payer ID
+
+        # If only one Medisoft ID is associated, assign it directly
+        if len(medisoft_ids) == 1:
+            try:
+                medisoft_id = int(medisoft_ids[0])
+                row['Ins1 Insurance ID'] = medisoft_id
+                MediLink_ConfigLoader.log("Assigned Medisoft ID '{}' to row number {} with Payer ID '{}'.".format(medisoft_id, csv_data.index(row) + 1, ins1_payer_id), level="DEBUG")
+            except ValueError as e:
+                MediLink_ConfigLoader.log("Error converting Medisoft ID '{}' to integer for Payer ID '{}': {}".format(medisoft_ids[0], ins1_payer_id, e), level="ERROR")
+                row['Ins1 Insurance ID'] = None
+            continue  # Move to the next row
+
+        # If multiple Medisoft IDs are associated, perform fuzzy matching
+        insurance_name = row.get('Primary Insurance', '').strip()
+        if not insurance_name:
+            MediLink_ConfigLoader.log("Row with Payer ID '{}' missing 'Primary Insurance', skipping assignment.".format(ins1_payer_id), level="WARNING")
+            continue  # Skip if insurance name is missing
+
+        best_medisoft_id = find_best_medisoft_id(insurance_name, medisoft_ids, medisoft_to_mains_names)
+
+        if best_medisoft_id:
+            row['Ins1 Insurance ID'] = best_medisoft_id
+            MediLink_ConfigLoader.log("Assigned Medisoft ID '{}' to row with Payer ID '{}' based on fuzzy match.".format(best_medisoft_id, ins1_payer_id), level="INFO")
+        else:
+            # Default to the first Medisoft ID if no good match is found
+            try:
+                default_medisoft_id = int(medisoft_ids[0])
+                row['Ins1 Insurance ID'] = default_medisoft_id
+                MediLink_ConfigLoader.log("No suitable match found. Defaulted to Medisoft ID '{}' for Payer ID '{}'.".format(default_medisoft_id, ins1_payer_id), level="INFO")
+            except ValueError as e:
+                MediLink_ConfigLoader.log("Error converting default Medisoft ID '{}' to integer for Payer ID '{}': {}".format(medisoft_ids[0], ins1_payer_id, e), level="ERROR")
+                row['Ins1 Insurance ID'] = None
+
+def update_insurance_ids(csv_data, config, crosswalk):
+    MediLink_ConfigLoader.log("Starting update_insurance_ids function.", level="DEBUG")
+    
+    # Create a dictionary to hold Medisoft IDs for each payer ID in the crosswalk
+    payer_id_to_medisoft = {}
+    MediLink_ConfigLoader.log("Initialized payer_id_to_medisoft dictionary.", level="DEBUG")
+    
+    # Populate the dictionary with data from the crosswalk
+    for payer_id, details in crosswalk.get('payer_id', {}).items():
+        medisoft_ids = details.get('medisoft_id', [])
+        # Filter out empty strings and take the first valid ID
+        medisoft_ids = [id for id in medisoft_ids if id]
+        payer_id_to_medisoft[payer_id] = int(medisoft_ids[0]) if medisoft_ids else None
+        MediLink_ConfigLoader.log("Processed Payer ID '{}': Medisoft IDs found: {}".format(payer_id, medisoft_ids), level="DEBUG")
+
+    # Process the csv_data
+    for row in csv_data:
+        ins1_payer_id = row.get('Ins1 Payer ID', '').strip()
+        MediLink_ConfigLoader.log("Processing row #{} with Ins1 Payer ID '{}'.".format(csv_data.index(row) + 1, ins1_payer_id), level="DEBUG")
+        
+        if ins1_payer_id not in payer_id_to_medisoft:
+            # Add placeholder entry for new payer ID
+            payer_id_to_medisoft[ins1_payer_id] = None  # No Medisoft ID available
+            crosswalk.setdefault('payer_id', {})[ins1_payer_id] = {
+                'medisoft_id': [],  # Placeholder for future Medisoft IDs
+                'medisoft_medicare_id': [],  # Placeholder for future Medicare IDs
+                'endpoint': None  # Placeholder for future endpoint
+            }
+            MediLink_ConfigLoader.log("Added placeholder entry for new Payer ID '{}'.".format(ins1_payer_id), level="INFO")
+        
+        # Assign the Medisoft ID to the row
+        row['Ins1 Insurance ID'] = payer_id_to_medisoft[ins1_payer_id]
+        MediLink_ConfigLoader.log("Assigned Medisoft ID '{}' to row with Ins1 Payer ID '{}'.".format(row['Ins1 Insurance ID'], ins1_payer_id), level="DEBUG")
 
-    # Define the procedure codes to diagnostic codes dictionary
-    procedure_to_diagnosis = {
-        "00142": ["H25.811", "H25.812", "H25.12", "H25.11", "T85.29XA"],
-        "00145": ["H43.01", "H43.02"],
-        "00140": ["H40.11X2", "H40.51X3"]
+def update_procedure_codes(csv_data, crosswalk): 
+    
+    # Get Medisoft shorthand dictionary from crosswalk and reverse it
+    diagnosis_to_medisoft = crosswalk.get('diagnosis_to_medisoft', {}) # BUG We need to be careful here in case we decide we need to change the crosswalk data specifically with regard to the T8/H usage.
+    medisoft_to_diagnosis = {v: k for k, v in diagnosis_to_medisoft.items()}
+
+    # Get procedure code to diagnosis dictionary from crosswalk and reverse it for easier lookup
+    diagnosis_to_procedure = {
+        diagnosis_code: procedure_code
+        for procedure_code, diagnosis_codes in crosswalk.get('procedure_to_diagnosis', {}).items()
+        for diagnosis_code in diagnosis_codes
     }
 
-    # Reverse the dictionary for easier lookup from diagnostic code to procedure code
-    diagnosis_to_procedure = {}
-    for procedure_code, diagnosis_codes in procedure_to_diagnosis.items():
-        for diagnosis_code in diagnosis_codes:
-            diagnosis_to_procedure[diagnosis_code] = procedure_code
-
     # Initialize counter for updated rows
     updated_count = 0
 
@@ -242,7 +352,7 @@ def update_procedure_codes(csv_data):
 def update_diagnosis_codes(csv_data):
     try:
         # Load configuration and crosswalk
-        config, _ = MediLink_ConfigLoader.load_configuration()
+        config, crosswalk = MediLink_ConfigLoader.load_configuration()
         
         # Extract the local storage path from the configuration
         local_storage_path = config['MediLink_Config']['local_storage_path']
@@ -250,48 +360,80 @@ def update_diagnosis_codes(csv_data):
         # Initialize a dictionary to hold diagnosis codes from all DOCX files
         all_patient_data = {}
 
-        # Iterate through all files in the specified directory
-        for filename in os.listdir(local_storage_path):
-            if filename.endswith(".docx"):
-                filepath = os.path.join(local_storage_path, filename)
-                MediLink_ConfigLoader.log("Processing DOCX file: {}".format(filepath), level="INFO")
-                try:
-                    patient_data = parse_docx(filepath)
-                    for patient_id, service_dates in patient_data.items():
-                        if patient_id not in all_patient_data:
-                            all_patient_data[patient_id] = {}
-                        for date_of_service, diagnosis_data in service_dates.items():
-                            all_patient_data[patient_id][date_of_service] = diagnosis_data
-                except Exception as e:
-                    MediLink_ConfigLoader.log("Error parsing DOCX file {}: {}".format(filepath, e), level="ERROR")
+        # Convert surgery dates in CSV data
+        convert_surgery_date(csv_data)
+        
+        # Extract all valid surgery dates from csv_data
+        surgery_dates = [row['Surgery Date'] for row in csv_data if row['Surgery Date'] != datetime.min]
+        
+        if not surgery_dates:
+            raise ValueError("No valid surgery dates found in csv_data.")
+        
+        # Determine the minimum and maximum surgery dates
+        min_surgery_date = min(surgery_dates)
+        max_surgery_date = max(surgery_dates)
+        
+        # Apply a ±5-day margin to the surgery dates
+        margin = timedelta(days=5)
+        threshold_start = min_surgery_date - margin
+        threshold_end = max_surgery_date + margin
+        
+        MediLink_ConfigLoader.log("Processing DOCX files modified between {} and {}.".format(threshold_start, threshold_end), level="INFO")
+
+        # Gather all relevant DOCX files in the specified directory
+        docx_files = [
+            os.path.join(local_storage_path, filename)
+            for filename in os.listdir(local_storage_path)
+            if filename.endswith(".docx")
+        ]
+
+        # Filter files based on modification time
+        valid_files = [
+            filepath for filepath in docx_files
+            if threshold_start <= datetime.fromtimestamp(os.path.getmtime(filepath)) <= threshold_end
+        ]
+
+        # Process valid DOCX files
+        for filepath in valid_files:
+            MediLink_ConfigLoader.log("Processing DOCX file: {}".format(filepath), level="INFO")
+            try:
+                patient_data = parse_docx(filepath, surgery_dates)  # Pass surgery_dates to parse_docx
+                for patient_id, service_dates in patient_data.items():
+                    if patient_id not in all_patient_data:
+                        all_patient_data[patient_id] = {}
+                    for date_of_service, diagnosis_data in service_dates.items():
+                        all_patient_data[patient_id][date_of_service] = diagnosis_data
+            except Exception as e:
+                MediLink_ConfigLoader.log("Error parsing DOCX file {}: {}".format(filepath, e), level="ERROR")
+
+        # Log if no valid files were found
+        if not valid_files:
+            MediLink_ConfigLoader.log("No valid DOCX files found within the modification time threshold.", level="INFO")
         
         # Debug logging for all_patient_data
-        MediLink_ConfigLoader.log("All patient data collected from DOCX files: {}".format(all_patient_data), level="INFO")
-
-        # Define the diagnosis to Medisoft shorthand dictionary
-        diagnosis_to_medisoft = {
-            "H25.811": "25811",
-            "H25.812": "25812",
-            "H25.12": "2512",
-            "H25.11": "2511",
-            "T85.29XA": "529XA",
-            "H43.01": "4301",
-            "H43.02": "4302",
-            "H40.11X2": "011X2",
-            "H40.51X3": "051X3",
-            "T85.398A": "5398A"
-        }
+        MediLink_ConfigLoader.log("All patient data collected from DOCX files: {}".format(all_patient_data), level="DEBUG")
         
-        # Convert surgery dates in CSV data
-        convert_surgery_date(csv_data)
+        # Extract patient IDs from csv_data for efficient matching
+        patient_ids_in_csv = {row.get('Patient ID', '').strip() for row in csv_data}
+
+        # Check if any patient data was collected
+        if not all_patient_data or not patient_ids_in_csv.intersection(all_patient_data.keys()):
+            MediLink_ConfigLoader.log("No patient data collected or no matching Patient IDs found. Skipping further processing.", level="INFO")
+            return  # Exit the function early if no data is available
+
+        # Get Medisoft shorthand dictionary from crosswalk.
+        diagnosis_to_medisoft = crosswalk.get('diagnosis_to_medisoft', {})
         
         # Initialize counter for updated rows
         updated_count = 0
 
         # Update the "Default Diagnosis #1" column in the CSV data
         for row_num, row in enumerate(csv_data, start=1):
-            MediLink_ConfigLoader.log("Processing row number {}.".format(row_num), level="INFO")
             patient_id = row.get('Patient ID', '').strip()
+            if patient_id not in patient_ids_in_csv:
+                continue  # Skip rows that do not match any patient ID
+
+            MediLink_ConfigLoader.log("Processing row number {}.".format(row_num), level="DEBUG")
             surgery_date = row.get('Surgery Date', '')
 
             # Convert surgery_date to string format for lookup
@@ -300,17 +442,19 @@ def update_diagnosis_codes(csv_data):
             else:
                 surgery_date_str = ''
 
-            MediLink_ConfigLoader.log("Patient ID: {}, Surgery Date: {}".format(patient_id, surgery_date_str), level="INFO")
+            MediLink_ConfigLoader.log("Patient ID: {}, Surgery Date: {}".format(patient_id, surgery_date_str), level="DEBUG")
 
             if patient_id in all_patient_data:
                 if surgery_date_str in all_patient_data[patient_id]:
                     diagnosis_code, left_or_right_eye, femto_yes_or_no = all_patient_data[patient_id][surgery_date_str]
-                    MediLink_ConfigLoader.log("Found diagnosis data for Patient ID: {}, Surgery Date: {}".format(patient_id, surgery_date_str), level="INFO")
+                    MediLink_ConfigLoader.log("Found diagnosis data for Patient ID: {}, Surgery Date: {}".format(patient_id, surgery_date_str), level="DEBUG")
                     
                     # Convert diagnosis code to Medisoft shorthand format.
-                    defaulted_code = diagnosis_code[1:].replace('.', '')[-5:] if diagnosis_code else ''
-                    medisoft_shorthand = diagnosis_to_medisoft.get(diagnosis_code, defaulted_code)
-                    MediLink_ConfigLoader.log("Converted diagnosis code to Medisoft shorthand: {}".format(medisoft_shorthand), level="INFO")
+                    medisoft_shorthand = diagnosis_to_medisoft.get(diagnosis_code, None)
+                    if medisoft_shorthand is None and diagnosis_code:
+                        defaulted_code = diagnosis_code.lstrip('H').lstrip('T8').replace('.', '')[-5:]
+                        medisoft_shorthand = defaulted_code
+                    MediLink_ConfigLoader.log("Converted diagnosis code to Medisoft shorthand: {}".format(medisoft_shorthand), level="DEBUG")
                     
                     row['Default Diagnosis #1'] = medisoft_shorthand
                     updated_count += 1
@@ -379,6 +523,7 @@ def load_insurance_data_from_mains(config):
     # TODO (Low) Performance: There probably needs to be a dictionary proxy for MAINS that gets updated.
     # Meh, this just has to be part of the new architecture plan where we make Medisoft a downstream 
     # recipient from the db.
+    # TODO (High) The Medisoft Medicare flag needs to be brought in here.
     mains_path = config['MAINS_MED_PATH']
     mains_slices = crosswalk['mains_mapping']['slices']
     
@@ -419,7 +564,7 @@ def load_insurance_data_from_mapat(config, crosswalk):
         
     return patient_id_to_insurance_id
 
-def parse_z_dat(z_dat_path, config):
+def parse_z_dat(z_dat_path, config): # Why is this in MediBot and not MediLink?
     """
     Parses the Z.dat file to map Patient IDs to Insurance Names using the provided fixed-width file format.
 
@@ -494,11 +639,12 @@ def load_historical_payer_to_patient_mappings(config):
                         
                         # Log the accumulated count for this CSV file
                         if patient_count > 0:
-                            MediLink_ConfigLoader.log("CSV file '{}' has {} Patient IDs with Payer IDs.".format(filename, patient_count))
+                            MediLink_ConfigLoader.log("CSV file '{}' has {} Patient IDs with Payer IDs.".format(filename, patient_count), level="DEBUG")
                         else:
-                            MediLink_ConfigLoader.log("CSV file '{}' is empty or does not have valid Patient ID or Payer ID mappings.".format(filename))
+                            MediLink_ConfigLoader.log("CSV file '{}' is empty or does not have valid Patient ID or Payer ID mappings.".format(filename), level="DEBUG")
                 except Exception as e:
                     print("Error processing file {}: {}".format(filename, e))
+                    MediLink_ConfigLoader.log("Error processing file '{}': {}".format(filename, e), level="ERROR")
     except FileNotFoundError as e:
         print("Error: {}".format(e))