medicafe 0.240419.2__py3-none-any.whl → 0.240517.0__py3-none-any.whl

MediBot/MediBot_Preprocessor.py

@@ -1,18 +1,24 @@
-import csv
-import subprocess
 import os
 import re
-from datetime import datetime
 from collections import OrderedDict # so that the field_mapping stays in order.
 import re
 import sys
+import argparse
+import MediBot_Crosswalk_Library
 
 # Add parent directory of the project to the Python path
 project_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
 sys.path.append(project_dir)
 
-from MediLink import MediLink_ConfigLoader
-from MediLink import MediLink_DataMgmt
+try: 
+    from MediLink import MediLink_ConfigLoader
+except ImportError:
+    import MediLink_ConfigLoader
+
+try:
+    import MediBot_Preprocessor_lib
+except ImportError:
+    from MediBot import MediBot_Preprocessor_lib
 
 """
 Preprocessing Enhancements
@@ -26,7 +32,7 @@ Data Integrity and Validation
 - [ ] Conduct a thorough CSV integrity check before processing to flag potential issues upfront.
 - [ ] Implement a mechanism to confirm the accuracy of entered data, potentially through a verification step or summary report.
 - [ ] Explore the possibility of integrating direct database queries for existing patient checks to streamline the process.
-- [ ] Automate the replacement of spaces with underscores ('_') in last names for Medicare entries, ensuring data consistency.
+- [ ] Automate the replacement of spaces with underscores ('_') in last names for Medicare entries.
 - [ ] Enhance CSV integrity checks to identify and report potential issues with data format, especially concerning insurance policy numbers and special character handling.
 
 Known Issues and Bugs
@@ -34,185 +40,88 @@ Known Issues and Bugs
 - [ ] Investigate the issue with Excel modifying long policy numbers in the CSV and provide guidance or a workaround.
 
 Future Work
+- [X] Check for PatientID number in La Forma Z to link back to Carol's table for mapping Medisoft insurance name to payerID and payer name and address.
+- [X] Check for PatientID to Medisoft custom insurance name mapping in MAPAT.
+- [X] Middle Names should all be single letters. Make sure it gets truncated before submitting.
 - [ ] Consolidate data from multiple sources (Provider_Notes.csv, Surgery_Schedule.csv, and Carols_CSV.csv) into a single table with Patient ID as the key, ensuring all data elements are aligned and duplicate entries are minimized.
-- [ ] Implement logic to verify and match Patient IDs across different files to ensure data integrity before consolidation.
+- [ ] Implement logic to verify and match Patient IDs across different files to ensure data integrity before consolidation. (Catching errors between source data)
 - [ ] Optimize the preprocessing of surgery dates and diagnosis codes for use in patient billing and scheduling systems.
-- [ ] This needs to be able to take in the Surgery Schedule doc and parse out a Patient ID : Diagnosis Code table
-- [ ] The Minutes & Cacncellation data with logic to consolidate into one table in memory.
-
-
-Future Work: crosswalk_update() automates the process of updating the crosswalk.json file with new Medisoft insurance information.
-
-Development Roadmap:
-1. Problem Statement:
-    - The need to update the crosswalk.json file arises whenever a new Medisoft insurance is discovered. Automation of this process is required for accuracy and efficiency.
-
-2. Identifying New Insurance:
-    - New Medisoft insurances are identified based on the payer ID number.
-    - The existence of the payer ID number is checked in the crosswalk.json under existing endpoints.
-
-3. Adding New Insurance:
-    - If the payer ID number does not exist in any endpoint, the tool prompts the user, assisted by endpoint APIs, to add the payer ID to a specific endpoint.
-    - The corresponding name from Carol's spreadsheet is used as the value for the new payer ID.
-
-4. Mapping to Main Insurance:
-    - The tool presents the user with a list of the top 5-7 insurances, scored higher on a fuzzy search or above a certain score.
-    - The user selects the appropriate insurance based on the identified Medisoft insurance, establishing the medisoft_insurance_to_payer_id relationship.
-
-5. Confirming Mapping:
-    - The tool implicitly establishes the insurance_to_endpoint_mapping based on the selected MediSoft name and endpoint.
-    - This step is confirmed or re-evaluated to ensure accuracy.
-
-6. User Interaction:
-    - Unrecognized payer IDs are presented to the user.
-    - Users can assign these payer IDs to MediSoft custom names individually.
-    - Grouping of payer IDs may be facilitated, especially for insurances like CIGNA with multiple addresses but few payer IDs.
-
-7. Handling Unavailable Payer IDs:
-    - An extra endpoint named "Fax/Mail or Other" is created to handle cases where the payer ID is unavailable.
-    - The tool retains payer IDs not existing in any endpoint, allowing users to assign them to the "Fax/Mail or Other" key in the crosswalk.
-
-8. Implementation Considerations:
-    - The tool should handle various scenarios, including checking for free payer IDs and determining the appropriate endpoint for assignment.
-    - Integration of API checks to verify payer ID availability and associated information is recommended.
-    - Validation mechanisms should be implemented to prevent incorrect mappings and ensure data integrity.
-
-NOTE: this needs to also pull from the CSV the listed address of the insruance.
-NOTE: La Forma Z can have the PatientID number which can link back to Carol's table which can then map the Medisoft insurance name to the payerID 
-and payer name and address when the insurance is already selected in Medisoft so the program can learn retroactively and would know the Medisoft # from
-the sequencing rather than trying to feed it from the beginning. so that'll be out of ["fixedWidthSlices"]["personal_slices"]["PATID"].
-NOTE: Also check MAPAT because maybe the PatientID to Medisoft custom insurance name might exist there enmasse + the PatientID to PayerID link from Carol's CSV
-gives us the Medisoft custom insurance name to Payer ID. Then, the endpoint mapping is the clearinghouse PayerID list (API?). MAPAT has the PatientID to Medisoft 
-insruance reference number which is the MAINS offset by 1 for the header. MAPAT has columns [159,162] for insurance and [195,200] for patient ID.
+- [ ] Read Surgery Schedule doc and parse out a Patient ID : Diagnosis Code table.
+- [ ] The Minutes & Cancellation data with logic to consolidate into one table in memory.
+- [ ] Dynamically list the endpoint for a new Payer ID via API or user interaction to update the crosswalk.json efficiently.
+- [ ] Pull listed addresses of insurance from the CSV. (Not really necessary)
+- [ ] Retroactively learn Medisoft insurance name and payerID from the provided data sources.
+
+Development Roadmap for crosswalk_update():
+- [X] Automation required for updating the crosswalk.json when new Medisoft insurance is discovered.
+- [X] New Medisoft insurances are identified based on the payer ID number.
+- [X] Check the existence of the payer ID in crosswalk.json under existing endpoints.
+- [X] Facilitate grouping of IDs for insurances like CIGNA with multiple addresses but few payer IDs.
+- [X] Retroactive learning based on selected insurances in Medisoft 
+- [ ] Prompt user via endpoint APIs to add new payer ID to an endpoint if it does not exist.
+- [ ] Retain payer IDs without Insurance ID for future assignments.
+- [ ] Check for free payer IDs and determine the appropriate endpoint for assignment.
+- [ ] Present unrecognized payer IDs with Carol's Insurance Name to users for assignment to Insurance ID. (Try API Call)
+- [ ] Integrate API checks to verify payer ID availability and related information.
+- [ ] Implement "Fax/Mail or Other" endpoint for unavailable payer IDs.
+- [ ] Present user with a list of top insurances for selection based on fuzzy search scores.
+- [ ] Establish payer ID to insurance ID relationship based on user selection.
+- [ ] Implicitly establish payer ID to endpoint mapping based on user selection.
+- [ ] Implement validation mechanisms to prevent incorrect mappings and ensure data integrity.
+- [ ] Considerations for extracting insurance addresses (if necessary)
+- [ ] Handle better the case where a payer_id doesn't exist (When Carol's CSV doesn't bring the Payer ID).
+        Maybe ask the user what the payer ID is for that patient? I dont know.
+- [ ] TODO (MED) Crosswalk (both initializing and updating) needs to pull AFTER the Preprocessor for Carol's CSV because
+        all that data lives in-memory and then gets corrections or replacements before being used so we need 
+        the post-correction data to be used to build and update the crosswalk.
 """
-
 # Load configuration
 # Should this also take args? Path for ./MediLink needed to be added for this to resolve
-config, _ = MediLink_ConfigLoader.load_configuration()
-
-class InitializationError(Exception):
-    def __init__(self, message):
-        self.message = message
-        super().__init__(self.message)
-
-def initialize(config):
-    global AHK_EXECUTABLE, CSV_FILE_PATH, field_mapping, page_end_markers
-    
-    try:
-        AHK_EXECUTABLE = config.get('AHK_EXECUTABLE', "")
-    except AttributeError:
-        raise InitializationError("Error: 'AHK_EXECUTABLE' not found in config.")
-    
-    try:
-        CSV_FILE_PATH = config.get('CSV_FILE_PATH', "")
-    except AttributeError:
-        raise InitializationError("Error: 'CSV_FILE_PATH' not found in config.")
-    
-    try:
-        field_mapping = OrderedDict(config.get('field_mapping', {}))
-    except AttributeError:
-        raise InitializationError("Error: 'field_mapping' not found in config.")
-    
-    try:
-        page_end_markers = config.get('page_end_markers', [])
-    except AttributeError:
-        raise InitializationError("Error: 'page_end_markers' not found in config.")
-
-
-def open_csv_for_editing(csv_file_path):
-    try:
-        # Open the CSV file in the default program
-        subprocess.run(['open' if os.name == 'posix' else 'start', csv_file_path], check=True, shell=True)
-        print("After saving the revised CSV, please re-run MediBot.")
-    except subprocess.CalledProcessError as e:
-        print("Failed to open CSV file:", e)
-        
-# Function to load and process CSV data
-def load_csv_data(csv_file_path):
-    try:
-        # Check if the file exists
-        if not os.path.exists(csv_file_path):
-            raise FileNotFoundError("***Error: CSV file '{}' not found.".format(csv_file_path))
-        
-        with open(csv_file_path, 'r') as csvfile:
-            reader = csv.DictReader(csvfile)
-            return [row for row in reader]  # Return a list of dictionaries
-    except FileNotFoundError as e:
-        print(e)  # Print the informative error message
-        print("Hint: Check if CSV file is located in the expected directory or specify a different path in config file.")
-        print("Please correct the issue and re-run MediBot.")
-        sys.exit(1)  # Halt the script
-    except IOError as e:
-        print("Error reading CSV file: {}. Please check the file path and permissions.".format(e))
-        sys.exit(1)  # Halt the script in case of other IO errors
+config, crosswalk = MediLink_ConfigLoader.load_configuration()
 
 # CSV Preprocessor built for Carol
-def preprocess_csv_data(csv_data):
+def preprocess_csv_data(csv_data, crosswalk):
     try:
-        # Filter out rows without a Patient ID
-        csv_data[:] = [row for row in csv_data if row.get('Patient ID', '').strip()]
+        # Add the "Ins1 Insurance ID" column to the CSV data.
+        # This initializes the column with empty values for each row.
+        MediLink_ConfigLoader.log("CSV Pre-processor: Adding 'Ins1 Insurance ID' column to the CSV data...", level="INFO")
+        MediBot_Preprocessor_lib.add_insurance_id_column(csv_data)
         
-        # Remove Patients (rows) that are Primary Insurance: 'AETNA', 'AETNA MEDICARE', or 'HUMANA MED HMO'.
-        csv_data[:] = [row for row in csv_data if row.get('Primary Insurance', '').strip() not in ['AETNA', 'AETNA MEDICARE', 'HUMANA MED HMO']]
-                
-        # Convert 'Surgery Date' to datetime objects for sorting
-        for row in csv_data:
-            try:
-                row['Surgery Date'] = datetime.strptime(row.get('Surgery Date', ''), '%m/%d/%Y')
-            except ValueError:
-                # Handle or log the error if the date is invalid
-                row['Surgery Date'] = datetime.min  # Assign a minimum datetime value for sorting purposes
-
-        # Initially sort the patients first by 'Surgery Date' and then by 'Patient Last' alphabetically
-        csv_data.sort(key=lambda x: (x['Surgery Date'], x.get('Patient Last', '').strip()))
+        # Filter out rows without a Patient ID and rows where the Primary Insurance
+        # is 'AETNA', 'AETNA MEDICARE', or 'HUMANA MED HMO'.
+        MediLink_ConfigLoader.log("CSV Pre-processor: Filtering out missing Patient IDs and 'AETNA', 'AETNA MEDICARE', or 'HUMANA MED HMO'...", level="INFO")
+        MediBot_Preprocessor_lib.filter_rows(csv_data)
         
-        # Deduplicate patient records based on Patient ID, keeping the entry with the earliest surgery date
-        unique_patients = {}
-        for row in csv_data:
-            patient_id = row.get('Patient ID')
-            if patient_id not in unique_patients or row['Surgery Date'] < unique_patients[patient_id]['Surgery Date']:
-                unique_patients[patient_id] = row
+        # Convert 'Surgery Date' from string format to datetime objects for sorting purposes.
+        # Sort the patients by 'Surgery Date' and then by 'Patient Last' name alphabetically.
+        # Deduplicate patient records based on Patient ID, keeping the entry with the earliest surgery date.
+        # Update the CSV data to include only unique patient records.
+        # Re-sort the CSV data after deduplication to ensure the correct order.
+        MediLink_ConfigLoader.log("CSV Pre-processor: Sorting and de-duplicating patient records...", level="INFO")
+        MediBot_Preprocessor_lib.convert_surgery_date(csv_data)
+        MediBot_Preprocessor_lib.sort_and_deduplicate(csv_data)
         
-        # Update csv_data to only include unique patient records
-        csv_data[:] = list(unique_patients.values())
-
-        # Re-sort the csv_data after deduplication to ensure correct order
-        csv_data.sort(key=lambda x: (x['Surgery Date'], x.get('Patient Last', '').strip()))
+        # Convert 'Surgery Date' back to string format if needed for further processing.
+        # Combine 'Patient First', 'Patient Middle', and 'Patient Last' into a single 'Patient Name' field.
+        # Combine 'Patient Address1' and 'Patient Address2' into a single 'Patient Street' field.
+        MediLink_ConfigLoader.log("CSV Pre-processor: Constructing Patient Name and Address for Medisoft...", level="INFO")
+        MediBot_Preprocessor_lib.combine_fields(csv_data)
         
-        # Maybe make a dataformat_library function for this? csv_data = format_preprocessor(csv_data)?
-        for row in csv_data:
-            # Convert 'Surgery Date' back to string format if needed for further processing (cleanup)
-            row['Surgery Date'] = row['Surgery Date'].strftime('%m/%d/%Y')
-            
-            # Combine name fields
-            first_name = row.get('Patient First', '').strip()
-            middle_name = row.get('Patient Middle', '').strip()
-            last_name = row.get('Patient Last', '').strip()
-            row['Patient Name'] = "{}, {} {}".format(last_name, first_name, middle_name).strip()
-
-            # Combine address fields
-            address1 = row.get('Patient Address1', '').strip()
-            address2 = row.get('Patient Address2', '').strip()
-            row['Patient Street'] = "{} {}".format(address1, address2).strip()
-            
-            # Probably make a data_format function for this:
-            # Define the replacements as a dictionary
-            replacements = {
-                '777777777': '',  # Replace '777777777' with an empty string
-                'RAILROAD MEDICARE': 'RAILROAD',  # Replace 'RAILROAD MEDICARE' with 'RAILROAD'
-                'AARP MEDICARE COMPLETE': 'AARP COMPLETE'  # Replace 'AARP MEDICARE COMPLETE' with 'AARP COMPLETE'
-            }
-
-            # Iterate over each key-value pair in the replacements dictionary
-            for old_value, new_value in replacements.items():
-                # Replace the old value with the new value if it exists in the row
-                if row.get('Patient SSN', '') == old_value:
-                    row['Patient SSN'] = new_value
-                elif row.get('Primary Insurance', '') == old_value:
-                    row['Primary Insurance'] = new_value
-
+        # Retrieve replacement values from the crosswalk.
+        # Iterate over each key-value pair in the replacements dictionary and replace the old value
+        # with the new value in the corresponding fields of each row.
+        MediLink_ConfigLoader.log("CSV Pre-processor: Applying mandatory replacements per Crosswalk...", level="INFO")
+        MediBot_Preprocessor_lib.apply_replacements(csv_data, crosswalk)
+        
+        # Update the "Ins1 Insurance ID" column based on the crosswalk and the "Ins1 Payer ID" column for each row.
+        # If the Payer ID is not found in the crosswalk, create a placeholder entry in the crosswalk and mark the row for review.
+        MediLink_ConfigLoader.log("CSV Pre-processor: Populating 'Ins1 Insurance ID' based on Crosswalk...", level="INFO")
+        MediBot_Preprocessor_lib.update_insurance_ids(csv_data, crosswalk)
+    
     except Exception as e:
-        print("An error occurred while pre-processing CSV data. Please repair the CSV directly and try again:", e)
+        message = "An error occurred while pre-processing CSV data. Please repair the CSV directly and try again: {}".format(e)
+        MediLink_ConfigLoader.log(message, level="ERROR")
+        print(message)
 
 def check_existing_patients(selected_patient_ids, MAPAT_MED_PATH):
     existing_patients = []
@@ -245,6 +154,9 @@ def intake_scan(csv_headers, field_mapping):
     missing_fields_warnings = []
     required_fields = config["required_fields"]
     
+    # MediLink_ConfigLoader.log("Intake Scan - Field Mapping: {}".format(field_mapping))
+    # MediLink_ConfigLoader.log("Intake Scan - CSV Headers: {}".format(csv_headers))
+    
     # Iterate over the Medisoft fields defined in field_mapping
     for medisoft_field in field_mapping.keys():
         for pattern in field_mapping[medisoft_field]:
@@ -252,32 +164,84 @@ def intake_scan(csv_headers, field_mapping):
             if matched_headers:
                 # Assuming the first matched header is the desired one
                 identified_fields[matched_headers[0]] = medisoft_field
+                # MediLink_ConfigLoader.log("Found Header: {}".format(identified_fields[matched_headers[0]]))
                 break
         else:
             # Check if the missing field is a required field before appending the warning
             if medisoft_field in required_fields:
                 missing_fields_warnings.append("WARNING: No matching CSV header found for Medisoft field '{0}'".format(medisoft_field))
    
-   #-----------------------
-   # CSV Integrity Check
-   #-----------------------
-   
-   # This section needs to be revamped further so that it can interpret the information from here and decide 
-   # if it's significant or not.
-   # e.g. If the 'Street' value:key is 'Address', then any warnings about City, State, Zip can be ignored. 
-   # Insurance Policy Numbers should be all alphanumeric with no other characters. 
-   # Make sure that the name field has at least one name under it (basically check for a blank or 
-   # partially blank csv with just a header)
-      
+    # CSV Integrity Checks
+    # Check for blank or partially blank CSV
+    if len(csv_headers) == 0 or all(header == "" for header in csv_headers):
+        missing_fields_warnings.append("WARNING: The CSV appears to be blank or contains only headers without data.")
+
     # Display the identified fields and missing fields warnings
-    #print("The following Medisoft fields have been identified in the CSV:\n")
+    #MediLink_ConfigLoader.log("The following Medisoft fields have been identified in the CSV:")
     #for header, medisoft_field in identified_fields.items():
-    #    print("{0} (CSV header: {1})".format(medisoft_field, header))
+    #    MediLink_ConfigLoader.log("{} (CSV header: {})".format(medisoft_field, header))
+
+    # This section interprets the information from identified_fields and decides if there are significant issues.
+    # e.g. If the 'Street' value:key is 'Address', then any warnings about City, State, Zip can be ignored.
+    for header, field in identified_fields.items():
+        # Insurance Policy Numbers should be all alphanumeric with no other characters. 
+        if 'Insurance Policy Number' in field:
+            policy_number = identified_fields.get(header)
+            if not bool(re.match("^[a-zA-Z0-9]*$", policy_number)):
+                missing_fields_warnings.append("WARNING: Insurance Policy Number '{}' contains invalid characters.".format(policy_number))
+        # Additional checks can be added as needed for other fields
+    
+    if missing_fields_warnings:
+        MediLink_ConfigLoader.log("\nSome required fields could not be matched:")
+        for warning in missing_fields_warnings:
+            MediLink_ConfigLoader.log(warning)
+
+    return identified_fields
+
+def main():
+    parser = argparse.ArgumentParser(description='Run MediLink Data Management Tasks')
+    parser.add_argument('--update-crosswalk', action='store_true',
+                        help='Run the crosswalk update independently')
+    parser.add_argument('--init-crosswalk', action='store_true',
+                        help='Initialize the crosswalk using historical data from MAPAT and Carols CSV')
+    parser.add_argument('--load-csv', action='store_true',
+                        help='Load and process CSV data')
+    parser.add_argument('--preprocess-csv', action='store_true',
+                        help='Preprocess CSV data based on specific rules')
+    parser.add_argument('--open-csv', action='store_true',
+                        help='Open CSV for manual editing')
+
+    args = parser.parse_args()
+
+    config, crosswalk = MediLink_ConfigLoader.load_configuration()
+    
+    # If no arguments provided, print usage instructions
+    if not any(vars(args).values()):
+        parser.print_help()
+        return
+
+    if args.update_crosswalk:
+        print("Updating the crosswalk...")
+        MediBot_Crosswalk_Library.crosswalk_update(config, crosswalk)
+
+    if args.init_crosswalk:
+        MediBot_Crosswalk_Library.initialize_crosswalk_from_mapat()
+
+    if args.load_csv:
+        print("Loading CSV data...")
+        csv_data = MediBot_Preprocessor_lib.load_csv_data(config['CSV_FILE_PATH'])
+        print("Loaded {} records from the CSV.".format(len(csv_data)))
+
+    if args.preprocess_csv:
+        if 'csv_data' in locals():
+            print("Preprocessing CSV data...")
+            preprocess_csv_data(csv_data, crosswalk)
+        else:
+            print("Error: CSV data needs to be loaded before preprocessing. Use --load-csv.")
     
-    #if missing_fields_warnings:
-    #    print("\nSome required fields could not be matched:")
-    #    for warning in missing_fields_warnings:
-    #        print(warning)  
+    if args.open_csv:
+        print("Opening CSV for editing...")
+        MediBot_Preprocessor_lib.open_csv_for_editing(config['CSV_FILE_PATH'])
 
-    #print("Debug - Identified fields mapping (intake scan):", identified_fields)
-    return identified_fields
+if __name__ == '__main__':
+    main()