medicafe 0.240809.0__py3-none-any.whl → 0.240925.9__py3-none-any.whl

MediLink/MediLink_DataMgmt.py

@@ -1,9 +1,6 @@
-# MediLink_DataMgmt.py
-import csv
-import os
+
+import csv, os, re, subprocess, time
 from datetime import datetime, timedelta
-import re
-import subprocess 
 
 # Need this for running Medibot and MediLink
 try:
@@ -167,64 +164,170 @@ def consolidate_csvs(source_directory, file_prefix="Consolidated", interactive=F
 def operate_winscp(operation_type, files, endpoint_config, local_storage_path, config):
     """
     General function to operate WinSCP for uploading or downloading files.
+    """
+    MediLink_ConfigLoader.log("Starting operate_winscp with operation_type: {}".format(operation_type))
+    
+    config = ensure_config_loaded(config)
+    winscp_path = get_winscp_path(config)
+    
+    if not os.path.isfile(winscp_path):
+        MediLink_ConfigLoader.log("WinSCP.com not found at {}".format(winscp_path), level="ERROR")
+        return []
 
-    :param operation_type: 'upload' or 'download'
-    :param files: List of files to upload or pattern for files to download.
-    :param endpoint_config: Dictionary containing endpoint configuration.
-    :param local_storage_path: Base local storage path for logs and files.
+    validate_endpoint_config(endpoint_config)
+    winscp_log_path = setup_logging(operation_type, local_storage_path)
 
-    # Example of how to call this function for uploads
-    upload_files = ['path/to/local/file1.txt', 'path/to/local/file2.txt']
-    upload_config = {
-        'session_name': 'MySession',
-        'remote_directory_up': '/remote/upload/path'
-    }
+    # Validate the local_storage_path and replace it if necessary
+    local_storage_path = validate_local_storage_path(local_storage_path, config)
 
-    operate_winscp('upload', upload_files, upload_config, 'path/to/local/storage', config)
+    remote_directory = get_remote_directory(endpoint_config, operation_type)
+    command = build_command(winscp_path, winscp_log_path, endpoint_config, remote_directory, operation_type, files, local_storage_path)
 
-    # Example of how to call this function for downloads
-    download_config = {
-        'session_name': 'MySession',
-        'remote_directory_down': '/remote/download/path'
-    }
+    if config.get("TestMode", True):
+        MediLink_ConfigLoader.log("Test mode is enabled. Simulating operation.")
+        return simulate_operation(operation_type, files, config)
+    
+    result = execute_winscp_command(command, operation_type, files, local_storage_path)
+    MediLink_ConfigLoader.log("[Execute WinSCP Command] Result: {}".format(result), level="DEBUG")
+    return result
 
-    operate_winscp('download', None, download_config, 'path/to/local/storage', config)
+def validate_local_storage_path(local_storage_path, config):
     """
-    # Setup paths
-    try:
-        # TODO (Easy / Config) Get this updated. ??
-        winscp_path = config['winscp_path']
-    except KeyError:
-        winscp_path = os.path.join(os.getcwd(), "Installers", "WinSCP-Portable", "WinSCP.com")
-    except Exception as e:
-        # Handle any other exceptions here
-        print("An error occurred while running WinSCP:", e)
-        winscp_path = None
+    Validates the local storage path and replaces it with outputFilePath from config if it contains spaces.
+    """
+    if ' ' in local_storage_path:
+        MediLink_ConfigLoader.log("Local storage path contains spaces, using outputFilePath from config.", level="WARN")
+        output_file_path = config.get('outputFilePath', None)
+        if not output_file_path:
+            raise ValueError("outputFilePath not found in config.")
+        return os.path.normpath(output_file_path)
+    return os.path.normpath(local_storage_path)
+
+def ensure_config_loaded(config):
+    MediLink_ConfigLoader.log("Ensuring configuration is loaded.")
+    if not config:
+        MediLink_ConfigLoader.log("Warning: No config passed to ensure_config_loaded. Re-loading config...")
+        config, _ = MediLink_ConfigLoader.load_configuration()
+    
+    # Check if config was successfully loaded
+    if not config or 'MediLink_Config' not in config:
+        MediLink_ConfigLoader.log("Failed to load the MediLink configuration. Config is None or missing 'MediLink_Config'.")
+        raise RuntimeError("Failed to load the MediLink configuration. Config is None or missing 'MediLink_Config'.")
+
+    # Check that 'endpoints' key exists within 'MediLink_Config'
+    if 'endpoints' not in config['MediLink_Config']:
+        MediLink_ConfigLoader.log("The loaded configuration is missing the 'endpoints' section.")
+        raise ValueError("The loaded configuration is missing the 'endpoints' section.")
+
+    # Additional checks can be added here to ensure all expected keys and structures are present
+    if 'local_storage_path' not in config['MediLink_Config']:
+        MediLink_ConfigLoader.log("The loaded configuration is missing the 'local_storage_path' setting.")
+        raise ValueError("The loaded configuration is missing the 'local_storage_path' setting.")
+
+    MediLink_ConfigLoader.log("Configuration loaded successfully.")
+    return config['MediLink_Config']  # Return the relevant part of the config for simplicity
+
+def get_winscp_path(config):
+    MediLink_ConfigLoader.log("Retrieving WinSCP path from provided config.")
+    
+    def find_winscp_path(cfg):
+        if 'winscp_path' in cfg:
+            # cfg is already 'MediLink_Config'
+            MediLink_ConfigLoader.log("Config provided directly as 'MediLink_Config'.")
+            return cfg.get('winscp_path')
+        else:
+            # cfg is the full configuration, retrieve 'MediLink_Config'
+            MediLink_ConfigLoader.log("Config provided as full configuration; accessing 'MediLink_Config'.")
+            medi_link_config = cfg.get('MediLink_Config', {})
+            return medi_link_config.get('winscp_path')
+
+    # Attempt to find the WinSCP path using the provided config
+    winscp_path = find_winscp_path(config)
+    
+    # If the path is not found, attempt to use default paths
+    if not winscp_path:
+        error_message = "WinSCP path not found in config. Attempting to use default paths."
+        # print(error_message)
+        MediLink_ConfigLoader.log(error_message)
         
-    if not os.path.isfile(winscp_path):
-        MediLink_ConfigLoader.log("WinSCP.com not found at {}".format(winscp_path))
-        return []
+        # Try the default paths
+        default_paths = [
+            os.path.join(os.getcwd(), "Installers", "WinSCP-Portable", "WinSCP.com"),
+            os.path.join(os.getcwd(), "Necessary Programs", "WinSCP-Portable", "WinSCP.com")
+        ]
+        
+        for path in default_paths:
+            if os.path.exists(path):
+                found_message = "WinSCP found at {}. Using this path.".format(path)
+                # print(found_message)
+                MediLink_ConfigLoader.log(found_message)
+                return path
+        
+        # If no valid path is found, attempt to reload the configuration
+        reload_message = "WinSCP not found in config or default paths. Reloading the entire configuration."
+        # print(reload_message)
+        MediLink_ConfigLoader.log(reload_message)
+        
+        try:
+            config, _ = MediLink_ConfigLoader.load_configuration()
+            winscp_path = find_winscp_path(config)
+            
+            if winscp_path:
+                success_message = "WinSCP path found after reloading configuration. Using this path."
+                # print(success_message)
+                MediLink_ConfigLoader.log(success_message)
+                return winscp_path
+            else:
+                raise FileNotFoundError("WinSCP path not found even after reloading configuration.")
+        
+        except Exception as e:
+            error_message = "Failed to reload configuration or find WinSCP path: {}. Exiting script.".format(e)
+            print(error_message)
+            MediLink_ConfigLoader.log(error_message)
+            raise FileNotFoundError(error_message)
+    
+    return winscp_path
 
-    # Setup logging
+def validate_endpoint_config(endpoint_config):
+    MediLink_ConfigLoader.log("Validating endpoint configuration.")
+    if not isinstance(endpoint_config, dict):
+        MediLink_ConfigLoader.log("Endpoint configuration object is invalid. Expected a dictionary, got: {}".format(type(endpoint_config)))
+        raise ValueError("Endpoint configuration object is invalid. Expected a dictionary, got: {}".format(type(endpoint_config)))
+
+def setup_logging(operation_type, local_storage_path):
+    MediLink_ConfigLoader.log("Setting up logging for operation type: {}".format(operation_type))
     log_filename = "winscp_upload.log" if operation_type == "upload" else "winscp_download.log"
-    winscp_log_path = os.path.join(local_storage_path, log_filename)
+    return os.path.join(local_storage_path, log_filename)
+
+def get_remote_directory(endpoint_config, operation_type):
+    MediLink_ConfigLoader.log("Getting remote directory for operation type: {}".format(operation_type))
+    if endpoint_config is None:
+        MediLink_ConfigLoader.log("Error: Endpoint configuration is None.")
+        raise ValueError("Endpoint configuration is None. Expected a dictionary with configuration details.")
+
+    if not isinstance(endpoint_config, dict):
+        MediLink_ConfigLoader.log("Error: Endpoint configuration is invalid. Expected a dictionary, got: {}".format(type(endpoint_config)))
+        raise TypeError("Endpoint configuration is invalid. Expected a dictionary, got: {}".format(type(endpoint_config)))
 
-    # Session and directory setup
     try:
-        session_name = endpoint_config.get('session_name', '')
         if operation_type == "upload":
-            remote_directory = endpoint_config['remote_directory_up']
+            return endpoint_config['remote_directory_up']
+        elif operation_type == "download":
+            return endpoint_config['remote_directory_down']
         else:
-            remote_directory = endpoint_config['remote_directory_down']
+            MediLink_ConfigLoader.log("Invalid operation type: {}. Expected 'upload' or 'download'.".format(operation_type))
+            raise ValueError("Invalid operation type: {}. Expected 'upload' or 'download'.".format(operation_type))
     except KeyError as e:
-        # Log the missing key information
-        missing_key = str(e)
-        message = "Critical Error: Endpoint config is missing key: {}".format(missing_key)
-        MediLink_ConfigLoader.log(message)
-        # Raise an exception to halt execution
-        raise RuntimeError("Configuration error: The endpoint configuration is missing definitions for the required remote directories. Please check the configuration and try again.")
-
-    # Command building
+        MediLink_ConfigLoader.log("Critical Error: Endpoint config is missing key: {}".format(e))
+        raise RuntimeError("Configuration error: Missing required remote directory in endpoint configuration.")
+
+def build_command(winscp_path, winscp_log_path, endpoint_config, remote_directory, operation_type, files, local_storage_path, newer_than=None, filemask=None):
+    # Log the operation type
+    MediLink_ConfigLoader.log("[Build Command] Building WinSCP command for operation type: {}".format(operation_type))
+
+    session_name = endpoint_config.get('session_name', '')
+
+    # Initial command structure
     command = [
         winscp_path,
         '/log=' + winscp_log_path,
@@ -235,66 +338,184 @@ def operate_winscp(operation_type, files, endpoint_config, local_storage_path, c
         'cd {}'.format(remote_directory)
     ]
 
-    # Add commands to WinSCP script
-    # BUG (Low SFTP) We really need to fix this path situation.
-    #  Unfortunately, this just needs to be a non-spaced path because WinSCP can't
-    #  handle the spaces. Also, Windows won't let me use shutil to move the files out of G:\ into C:\ and it it wants an administrator security 
-    #  check or verification thing for me to even move the file by hand so that doesn't work either. 
-    #  command.append("put {}".format("C:\\Z_optumedi_04161742.txt"))
-    if operation_type == "upload":
-        for file_path in files:
-            normalized_path = os.path.normpath(file_path)
-            command.append("put {}".format(normalized_path))
-    else:
-        command.append('get *')  # Adjust pattern as needed
-
-    command += ['close', 'exit']
-
-    # Check if TestMode is enabled in the configuration
-    if config.get("MediLink_Config", {}).get("TestMode", True):
-        # TestMode is enabled, do not execute the command
-        print("Test Mode is enabled! WinSCP Command not executed.")
-        MediLink_ConfigLoader.log("Test Mode is enabled! WinSCP Command not executed.")
-        MediLink_ConfigLoader.log("TEST MODE: Simulating WinSCP {} File List.".format(operation_type))
-        uploaded_files = []
-        if files is not None:  # Check if files is not None
-            for file_path in files:
-                normalized_path = os.path.normpath(file_path)
-                if os.path.exists(normalized_path):  # Check if the file exists before appending
-                    uploaded_files.append(normalized_path)
+    try:
+        # Handle upload operation
+        if operation_type == "upload":
+            if not files:
+                MediLink_ConfigLoader.log("Error: No files provided for upload operation.", level="ERROR")
+                raise ValueError("No files provided for upload operation.")
+
+            put_commands = []
+            for f in files:
+                # Normalize the path
+                normalized_path = os.path.normpath(f)
+                original_path = normalized_path  # Keep for logging
+
+                # Remove leading slash if present
+                if normalized_path.startswith('\\') or normalized_path.startswith('/'):
+                    normalized_path = normalized_path.lstrip('\\/')
+                    MediLink_ConfigLoader.log("Removed leading slash from path: {}".format(original_path), level="DEBUG")
+
+                # Remove trailing slash if present
+                if normalized_path.endswith('\\') or normalized_path.endswith('/'):
+                    normalized_path = normalized_path.rstrip('\\/')
+                    MediLink_ConfigLoader.log("Removed trailing slash from path: {}".format(original_path), level="DEBUG")
+
+                # Determine if quotes are necessary (e.g., if path contains spaces)
+                if ' ' in normalized_path:
+                    put_command = 'put "{}"'.format(normalized_path)
+                    MediLink_ConfigLoader.log("Constructed put command with quotes: {}".format(put_command), level="DEBUG")
+                else:
+                    put_command = 'put {}'.format(normalized_path)
+                    MediLink_ConfigLoader.log("Constructed put command without quotes: {}".format(put_command), level="DEBUG")
+
+                put_commands.append(put_command)
+            command += put_commands
+
+        # Handle download operation
+        elif operation_type == "download":
+            lcd_path = os.path.normpath(local_storage_path)
+            original_lcd_path = lcd_path  # Keep for logging
+
+            # Remove leading slash if present
+            if lcd_path.startswith('\\') or lcd_path.startswith('/'):
+                lcd_path = lcd_path.lstrip('\\/')
+                MediLink_ConfigLoader.log("Removed leading slash from local storage path: {}".format(original_lcd_path), level="DEBUG")
+
+            # Remove trailing slash if present
+            if lcd_path.endswith('\\') or lcd_path.endswith('/'):
+                lcd_path = lcd_path.rstrip('\\/')
+                MediLink_ConfigLoader.log("Removed trailing slash from local storage path: {}".format(original_lcd_path), level="DEBUG")
+
+            # Determine if quotes are necessary (e.g., if path contains spaces)
+            if ' ' in lcd_path:
+                lcd_command = 'lcd "{}"'.format(lcd_path)
+                MediLink_ConfigLoader.log("Constructed lcd command with quotes: {}".format(lcd_command), level="DEBUG")
+            else:
+                lcd_command = 'lcd {}'.format(lcd_path)
+                MediLink_ConfigLoader.log("Constructed lcd command without quotes: {}".format(lcd_command), level="DEBUG")
+
+            command.append(lcd_command)
+
+            # Handle filemask input
+            if filemask:
+                # TODO: Implement logic to translate filemask into WinSCP syntax
+                # This should handle cases where filemask is a list, JSON, dictionary, or None.
+                # Example: Convert to a string like "*.{ext1}|*.{ext2}|*.{ext3}".
+                if isinstance(filemask, list):
+                    filemask_str = '|'.join(['*.' + ext for ext in filemask])
+                elif isinstance(filemask, dict):
+                    filemask_str = '|'.join(['*.' + ext for ext in filemask.keys()])
+                elif isinstance(filemask, str):
+                    filemask_str = filemask  # Assume it's already in the correct format
                 else:
-                    MediLink_ConfigLoader.log("TEST MODE: Failed to {} file: {} does not exist.".format(operation_type, normalized_path))
+                    filemask_str = '*'  # Default to all files if filemask is None or unsupported type
+            else:
+                filemask_str = '*'  # Default to all files if filemask is None
+
+            # Use synchronize command for efficient downloading
+            if newer_than:
+                command.append('synchronize local -filemask="{}" -newerthan={}'.format(filemask_str, newer_than))
+            else:
+                command.append('synchronize local -filemask="{}"'.format(filemask_str))
+
+        # Close and exit commands
+        command += ['close', 'exit']
+        MediLink_ConfigLoader.log("[Build Command] WinSCP command: {}".format(command))
+        return command
+
+    except Exception as e:
+        MediLink_ConfigLoader.log("Error in build_command: {}. Reverting to original implementation.".format(e), level="ERROR")
+
+        # Fallback to original implementation
+        # Handle upload operation
+        if operation_type == "upload":
+            if not files:
+                MediLink_ConfigLoader.log("Error: No files provided for upload operation.", level="ERROR")
+                raise ValueError("No files provided for upload operation.")
+            command.extend(["put {}".format(os.path.normpath(file_path)) for file_path in files])
+
+        # Handle download operation
         else:
-            MediLink_ConfigLoader.log("TEST MODE: No files to upload.")
-        return uploaded_files if files is not None else []
+            command.append('get *')
+
+        # Close and exit commands
+        command.extend(['close', 'exit'])
+        MediLink_ConfigLoader.log("[Build Command] Original WinSCP command: {}".format(command))
+        return command
+
+def simulate_operation(operation_type, files, config):
+    MediLink_ConfigLoader.log("Test Mode is enabled! Simulating WinSCP {} operation.".format(operation_type))
+    
+    if operation_type == 'upload' and files:
+        MediLink_ConfigLoader.log("Simulating 3 second delay for upload operation for files: {}".format(files))
+        time.sleep(3)
+        return [os.path.normpath(file) for file in files if os.path.exists(file)]
+    elif operation_type == 'download':
+        MediLink_ConfigLoader.log("Simulating 3 second delay for download operation. No files to download in test mode.")
+        time.sleep(3)
+        return []
     else:
-        # TestMode is not enabled, execute the command
+        MediLink_ConfigLoader.log("Invalid operation type during simulation: {}".format(operation_type))
+        return []
+
+def execute_winscp_command(command, operation_type, files, local_storage_path):
+    """
+    Execute the WinSCP command for the specified operation type.
+    """
+    MediLink_ConfigLoader.log("Executing WinSCP command for operation type: {}".format(operation_type))
+    
+    try:
         process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False)
         stdout, stderr = process.communicate()
-    
-    if process.returncode == 0: # BUG Does this work as intended?
-        MediLink_ConfigLoader.log("WinSCP {} attempted.".format(operation_type))
-        # Construct a list of downloaded files if operation_type is 'download'
+    except Exception as e:
+        MediLink_ConfigLoader.log("Error occurred while executing WinSCP command: {}".format(e), level="ERROR")
+        return []  # Return an empty list instead of None
+
+    if process.returncode == 0:
+        MediLink_ConfigLoader.log("WinSCP {} operation completed successfully.".format(operation_type))
+
         if operation_type == 'download':
-            downloaded_files = []
-            for root, dirs, files in os.walk(local_storage_path):
-                for file in files:
-                    downloaded_files.append(os.path.join(root, file))
+            downloaded_files = list_downloaded_files(local_storage_path) # BUG This isn't behaving correctly because the local_storage_path isn't where winscp is dumping the files
+            MediLink_ConfigLoader.log("Files currently located in local_storage_path: {}".format(downloaded_files), level="DEBUG")
+
+            if not downloaded_files:
+                MediLink_ConfigLoader.log("No files were downloaded or an error occurred during the listing process.", level="WARNING")
             return downloaded_files
-        
-        if operation_type == 'upload':
-            # Return a list of uploaded files
-            uploaded_files = []
-            for file_path in files:
-                normalized_path = os.path.normpath(file_path)
-                if os.path.exists(normalized_path):  # Check if the file exists before appending
-                    uploaded_files.append(normalized_path)
-                else:
-                    MediLink_ConfigLoader.log("Failed to upload file: {} does not exist.".format(normalized_path))
+
+        elif operation_type == 'upload':
+            uploaded_files = [os.path.normpath(file) for file in files if os.path.exists(file)]
+            MediLink_ConfigLoader.log("Uploaded files: {}".format(uploaded_files), level="DEBUG")
             return uploaded_files
     else:
-        MediLink_ConfigLoader.log("Failed to {} files. Details: {}".format(operation_type, stderr.decode('utf-8')))
-        return []  # Return empty list to indicate failure. BUG check to make sure this doesn't break something else.
+        error_message = stderr.decode('utf-8').strip()
+        MediLink_ConfigLoader.log("Failed to {} files. Exit code: {}. Details: {}".format(
+            operation_type, process.returncode, error_message), level="ERROR")
+        return []  # Return an empty list instead of None
+
+def list_downloaded_files(local_storage_path):
+
+    MediLink_ConfigLoader.log("Listing downloaded files in local storage path: {}".format(local_storage_path))
+    
+    # Initialize an empty list to hold file paths
+    downloaded_files = []
+
+    try:
+        # Walk through the directory and collect all file paths
+        for root, _, files in os.walk(local_storage_path):
+            for file in files:
+                file_path = os.path.join(root, file)
+                downloaded_files.append(file_path)
+                MediLink_ConfigLoader.log("File found: {}".format(file_path), level="DEBUG")
+        
+        if not downloaded_files:
+            MediLink_ConfigLoader.log("No files found in the directory: {}".format(local_storage_path), level="WARNING")
+
+    except Exception as e:
+        MediLink_ConfigLoader.log("Error occurred while listing files in {}: {}".format(local_storage_path, e), level="ERROR")
+
+    # Ensure that the function always returns a list
+    return downloaded_files
 
 def detect_new_files(directory_path, file_extension='.DAT'):
     """