dragon-ml-toolbox 19.13.0__py3-none-any.whl → 20.0.0__py3-none-any.whl

ml_tools/IO_tools/_IO_loggers.py

@@ -0,0 +1,235 @@
+import json
+import csv
+import traceback
+from pathlib import Path
+from typing import Union, Any, Literal
+from datetime import datetime
+
+from ..path_manager import sanitize_filename, make_fullpath
+from .._core import get_logger
+
+
+_LOGGER = get_logger("IO logger")
+
+
+__all__ = [
+    "custom_logger",
+    "train_logger"
+]
+
+
+def custom_logger(
+    data: Union[
+        list[Any],
+        dict[Any, Any],
+        str,
+        BaseException
+    ],
+    save_directory: Union[str, Path],
+    log_name: str,
+    add_timestamp: bool=True,
+    dict_as: Literal['auto', 'json', 'csv'] = 'auto',
+) -> None:
+    """
+    Logs various data types to corresponding output formats:
+
+    - list[Any]                    → .txt
+        Each element is written on a new line.
+
+    - dict[str, list[Any]]        → .csv    (if dict_as='auto' or 'csv')
+        Dictionary is treated as tabular data; keys become columns, values become rows.
+
+    - dict[str, scalar]           → .json   (if dict_as='auto' or 'json')
+        Dictionary is treated as structured data and serialized as JSON.
+
+    - str                         → .log
+        Plain text string is written to a .log file.
+
+    - BaseException               → .log
+        Full traceback is logged for debugging purposes.
+
+    Args:
+        data (Any): The data to be logged. Must be one of the supported types.
+        save_directory (str | Path): Directory where the log will be saved. Created if it does not exist.
+        log_name (str): Base name for the log file.
+        add_timestamp (bool): Whether to add a timestamp to the filename.
+        dict_as ('auto'|'json'|'csv'): 
+            - 'auto': Guesses format (JSON or CSV) based on dictionary content.
+            - 'json': Forces .json format for any dictionary.
+            - 'csv': Forces .csv format. Will fail if dict values are not all lists.
+
+    Raises:
+        ValueError: If the data type is unsupported.
+    """
+    try:
+        if not isinstance(data, BaseException) and not data:
+            _LOGGER.warning("Empty data received. No log file will be saved.")
+            return
+        
+        save_path = make_fullpath(save_directory, make=True)
+        
+        sanitized_log_name = sanitize_filename(log_name)
+        
+        if add_timestamp:
+            timestamp = datetime.now().strftime(r"%Y%m%d_%H%M%S")
+            base_path = save_path / f"{sanitized_log_name}_{timestamp}"
+        else:
+            base_path = save_path / sanitized_log_name
+        
+        # Router
+        if isinstance(data, list):
+            _log_list_to_txt(data, base_path.with_suffix(".txt"))
+
+        elif isinstance(data, dict):
+            if dict_as == 'json':
+                _log_dict_to_json(data, base_path.with_suffix(".json"))
+            
+            elif dict_as == 'csv':
+                # This will raise a ValueError if data is not all lists
+                _log_dict_to_csv(data, base_path.with_suffix(".csv"))
+            
+            else: # 'auto' mode
+                if all(isinstance(v, list) for v in data.values()):
+                    _log_dict_to_csv(data, base_path.with_suffix(".csv"))
+                else:
+                    _log_dict_to_json(data, base_path.with_suffix(".json"))
+
+        elif isinstance(data, str):
+            _log_string_to_log(data, base_path.with_suffix(".log"))
+
+        elif isinstance(data, BaseException):
+            _log_exception_to_log(data, base_path.with_suffix(".log"))
+
+        else:
+            _LOGGER.error("Unsupported data type. Must be list, dict, str, or BaseException.")
+            raise ValueError()
+
+        _LOGGER.info(f"Log saved as: '{base_path.name}'")
+
+    except Exception:
+        _LOGGER.exception(f"Log not saved.")
+
+
+def _log_list_to_txt(data: list[Any], path: Path) -> None:
+    log_lines = []
+    for item in data:
+        try:
+            log_lines.append(str(item).strip())
+        except Exception:
+            log_lines.append(f"(unrepresentable item of type {type(item)})")
+
+    with open(path, 'w', encoding='utf-8') as f:
+        f.write('\n'.join(log_lines))
+
+
+def _log_dict_to_csv(data: dict[Any, list[Any]], path: Path) -> None:
+    sanitized_dict = {}
+    max_length = max(len(v) for v in data.values()) if data else 0
+
+    for key, value in data.items():
+        if not isinstance(value, list):
+            _LOGGER.error(f"Dictionary value for key '{key}' must be a list.")
+            raise ValueError()
+        
+        sanitized_key = str(key).strip().replace('\n', '_').replace('\r', '_')
+        padded_value = value + [None] * (max_length - len(value))
+        sanitized_dict[sanitized_key] = padded_value
+
+    # The `newline=''` argument is important to prevent extra blank rows
+    with open(path, 'w', newline='', encoding='utf-8') as csv_file:
+        writer = csv.writer(csv_file)
+
+        # 1. Write the header row from the sanitized dictionary keys
+        header = list(sanitized_dict.keys())
+        writer.writerow(header)
+
+        # 2. Transpose columns to rows and write them
+        # zip(*sanitized_dict.values()) elegantly converts the column data
+        # (lists in the dict) into row-by-row tuples.
+        rows_to_write = zip(*sanitized_dict.values())
+        writer.writerows(rows_to_write)
+
+
+def _log_string_to_log(data: str, path: Path) -> None:
+    with open(path, 'w', encoding='utf-8') as f:
+        f.write(data.strip() + '\n')
+
+
+def _log_exception_to_log(exc: BaseException, path: Path) -> None:
+    with open(path, 'w', encoding='utf-8') as f:
+        f.write("Exception occurred:\n")
+        traceback.print_exception(type(exc), exc, exc.__traceback__, file=f)
+
+
+def _log_dict_to_json(data: dict[Any, Any], path: Path) -> None:
+    with open(path, 'w', encoding='utf-8') as f:
+        json.dump(data, f, indent=4, ensure_ascii=False)        
+
+
+def train_logger(train_config: Union[dict, Any],
+                 model_parameters: Union[dict, Any],
+                 train_history: Union[dict, None],
+                 save_directory: Union[str, Path]):
+    """
+    Logs training data to JSON, adding a timestamp to the filename.
+    
+    Args:
+        train_config (dict | Any): Training configuration parameters. If object, must have a `.to_log()` method returning a dict.
+        model_parameters (dict | Any): Model parameters. If object, must have a `.to_log()` method returning a dict.
+        train_history (dict | None): Training history log.
+        save_directory (str | Path): Directory to save the log file.
+    """
+    # train_config should be a dict or a custom object with the ".to_log()" method
+    if not isinstance(train_config, dict):
+        if hasattr(train_config, "to_log") and callable(getattr(train_config, "to_log")):
+            train_config_dict: dict = train_config.to_log()
+            if not isinstance(train_config_dict, dict):
+                _LOGGER.error("'train_config.to_log()' did not return a dictionary.")
+                raise ValueError()
+        else:
+            _LOGGER.error("'train_config' must be a dict or an object with a 'to_log()' method.")
+            raise ValueError()
+    else:
+        # check for empty dict
+        if not train_config:
+            _LOGGER.error("'train_config' dictionary is empty.")
+            raise ValueError()
+        
+        train_config_dict = train_config
+        
+    # model_parameters should be a dict or a custom object with the ".to_log()" method
+    if not isinstance(model_parameters, dict):
+        if hasattr(model_parameters, "to_log") and callable(getattr(model_parameters, "to_log")):
+            model_parameters_dict: dict = model_parameters.to_log()
+            if not isinstance(model_parameters_dict, dict):
+                _LOGGER.error("'model_parameters.to_log()' did not return a dictionary.")
+                raise ValueError()
+        else:
+            _LOGGER.error("'model_parameters' must be a dict or an object with a 'to_log()' method.")
+            raise ValueError()
+    else:
+        # check for empty dict
+        if not model_parameters:
+            _LOGGER.error("'model_parameters' dictionary is empty.")
+            raise ValueError()
+        
+        model_parameters_dict = model_parameters
+    
+    # make base dictionary
+    data: dict = train_config_dict | model_parameters_dict
+    
+    # add training history if provided and is not empty
+    if train_history is not None:
+        if not train_history:
+            _LOGGER.error("'train_history' dictionary was provided but is empty.")
+            raise ValueError()
+        data.update(train_history)
+    
+    custom_logger(
+        data=data,
+        save_directory=save_directory,
+        log_name="Training_Log",
+        add_timestamp=True,
+        dict_as='json'
+    )
+

ml_tools/IO_tools/_IO_save_load.py

@@ -0,0 +1,151 @@
+from typing import Any, Union, Literal, overload
+from pathlib import Path
+import json
+
+from ..path_manager import sanitize_filename, make_fullpath
+from .._core import get_logger
+
+from ._IO_utils import _RobustEncoder
+
+
+_LOGGER = get_logger("IO Save/Load")
+
+
+__all__ = [
+    "save_json",
+    "load_json",
+    "save_list_strings",
+    "load_list_strings"
+]
+
+
+def save_json(
+    data: Union[dict[Any, Any], list[Any]],
+    directory: Union[str, Path],
+    filename: str,
+    verbose: bool = True
+) -> None:
+    """
+    Saves a dictionary or list as a JSON file.
+
+    Args:
+        data (dict | list): The data to save.
+        directory (str | Path): The directory to save the file in.
+        filename (str): The name of the file (extension .json will be added if missing).
+        verbose (bool): Whether to log success messages.
+    """
+    target_dir = make_fullpath(directory, make=True, enforce="directory")
+    sanitized_name = sanitize_filename(filename)
+
+    if not sanitized_name.endswith(".json"):
+        sanitized_name += ".json"
+
+    full_path = target_dir / sanitized_name
+
+    try:
+        with open(full_path, 'w', encoding='utf-8') as f:
+            # Using _RobustEncoder ensures compatibility with non-standard types (like 'type' objects)
+            json.dump(data, f, indent=4, ensure_ascii=False, cls=_RobustEncoder)
+
+        if verbose:
+            _LOGGER.info(f"JSON file saved as '{full_path.name}'.")
+
+    except Exception as e:
+        _LOGGER.error(f"Failed to save JSON to '{full_path}': {e}")
+        raise
+
+
+# 1. Define Overloads (for the type checker)
+@overload
+def load_json(
+    file_path: Union[str, Path], 
+    expected_type: Literal["dict"] = "dict",
+    verbose: bool = True
+) -> dict[Any, Any]: ...
+
+@overload
+def load_json(
+    file_path: Union[str, Path], 
+    expected_type: Literal["list"],
+    verbose: bool = True
+) -> list[Any]: ...
+
+
+def load_json(
+    file_path: Union[str, Path], 
+    expected_type: Literal["dict", "list"] = "dict",
+    verbose: bool = True
+) -> Union[dict[Any, Any], list[Any]]:
+    """
+    Loads a JSON file.
+
+    Args:
+        file_path (str | Path): The path to the JSON file.
+        expected_type ('dict' | 'list'): strict check for the root type of the JSON.
+        verbose (bool): Whether to log success/failure messages.
+
+    Returns:
+        dict | list: The loaded JSON data.
+    """
+    target_path = make_fullpath(file_path, enforce="file")
+    
+    # Map string literals to actual python types
+    type_map = {"dict": dict, "list": list}
+    target_type = type_map.get(expected_type, dict)
+
+    try:
+        with open(target_path, 'r', encoding='utf-8') as f:
+            data = json.load(f)
+        
+        if not isinstance(data, target_type):
+            _LOGGER.error(f"JSON root is type {type(data)}, expected {expected_type}.")
+            raise ValueError()
+
+        if verbose:
+            _LOGGER.info(f"Loaded JSON data from '{target_path.name}'.")
+        
+        return data
+
+    except json.JSONDecodeError as e:
+        _LOGGER.error(f"Failed to decode JSON from '{target_path}': {e.msg}")
+        raise ValueError()
+        
+    except Exception as e:
+        _LOGGER.error(f"Error loading JSON from '{target_path}': {e}")
+        raise
+
+
+def save_list_strings(list_strings: list[str], directory: Union[str,Path], filename: str, verbose: bool=True):
+    """Saves a list of strings as a text file."""
+    target_dir = make_fullpath(directory, make=True, enforce="directory")
+    sanitized_name = sanitize_filename(filename)
+    
+    if not sanitized_name.endswith(".txt"):
+        sanitized_name = sanitized_name + ".txt"
+    
+    full_path = target_dir / sanitized_name
+    with open(full_path, 'w') as f:
+        for string_data in list_strings:
+            f.write(f"{string_data}\n")
+    
+    if verbose:
+        _LOGGER.info(f"Text file saved as '{full_path.name}'.")
+
+
+def load_list_strings(text_file: Union[str,Path], verbose: bool=True) -> list[str]:
+    """Loads a text file as a list of strings."""
+    target_path = make_fullpath(text_file, enforce="file")
+    loaded_strings = []
+
+    with open(target_path, 'r') as f:
+        loaded_strings = [line.strip() for line in f]
+    
+    if len(loaded_strings) == 0:
+        _LOGGER.error("The text file is empty.")
+        raise ValueError()
+    
+    if verbose:
+        _LOGGER.info(f"Loaded '{target_path.name}' as list of strings.")
+        
+    return loaded_strings
+

ml_tools/IO_tools/_IO_utils.py

@@ -0,0 +1,140 @@
+import json
+from collections import Counter
+from itertools import zip_longest
+from pathlib import Path
+from typing import Union
+
+from ..path_manager import make_fullpath
+from .._core import get_logger
+
+
+_LOGGER = get_logger("IO tools")
+
+
+__all__ = [
+    "compare_lists",
+    "_RobustEncoder"
+]
+
+
+class _RobustEncoder(json.JSONEncoder):
+    """
+    Custom JSON encoder to handle non-serializable objects.
+
+    This handles:
+    1.  `type` objects (e.g., <class 'int'>) which result from
+        `check_type_only=True`.
+    2.  Any other custom class or object by falling back to its
+        string representation.
+    """
+    def default(self, o):
+        if isinstance(o, type):
+            return str(o)
+        try:
+            return super().default(o)
+        except TypeError:
+            return str(o)
+
+
+def compare_lists(
+    list_A: list,
+    list_B: list,
+    save_dir: Union[str, Path],
+    strict: bool = False,
+    check_type_only: bool = False
+) -> dict:
+    """
+    Compares two lists and saves a JSON report of the differences.
+
+    Args:
+        list_A (list): The first list to compare.
+        list_B (list): The second list to compare.
+        save_dir (str | Path): The directory where the resulting report will be saved.
+        strict (bool):
+            - If False: Performs a "bag" comparison. Order does not matter, but duplicates do.
+            - If True: Performs a strict, positional comparison.
+            
+        check_type_only (bool):
+            - If False: Compares items using `==` (`__eq__` operator).
+            - If True: Compares only the `type()` of the items.
+
+    Returns:
+        dict: A dictionary detailing the differences. (saved to `save_dir`).
+    """
+    MISSING_A_KEY = "missing_in_A"
+    MISSING_B_KEY = "missing_in_B"
+    MISMATCH_KEY = "mismatch"
+    
+    results: dict[str, list] = {MISSING_A_KEY: [], MISSING_B_KEY: []}
+    
+    # make directory
+    save_path = make_fullpath(input_path=save_dir, make=True, enforce="directory")
+
+    if strict:
+        # --- STRICT (Positional) Mode ---
+        results[MISMATCH_KEY] = []
+        sentinel = object()
+
+        if check_type_only:
+            compare_func = lambda a, b: type(a) == type(b)
+        else:
+            compare_func = lambda a, b: a == b
+
+        for index, (item_a, item_b) in enumerate(
+            zip_longest(list_A, list_B, fillvalue=sentinel)
+        ):
+            if item_a is sentinel:
+                results[MISSING_A_KEY].append({"index": index, "item": item_b})
+            elif item_b is sentinel:
+                results[MISSING_B_KEY].append({"index": index, "item": item_a})
+            elif not compare_func(item_a, item_b):
+                results[MISMATCH_KEY].append(
+                    {
+                        "index": index,
+                        "list_A_item": item_a,
+                        "list_B_item": item_b,
+                    }
+                )
+
+    else:
+        # --- NON-STRICT (Bag) Mode ---
+        if check_type_only:
+            # Types are hashable, we can use Counter (O(N))
+            types_A_counts = Counter(type(item) for item in list_A)
+            types_B_counts = Counter(type(item) for item in list_B)
+
+            diff_A_B = types_A_counts - types_B_counts
+            for item_type, count in diff_A_B.items():
+                results[MISSING_B_KEY].extend([item_type] * count)
+
+            diff_B_A = types_B_counts - types_A_counts
+            for item_type, count in diff_B_A.items():
+                results[MISSING_A_KEY].extend([item_type] * count)
+
+        else:
+            # Items may be unhashable. Use O(N*M) .remove() method
+            temp_B = list(list_B)
+            missing_in_B = []
+
+            for item_a in list_A:
+                try:
+                    temp_B.remove(item_a)
+                except ValueError:
+                    missing_in_B.append(item_a)
+            
+            results[MISSING_A_KEY] = temp_B
+            results[MISSING_B_KEY] = missing_in_B
+
+    # --- Save the Report ---
+    try:
+        full_path = save_path / "list_comparison.json"
+
+        # Write the report dictionary to the JSON file
+        with open(full_path, 'w', encoding='utf-8') as f:
+            json.dump(results, f, indent=4, cls=_RobustEncoder)
+            
+    except Exception as e:
+        _LOGGER.error(f"Failed to save comparison report to {save_path}: \n{e}")
+
+    return results
+

ml_tools/{IO_tools.py → IO_tools/__init__.py}

@@ -1,14 +1,22 @@
-from ._core._IO_tools import (
+from ._IO_utils import (
+    compare_lists,
+)
+
+from ._IO_loggers import (
     custom_logger,
     train_logger,
-    save_list_strings,
-    load_list_strings,
+)
+
+from ._IO_save_load import (
     save_json,
     load_json,
-    compare_lists,
-    info
+    save_list_strings,
+    load_list_strings,
 )
 
+from ._imprimir import info
+
+
 __all__ = [
     "custom_logger",
     "train_logger",

ml_tools/IO_tools/_imprimir.py

@@ -0,0 +1,14 @@
+from .._core import _imprimir_disponibles
+
+_GRUPOS = [
+    "custom_logger",
+    "train_logger",
+    "save_json",
+    "load_json",
+    "save_list_strings",
+    "load_list_strings",
+    "compare_lists"
+]
+
+def info():
+    _imprimir_disponibles(_GRUPOS)

ml_tools/MICE/_MICE_imputation.py

@@ -0,0 +1,132 @@
+import pandas as pd
+import miceforest as mf
+from pathlib import Path
+from typing import Optional, Union
+
+from ..utilities import load_dataframe
+from ..math_utilities import threshold_binary_values
+
+from ..path_manager import make_fullpath, list_csv_paths
+from .._core import get_logger
+
+from ._dragon_mice import (
+    _save_imputed_datasets,
+    get_convergence_diagnostic,
+    get_imputed_distributions,
+)
+
+_LOGGER = get_logger("MICE")
+
+
+__all__ = [
+    "apply_mice",
+    "run_mice_pipeline",
+]
+
+
+def apply_mice(df: pd.DataFrame, df_name: str, binary_columns: Optional[list[str]]=None, resulting_datasets: int=1, iterations: int=20, random_state: int=101):
+    
+    # Initialize kernel with number of imputed datasets to generate
+    kernel = mf.ImputationKernel(
+        data=df,
+        num_datasets=resulting_datasets,
+        random_state=random_state
+    )
+    
+    _LOGGER.info("➡️ MICE imputation running...")
+    
+    # Perform MICE with n iterations per dataset
+    kernel.mice(iterations)
+    
+    # Retrieve the imputed datasets 
+    imputed_datasets = [kernel.complete_data(dataset=i) for i in range(resulting_datasets)]
+    
+    if imputed_datasets is None or len(imputed_datasets) == 0:
+        _LOGGER.error("No imputed datasets were generated. Check the MICE process.")
+        raise ValueError()
+    
+    # threshold binary columns
+    if binary_columns is not None:
+        invalid_binary_columns = set(binary_columns) - set(df.columns)
+        if invalid_binary_columns:
+            _LOGGER.warning(f"These 'binary columns' are not in the dataset:")
+            for invalid_binary_col in invalid_binary_columns:
+                print(f"  - {invalid_binary_col}")
+        valid_binary_columns = [col for col in binary_columns if col not in invalid_binary_columns]
+        for imputed_df in imputed_datasets:
+            for binary_column_name in valid_binary_columns:
+                imputed_df[binary_column_name] = threshold_binary_values(imputed_df[binary_column_name]) # type: ignore
+            
+    if resulting_datasets == 1:
+        imputed_dataset_names = [f"{df_name}_MICE"]
+    else:
+        imputed_dataset_names = [f"{df_name}_MICE_{i+1}" for i in range(resulting_datasets)]
+    
+    # Ensure indexes match
+    for imputed_df, subname in zip(imputed_datasets, imputed_dataset_names):
+        assert imputed_df.shape[0] == df.shape[0], f"❌ Row count mismatch in dataset {subname}" # type: ignore
+        assert all(imputed_df.index == df.index), f"❌ Index mismatch in dataset {subname}" # type: ignore
+    # print("✅ All imputed datasets match the original DataFrame indexes.")
+    
+    _LOGGER.info("MICE imputation complete.")
+    
+    return kernel, imputed_datasets, imputed_dataset_names
+
+
+#Get names of features that had missing values before imputation
+def _get_na_column_names(df: pd.DataFrame):
+    return [col for col in df.columns if df[col].isna().any()]
+
+
+def run_mice_pipeline(df_path_or_dir: Union[str,Path], target_columns: list[str], 
+                      save_datasets_dir: Union[str,Path], save_metrics_dir: Union[str,Path], 
+                      binary_columns: Optional[list[str]]=None,
+                      resulting_datasets: int=1, 
+                      iterations: int=20, 
+                      random_state: int=101):
+    """
+    DEPRECATED: Use DragonMICE class instead.
+    
+    Call functions in sequence for each dataset in the provided path or directory:
+        1. Load dataframe
+        2. Apply MICE
+        3. Save imputed dataset(s)
+        4. Save convergence metrics
+        5. Save distribution metrics
+        
+    Target columns must be skipped from the imputation. Binary columns will be thresholded after imputation.
+    """
+    # Check paths
+    save_datasets_path = make_fullpath(save_datasets_dir, make=True)
+    save_metrics_path = make_fullpath(save_metrics_dir, make=True)
+    
+    input_path = make_fullpath(df_path_or_dir)
+    if input_path.is_file():
+        all_file_paths = [input_path]
+    else:
+        all_file_paths = list(list_csv_paths(input_path, raise_on_empty=True).values())
+    
+    for df_path in all_file_paths:
+        df: pd.DataFrame
+        df, df_name = load_dataframe(df_path=df_path, kind="pandas") # type: ignore
+        
+        df, df_targets = _skip_targets(df, target_columns)
+        
+        kernel, imputed_datasets, imputed_dataset_names = apply_mice(df=df, df_name=df_name, binary_columns=binary_columns, resulting_datasets=resulting_datasets, iterations=iterations, random_state=random_state)
+        
+        _save_imputed_datasets(save_dir=save_datasets_path, imputed_datasets=imputed_datasets, df_targets=df_targets, imputed_dataset_names=imputed_dataset_names)
+        
+        imputed_column_names = _get_na_column_names(df=df)
+        
+        get_convergence_diagnostic(kernel=kernel, imputed_dataset_names=imputed_dataset_names, column_names=imputed_column_names, root_dir=save_metrics_path)
+        
+        get_imputed_distributions(kernel=kernel, df_name=df_name, root_dir=save_metrics_path, column_names=imputed_column_names)
+
+
+def _skip_targets(df: pd.DataFrame, target_cols: list[str]):
+    valid_targets = [col for col in target_cols if col in df.columns]
+    df_targets = df[valid_targets]
+    df_feats = df.drop(columns=valid_targets)
+    return df_feats, df_targets
+
+