congrads 0.2.0__py3-none-any.whl → 1.0.2__py3-none-any.whl

congrads/utils.py

@@ -1,9 +1,148 @@
-import pandas as pd
+"""
+This module provides utility functions and classes for managing data,
+logging, and preprocessing in machine learning workflows. The functionalities
+include logging key-value pairs to CSV files, splitting datasets into
+training, validation, and test sets, preprocessing functions for various 
+data sets and validator functions for type checking.
+"""
+
+import os
+
 import numpy as np
-from torch.utils.data import Dataset, random_split, DataLoader
+import pandas as pd
+import torch
+from torch import Generator
+from torch.utils.data import DataLoader, Dataset, random_split
+
+
+class CSVLogger:
+    """
+    A utility class for logging key-value pairs to a CSV file, organized by
+    epochs. Supports merging with existing logs or overwriting them.
+
+    Args:
+        file_path (str): The path to the CSV file for logging.
+        overwrite (bool): If True, overwrites any existing file at
+            the file_path.
+        merge (bool): If True, merges new values with existing data
+            in the file.
+
+    Raises:
+        ValueError: If both overwrite and merge are True.
+        FileExistsError: If the file already exists and neither
+            overwrite nor merge is True.
+    """
+
+    def __init__(
+        self, file_path: str, overwrite: bool = False, merge: bool = True
+    ):
+        """
+        Initializes the CSVLogger.
+        """
+
+        self.file_path = file_path
+        self.values: dict[tuple[int, str], float] = {}
+
+        if merge and overwrite:
+            raise ValueError(
+                "The attributes overwrite and merge cannot be True at the "
+                "same time. Either specify overwrite=True or merge=True."
+            )
+
+        if not os.path.exists(file_path):
+            pass
+        elif merge:
+            self.load()
+        elif overwrite:
+            pass
+        else:
+            raise FileExistsError(
+                f"A CSV file already exists at {file_path}. Specify "
+                "CSVLogger(..., overwrite=True) to overwrite the file."
+            )
+
+    def add_value(self, name: str, value: float, epoch: int):
+        """
+        Adds a value to the logger for a specific epoch and name.
+
+        Args:
+            name (str): The name of the metric or value to log.
+            value (float): The value to log.
+            epoch (int): The epoch associated with the value.
+        """
+
+        self.values[epoch, name] = value
+
+    def save(self):
+        """
+        Saves the logged values to the specified CSV file.
+
+        If the file exists and merge is enabled, merges the current data
+        with the existing file.
+        """
 
+        data = self.to_dataframe(self.values)
+        data.to_csv(self.file_path, index=False)
 
-def splitDataLoaders(
+    def load(self):
+        """
+        Loads data from the CSV file into the logger.
+
+        Converts the CSV data into the internal dictionary format for
+        further updates or operations.
+        """
+
+        df = pd.read_csv(self.file_path)
+        self.values = self.to_dict(df)
+
+    @staticmethod
+    def to_dataframe(values: dict[tuple[int, str], float]) -> pd.DataFrame:
+        """
+        Converts a dictionary of values into a DataFrame.
+
+        Args:
+            values (dict[tuple[int, str], float]): A dictionary of values
+                keyed by (epoch, name).
+
+        Returns:
+            pd.DataFrame: A DataFrame where epochs are rows, names are
+            columns, and values are the cell data.
+        """
+
+        # Convert to a DataFrame
+        df = pd.DataFrame.from_dict(values, orient="index", columns=["value"])
+
+        # Reset the index to separate epoch and name into columns
+        df.index = pd.MultiIndex.from_tuples(df.index, names=["epoch", "name"])
+        df = df.reset_index()
+
+        # Pivot the DataFrame so epochs are rows and names are columns
+        result = df.pivot(index="epoch", columns="name", values="value")
+
+        # Optional: Reset the column names for a cleaner look
+        result = result.reset_index().rename_axis(columns=None)
+
+        return result
+
+    @staticmethod
+    def to_dict(df: pd.DataFrame) -> dict[tuple[int, str], float]:
+        """
+        Converts a DataFrame with epochs as rows and names as columns
+        back into a dictionary of the format {(epoch, name): value}.
+        """
+        # Set the epoch column as the index (if not already)
+        df = df.set_index("epoch")
+
+        # Stack the DataFrame to create a multi-index series
+        stacked = df.stack()
+
+        # Convert the multi-index series to a dictionary
+        result = stacked.to_dict()
+
+        return result
+
+
+def split_data_loaders(
     data: Dataset,
     loader_args: dict = None,
     train_loader_args: dict = None,
@@ -12,7 +151,40 @@ def splitDataLoaders(
     train_size: float = 0.8,
     valid_size: float = 0.1,
     test_size: float = 0.1,
+    split_generator: Generator = None,
 ) -> tuple[DataLoader, DataLoader, DataLoader]:
+    """
+    Splits a dataset into training, validation, and test sets,
+    and returns corresponding DataLoader objects.
+
+    Args:
+        data (Dataset): The dataset to be split.
+        loader_args (dict, optional): Default DataLoader arguments, merges
+            with loader-specific arguments, overlapping keys from
+            loader-specific arguments are superseded.
+        train_loader_args (dict, optional): Training DataLoader arguments,
+            merges with `loader_args`, overriding overlapping keys.
+        valid_loader_args (dict, optional): Validation DataLoader arguments,
+            merges with `loader_args`, overriding overlapping keys.
+        test_loader_args (dict, optional): Test DataLoader arguments,
+            merges with `loader_args`, overriding overlapping keys.
+        train_size (float, optional): Proportion of data to be used for
+            training. Defaults to 0.8.
+        valid_size (float, optional): Proportion of data to be used for
+            validation. Defaults to 0.1.
+        test_size (float, optional): Proportion of data to be used for
+            testing. Defaults to 0.1.
+        split_generator (Generator, optional): Optional random seed generator
+            to control the splitting of the dataset.
+
+    Returns:
+        tuple: A tuple containing three DataLoader objects: one for the
+        training, validation and test set.
+
+    Raises:
+        ValueError: If the train_size, valid_size, and test_size are not
+            between 0 and 1, or if their sum does not equal 1.
+    """
 
     # Validate split sizes
     if not (0 < train_size < 1 and 0 < valid_size < 1 and 0 < test_size < 1):
@@ -23,19 +195,22 @@ def splitDataLoaders(
         raise ValueError("train_size, valid_size, and test_size must sum to 1.")
 
     # Perform the splits
-    train_val_data, test_data = random_split(data, [1 - test_size, test_size])
+    train_val_data, test_data = random_split(
+        data, [1 - test_size, test_size], generator=split_generator
+    )
     train_data, valid_data = random_split(
         train_val_data,
         [
             train_size / (1 - test_size),
             valid_size / (1 - test_size),
         ],
+        generator=split_generator,
     )
 
     # Set default arguments for each loader
-    train_loader_args = train_loader_args or loader_args or {}
-    valid_loader_args = valid_loader_args or loader_args or {}
-    test_loader_args = test_loader_args or loader_args or {}
+    train_loader_args = dict(loader_args or {}, **(train_loader_args or {}))
+    valid_loader_args = dict(loader_args or {}, **(valid_loader_args or {}))
+    test_loader_args = dict(loader_args or {}, **(test_loader_args or {}))
 
     # Create the DataLoaders
     train_generator = DataLoader(train_data, **train_loader_args)
@@ -45,18 +220,43 @@ def splitDataLoaders(
     return train_generator, valid_generator, test_generator
 
 
+# pylint: disable-next=invalid-name
 def preprocess_BiasCorrection(df: pd.DataFrame) -> pd.DataFrame:
+    """
+    Preprocesses the given dataframe for bias correction by
+    performing a series of transformations.
+
+    The function sequentially:
+
+    - Drops rows with missing values.
+    - Converts a date string to datetime format and adds year, month,
+      and day columns.
+    - Normalizes the columns with specific logic for input and output variables.
+    - Adds a multi-index indicating which columns are input or output variables.
+    - Samples 2500 examples from the dataset without replacement.
+
+    Args:
+        df (pd.DataFrame): The input dataframe containing the data
+            to be processed.
+
+    Returns:
+        pd.DataFrame: The processed dataframe after applying
+        the transformations.
+    """
 
     def date_to_datetime(df: pd.DataFrame) -> pd.DataFrame:
-        """Transform the string that denotes the date to the datetime format in pandas."""
+        """Transform the string that denotes the date to
+        the datetime format in pandas."""
         # make copy of dataframe
         df_temp = df.copy()
-        # add new column at the front where the date string is transformed to the datetime format
+        # add new column at the front where the date string is
+        # transformed to the datetime format
         df_temp.insert(0, "DateTransformed", pd.to_datetime(df_temp["Date"]))
         return df_temp
 
     def add_year(df: pd.DataFrame) -> pd.DataFrame:
-        """Extract the year from the datetime cell and add it as a new column to the dataframe at the front."""
+        """Extract the year from the datetime cell and add it
+        as a new column to the dataframe at the front."""
         # make copy of dataframe
         df_temp = df.copy()
         # extract year and add new column at the front containing these numbers
@@ -64,7 +264,8 @@ def preprocess_BiasCorrection(df: pd.DataFrame) -> pd.DataFrame:
         return df_temp
 
     def add_month(df: pd.DataFrame) -> pd.DataFrame:
-        """Extract the month from the datetime cell and add it as a new column to the dataframe at the front."""
+        """Extract the month from the datetime cell and add it
+        as a new column to the dataframe at the front."""
         # make copy of dataframe
         df_temp = df.copy()
         # extract month and add new column at index 1 containing these numbers
@@ -72,7 +273,8 @@ def preprocess_BiasCorrection(df: pd.DataFrame) -> pd.DataFrame:
         return df_temp
 
     def add_day(df: pd.DataFrame) -> pd.DataFrame:
-        """Extract the day from the datetime cell and add it as a new column to the dataframe at the front."""
+        """Extract the day from the datetime cell and add it
+        as a new column to the dataframe at the front."""
         # make copy of dataframe
         df_temp = df.copy()
         # extract day and add new column at index 2 containing these numbers
@@ -80,35 +282,46 @@ def preprocess_BiasCorrection(df: pd.DataFrame) -> pd.DataFrame:
         return df_temp
 
     def add_input_output_temperature(df: pd.DataFrame) -> pd.DataFrame:
-        """Add a multiindex denoting if the column is an input or output variable."""
+        """Add a multiindex denoting if the column
+        is an input or output variable."""
         # copy the dataframe
         temp_df = df.copy()
         # extract all the column names
         column_names = temp_df.columns.tolist()
-        # only the last 2 columns are output variables, all others are input variables. So make list of corresponding lengths of 'Input' and 'Output'
+        # only the last 2 columns are output variables, all others are input
+        # variables. So make list of corresponding lengths of
+        # 'Input' and 'Output'
         input_list = ["Input"] * (len(column_names) - 2)
         output_list = ["Output"] * 2
         # concat both lists
         input_output_list = input_list + output_list
-        # define multi index for attaching this 'Input' and 'Output' list with the column names already existing
-        multiindex_bias = pd.MultiIndex.from_arrays([input_output_list, column_names])
+        # define multi index for attaching this 'Input' and 'Output' list with
+        # the column names already existing
+        multiindex_bias = pd.MultiIndex.from_arrays(
+            [input_output_list, column_names]
+        )
         # transpose such that index can be adjusted to multi index
         new_df = pd.DataFrame(df.transpose().to_numpy(), index=multiindex_bias)
-        # transpose back such that columns are the same as before except with different labels
+        # transpose back such that columns are the same as before
+        # except with different labels
         return new_df.transpose()
 
     def normalize_columns_bias(df: pd.DataFrame) -> pd.DataFrame:
-        """Normalize the columns for the bias correction dataset. This is different from normalizing all the columns separately because the
-        upper and lower bounds for the output variables are assumed to be the same."""
+        """Normalize the columns for the bias correction dataset.
+        This is different from normalizing all the columns separately
+        because the upper and lower bounds for the output variables
+        are assumed to be the same."""
         # copy the dataframe
         temp_df = df.copy()
         # normalize each column
         for feature_name in df.columns:
-            # the output columns are normalized using the same upper and lower bound for more efficient check of the inequality
+            # the output columns are normalized using the same upper and
+            # lower bound for more efficient check of the inequality
             if feature_name == "Next_Tmax" or feature_name == "Next_Tmin":
                 max_value = 38.9
                 min_value = 11.3
-            # the input columns are normalized using their respective upper and lower bounds
+            # the input columns are normalized using their respective
+            # upper and lower bounds
             else:
                 max_value = df[feature_name].max()
                 min_value = df[feature_name].min()
@@ -120,7 +333,9 @@ def preprocess_BiasCorrection(df: pd.DataFrame) -> pd.DataFrame:
     def sample_2500_examples(df: pd.DataFrame) -> pd.DataFrame:
         """Sample 2500 examples from the dataframe without replacement."""
         temp_df = df.copy()
-        sample_df = temp_df.sample(n=2500, replace=False, random_state=3, axis=0)
+        sample_df = temp_df.sample(
+            n=2500, replace=False, random_state=3, axis=0
+        )
         return sample_df
 
     return (
@@ -140,18 +355,47 @@ def preprocess_BiasCorrection(df: pd.DataFrame) -> pd.DataFrame:
         .astype("float32")
         # normalize columns
         .pipe(normalize_columns_bias)
-        # add multi index indicating which columns are corresponding to input and output variables
+        # add multi index indicating which columns are corresponding
+        # to input and output variables
         .pipe(add_input_output_temperature)
         # sample 2500 examples out of the dataset
         .pipe(sample_2500_examples)
     )
 
 
-def preprocess_FiniteIncome(df: pd.DataFrame) -> pd.DataFrame:
+# pylint: disable-next=invalid-name
+def preprocess_FamilyIncome(df: pd.DataFrame) -> pd.DataFrame:
+    """
+    Preprocesses the given Family Income dataframe by applying a
+    series of transformations and constraints.
+
+    The function sequentially:
+
+    - Drops rows with missing values.
+    - Converts object columns to appropriate data types and
+      removes string columns.
+    - Removes certain unnecessary columns like
+      'Agricultural Household indicator' and related features.
+    - Adds labels to columns indicating whether they are
+      input or output variables.
+    - Normalizes the columns individually.
+    - Checks and removes rows that do not satisfy predefined constraints
+      (household income > expenses, food expenses > sub-expenses).
+    - Samples 2500 examples from the dataset without replacement.
+
+    Args:
+        df (pd.DataFrame): The input Family Income dataframe containing
+            the data to be processed.
+
+    Returns:
+        pd.DataFrame: The processed dataframe after applying the
+        transformations and constraints.
+    """
 
     def normalize_columns_income(df: pd.DataFrame) -> pd.DataFrame:
-        """Normalize the columns for the Family Income dataframe. This can also be applied to other dataframes because this function normalizes
-        all columns individually."""
+        """Normalize the columns for the Family Income dataframe.
+        This can also be applied to other dataframes because this
+        function normalizes all columns individually."""
         # copy the dataframe
         temp_df = df.copy()
         # normalize each column
@@ -164,9 +408,12 @@ def preprocess_FiniteIncome(df: pd.DataFrame) -> pd.DataFrame:
         return temp_df
 
     def check_constraints_income(df: pd.DataFrame) -> pd.DataFrame:
-        """Check if all the constraints are satisfied for the dataframe and remove the examples that do not satisfy the constraint. This
-        function only works for the Family Income dataset and the constraints are that the household income is larger than all the expenses
-        and the food expense is larger than the sum of the other (more detailed) food expenses.
+        """Check if all the constraints are satisfied for the dataframe
+        and remove the examples that do not satisfy the constraint.
+        This function only works for the Family Income dataset and the
+        constraints are that the household income is larger than all the
+        expenses and the food expense is larger than the sum of
+        the other (more detailed) food expenses.
         """
         temp_df = df.copy()
         # check that household income is larger than expenses in the output
@@ -174,7 +421,9 @@ def preprocess_FiniteIncome(df: pd.DataFrame) -> pd.DataFrame:
         income_array = np.add(
             np.multiply(
                 input_array[:, [0, 1]],
-                np.subtract(np.asarray([11815988, 9234485]), np.asarray([11285, 0])),
+                np.subtract(
+                    np.asarray([11815988, 9234485]), np.asarray([11285, 0])
+                ),
             ),
             np.asarray([11285, 0]),
         )
@@ -244,28 +493,38 @@ def preprocess_FiniteIncome(df: pd.DataFrame) -> pd.DataFrame:
         return satisfied_constraints_df
 
     def add_input_output_family_income(df: pd.DataFrame) -> pd.DataFrame:
-        """Add a multiindex denoting if the column is an input or output variable."""
+        """Add a multiindex denoting if the column is
+        an input or output variable."""
+
         # copy the dataframe
         temp_df = df.copy()
         # extract all the column names
         column_names = temp_df.columns.tolist()
-        # the 2nd-9th columns correspond to output variables and all others to input variables. So make list of corresponding lengths of 'Input' and 'Output'
+        # the 2nd-9th columns correspond to output variables and all
+        # others to input variables. So make list of corresponding
+        # lengths of 'Input' and 'Output'
         input_list_start = ["Input"]
         input_list_end = ["Input"] * (len(column_names) - 9)
         output_list = ["Output"] * 8
         # concat both lists
         input_output_list = input_list_start + output_list + input_list_end
-        # define multi index for attaching this 'Input' and 'Output' list with the column names already existing
-        multiindex_bias = pd.MultiIndex.from_arrays([input_output_list, column_names])
+        # define multi index for attaching this 'Input' and
+        # 'Output' list with the column names already existing
+        multiindex_bias = pd.MultiIndex.from_arrays(
+            [input_output_list, column_names]
+        )
         # transpose such that index can be adjusted to multi index
         new_df = pd.DataFrame(df.transpose().to_numpy(), index=multiindex_bias)
-        # transpose back such that columns are the same as before except with different labels
+        # transpose back such that columns are the same as
+        # before except with different labels
         return new_df.transpose()
 
     def sample_2500_examples(df: pd.DataFrame) -> pd.DataFrame:
         """Sample 2500 examples from the dataframe without replacement."""
         temp_df = df.copy()
-        sample_df = temp_df.sample(n=2500, replace=False, random_state=3, axis=0)
+        sample_df = temp_df.sample(
+            n=2500, replace=False, random_state=3, axis=0
+        )
         return sample_df
 
     return (
@@ -273,13 +532,17 @@ def preprocess_FiniteIncome(df: pd.DataFrame) -> pd.DataFrame:
         df.dropna(how="any")
         # convert object to fitting dtype
         .convert_dtypes()
-        # remove all strings (no other dtypes are present except for integers and floats)
+        # remove all strings (no other dtypes are present
+        # except for integers and floats)
         .select_dtypes(exclude=["string"])
         # transform all numbers to same dtype
         .astype("float32")
-        # drop column with label Agricultural Household indicator because this is not really a numerical input but rather a categorical/classification
+        # drop column with label Agricultural Household indicator
+        # because this is not really a numerical input but
+        # rather a categorical/classification
         .drop(["Agricultural Household indicator"], axis=1, inplace=False)
-        # this column is dropped because it depends on Agricultural Household indicator
+        # this column is dropped because it depends on
+        # Agricultural Household indicator
         .drop(["Crop Farming and Gardening expenses"], axis=1, inplace=False)
         # use 8 output variables and 24 input variables
         .drop(
@@ -308,3 +571,140 @@ def preprocess_FiniteIncome(df: pd.DataFrame) -> pd.DataFrame:
         # sample 2500 examples
         .pipe(sample_2500_examples)
     )
+
+
+def validate_type(name, value, expected_types, allow_none=False):
+    """
+    Validate that a value is of the specified type(s).
+
+    Args:
+        name (str): Name of the argument for error messages.
+        value: Value to validate.
+        expected_types (type or tuple of types): Expected type(s) for the value.
+        allow_none (bool): Whether to allow the value to be None.
+            Defaults to False.
+
+    Raises:
+        TypeError: If the value is not of the expected type(s).
+    """
+
+    if value is None:
+        if not allow_none:
+            raise TypeError(f"Argument {name} cannot be None.")
+        return
+
+    if not isinstance(value, expected_types):
+        raise TypeError(
+            f"Argument {name} '{str(value)}' is not supported. "
+            f"Only values of type {str(expected_types)} are allowed."
+        )
+
+
+def validate_iterable(
+    name,
+    value,
+    expected_element_types,
+    allowed_iterables=(list, set),
+    allow_none=False,
+):
+    """
+    Validate that a value is an iterable (e.g., list, set) with elements of
+    the specified type(s).
+
+    Args:
+        name (str): Name of the argument for error messages.
+        value: Value to validate.
+        expected_element_types (type or tuple of types): Expected type(s)
+            for the elements.
+        allowed_iterables (tuple of types): Iterable types that are
+            allowed (default: list and set).
+        allow_none (bool): Whether to allow the value to be None.
+            Defaults to False.
+
+    Raises:
+        TypeError: If the value is not an allowed iterable type or if
+            any element is not of the expected type(s).
+    """
+
+    if value is None:
+        if not allow_none:
+            raise TypeError(f"Argument {name} cannot be None.")
+        return
+
+    if not isinstance(value, allowed_iterables):
+        raise TypeError(
+            f"Argument {name} '{str(value)}' is not supported. "
+            f"Only values of type {str(allowed_iterables)} are allowed."
+        )
+    if not all(
+        isinstance(element, expected_element_types) for element in value
+    ):
+        raise TypeError(
+            f"Invalid elements in {name} '{str(value)}'. "
+            f"Only elements of type {str(expected_element_types)} are allowed."
+        )
+
+
+def validate_comparator_pytorch(name, value):
+    """
+    Validate that a value is a callable PyTorch comparator function.
+
+    Args:
+        name (str): Name of the argument for error messages.
+        value: Value to validate.
+
+    Raises:
+        TypeError: If the value is not callable or not a PyTorch comparator.
+    """
+
+    # List of valid PyTorch comparator functions
+    pytorch_comparators = {torch.gt, torch.lt, torch.ge, torch.le}
+
+    # Check if value is callable and if it's one of
+    # the PyTorch comparator functions
+    if not callable(value):
+        raise TypeError(
+            f"Argument {name} '{str(value)}' is not supported. "
+            "Only callable functions are allowed."
+        )
+
+    if value not in pytorch_comparators:
+        raise TypeError(
+            f"Argument {name} '{str(value)}' is not a valid PyTorch comparator "
+            "function. Only PyTorch functions like torch.gt, torch.lt, "
+            "torch.ge, torch.le are allowed."
+        )
+
+
+def validate_callable(name, value, allow_none=False):
+    """
+    Validate that a value is callable function.
+
+    Args:
+        name (str): Name of the argument for error messages.
+        value: Value to validate.
+        allow_none (bool): Whether to allow the value to be None.
+            Defaults to False.
+
+    Raises:
+        TypeError: If the value is not callable.
+    """
+
+    if value is None:
+        if not allow_none:
+            raise TypeError(f"Argument {name} cannot be None.")
+        return
+
+    if not callable(value):
+        raise TypeError(
+            f"Argument {name} '{str(value)}' is not supported. "
+            "Only callable functions are allowed."
+        )
+
+
+def validate_loaders() -> None:
+    """
+    TODO: implement function
+    """
+    # TODO complete
+    pass