congrads 0.2.0__py3-none-any.whl → 0.3.0__py3-none-any.whl

congrads/{utils.py → utils/preprocessors.py}

@@ -1,57 +1,45 @@
-import pandas as pd
+"""Preprocessing functions for various datasets.
+
+This module provides preprocessing pipelines for multiple datasets:
+- BiasCorrection: Temperature bias correction dataset
+- FamilyIncome: Family income and expenses dataset
+- AdultCensusIncome: Adult Census Income dataset
+
+Each preprocessing function applies appropriate transformations including
+normalization, feature engineering, constraint filtering, and sampling.
+"""
+
 import numpy as np
-from torch.utils.data import Dataset, random_split, DataLoader
-
-
-def splitDataLoaders(
-    data: Dataset,
-    loader_args: dict = None,
-    train_loader_args: dict = None,
-    valid_loader_args: dict = None,
-    test_loader_args: dict = None,
-    train_size: float = 0.8,
-    valid_size: float = 0.1,
-    test_size: float = 0.1,
-) -> tuple[DataLoader, DataLoader, DataLoader]:
-
-    # Validate split sizes
-    if not (0 < train_size < 1 and 0 < valid_size < 1 and 0 < test_size < 1):
-        raise ValueError(
-            "train_size, valid_size, and test_size must be between 0 and 1."
-        )
-    if not abs(train_size + valid_size + test_size - 1.0) < 1e-6:
-        raise ValueError("train_size, valid_size, and test_size must sum to 1.")
-
-    # Perform the splits
-    train_val_data, test_data = random_split(data, [1 - test_size, test_size])
-    train_data, valid_data = random_split(
-        train_val_data,
-        [
-            train_size / (1 - test_size),
-            valid_size / (1 - test_size),
-        ],
-    )
+import pandas as pd
+
 
-    # Set default arguments for each loader
-    train_loader_args = train_loader_args or loader_args or {}
-    valid_loader_args = valid_loader_args or loader_args or {}
-    test_loader_args = test_loader_args or loader_args or {}
+def preprocess_BiasCorrection(df: pd.DataFrame) -> pd.DataFrame:  # noqa: N802
+    """Preprocesses the given dataframe for bias correction by performing a series of transformations.
 
-    # Create the DataLoaders
-    train_generator = DataLoader(train_data, **train_loader_args)
-    valid_generator = DataLoader(valid_data, **valid_loader_args)
-    test_generator = DataLoader(test_data, **test_loader_args)
+    The function sequentially:
 
-    return train_generator, valid_generator, test_generator
+    - Drops rows with missing values.
+    - Converts a date string to datetime format and adds year, month,
+      and day columns.
+    - Normalizes the columns with specific logic for input and output variables.
+    - Adds a multi-index indicating which columns are input or output variables.
+    - Samples 2500 examples from the dataset without replacement.
 
+    Args:
+        df (pd.DataFrame): The input dataframe containing the data
+            to be processed.
 
-def preprocess_BiasCorrection(df: pd.DataFrame) -> pd.DataFrame:
+    Returns:
+        pd.DataFrame: The processed dataframe after applying
+        the transformations.
+    """
 
     def date_to_datetime(df: pd.DataFrame) -> pd.DataFrame:
         """Transform the string that denotes the date to the datetime format in pandas."""
         # make copy of dataframe
         df_temp = df.copy()
-        # add new column at the front where the date string is transformed to the datetime format
+        # add new column at the front where the date string is
+        # transformed to the datetime format
         df_temp.insert(0, "DateTransformed", pd.to_datetime(df_temp["Date"]))
         return df_temp
 
@@ -85,36 +73,44 @@ def preprocess_BiasCorrection(df: pd.DataFrame) -> pd.DataFrame:
         temp_df = df.copy()
         # extract all the column names
         column_names = temp_df.columns.tolist()
-        # only the last 2 columns are output variables, all others are input variables. So make list of corresponding lengths of 'Input' and 'Output'
+        # only the last 2 columns are output variables, all others are input
+        # variables. So make list of corresponding lengths of
+        # 'Input' and 'Output'
         input_list = ["Input"] * (len(column_names) - 2)
         output_list = ["Output"] * 2
         # concat both lists
         input_output_list = input_list + output_list
-        # define multi index for attaching this 'Input' and 'Output' list with the column names already existing
+        # define multi index for attaching this 'Input' and 'Output' list with
+        # the column names already existing
         multiindex_bias = pd.MultiIndex.from_arrays([input_output_list, column_names])
         # transpose such that index can be adjusted to multi index
         new_df = pd.DataFrame(df.transpose().to_numpy(), index=multiindex_bias)
-        # transpose back such that columns are the same as before except with different labels
+        # transpose back such that columns are the same as before
+        # except with different labels
         return new_df.transpose()
 
     def normalize_columns_bias(df: pd.DataFrame) -> pd.DataFrame:
-        """Normalize the columns for the bias correction dataset. This is different from normalizing all the columns separately because the
-        upper and lower bounds for the output variables are assumed to be the same."""
+        """Normalize the columns for the bias correction dataset.
+
+        This is different from normalizing all the columns separately
+        because the upper and lower bounds for the output variables
+        are assumed to be the same.
+        """
         # copy the dataframe
         temp_df = df.copy()
         # normalize each column
         for feature_name in df.columns:
-            # the output columns are normalized using the same upper and lower bound for more efficient check of the inequality
+            # the output columns are normalized using the same upper and
+            # lower bound for more efficient check of the inequality
             if feature_name == "Next_Tmax" or feature_name == "Next_Tmin":
                 max_value = 38.9
                 min_value = 11.3
-            # the input columns are normalized using their respective upper and lower bounds
+            # the input columns are normalized using their respective
+            # upper and lower bounds
             else:
                 max_value = df[feature_name].max()
                 min_value = df[feature_name].min()
-            temp_df[feature_name] = (df[feature_name] - min_value) / (
-                max_value - min_value
-            )
+            temp_df[feature_name] = (df[feature_name] - min_value) / (max_value - min_value)
         return temp_df
 
     def sample_2500_examples(df: pd.DataFrame) -> pd.DataFrame:
@@ -140,33 +136,78 @@ def preprocess_BiasCorrection(df: pd.DataFrame) -> pd.DataFrame:
         .astype("float32")
         # normalize columns
         .pipe(normalize_columns_bias)
-        # add multi index indicating which columns are corresponding to input and output variables
+        # add multi index indicating which columns are corresponding
+        # to input and output variables
         .pipe(add_input_output_temperature)
         # sample 2500 examples out of the dataset
         .pipe(sample_2500_examples)
     )
 
 
-def preprocess_FiniteIncome(df: pd.DataFrame) -> pd.DataFrame:
+def preprocess_FamilyIncome(df: pd.DataFrame) -> pd.DataFrame:  # noqa: N802
+    """Preprocesses the given Family Income dataframe.
+
+    The function sequentially:
+
+    - Drops rows with missing values.
+    - Converts object columns to appropriate data types and
+      removes string columns.
+    - Removes certain unnecessary columns like
+      'Agricultural Household indicator' and related features.
+    - Adds labels to columns indicating whether they are
+      input or output variables.
+    - Normalizes the columns individually.
+    - Checks and removes rows that do not satisfy predefined constraints
+      (household income > expenses, food expenses > sub-expenses).
+    - Samples 2500 examples from the dataset without replacement.
+
+    Args:
+        df (pd.DataFrame): The input Family Income dataframe containing
+            the data to be processed.
+
+    Returns:
+        pd.DataFrame: The processed dataframe after applying the
+        transformations and constraints.
+    """
 
     def normalize_columns_income(df: pd.DataFrame) -> pd.DataFrame:
-        """Normalize the columns for the Family Income dataframe. This can also be applied to other dataframes because this function normalizes
-        all columns individually."""
+        """Normalize each column of the dataframe independently.
+
+        This function scales each column to have values between 0 and 1
+        (or another standard normalization, depending on implementation),
+        making it suitable for numerical processing. While designed for
+        the Family Income dataset, it can be applied to any dataframe
+        with numeric columns.
+
+        Args:
+            df (pd.DataFrame): Input dataframe to normalize.
+
+        Returns:
+            pd.DataFrame: Dataframe with each column normalized independently.
+        """
         # copy the dataframe
         temp_df = df.copy()
         # normalize each column
         for feature_name in df.columns:
             max_value = df[feature_name].max()
             min_value = df[feature_name].min()
-            temp_df[feature_name] = (df[feature_name] - min_value) / (
-                max_value - min_value
-            )
+            temp_df[feature_name] = (df[feature_name] - min_value) / (max_value - min_value)
         return temp_df
 
     def check_constraints_income(df: pd.DataFrame) -> pd.DataFrame:
-        """Check if all the constraints are satisfied for the dataframe and remove the examples that do not satisfy the constraint. This
-        function only works for the Family Income dataset and the constraints are that the household income is larger than all the expenses
-        and the food expense is larger than the sum of the other (more detailed) food expenses.
+        """Filter rows that violate income-related constraints.
+
+        This function is specific to the Family Income dataset. It removes rows
+        that do not satisfy the following constraints:
+            1. Household income must be greater than all expenses.
+            2. Food expense must be greater than the sum of detailed food expenses.
+
+        Args:
+            df (pd.DataFrame): Input dataframe containing income and expense data.
+
+        Returns:
+            pd.DataFrame: Filtered dataframe containing only rows that satisfy
+            all constraints.
         """
         temp_df = df.copy()
         # check that household income is larger than expenses in the output
@@ -234,9 +275,7 @@ def preprocess_FiniteIncome(df: pd.DataFrame) -> pd.DataFrame:
         food_mul_expense_array = expense_reduced_array[:, [1, 2, 3]]
         food_mul_expense_array_sum = np.sum(food_mul_expense_array, axis=1)
         food_expense_array = expense_reduced_array[:, 0]
-        sanity_check_array = np.greater_equal(
-            food_expense_array, food_mul_expense_array_sum
-        )
+        sanity_check_array = np.greater_equal(food_expense_array, food_mul_expense_array_sum)
         drop_reduction["Unimportant"] = sanity_check_array.tolist()
         new_reduction = drop_reduction[drop_reduction.Unimportant]
         satisfied_constraints_df = new_reduction.drop("Unimportant", axis=1)
@@ -249,17 +288,21 @@ def preprocess_FiniteIncome(df: pd.DataFrame) -> pd.DataFrame:
         temp_df = df.copy()
         # extract all the column names
         column_names = temp_df.columns.tolist()
-        # the 2nd-9th columns correspond to output variables and all others to input variables. So make list of corresponding lengths of 'Input' and 'Output'
+        # the 2nd-9th columns correspond to output variables and all
+        # others to input variables. So make list of corresponding
+        # lengths of 'Input' and 'Output'
         input_list_start = ["Input"]
         input_list_end = ["Input"] * (len(column_names) - 9)
         output_list = ["Output"] * 8
         # concat both lists
         input_output_list = input_list_start + output_list + input_list_end
-        # define multi index for attaching this 'Input' and 'Output' list with the column names already existing
+        # define multi index for attaching this 'Input' and
+        # 'Output' list with the column names already existing
         multiindex_bias = pd.MultiIndex.from_arrays([input_output_list, column_names])
         # transpose such that index can be adjusted to multi index
         new_df = pd.DataFrame(df.transpose().to_numpy(), index=multiindex_bias)
-        # transpose back such that columns are the same as before except with different labels
+        # transpose back such that columns are the same as
+        # before except with different labels
         return new_df.transpose()
 
     def sample_2500_examples(df: pd.DataFrame) -> pd.DataFrame:
@@ -273,13 +316,17 @@ def preprocess_FiniteIncome(df: pd.DataFrame) -> pd.DataFrame:
         df.dropna(how="any")
         # convert object to fitting dtype
         .convert_dtypes()
-        # remove all strings (no other dtypes are present except for integers and floats)
+        # remove all strings (no other dtypes are present
+        # except for integers and floats)
         .select_dtypes(exclude=["string"])
         # transform all numbers to same dtype
         .astype("float32")
-        # drop column with label Agricultural Household indicator because this is not really a numerical input but rather a categorical/classification
+        # drop column with label Agricultural Household indicator
+        # because this is not really a numerical input but
+        # rather a categorical/classification
         .drop(["Agricultural Household indicator"], axis=1, inplace=False)
-        # this column is dropped because it depends on Agricultural Household indicator
+        # this column is dropped because it depends on
+        # Agricultural Household indicator
         .drop(["Crop Farming and Gardening expenses"], axis=1, inplace=False)
         # use 8 output variables and 24 input variables
         .drop(
@@ -308,3 +355,85 @@ def preprocess_FiniteIncome(df: pd.DataFrame) -> pd.DataFrame:
         # sample 2500 examples
         .pipe(sample_2500_examples)
     )
+
+
+def preprocess_AdultCensusIncome(df: pd.DataFrame) -> pd.DataFrame:  # noqa: N802
+    """Preprocesses the Adult Census Income dataset for PyTorch ML.
+
+    Sequential steps:
+    - Drop rows with missing values.
+    - Encode categorical variables to integer labels.
+    - Map the target 'income' column to 0/1.
+    - Convert all data to float32.
+    - Add a multiindex to denote Input vs Output columns.
+
+    Args:
+        df (pd.DataFrame): Raw dataframe containing Adult Census Income data.
+
+    Returns:
+        pd.DataFrame: Preprocessed dataframe.
+    """
+
+    def drop_missing(df: pd.DataFrame) -> pd.DataFrame:
+        """Drop rows with any missing values."""
+        return df.dropna(how="any")
+
+    def drop_columns(df: pd.DataFrame) -> pd.DataFrame:
+        return df.drop(columns=["fnlwgt", "education.num"], errors="ignore")
+
+    def label_encode_column(series: pd.Series, col_name: str = None) -> pd.Series:
+        """Encode a pandas Series of categorical strings into integers."""
+        categories = series.dropna().unique().tolist()
+        cat_to_int = {cat: i for i, cat in enumerate(categories)}
+        if col_name:
+            print(f"Column '{col_name}' encoding:")
+            for cat, idx in cat_to_int.items():
+                print(f"  {cat} -> {idx}")
+        return series.map(cat_to_int).astype(int)
+
+    def encode_categorical(df: pd.DataFrame) -> pd.DataFrame:
+        """Convert categorical string columns to integer labels using label_encode_column."""
+        df_temp = df.copy()
+        categorical_cols = [
+            "workclass",
+            "education",
+            "marital.status",
+            "occupation",
+            "relationship",
+            "race",
+            "sex",
+            "native.country",
+        ]
+        for col in categorical_cols:
+            df_temp[col] = label_encode_column(df_temp[col].astype(str), col_name=col)
+        return df_temp
+
+    def map_target(df: pd.DataFrame) -> pd.DataFrame:
+        """Map income column to 0 (<=50K) and 1 (>50K)."""
+        df_temp = df.copy()
+        df_temp["income"] = df_temp["income"].map({"<=50K": 0, ">50K": 1})
+        return df_temp
+
+    def convert_float32(df: pd.DataFrame) -> pd.DataFrame:
+        """Convert all data to float32 for PyTorch compatibility."""
+        return df.astype("float32")
+
+    def add_input_output_index(df: pd.DataFrame) -> pd.DataFrame:
+        """Add a multiindex indicating input and output columns."""
+        temp_df = df.copy()
+        column_names = temp_df.columns.tolist()
+        # Only the 'income' column is output
+        input_list = ["Input"] * (len(column_names) - 1)
+        output_list = ["Output"]
+        multiindex_list = input_list + output_list
+        multiindex = pd.MultiIndex.from_arrays([multiindex_list, column_names])
+        return pd.DataFrame(temp_df.to_numpy(), columns=multiindex)
+
+    return (
+        df.pipe(drop_missing)
+        .pipe(drop_columns)
+        .pipe(encode_categorical)
+        .pipe(map_target)
+        .pipe(convert_float32)
+        .pipe(add_input_output_index)
+    )