data-auditor 0.1.0__tar.gz

data_auditor-0.1.0/PKG-INFO

@@ -0,0 +1,15 @@
+Metadata-Version: 2.4
+Name: data-auditor
+Version: 0.1.0
+Summary: A modular profiling and preprocessing library for tabular ML datasets
+Author-email: miriamspsantos <miriam.santos@fc.up.pt>
+License: MIT
+Project-URL: Homepage, https://github.com/miriamspsantos/data-auditor
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: pandas
+Requires-Dist: scikit-learn
+Requires-Dist: matplotlib
+Requires-Dist: seaborn
+
+data-auditor: 0.1.0

data_auditor-0.1.0/README.md

@@ -0,0 +1 @@
+data-auditor: 0.1.0

data_auditor-0.1.0/pyproject.toml

@@ -0,0 +1,23 @@
+[build-system]
+requires = ["setuptools>=61.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "data-auditor"
+version = "0.1.0"
+description = "A modular profiling and preprocessing library for tabular ML datasets"
+authors = [
+    { name="miriamspsantos", email="miriam.santos@fc.up.pt" }
+]
+license = {text = "MIT"}
+readme = "README.md"
+requires-python = ">=3.9"
+dependencies = [
+    "pandas",
+    "scikit-learn",
+    "matplotlib",
+    "seaborn"
+]
+
+[project.urls]
+"Homepage" = "https://github.com/miriamspsantos/data-auditor"

data_auditor-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

data_auditor-0.1.0/src/data_auditor.egg-info/PKG-INFO

@@ -0,0 +1,15 @@
+Metadata-Version: 2.4
+Name: data-auditor
+Version: 0.1.0
+Summary: A modular profiling and preprocessing library for tabular ML datasets
+Author-email: miriamspsantos <miriam.santos@fc.up.pt>
+License: MIT
+Project-URL: Homepage, https://github.com/miriamspsantos/data-auditor
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: pandas
+Requires-Dist: scikit-learn
+Requires-Dist: matplotlib
+Requires-Dist: seaborn
+
+data-auditor: 0.1.0

data_auditor-0.1.0/src/data_auditor.egg-info/SOURCES.txt

@@ -0,0 +1,14 @@
+README.md
+pyproject.toml
+src/fairness.py
+src/preprocessing.py
+src/profiler.py
+src/utils.py
+src/data_auditor.egg-info/PKG-INFO
+src/data_auditor.egg-info/SOURCES.txt
+src/data_auditor.egg-info/dependency_links.txt
+src/data_auditor.egg-info/requires.txt
+src/data_auditor.egg-info/top_level.txt
+src/encoders/custom_transformer.py
+src/encoders/onehot.py
+src/encoders/ordinal.py

data_auditor-0.1.0/src/data_auditor.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

data_auditor-0.1.0/src/data_auditor.egg-info/requires.txt

@@ -0,0 +1,4 @@
+pandas
+scikit-learn
+matplotlib
+seaborn

data_auditor-0.1.0/src/data_auditor.egg-info/top_level.txt

@@ -0,0 +1,5 @@
+encoders
+fairness
+preprocessing
+profiler
+utils

data_auditor-0.1.0/src/encoders/custom_transformer.py

@@ -0,0 +1,42 @@
+from numpy.random import randint
+from sklearn.base import BaseEstimator, TransformerMixin
+
+#from encoders.ordinal import ProfilerOrdinalEncoder
+
+
+class CustomTransformer(BaseEstimator, TransformerMixin):
+    """Create custom transformer for learning purposes.
+       https://www.andrewvillazon.com/custom-scikit-learn-transformers/
+    
+    """
+
+    def fit(self, X, y=None):
+        return self
+    
+    def transform(self, X, y=None):
+        # perform arbitrary transformation
+        X["random_int"] = randint(0, 10, X.shape[0])
+        return X
+    
+
+class MultiplyColumns(BaseEstimator, TransformerMixin):
+    def __init__(self, by=1, columns=None):
+        self.by = by
+        self.columns = columns
+
+    def fit(self, X, y=None):
+        return self
+    
+    def transform(self, X, y=None):
+        cols_to_transform = list(X.columns)
+
+        if self.columns:
+            cols_to_transform = self.columns
+
+        X[cols_to_transform] = X[cols_to_transform] * self.by
+
+        return X
+
+    
+
+    

data_auditor-0.1.0/src/encoders/onehot.py

@@ -0,0 +1,144 @@
+import pandas as pd
+import numpy as np
+from profiler import DataProfiler
+from sklearn.base import BaseEstimator, TransformerMixin
+import copy
+
+
+class ProfilerOneHotEncoder(BaseEstimator, TransformerMixin):
+    def __init__(self, profiler:DataProfiler=None, columns=None, drop=None, handle_unknown="use_nan"):
+        """Perform One-Hot-Encoding
+        
+        Parameters
+        ----------
+        profiler : DataProfiler, optional
+            An instance of DataProfiler, for saving mappings.
+            Optional so that it works standalone too.
+        columns : list of str, optional
+            Columns to apply the encoding to. If None, encodes all columns.
+            Use columns for flexible control (standalone pipelines).
+        drop : {"if_binary", "last"}
+            How to drop one of the categories per feature. 
+            "if_binary" removes 1 category from binary columns.
+            "last" keeps only n-1 columns for nominal data.
+        handle_unknown : {"use_nan", "error"}
+            How to handle unknown categories during transform.
+        """
+        self.profiler = profiler
+        self.columns = columns
+        self.handle_unknown = handle_unknown
+        self.drop = drop
+
+        self.mappings = {} # {col: [cat1, cat2, cat3]}, e.g., {'Gender': ['F', 'M']}
+        self.inverse_mappings = {}
+
+    def get_encoder_type(self):
+        return "OHE"
+    
+    def fit(self, X, y=None):
+        """Scan columns and build mappings."""
+        X = pd.DataFrame(X)
+
+        cols_to_map = self.columns if self.columns else list(X.columns)
+    
+        for col in cols_to_map:
+            categories = pd.Series(X[col].dropna().unique()).sort_values().tolist() # deterministic encoding
+            self.mappings[col] = categories     # save to rebuild at transform time
+
+            # pre-save inverse mappings
+            for category in categories:
+                onehot_col = f"{col}_{category}"
+                self.inverse_mappings[onehot_col] = (col, category)
+        
+        # Update profiler
+        if self.profiler is not None:
+            self.profiler._set_feature_mappings(map=self.mappings, inverse_map=self.inverse_mappings)
+        
+        return self
+    
+    def transform(self, X, y=None):
+        """Applies mappings to create new one hot columns."""
+        X = pd.DataFrame(X).copy()
+        
+        cols_to_map = self.columns if self.columns else list(X.columns)
+    
+        for col in cols_to_map:
+            if col not in self.mappings:
+                raise KeyError(f"Error: No mapping found for column: {col}")
+                        
+            if ProfilerOneHotEncoder._value_not_in_mappings(X, col, self.mappings):
+                if self.handle_unknown == "error":
+                    raise ValueError(f"Error: Category not in mappings: {col}")
+                elif self.handle_unknown != "use_nan":
+                    raise ValueError(f"Error: Invalid handle_unknown option: {self.handle_unknown}")
+
+            X[col] = X[col].where(X[col].isin(self.mappings[col]))
+
+            cats_to_encode = self.mappings[col][:] # shallow copy, new list with same els
+            
+            if self.drop is not None:
+                if self.drop == "if_binary":
+                    if len(cats_to_encode) == 2:
+                        del cats_to_encode[-1] # drop 1 if binary
+                elif self.drop == "last":
+                    if len(cats_to_encode) >= 2:
+                        del cats_to_encode[-1] # drop last category, keep n-1
+                else:
+                    raise ValueError(f"Error: Invalid drop option: {self.drop}")
+
+            for cat in cats_to_encode:
+                ohe_col = f"{col}_{cat}"
+                X[ohe_col] = ((X[col] == cat).astype(int)).where(X[col].notnull(), other=np.nan)
+                
+        # drop original columns
+        X.drop(columns=cols_to_map, inplace=True)
+        return X
+    
+    def inverse_transform(self, X, y=None):
+        """Reverse one hot vectors to original categories."""
+        X = pd.DataFrame(X).copy()
+
+        col_groups = {}
+        
+        # Go through inverse_mappings and group ohe columns
+        # inverse_mappings: {'Gender_F': ('Gender', 'F'), 'Gender_M': ('Gender', 'M')}
+        # col_groups = {'Gender': [('Gender_F', 'F'), ('Gender_M', 'M')]}
+        for ohe_col, (orig_col, cat) in self.inverse_mappings.items():
+            if orig_col not in col_groups.keys():
+                col_groups[orig_col] = []
+            col_groups[orig_col].append((ohe_col, cat))
+            
+        result = pd.DataFrame(index=X.index) # empty dataframe with same index (row labels) as X
+
+        for orig_col in col_groups.keys():
+            result[orig_col] = X.apply(
+                ProfilerOneHotEncoder.decode_row, 
+                axis=1,
+                args=(col_groups[orig_col], self.drop, self.mappings[orig_col])
+            )
+
+        ohe_cols = list(self.inverse_mappings.keys())
+        X.drop(columns=[col for col in ohe_cols if col in X.columns], inplace=True)
+        
+        X = pd.concat([X, result], axis=1)
+
+        return X
+
+    @staticmethod
+    def _value_not_in_mappings(df, col:str, mappings:list):
+        """Checks whether the values of a column exist in mappings.
+        Supports either mapping direction.
+        """
+        for category in list(df[col].dropna().unique()):
+            if category not in mappings[col]:
+                return True
+
+    @staticmethod
+    def decode_row(row, ohe_list, drop, mappings):
+        for ohe_col, category in ohe_list:
+            if ohe_col in row:
+                if row[ohe_col] == 1: return category
+                elif pd.isna(row[ohe_col]): return np.nan
+        if drop in ["if_binary", "last"]:
+            return mappings[-1]
+        return np.nan

data_auditor-0.1.0/src/encoders/ordinal.py

@@ -0,0 +1,108 @@
+import pandas as pd
+import numpy as np
+from sklearn.base import BaseEstimator, TransformerMixin
+from profiler import DataProfiler
+
+
+class ProfilerOrdinalEncoder(BaseEstimator, TransformerMixin):
+    def __init__(self, profiler:DataProfiler=None, columns=None, categories=None, handle_unknown="use_nan"):
+        """Perform ordinal encoding.
+        https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.OrdinalEncoder.html
+        
+        Parameters
+        ----------
+        profiler : DataProfiler, optional
+            An instance of DataProfiler, for saving mappings.
+            Optional so that it works standalone too.
+        columns : list of str, optional
+            Columns to apply the encoding to. If None, encodes all columns.
+            Use columns for flexible control (standalone pipelines).
+        categories: list of list, optional
+            Defined the expected ordered categories for the ith column.
+        handle_unknown : {"use_nan", "error"}
+            How to handle unknown categories during transform.
+        """
+        self.profiler = profiler
+        self.columns = columns
+        self.categories = categories
+        self.handle_unknown = handle_unknown
+
+        # provide scaffolding to save mappings during fit()
+        self.mappings = {} # {col: {label: int}}
+        self.inverse_mappings = {} # {col: {int: label}}
+
+    def get_encoder_type(self):
+        return "Ordinal Encoder"
+
+    def fit(self, X, y=None):
+        """Scan columns and build mappings."""
+        X = pd.DataFrame(X)
+
+        cols_to_map = self.columns if self.columns else list(X.columns)
+
+        for i, col in enumerate(cols_to_map):
+            if self.categories and i < len(self.categories):
+                ordered_values = self.categories[i]
+            else:
+                ordered_values = pd.Series(X[col].dropna().unique()).sort_values().tolist() # deterministic encoding
+            
+            mapping = {val: idx for idx, val in enumerate(ordered_values)}
+            inverse_mapping = {idx: val for val, idx in mapping.items()}
+        
+            self.mappings[col] = mapping
+            self.inverse_mappings[col] = inverse_mapping
+
+        # Update profiler
+        if self.profiler is not None:
+            self.profiler._set_feature_mappings(map=self.mappings, inverse_map=self.inverse_mappings)
+        
+        return self # always return self with a fit()
+    
+    def transform(self, X, y=None):
+        """Apply mappings to turn labels into numerical values."""
+        X = pd.DataFrame(X).copy()
+
+        cols_to_map = self.columns if self.columns else list(X.columns)
+        
+        for col in cols_to_map:
+            if col not in self.mappings:
+                raise KeyError(f"Error: No mapping found for column: {col}")
+            
+            if ProfilerOrdinalEncoder._value_not_in_mappings(X, col, self.mappings):
+                if self.handle_unknown == "error":
+                    raise ValueError(f"Error: Category not in mappings: {col}")
+                elif self.handle_unknown != "use_nan":
+                    raise ValueError(f"Error: Invalid handle_unknown option: {self.handle_unknown}")
+                
+            X[col] = X[col].map(self.mappings[col]) # this will type-cast to float if there are NaN
+    
+        return X
+        
+    def inverse_transform(self, X, y=None):
+        """Reverse integers back to original labels."""
+        X = pd.DataFrame(X).copy()
+
+        cols_to_map = self.columns if self.columns else list(X.columns)
+        
+        for col in cols_to_map:
+            if col not in self.inverse_mappings:
+                raise KeyError(f"Error: No inverse mapping found for column: {col}")
+            
+            if ProfilerOrdinalEncoder._value_not_in_mappings(X, col, self.inverse_mappings):
+                if self.handle_unknown == "error":
+                    raise ValueError(f"Error: Category not in mappings: {col}")
+                elif self.handle_unknown != "use_nan":
+                    raise ValueError(f"Error: Invalid handle_unknown option: {self.handle_unknown}")
+                
+            X[col] = X[col].map(self.inverse_mappings[col])
+    
+        return X
+
+    @staticmethod
+    def _value_not_in_mappings(df, col:str, mappings:list):
+        """Checks whether the values of a column exist in mappings.
+        Supports either mapping direction.
+        """
+        for category in list(df[col].dropna().unique()):
+            if category not in list(mappings[col].keys()):
+                return True

data_auditor-0.1.0/src/fairness.py

@@ -0,0 +1,161 @@
+import pandas as pd
+import seaborn as sns
+import numpy as np
+import matplotlib.pyplot as plt
+from profiler import DataProfiler
+from itertools import combinations
+from typing import Union
+
+class FairnessEngine:
+    def __init__(self, df, sensitive_features:list[str], target:str=None):
+        self.df = df
+        #TODO: maybe check if these features exist in df and if they are categorical
+        self.sensitive_features = sensitive_features
+        self.target = target
+        self.profiler = DataProfiler(df=df, target=target)
+        self.sensitive_groups = FairnessEngine.intersectional_fairness_groups(sensitive_features)
+        self.fairness_df = FairnessEngine.create_fairness_df(df=self.df, 
+                                                                      sensitive_groups=self.sensitive_groups)
+        self.fairness_profiler = DataProfiler(df=self.fairness_df, target=target)
+     
+    def group_proportions_table(self, group:str, plot_tab=False) -> pd.DataFrame: 
+        """Creates a table of group distribution.
+        Group can be a combination of sensitive features, given by
+        self.sensitive_groups.
+        """
+        if group in self.sensitive_groups.keys():
+            sensitive_features = self.sensitive_groups[group]
+            rows_index = sensitive_features[0] if len(sensitive_features) > 1 else None
+            cols = sensitive_features[1:] if rows_index else sensitive_features
+            
+        pivot = pd.pivot_table(
+            self.df,
+            index=rows_index,
+            columns=cols,
+            aggfunc="size",
+            fill_value=0
+        )
+
+        pivot_percent = round(pivot / pivot.values.sum() * 100, 2)
+
+        if plot_tab:
+                self.profiler.plot_pivot_table(
+                    index=rows_index,
+                    columns=cols,
+                    aggfunc="size", 
+                    title="Proportions by Group",
+                    plot_percent=True, 
+                    figsize=(pivot_percent.shape[1] * 2, pivot_percent.shape[0] * 2))
+
+        return pivot_percent
+
+    def target_rate_by_group_table(self, group:str, target:str=None, plot_tab=False) -> pd.DataFrame:
+        """Creates a table of target distribution by group.
+        Group can be a combination of sensitive features, given by
+        self.sensitive_groups.
+        """
+        #TODO: add protections for unknown group or target
+        target = self.target if target is None else target 
+            
+        if group in self.sensitive_groups.keys() and target:
+            pivot = pd.pivot_table(
+                self.df, 
+                index=target,
+                columns=self.sensitive_groups[group],
+                aggfunc="size",
+                fill_value=0
+            )
+
+            pivot_rate = round(pivot / pivot.values.sum() * 100, 2)
+
+            if plot_tab:
+                self.profiler.plot_pivot_table(
+                    index=target,
+                    columns=self.sensitive_groups[group],
+                    aggfunc="size", 
+                    title="Proportions by Group",
+                    plot_percent=True, 
+                    figsize=(pivot_rate.shape[1] * 2, pivot_rate.shape[0] * 2))
+
+            return pivot_rate
+
+    def missing_data_disparity_imbalance(self):
+        """Calculates the disparity (imbalance ratio) by feature
+        for each group or group aggregation. 
+        Returns a dictionary for all groups and all features.
+        """
+        pass
+
+    def plot_sensitive_groups(self, groups:Union[str, list], target=None, subplot_size=(9, 4)):
+        """Plots the distribution of sensitive groups. If target is given, 
+        the distribution will be color-coded. Parameter groups can be a str of specific group,
+        a list of groups, or "all" (in this case it plots all sensitive groups).
+        """
+    
+        all_groups = self.get_sensitive_groups()
+
+        if groups == "all": groups = all_groups
+        elif isinstance(groups, str): groups=[groups]
+        elif not (isinstance(groups, list) and all(g in all_groups for g in groups)): 
+             raise ValueError("Error: Unknown group passed as argument.")
+        
+        self.fairness_profiler.plot_mult_histogram(
+                col_names=groups,
+                hue=target,
+                cols=1,
+                subplot_size=subplot_size,
+        )
+
+    def plot_features_by_group(self, col_names:list, groups:Union[str, list], subplot_size=(9, 4)):
+        
+        all_groups = self.get_sensitive_groups()
+
+        if groups == "all": groups = all_groups
+        elif isinstance(groups, str): groups=[groups]
+        elif not (isinstance(groups, list) and all(g in all_groups for g in groups)): 
+             raise ValueError("Error: Unknown group passed as argument.")
+        
+        if isinstance(col_names, str): col_names=[col_names]
+
+        for group in groups:
+            print(col_names)
+            print(group)
+            self.fairness_profiler.plot_mult_histogram(col_names=col_names, hue=group)
+
+    def get_sensitive_groups(self):
+        return list(self.sensitive_groups.keys())
+    
+    def get_sensitive_features(self):
+        return self.sensitive_features
+
+
+    @staticmethod
+    def intersectional_fairness_groups(sensitive_features:list[str]) -> dict:
+        """Determines interseccional sensitive groups.
+        For each sensitive group, returns the features that
+        create it.
+        """
+        res = []
+        for i in range(1, len(sensitive_features) + 1):
+            c = list(combinations(sensitive_features, i))
+            res.extend(c)
+        group_lst = ["_".join(pairs) for pairs in res]
+
+        intersectional_groups = {}
+        for g in group_lst:
+            intersectional_groups[g] = g.split("_")
+        
+        return intersectional_groups
+    
+    @staticmethod
+    def create_fairness_df(df: pd.DataFrame, sensitive_groups: dict):
+        """Creates a copy of the original dataframe and adds columns
+        corresponding to the combination of sensitive features.
+        """
+        fairness_df = df.copy()
+        for group, features in sensitive_groups.items():
+            if len(features) > 1:
+                fairness_df[group] = fairness_df[features].agg(lambda x: "_".join(x.dropna().astype(str)) 
+                                             if x.notna().all() else np.nan, axis=1)
+        return fairness_df
+

data_auditor-0.1.0/src/preprocessing.py

@@ -0,0 +1,165 @@
+from profiler import DataProfiler
+import copy
+from typing import Callable, Optional
+from encoders.ordinal import ProfilerOrdinalEncoder
+from encoders.onehot import ProfilerOneHotEncoder
+from pandas import DataFrame
+from sklearn.compose import ColumnTransformer
+from sklearn.preprocessing import MinMaxScaler
+
+# And functions to perform MinMax, Scaling, etc to each of them
+# Note sure how Pipelines work and whether we'd like to incorporate them here, probably we won't
+
+# Decorators
+def track_changes(func: Callable):
+    def wrapper(self, *args, **kwargs):
+        result = func(self, *args, **kwargs)
+        self.change_logs.append(f"{func.__name__}")
+        return result   # can't be swallowed by decorator to allow chaining
+    return wrapper
+
+
+class DataPreprocessor:
+    def __init__(self, profiler: DataProfiler):
+        self.profiler = profiler    # shared reference
+        #self.df = copy.deepcopy(profiler.df)
+        self.df = profiler.df.copy()
+        self.encoders = {}
+        self.column_transformers = {}
+        self.change_logs = []
+        self.log_details = []
+    
+    @track_changes
+    def delete_features(self, col_names: list):
+        """Delete specific features from the dataset and update profiler."""
+        if self.profiler.target in col_names:
+            self.profiler.target = None
+        self.df.drop(columns=col_names, inplace=True) # modify directly, do not create new dataframe
+        self.profiler.df = self.df
+        self.profiler._refresh()
+        log_msg = f"Deleted: {col_names}"
+        self.log_details.append(log_msg)
+        return self # allows for method chaining (e.g. processor.delete_features().scale_numericals())
+                    # without return self, you need to call separately: 
+                    # processor.delete_features()
+                    # processor.scale_numericals()
+        #TODO: Maybe delete corresponding feature mappings, if feature gets deleted? 
+        # (self.mapping and profiler.mappings?)
+
+    @track_changes
+    def set_semantic_feature_types(self,
+                          continuous_cols: Optional[list]=None, 
+                          ordinal_cols: Optional[list]=None, 
+                          nominal_cols: Optional[list]=None, 
+                          binary_cols: Optional[list]=None):
+        """Defines continuous, ordinal, nominal, and binary features."""
+        self.profiler._set_semantic_feature_types(
+            cont_cols = continuous_cols,
+            ord_cols = ordinal_cols,
+            nom_cols = nominal_cols,
+            bin_cols = binary_cols)
+        log_msg = ""
+        if continuous_cols: log_msg += f"Continuous: {continuous_cols}, "
+        if ordinal_cols: log_msg += f"Ordinal: {ordinal_cols} "
+        if nominal_cols: log_msg += f"Nominal: {nominal_cols} "
+        if binary_cols: log_msg += f"Binary: {binary_cols} "
+        self.log_details.append(log_msg)
+        return self
+    
+    @track_changes
+    def change_feature_types(self, num_cols:list[str]=None, cat_cols:list[str]=None):
+        self.profiler._override_feature_types(numeric=num_cols, categorical=cat_cols)
+        self.profiler._refresh()
+        log_msg = ""
+        if num_cols: log_msg += f"Num Cols: {num_cols}"
+        if cat_cols: log_msg += f"Cat Cols: {cat_cols}"
+        self.log_details.append(log_msg)
+        return self
+    
+    def print_log_report(self):
+        print("===== Track Changes =====")
+        for i in range(len(self.change_logs)):
+            print(f"Method: {self.change_logs[i]}. {self.log_details[i]}")
+
+    @track_changes
+    def ohe_transform(self, profiler=None, columns:list[str]=None, drop=None, handle_unknown="use_nan"):
+        ohe = ProfilerOneHotEncoder(
+            profiler=profiler if profiler is not None else self.profiler,
+            columns=columns,
+            drop=drop,
+            handle_unknown=handle_unknown
+        )
+        
+        self.df = ohe.fit_transform(self.df)
+        self.profiler.df = self.df
+        key = f"OHE_{'_'.join(ohe.columns)}"    
+        self.encoders[key] = ohe
+        self.profiler._refresh()
+        log_msg = f"OHE Encoding: {ohe.columns}"
+        self.log_details.append(log_msg)
+        return self
+    
+    @track_changes
+    def ordinal_transform(self, profiler=None, columns:list[str]=None, categories=None, handle_unknown="use_nan"):
+        oe = ProfilerOrdinalEncoder(
+            profiler=profiler if profiler is not None else self.profiler,
+            columns=columns,
+            categories=categories,
+            handle_unknown=handle_unknown
+        )
+        
+        self.df = oe.fit_transform(self.df)
+        self.profiler.df = self.df
+        key = f"OrdinalEnc_{'_'.join(oe.columns)}"    
+        self.encoders[key] = oe
+        self.profiler._refresh()
+        log_msg = f"Ordinal Encoding: {oe.columns}"
+        self.log_details.append(log_msg)
+        return self
+    
+    @track_changes
+    def encoder_inverse_transform(self, encoder_key:str):
+        encoder = self.encoders[encoder_key]
+        self.df = encoder.inverse_transform(self.df)
+        self.profiler.df = self.df
+        self.profiler._refresh()
+        encoder_type = encoder.get_encoder_type()
+        log_msg = f"{encoder_type} Inverse Transform: {encoder.columns}"
+        self.log_details.append(log_msg)
+        return self
+    
+    @track_changes
+    def auto_col_transformer(self):
+        """Automatically build ColumnTransformer based on profiler metadata."""
+
+        transformer_constructor = {
+            "cont_cols": lambda: ("scale_continuous", MinMaxScaler(), getattr(self.profiler, "cont_cols", [])),
+            "ord_cols": lambda: ("encode_ordinal", ProfilerOrdinalEncoder(profiler=self.profiler), getattr(self.profiler, "ord_cols", [])),
+            "nom_cols": lambda: ("encode_nominal", ProfilerOneHotEncoder(profiler=self.profiler), getattr(self.profiler, "nom_cols", [])),
+            "bin_cols": lambda: ("encode_binary", ProfilerOneHotEncoder(profiler=self.profiler), getattr(self.profiler, "bin_cols", []))
+        }
+
+        transformers = []
+        log_transformers = ""
+
+        for constructor in transformer_constructor.values():
+            name, transformer, cols = constructor() # call lambda function to return tuple
+            if cols:
+                transformers.append((name, transformer, cols))
+                log_transformers += f"{name}"
+      
+        if not transformers:
+            raise Exception("Error: No transformer was created.")
+        
+
+        ct = ColumnTransformer(transformers=transformers, remainder="passthrough")
+        log_msg = "Automatic Column Transformer Created: " + log_transformers.rstrip("_")
+        self.log_details.append(log_msg)
+        key = f"ColumnTransformer_{log_transformers.rstrip("_")}"    
+        self.column_transformers[key] = ct
+        
+        return ct
+        
+    def get_encoders(self):
+        return self.encoders
+

data_auditor-0.1.0/src/profiler.py

@@ -0,0 +1,322 @@
+import pandas as pd
+import seaborn as sns
+import numpy as np
+import matplotlib.pyplot as plt
+import math
+
+class DataProfiler:
+    """Organizes metadata and offers visual diagnostics on tabular datasets."""
+    def __init__(self, df: pd.DataFrame, target: str = None):
+        self.df = df
+        self.target = target
+        self.analysis_summary = {}
+        self.metadata_summary = {}
+        self._feature_overrides = {
+            "num_cols": set(),
+            "cat_cols": set()
+        }
+        self.cont_cols = None
+        self.ord_cols = None
+        self.nom_cols = None
+        self.bin_cols = None
+        self._analyze()
+    
+    def get_data(self):
+        return self.df
+    
+    @property
+    def summary(self):
+        return {**self.analysis_summary, **self.metadata_summary}
+    
+    def print_report(self):
+        # TODO: add all, metadata, analysis
+        for k,v in self.summary.items():
+            print(f"{k}: {v}")
+
+    def plot_histogram(self, x: str, hue=None, kde=None, multiple="layer", shrink=1, title=None, figsize=(6, 4)):
+        """Plot basic histogram.
+        
+        Parameters
+        ----------
+        x: str
+            The name of the column to plot the histogram
+        kde: bool, optional
+            If True, computes kernel density and shows on the plot
+        hue: str, optional
+            Vector or key in self.df. Semantic feature to determine the color of plot elements.
+        multiple: {"layer", "dodge", "stack", "fill"}
+            Approach to resolve multiple elements when semantic mapping creates subsets.
+        """
+        if multiple == "dodge": shrink = 0.8
+        plt.figure(figsize=figsize)
+        sns.histplot(data=self.df, x=x, hue=hue, kde=kde, multiple=multiple, shrink=shrink)
+        if not title:
+            title = f"Histogram of {x}"
+            if hue: title += f" based on {hue}"
+        plt.title(title)
+        plt.tight_layout()
+        plt.show()
+
+    def plot_scatter(self, x: str, y: str, hue=None, style=None, title=None, x_legend=None, y_legend=None):
+        """Plot basic scatterplot.
+        
+        Parameters
+        ----------
+        x: str
+            The name of the column on the xx axis
+        y: str
+            The name of the column on the yy axis
+        hue: str, optional
+            Grouping column to determine the color of plot elements.
+        style: str, optional
+            Grouping column that will produce different markers.
+        
+        """
+        plt.figure(figsize=(6, 4))
+        palette = "deep" if hue else None
+        sns.scatterplot(self.df, x=x, y=y, hue=hue, style=style, palette=palette)
+        plt.xlabel(x_legend if x_legend else x)
+        plt.ylabel(y_legend if y_legend else y)
+        text_title = f"{x} versus {y}"
+        if hue: text_title += f" by {hue}"
+        plt.title(title if title else f"{x} versus {y}")
+        plt.tight_layout()
+        plt.show()
+
+    def plot_pivot_table(self, index: str, columns: str, aggfunc="mean", title:str=None, values:str=None, figsize:tuple=(6,4), plot_percent=False):
+        """Plot heatmap based on pivot table.
+
+        Parameters
+        ----------
+        values: str
+            Column to aggregate.
+        index: str
+            Keys to group by on the pivot table.
+        columns: str
+            Keys to group by on the pivot table.
+        aggfunc: {"mean", "max", "min"}
+            Function to calculate aggregates.
+        """
+        fill_value = 0 if aggfunc == "size" else None
+
+        pivot_df = self.df.pivot_table(
+            values=values,
+            index=index,
+            columns=columns,
+            aggfunc = aggfunc,
+            fill_value=fill_value
+        )
+
+        if aggfunc == "size" and plot_percent:
+            pivot_df = round(pivot_df / pivot_df.values.sum() * 100, 2)
+
+        plt.figure(figsize=figsize)
+        ax = sns.heatmap(pivot_df, annot=True, fmt=".1f", linewidth=.5, cbar_kws={"label": f"{f"{aggfunc}_{values}" if values else aggfunc}"})
+        ax.set(xlabel=columns, ylabel=index)
+        ax.xaxis.tick_top()
+        ax.xaxis.set_label_position('top') 
+        plt.title(title if title else f"{values if values else aggfunc} by {index} and {columns}")
+        plt.show()
+
+    def plot_mult_histogram(self, col_names: list, hue=None, cols=2, subplot_size=(6, 4)):
+        """Plot multiple histograms in a grid layout.
+
+        Parameters
+        ----------
+        col_names: list
+            List of column names to plot.
+        hue: str, optional
+            Column used to group histograms.
+        cols: int
+            Number of columns in the grid layout.
+        figsize: tuple
+            Size of the figure.
+        """
+        n = len(col_names)
+        rows = math.ceil(n/cols)
+
+        fig_width, fig_height = subplot_size[0] * cols, subplot_size[1] * rows 
+        
+        _ , axes = plt.subplots(rows, cols, figsize=(fig_width, fig_height))
+        axes = np.atleast_1d(axes)
+        axes = axes.flatten()
+
+        for i, col in enumerate(col_names):
+            ax = axes[i]
+            if col in self.cat_cols: sns.histplot(self.df, x=col, hue=hue, multiple="dodge", shrink=0.8, ax=ax)
+            else: sns.histplot(self.df, x=col, hue=hue, ax=ax)
+            title_text = f"{col}"
+            if hue: title_text += f" by {hue}"
+            ax.set_title(title_text)
+
+        # Remove extra axis 
+        for j in range(i + 1, len(axes)):
+            axes[j].set_visible(False)
+
+        plt.tight_layout()
+        plt.show()
+
+    def plot_mult_scatter(self, col_names: list, fixed_col=None, hue=None, style=None, cols=2, figsize=(12, 4)):
+        """Plot all scatter, by hue, using numerical features.
+
+        Parameters
+        ----------
+        col_names: list
+            List of column names to plot.
+        hue: str, optional
+            Column used to group histograms.
+        cols: int
+            Number of columns in the grid layout.
+        figsize: tuple
+            Size of the figure.
+        """
+        if fixed_col: 
+            combinations = set()
+            col_names.remove(fixed_col)
+            for i in col_names:
+                combinations.add((fixed_col, i))
+        else: combinations = DataProfiler.combinatorial_k_2(col_names)
+
+        n = len(combinations)
+        rows = math.ceil(n/cols)
+        palette = "deep" if hue else None
+
+        _ , axes = plt.subplots(rows, cols, figsize=(figsize[0], figsize[1]*rows))
+        axes = axes.flatten()
+
+        for i, combo in enumerate(combinations):
+            ax = axes[i]
+            if combo[0] in self.num_cols and combo[1] in self.num_cols:
+                sns.scatterplot(self.df, x=combo[0], y=combo[1], hue=hue, style=style, palette=palette, ax=ax)
+                title_text = f"{combo[0]} versus {combo[1]}"
+                if hue: title_text += f" by {hue}"
+                ax.set_title(title_text)
+            else: raise ValueError("One or more features is not quantitative.") # TODO: raise specific error
+            
+        # Remove extra axis 
+        for j in range(i + 1, len(axes)):
+            axes[j].set_visible(False)
+
+        plt.tight_layout()
+        plt.show()
+        
+    def plot_correlation(self):
+        """Plot correlation matrix for numerical features.
+        Consider ordinal and othey types?
+        """
+        pass
+
+    def _analyze(self):
+        df = self.df
+        self.n_samples, self.n_features = df.shape
+
+        # update num and cat cols (check if they were user-defined as well)
+        num_cols = set(df.select_dtypes(include=["int64", "float64"]).columns.tolist())
+        cat_cols = set(df.select_dtypes(include=["object", "bool"]).columns.tolist())
+
+        num_cols.update(self._feature_overrides["num_cols"])
+        cat_cols.update(self._feature_overrides["cat_cols"])
+
+        num_cols.difference_update(self._feature_overrides["cat_cols"])
+        cat_cols.difference_update(self._feature_overrides["num_cols"])
+
+        self.num_cols = sorted(list(num_cols))
+        self.cat_cols = sorted(list(cat_cols))
+
+        self.cols = self.num_cols + self.cat_cols
+        if self.target: self.cat_cols.remove(self.target)
+        self.mv_cols = df.columns[df.isna().any()].to_list()
+        self.mv_percentage = round(df.isna().sum().sum() / df.size * 100, 2)
+        self.duplicate_rows = df.duplicated().sum()
+
+        if self.target and self.target in df.columns:
+            self.target_distribution = round(df[self.target].value_counts(normalize=True), 2).to_dict()
+        else:
+            self.target_distribution = {}
+
+        self.analysis_summary = {
+            "n_samples": self.n_samples,
+            "n_features": self.n_features,
+            "target": self.target,
+            "num_cols": self.num_cols,
+            "cat_cols": self.cat_cols,
+            "mv_cols": self.mv_cols,
+            "mv_percentage": self.mv_percentage,
+            "duplicate_rows": self.duplicate_rows,
+            "target_distribution": self.target_distribution,
+        }
+
+    def _refresh(self):
+        self._analyze()
+
+    def reset_feature_type_overrides(self):
+        """Clean all overrides to num and cat columns."""
+        self._feature_overrides = {"num_cols": set(), "cat_cols": set()}
+        self._refresh()
+
+    def _set_semantic_feature_types(self, cont_cols=None, ord_cols=None, nom_cols=None, bin_cols=None):
+        """Sets semantic feature types to the profiler.
+            If a new setting occurs, it will override the previous.
+        """
+        
+        feature_roles = {
+            "continuous": cont_cols,
+            "ordinal": ord_cols,
+            "nominal": nom_cols,
+            "binary": bin_cols
+        }
+
+        for role, cols in feature_roles.items():
+            if cols is not None:
+                setattr(self, role, cols)
+                self.metadata_summary[f"{role}_cols"] = cols
+
+    def _override_feature_types(self, numeric=None, categorical=None):
+        if numeric:
+            self._feature_overrides["num_cols"].update(numeric)
+            self._feature_overrides["cat_cols"].difference(numeric)
+        if categorical:
+            self._feature_overrides["cat_cols"].update(categorical)
+            self._feature_overrides["num_cols"].difference_update(categorical)
+
+    def _set_feature_mappings(self, map: dict, inverse_map: dict=None):
+        if not hasattr(self, "feature_mappings"):
+            self.feature_mappings = {}
+        self.feature_mappings.update(map)
+        self.metadata_summary["feature_mappings"] = self.feature_mappings
+        if inverse_map:
+            if not hasattr(self, "inverse_feature_mappings"):
+                self.inverse_feature_mappings = {}
+            self.inverse_feature_mappings.update(inverse_map)
+            self.metadata_summary["inverse_mappings"] = self.inverse_feature_mappings        
+
+    @staticmethod
+    def combinatorial_k_2(n_cols: list):
+        comb = set()
+        for i in n_cols:
+            for j in n_cols:
+                if i != j and (j, i) not in comb:
+                    comb.add((i, j))
+        return comb
+    
+    # TODO: def save_plot(self, plot_func, *args, **kwargs):
+
+    # TODO: get_data()?
+
+# Given this code, how could I use it to plot all features?
+# How can I determine feature types? And change them?
+# Can I provide alternative maps? map and reverse mapping?
+
+class MissingDataProfiler:
+    pass
+    # TODO: Get basic stats on missing data and visualization
+
+
+
+class DataVisualizer:
+    pass
+
+
+
+
+

data_auditor-0.1.0/src/utils.py

@@ -0,0 +1,12 @@
+import pandas as pd
+
+def load_data(path: str) -> pd.DataFrame:
+    df = pd.read_csv(path, na_values="?")
+    df.columns = (
+        df.columns.str.strip()
+        .str.lower()
+        .str.replace(" ", "_")
+        )
+    for col in df.select_dtypes(include=["object", "string"]).columns:
+        df[col] = df[col].str.strip().str.lower()
+    return df