model-auditor 0.1.0__tar.gz

model_auditor-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,32 @@
+name: Upload Python Package to PyPI when a Release is Created
+
+on:
+  release:
+    types: [created]
+
+jobs:
+  pypi-publish:
+    name: Publish release to PyPI
+    runs-on: ubuntu-latest
+    environment:
+      name: release
+      url: https://pypi.org/p/model-auditor
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.x"
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install --user --upgrade build
+          pip install setuptools wheel
+      - name: Build package
+        run: |
+          python -m build
+      - name: Publish package distributions to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

model_auditor-0.1.0/.gitignore

@@ -0,0 +1,3 @@
+__pycache__/
+*.egg-info
+.ipynb-checkpoints

model_auditor-0.1.0/PKG-INFO

@@ -0,0 +1,6 @@
+Metadata-Version: 2.4
+Name: model-auditor
+Version: 0.1.0
+Requires-Dist: pandas>=2.2
+Requires-Dist: numpy>=2.1
+Requires-Dist: scikit-learn>=1.5

model_auditor-0.1.0/README.md

@@ -0,0 +1,3 @@
+# Model Auditor [Pre-Alpha]
+
+### Beatrice BM

model_auditor-0.1.0/model_auditor/__init__.py

@@ -0,0 +1 @@
+from model_auditor.core import Auditor

model_auditor-0.1.0/model_auditor/core.py

@@ -0,0 +1,336 @@
+from typing import Optional, Type, Union, Callable
+import pandas as pd
+import numpy as np
+from numpy.typing import NDArray
+from sklearn.metrics import roc_curve
+from tqdm.auto import tqdm
+
+from model_auditor.metric_inputs import AuditorMetricInput
+from model_auditor.metrics import AuditorMetric
+from model_auditor.schemas import (
+    AuditorFeature,
+    AuditorScore,
+    AuditorOutcome,
+    FeatureEvaluation,
+    ScoreEvaluation,
+)
+from model_auditor.utils import collect_metric_inputs
+
+
+class Auditor:
+    def __init__(
+        self,
+        data: Optional[pd.DataFrame] = None,
+        features: Optional[list[AuditorFeature]] = None,
+        scores: Optional[list[AuditorScore]] = None,
+        outcome: Optional[AuditorOutcome] = None,
+        metrics: Optional[list[AuditorMetric]] = None,
+    ) -> None:
+        # initialize data
+        self.data: Optional[pd.DataFrame] = None if data is None else data.copy()
+
+        # initialize features
+        self.features: dict[str, AuditorFeature] = dict()
+        if features is not None:
+            for feature in features:
+                self.add_feature(**vars(feature))
+
+        # initialize scores
+        self.scores: dict[str, AuditorScore] = dict()
+        if scores is not None:
+            for score in scores:
+                self.add_score(**vars(score))
+
+        # initialize outcome
+        if outcome is not None:
+            self.add_outcome(**vars(outcome))
+
+        # initialize metrics
+        self.metrics: list[AuditorScore] = list()
+        if metrics is not None:
+            self.metrics = metrics
+
+        # initialize attrs for later
+        self._inputs: list[Type[AuditorMetricInput]] = list()
+        self._evaluations: list = list()
+        self.n_bootstraps: int = 1000
+
+    def add_data(self, data: pd.DataFrame) -> None:
+        """
+        Method to add a dataframe to the auditor
+
+        Args:
+            data (pd.DataFrame): Full dataframe which will be subset for subgroup evaluation
+        """
+        self.data = data.copy()
+
+    def add_feature(
+        self, name: str, label: Optional[str] = None, levels: Optional[list[any]] = None
+    ) -> None:
+        """
+        Method to add a feature to the auditor. Equivalent to a grouping variable in
+        packages like tableone, the score variable will be stratified by this feature
+
+        Args:
+            name (str): Column name for the feature.
+            label (Optional[str], optional): Optional label for the feature. Defaults to None.
+            levels (Optional[list[any]], optional): Valid levels to consider for the feature,
+            by default (when set to None) all levels will be considered. Defaults to None.
+            [Not currently implemented]
+        """
+        feature = AuditorFeature(
+            name=name,
+            label=label,
+            levels=levels,
+        )
+        self.features[feature.name] = feature
+
+    def add_score(
+        self, name: str, label: Optional[str] = None, threshold: Optional[float] = None
+    ) -> None:
+        """
+        Method to add a score to the auditor. Expects a continuous feature which will
+        be used to calculate metrics and confidence intervals
+
+        Args:
+            name (str): Column name for the score.
+            label (Optional[str], optional): Optional label for the score. Defaults to None.
+            threshold (Optional[float], optional): Threshold used to binarize the score column.
+            Defaults to None and can be optimized using the Youden index or updated separately later.
+        """
+        score = AuditorScore(
+            name=name,
+            label=label,
+            threshold=threshold,
+        )
+        self.scores[score.name] = score
+
+    def add_outcome(self, name: str, mapping: Optional[dict[any, int]] = None) -> None:
+        if self.data is None:
+            raise ValueError("Please add data with .add_data() first")
+
+        if mapping is not None:
+            self.data["_truth"] = self.data[name].map(mapping)
+        else:
+            self.data["_truth"] = self.data[name]
+
+    def optimize_score_threshold(self, score_name: str) -> float:
+        """
+        Method to optimize the decision threshold for a score based on the Youden index.
+
+        Args:
+            score_name (str): Name of the target score
+
+        Raises:
+            ValueError: If no scores have been defined with .add_score() first
+            ValueError: If no data has been added with .add_data() first
+            ValueError: If no outcome variable has been defined with .add_outcome() first
+
+        Returns:
+            float: Optimal threshold identified
+        """
+        if len(self.scores) == 0:
+            raise ValueError("Please define at least one score first")
+        if self.data is None:
+            raise ValueError("Please add data with .add_data() first")
+        elif "_truth" not in self.data.columns.tolist():
+            raise ValueError(
+                "Please define an outcome variable data with .add_outcome() first"
+            )
+
+        # throws an error if the score has not been defined
+        score: AuditorScore = self.scores[score_name]
+        score_list: list[float] = self.data[score.name].astype(float).tolist()
+
+        # otherwise the target score will be the single item in the list
+        truth_list: list[float] = self.data["_truth"].astype(float).tolist()
+
+        # calculate optimal threshold
+        fpr, tpr, thresholds = roc_curve(truth_list, score_list)
+        idx: int = np.argmax(tpr - fpr).astype(int)
+        optimal_threshold: float = thresholds[idx]
+
+        print(f"Optimal threshold for '{score.name}' found at: {optimal_threshold}")
+        return optimal_threshold
+
+    def set_metrics(self, metrics: list[AuditorMetric]) -> None:
+        """
+        Method to define the metrics the auditor will use during evaluation of score variables.
+
+        Args:
+            metrics (list[AuditorMetric]): A list of metrics classes following the AuditorMetric
+            protocol (pre-made metrics listed in model_auditor.metrics)
+        """
+        self.metrics: list[AuditorMetric] = metrics
+
+    def _collect_inputs(self) -> None:
+        """
+        Collects the minimum set of metric inputs necessary for evaluation
+        (based on the metrics defined in self.metrics with the .define_metrics() method)
+        """
+        inputs_set: set[str] = set()
+        for metric in self.metrics:
+            inputs_set.update(metric.inputs)
+
+        inputs_dict: dict[str, Type[AuditorMetricInput]] = collect_metric_inputs()
+
+        # reinit self._inputs and add all necessary inputs to it
+        self._inputs: list[Type[AuditorMetricInput]] = list()
+        for input_name in list(inputs_set):
+            if input_name not in ["_truth", "_pred"]:
+                self._inputs.append(inputs_dict[input_name])
+
+    def _apply_inputs(self, data: pd.DataFrame) -> pd.DataFrame:
+        """
+        Method to apply the metric input functions (collected with .collect_inputs())
+        to the target data to prepare it for metric calculation
+
+        Args:
+            data (pd.DataFrame): Dataframe to add input columns to
+
+        Returns:
+            pd.DataFrame: Transformed dataframe with metric input columns
+        """
+        for input_type in self._inputs:
+            metric_input = input_type()
+            data: pd.DataFrame = metric_input.data_transform(data)
+
+        return data
+
+    def _binarize(self, score_data: pd.Series, threshold: float) -> pd.Series:
+        return (score_data >= threshold).astype(int)
+
+    def evaluate(self, score_name: str, threshold: Optional[float] = None):
+        if self.data is None:
+            raise ValueError("Please add data with .add_data() first")
+
+        if len(self.metrics) == 0:
+            raise ValueError(
+                "Please define at least one metric with .set_metrics() first"
+            )
+
+        # get score
+        score: AuditorScore = self.scores[score_name]
+
+        if (threshold is None) & (score.threshold is None):
+            raise ValueError(
+                "Threshold must be defined in score object or passed to .evaluate_score()"
+            )
+        elif threshold is None:
+            threshold = score.threshold
+
+        # collect metric inputs to prep for evaluation
+        self._collect_inputs()
+
+        # get the list of columns to retain in the data
+        column_list: list[str] = [*self.features.keys(), "_truth"]
+
+        # copy a slice of the dataframe
+        data_slice: pd.DataFrame = self.data.loc[:, column_list]
+        data_slice["_pred"] = self.data[score.name]
+        data_slice["_binary_pred"] = self._binarize(score_data=data_slice["_pred"], threshold=threshold)  # type: ignore
+        data_slice = self._apply_inputs(data=data_slice)
+
+        # create an 'Overall' feature which will be used to calculate metrics on the full data
+        data_slice["overall"] = "Overall"
+        eval_features: dict[str, AuditorFeature] = {
+            "overall": AuditorFeature(
+                name="overall",
+                label="Overall",
+            )
+        }
+        eval_features.update(**self.features)
+
+        score_eval: ScoreEvaluation = ScoreEvaluation(
+            name=score.name,
+            label=score.label if score.label is not None else score.name,
+        )
+        with tqdm(
+            eval_features.values(), position=0, leave=True, desc="Features"
+        ) as pbar:
+            for feature in pbar:
+                pbar.set_postfix({"name": feature.name})
+
+                # e.g. {"f1": {'levelA': 0.2, 'levelB': 0.4}, ... }
+                feature_eval: FeatureEvaluation = self._evaluate_feature(
+                    data=data_slice, feature=feature
+                )
+                score_eval.features[feature.name] = feature_eval
+
+        return score_eval
+
+    def _evaluate_feature(
+        self, data: pd.DataFrame, feature: AuditorFeature
+    ) -> FeatureEvaluation:
+        with tqdm(range(2), position=1, desc="Stages", leave=False) as feature_pbar:
+            feature_pbar.set_postfix({"stage": "Evaluating metrics"})
+
+            # cast feature levels to string
+            data[feature.name] = data[feature.name].astype(str)
+
+            # then group the df by this feature (so each group contains one
+            # unique level of the data) and get all metrics for each
+            feature_groups = data.groupby(feature.name)
+
+            # e.g. {"f1": {'levelA': 0.2, 'levelB': 0.4}, ... }
+            feature_eval: FeatureEvaluation = FeatureEvaluation(
+                name=feature.name,
+                label=feature.label if feature.label is not None else feature.name,
+            )
+            for metric in tqdm(self.metrics, position=2, desc="Metrics", leave=False):
+                # gets a dict with the current metric calculated for levels of the feature
+                # e.g. {levelA: 0.5, levelB: 0.5}
+                level_eval_dict = feature_groups.apply(metric.data_call).to_dict()
+
+                feature_eval.update(
+                    metric_name=metric.name,
+                    metric_label=metric.label,
+                    data=level_eval_dict,
+                )
+
+            feature_pbar.update(1)
+            feature_pbar.set_postfix({"stage": "Evaluating intervals"})
+            # if calculating confidence intervals, do that here
+            if self.n_bootstraps is not None:
+                for level_name, level_data in tqdm(
+                    feature_groups, position=2, desc="Bootstrap Levels", leave=False
+                ):
+                    # calculate confidence intervals for eligible metrics for the current feature level
+                    level_metric_intervals: dict[str, tuple[float, float]] = (
+                        self._evaluate_confidence_interval(data=level_data)
+                    )
+                    # register the calculated intervals
+                    feature_eval.update_intervals(
+                        level_name=str(level_name),
+                        metric_intervals=level_metric_intervals,
+                    )
+            feature_pbar.update(1)
+
+        return feature_eval
+
+    def _evaluate_confidence_interval(
+        self, data: pd.DataFrame
+    ) -> dict[str, tuple[float, float]]:
+        n: int = len(data)
+
+        bootstrap_results: dict[str, NDArray[np.float64]] = dict()
+        for metric in self.metrics:
+            if metric.ci_eligible:
+                bootstrap_results[metric.name] = np.empty(shape=(self.n_bootstraps), dtype=np.float64)
+
+        # sample n_bootstrap times with replacement
+        for i in range(self.n_bootstraps):
+            boot_data: pd.DataFrame = data.sample(n, replace=True)
+
+            # calculate metrics on current bootstrap data
+            for metric in self.metrics:
+                if metric.ci_eligible:
+                    bootstrap_results[metric.name][i] = metric.data_call(boot_data)
+
+        metric_intervals: dict[str, tuple[float, float]] = dict()
+        for metric_name, bootstrap_array in bootstrap_results.items():
+            # get 95% confidence bounds for metric
+            lower, upper = np.percentile(bootstrap_array, [2.5, 97.5])
+            metric_intervals[metric_name] = (lower, upper)
+
+        return metric_intervals

model_auditor-0.1.0/model_auditor/metric_inputs.py

@@ -0,0 +1,58 @@
+import pandas as pd
+from typing import Protocol, Union, runtime_checkable
+
+
+@runtime_checkable
+class AuditorMetricInput(Protocol):
+    name: str
+    label: str
+    inputs: list[str]
+
+    def row_call(self, row: pd.Series) -> Union[int, float]:
+        """
+        method called on each row of a dataframe to calculate a metric
+        """
+        raise NotImplementedError
+
+    def data_transform(self, data: pd.DataFrame) -> pd.DataFrame:
+        """
+        method called on a dataframe to add a metric input column inplace
+        """
+        data[self.name] = data.apply(self.row_call, axis=1)
+        return data
+
+
+class TruePositives(AuditorMetricInput):
+    name: str = "tp"
+    label: str = "TP"
+    inputs: list[str] = ["_truth", "_binary_pred"]
+
+    def row_call(self, row: pd.Series) -> int:
+        return int((row["_truth"] == 1.0) & (row["_binary_pred"] == 1.0))
+
+
+class FalsePositives(AuditorMetricInput):
+    name: str = "fp"
+    label: str = "FP"
+    inputs: list[str] = ["_truth", "_binary_pred"]
+
+    def row_call(self, row: pd.Series) -> int:
+        return int((row["_truth"] == 0.0) & (row["_binary_pred"] == 1.0))
+
+
+class TrueNegatives(AuditorMetricInput):
+    name: str = "tn"
+    label: str = "TN"
+    inputs: list[str] = ["_truth", "_binary_pred"]
+
+    def row_call(self, row: pd.Series) -> int:
+        return int((row["_truth"] == 0.0) & (row["_binary_pred"] == 0.0))
+
+
+class FalseNegatives(AuditorMetricInput):
+    name: str = "fn"
+    label: str = "FN"
+    inputs: list[str] = ["_truth", "_binary_pred"]
+
+    def row_call(self, row: pd.Series) -> int:
+        return int((row["_truth"] == 1.0) & (row["_binary_pred"] == 0.0))

model_auditor-0.1.0/model_auditor/metrics.py

@@ -0,0 +1,250 @@
+from typing import Protocol, Union, Optional
+import pandas as pd
+import numpy as np
+from sklearn.metrics import average_precision_score, roc_auc_score
+
+
+class AuditorMetric(Protocol):
+    name: str
+    label: str
+    inputs: list[str]
+    ci_eligible: bool
+
+    def data_call(self, data: pd.DataFrame) -> Union[float, int]:
+        """
+        method called on a dataframe to calculate a metric
+        """
+        raise NotImplementedError
+
+
+class Sensitivity(AuditorMetric):
+    name: str = "sensitivity"
+    label: str = "Sensitivity"
+    inputs: list[str] = ["tp", "fn"]
+    ci_eligible: bool = True
+
+    def data_call(self, data: pd.DataFrame, eps: float = 1e-8) -> float:
+        n_tp: int = data["tp"].sum()
+        n_fn: int = data["fn"].sum()
+        return n_tp / (n_tp + n_fn + eps)
+
+
+class Specificity(AuditorMetric):
+    name: str = "specificity"
+    label: str = "Specificity"
+    inputs: list[str] = ["tn", "fp"]
+    ci_eligible: bool = True
+
+    def data_call(self, data: pd.DataFrame, eps: float = 1e-8) -> float:
+        n_tn: int = data["tn"].sum()
+        n_fp: int = data["fp"].sum()
+        return n_tn / (n_tn + n_fp + eps)
+
+
+class Precision(AuditorMetric):
+    name: str = "precision"
+    label: str = "Precision"
+    inputs: list[str] = ["tp", "fp"]
+    ci_eligible: bool = True
+
+    def data_call(self, data: pd.DataFrame, eps: float = 1e-8) -> float:
+        n_tp: int = data["tp"].sum()
+        n_fp: int = data["fp"].sum()
+        return n_tp / (n_tp + n_fp + eps)
+
+
+class Recall(AuditorMetric):
+    name: str = "recall"
+    label: str = "Recall"
+    inputs: list[str] = ["tp", "fn"]
+    ci_eligible: bool = True
+
+    def data_call(self, data: pd.DataFrame, eps: float = 1e-8) -> float:
+        n_tp: int = data["tp"].sum()
+        n_fn: int = data["fn"].sum()
+        return n_tp / (n_tp + n_fn + eps)
+
+
+class F1Score(AuditorMetric):
+    name: str = "f1"
+    label: str = "F1 Score"
+    inputs: list[str] = ["tp", "fp", "fn"]
+    ci_eligible: bool = True
+
+    def data_call(self, data: pd.DataFrame, eps: float = 1e-8) -> float:
+        # Recalculate to avoid dependency on ordering of metrics
+        precision = Precision().data_call(data)
+        recall = Recall().data_call(data)
+        return 2 * (precision * recall) / (precision + recall + eps)
+
+
+class AUROC(AuditorMetric):
+    name: str = "auroc"
+    label: str = "AUROC"
+    inputs: list[str] = ["_truth", "_pred"]
+    ci_eligible: bool = True
+
+    def data_call(self, data: pd.DataFrame) -> float:
+        try:
+            return float(roc_auc_score(data["_truth"], data["_pred"]))
+        except ValueError:
+            return 0.0
+
+
+class AUPRC(AuditorMetric):
+    name: str = "auprc"
+    label: str = "AUPRC"
+    inputs: list[str] = ["_truth", "_pred"]
+    ci_eligible: bool = True
+
+    def data_call(self, data: pd.DataFrame) -> float:
+        try:
+            return float(average_precision_score(data["_truth"], data["_pred"]))
+        except ValueError:
+            return 0.0
+
+
+class MatthewsCorrelationCoefficient(AuditorMetric):
+    name: str = "mcc"
+    label: str = "Matthews Correlation Coefficient"
+    inputs: list[str] = ["tp", "tn", "fp", "fn"]
+    ci_eligible: bool = True
+
+    def data_call(self, data: pd.DataFrame, eps: float = 1e-8) -> float:
+        tp = data["tp"].sum()
+        tn = data["tn"].sum()
+        fp = data["fp"].sum()
+        fn = data["fn"].sum()
+
+        numerator = (tp * tn) - (fp * fn)
+        denominator = np.sqrt((tp + fp) * (tp + fn) * (tn + fp) * (tn + fn))
+
+        if denominator == 0:
+            return 0.0
+        return numerator / (denominator + eps)
+
+
+class FBetaScore(AuditorMetric):
+    name: str = "fbeta"
+    label: str = "F-beta Score"
+    inputs: list[str] = ["precision", "recall"]
+    ci_eligible: bool = True
+
+    def __init__(self, beta: float = 1.0):
+        self.beta = beta
+        self.name = f"f{beta:.1f}".replace(".", "_")  # e.g., "f0_5" or "f2_0"
+        self.label = f"F{beta:.1f} Score"
+
+    def data_call(self, data: pd.DataFrame) -> float:
+        precision = Precision().data_call(data)
+        recall = Recall().data_call(data)
+        beta_sq = self.beta**2
+
+        if precision + recall == 0:
+            return 0.0
+
+        return (1 + beta_sq) * (precision * recall) / ((beta_sq * precision) + recall)
+
+
+class TPR(Sensitivity):
+    name: str = "tpr"
+    label: str = "TPR"
+
+
+class TNR(Specificity):
+    name: str = "tnr"
+    label: str = "TNR"
+
+
+class FPR(AuditorMetric):
+    name: str = "fpr"
+    label: str = "FPR"
+    inputs: list[str] = ["fp", "tn"]
+    ci_eligible: bool = True
+
+    def data_call(self, data: pd.DataFrame, eps: float = 1e-8) -> float:
+        n_fp: int = data["fp"].sum()
+        n_tn: int = data["tn"].sum()
+        return n_fp / (n_fp + n_tn + eps)
+
+
+class FNR(AuditorMetric):
+    name: str = "fnr"
+    label: str = "FNR"
+    inputs: list[str] = ["fn", "tp"]
+    ci_eligible: bool = True
+
+    def data_call(self, data: pd.DataFrame, eps: float = 1e-8) -> float:
+        n_fn: int = data["fn"].sum()
+        n_tp: int = data["tp"].sum()
+        return n_fn / (n_fn + n_tp + eps)
+
+
+class nData(AuditorMetric):
+    name: str = "n"
+    label: str = "N"
+    inputs: list[str] = []
+    ci_eligible: bool = False
+
+    def data_call(self, data: pd.DataFrame) -> int:
+        return len(data)
+
+
+class nTP(AuditorMetric):
+    name: str = "n_tp"
+    label: str = "TP"
+    inputs: list[str] = ['tp']
+    ci_eligible: bool = False
+
+    def data_call(self, data: pd.DataFrame) -> int:
+        return data['tp'].sum()
+    
+
+class nTN(AuditorMetric):
+    name: str = "n_tn"
+    label: str = "TN"
+    inputs: list[str] = ['tn']
+    ci_eligible: bool = False
+
+    def data_call(self, data: pd.DataFrame) -> int:
+        return data['tn'].sum()
+  
+
+class nFP(AuditorMetric):
+    name: str = "n_fp"
+    label: str = "FP"
+    inputs: list[str] = ['fp']
+    ci_eligible: bool = False
+
+    def data_call(self, data: pd.DataFrame) -> int:
+        return data['fp'].sum()
+    
+
+class nFN(AuditorMetric):
+    name: str = "n_fn"
+    label: str = "FN"
+    inputs: list[str] = ['fn']
+    ci_eligible: bool = False
+
+    def data_call(self, data: pd.DataFrame) -> int:
+        return data['fn'].sum()
+
+
+class nPositive(AuditorMetric):
+    name: str = "n_pos"
+    label: str = "Pos."
+    inputs: list[str] = ['_truth']
+    ci_eligible: bool = False
+
+    def data_call(self, data: pd.DataFrame) -> int:
+        return (data['_truth'] == 1).astype(int).sum()
+    
+
+class nNegative(AuditorMetric):
+    name: str = "n_neg"
+    label: str = "Neg."
+    inputs: list[str] = ['_truth']
+    ci_eligible: bool = False
+
+    def data_call(self, data: pd.DataFrame) -> int:
+        return (data['_truth'] == 0).astype(int).sum()

model_auditor-0.1.0/model_auditor/plotting/__init__.py

@@ -0,0 +1 @@
+from model_auditor.plotting.plotters import HierarchyPlotter

model_auditor-0.1.0/model_auditor/plotting/plotters.py

@@ -0,0 +1,228 @@
+from typing import Optional, Union, Callable
+import pandas as pd
+
+from model_auditor.schemas import AuditorScore, AuditorOutcome
+from model_auditor.plotting.schemas import Hierarchy, HLevel, HItem, PlotterData
+
+
+class HierarchyPlotter:
+    def __init__(self) -> None:
+        self.features: Optional[Hierarchy] = None  # type: ignore
+        self.data: Optional[pd.DataFrame] = None
+        self.aggregator: Union[str, Callable] = "median"
+
+        self.score: Optional[AuditorScore] = None
+        self.outcome: Optional[AuditorOutcome] = None
+
+    def set_data(self, data: pd.DataFrame) -> None:
+        """Set data for the plotter
+
+        Args:
+            data (pd.DataFrame): Data used to build the plot
+        """
+        self.data = data
+
+    def set_features(self, features: Union[Hierarchy, list[str]]) -> None:
+        """Set the feature hierarchy for the plotter
+
+        Args:
+            features (Union[Hierarchy, list[str]]): Expects a list of strings
+            (column names corresponding to the data provided) or a predefined
+            custom Hierarchy object
+
+        Raises:
+            ValueError: Raised if something other than a list of Hierarchy
+            object was passed
+        """
+        # flat hierarchy
+        if isinstance(features, list):
+            self.features: Hierarchy = Hierarchy()
+            for feature in features:
+                self.features.levels.append(HLevel([HItem(name=feature)]))
+        # complex/custom hierarchy
+        elif isinstance(features, Hierarchy):
+            self.features = features
+        else:
+            raise ValueError(
+                "unrecognized type for features, please pass a list of strings or a predefined Hierarchy() object"
+            )
+
+    def set_aggregator(self, method: Union[str, Callable]) -> None:
+        """Sets the aggregator used to color the plot cells
+
+        Args:
+            method (Union[str, Callable]): Expects a string corresponding to a
+            predefined aggregator for the .agg() pandas method, or a function
+            that takes the score column as a series and outputs some float
+        """
+        self.aggregator = method
+
+    def set_score(
+        self, name: str, label: Optional[str] = None, threshold: Optional[float] = None
+    ) -> None:
+        """Sets the score column used by the plotter
+
+        Args:
+            name (str): Name of the score column
+            label (Optional[str], optional): Label of the score column. Defaults to None
+            (plot will just use the column name).
+            threshold (Optional[float], optional): Threshold to binarize the score column.
+            Defaults to None (currently unused).
+        """
+        self.score = AuditorScore(
+            name=name, label=label if label is not None else name, threshold=threshold
+        )
+
+    def compile(self, container: str) -> PlotterData:
+        """Compiles the data for the plotter based on the defined HierarchyPlotter parameters
+
+        Args:
+            container (str): Name of the plot container trace
+
+        Raises:
+            ValueError: If features have not been set with .set_features() first
+            ValueError: If a score has not been set with .set_score() first
+
+        Returns:
+            PlotterData: Returns the formatted plotter data TODO: wrap this internally
+        """
+        if self.features is None:
+            raise ValueError("Please set features with .set_features() first")
+
+        if self.score is None:
+            raise ValueError("Please set a score variable with .set_score() first!")
+
+        datasource = self._prepare_datasource()
+        data = PlotterData()
+
+        if isinstance(self.aggregator, str):
+            container_agg: float = (
+                datasource[self.score.name].agg(self.aggregator).item()
+            )
+        else:
+            container_agg: float = self.aggregator(datasource)
+
+        data.add(
+            label=container,
+            id=container,
+            parent="",
+            value=len(datasource),
+            color=container_agg,
+        )
+
+        return self._recursive_record(
+            data=data, datasource=datasource, parent_id=container, idx=0
+        )
+
+    def _recursive_record(
+        self, data: PlotterData, datasource: pd.DataFrame, parent_id: str, idx: int
+    ):
+        """Recursive internal function used to compile data for the plotter"""
+        level: HLevel = self.features.levels[idx]
+        # init a list to track valid features for this level
+        level_features: list[HItem] = []
+        for item in level.items:
+            # if the item has no query then include it
+            if item.query is None:
+                level_features.append(item)
+
+            # otherwise, include it if the feature query evaluates to true for the *entire* datasource
+            elif all(datasource.eval(item.query).tolist()): # type: ignore
+                level_features.append(item)
+
+        # if this level has only 1 valid item, consider it the feature
+        if len(level_features) == 1:
+            feature = level_features[0]
+
+        # otherwise, if this level has >1 valid item, concatenate them into a temp derived feature
+        elif len(level_features) > 1:
+            datasource.loc[:, '_temp_feature'] = (
+                datasource[[i.name for i in level_features]]
+                    .apply(lambda row: " & ".join(row.values.astype(str)), axis=1)
+            )
+
+            feature = HItem(name="_temp_feature")
+
+        # if this level has 0 valid items, return
+        else:
+            return data
+            
+        # group the df by the current feature and get its frequency and agg metric
+        assert isinstance(
+            self.score, AuditorScore
+        )  # handled by the wrapper but here for type hinting
+
+        count_dict: dict[str, int] = (
+            datasource.groupby(feature.name, as_index=True, observed=False)[self.score.name]
+            .agg("count")
+            .to_dict()
+        )
+
+        if isinstance(self.aggregator, str):
+            # built-in aggregators
+            agg_dict: dict[str, float] = (
+                datasource.groupby(feature.name, as_index=True, observed=False)[self.score.name]
+                .agg(self.aggregator)
+                .to_dict()
+            )
+        else:
+            # custom aggregators (pass entire df here instead of just the score series)
+            agg_dict: dict = (
+                datasource.groupby(feature.name, as_index=True, observed=False)
+                .apply(self.aggregator)
+                .to_dict()
+            )
+
+        # extract the count dict keys to get the levels for the current feature
+        feature_levels: list[str] = list(count_dict.keys())
+        # format the parent_id + feature levels into trace identifiers
+        id_dict: dict[str, str] = {
+            feature_level: f"{parent_id}${feature_level}"
+            for feature_level in count_dict.keys()
+        }
+
+        for feature_level in feature_levels:
+            # add the current feature data
+            data.add(
+                label=feature_level,
+                id=id_dict[feature_level],
+                parent=parent_id,
+                value=count_dict[feature_level],
+                color=agg_dict[feature_level],
+            )
+
+            # if this isn't the last feature in the stack, get the subset of data for this feature and recurse
+            if idx < (len(self.features.levels) - 1):
+                data = self._recursive_record(
+                    data=data,
+                    datasource=datasource.loc[datasource[feature.name] == feature_level, :].copy(),
+                    parent_id=id_dict[feature_level],
+                    idx=idx + 1,
+                )
+
+        return data
+
+    def _prepare_datasource(self) -> pd.DataFrame:
+        """Internal function used to prepare the datasource for plotting
+
+        Raises:
+            ValueError: If data has not been added with .set_data() first
+
+        Returns:
+            pd.DataFrame: Prepared data source
+        """
+        if self.data is None:
+            raise ValueError("Please set data with .set_data() first")
+
+        data = self.data.copy()
+        if self.score is not None:
+            data["_pred"] = self.score.name
+        else:
+            print("no score set")
+
+        if self.outcome is not None:
+            data["_outcome"] = self.outcome.name
+        else:
+            print("no outcome set")
+
+        return data

model_auditor-0.1.0/model_auditor/plotting/schemas.py

@@ -0,0 +1,40 @@
+from typing import Optional
+from dataclasses import dataclass, field
+
+
+@dataclass
+class PlotterData:
+    labels: list = field(default_factory=list)
+    ids: list = field(default_factory=list)
+    parents: list = field(default_factory=list)
+    values: list = field(default_factory=list)
+    colors: list = field(default_factory=list)
+
+    def add(self, label: str, id: str, parent: str, value: int, color: float) -> None:
+        self.labels.append(label)
+        self.ids.append(id)
+        self.parents.append(parent)
+        self.values.append(value)
+        self.colors.append(color)
+
+
+@dataclass
+class HItem:
+    """Hierarchy Item"""
+
+    name: str
+    query: Optional[str] = None
+
+
+@dataclass
+class HLevel:
+    """Hierarchy level"""
+
+    items: list[HItem] = field(default_factory=list)
+
+
+@dataclass
+class Hierarchy:
+    """Hierarchy container"""
+
+    levels: list[HLevel] = field(default_factory=list)

model_auditor-0.1.0/model_auditor/schemas.py

@@ -0,0 +1,160 @@
+from typing import Optional, Union
+from dataclasses import dataclass, field
+
+import pandas as pd
+
+
+@dataclass
+class LevelMetric:
+    """
+    Object to store the evaluation results for one metric of one level of a feature.
+    (for example, AUC for one category of finding)
+
+    Args:
+        name (str): Name of the current feature level metric
+        score (Union[float, int]): Score for the current feature level metric
+        interval (tuple[float, float], optional): Optional lower and upper confidence
+        bounds for the current feature level metric (defaults to None)
+    """
+
+    name: str
+    label: str
+    score: Union[float, int]
+    interval: Optional[tuple[float, float]] = None
+
+
+@dataclass
+class LevelEvaluation:
+    """
+    Object to store the evaluation results for one level of a feature
+    (for example, all metrics for one category of finding).
+
+    Args:
+        name (str): Name of the current feature level
+        metrics (dict[str, LevelMetric]): Metrics for the current feature level
+        (defaults to an empty dict)
+    """
+
+    name: str
+    metrics: dict[str, LevelMetric] = field(default_factory=dict)
+
+    def update(self, metric_name: str, metric_label: str, metric_score: float) -> None:
+        self.metrics[metric_name] = LevelMetric(
+            name=metric_name, label=metric_label, score=metric_score
+        )
+
+    def update_intervals(self, metric_intervals: dict[str, tuple[float, float]]):
+        for metric_name, confidence_interval in metric_intervals.items():
+            self.metrics[metric_name].interval = confidence_interval
+
+    def to_dataframe(self, n_decimals: int = 3, add_index: bool = False, metric_labels: bool = False):
+        metric_data: dict[str, str] = dict()
+        for metric in self.metrics.values():
+            # get the key name for the current metric (label if metric_labels is True)
+            metric_key: str = metric.label if metric_labels else metric.name
+            
+            if metric.interval is not None:
+                metric_data[metric_key] = (
+                    f"{metric.score:.{n_decimals}f} ({metric.interval[0]:.{n_decimals}f}, {metric.interval[1]:.{n_decimals}f})"
+                )
+            elif isinstance(metric.score, float):
+                metric_data[metric_key] = f"{metric.score:.{n_decimals}f}"
+            else:
+                # integer scores (default to comma delimited for now)
+                metric_data[metric_key] = f"{metric.score:,}"
+
+        return pd.DataFrame(metric_data, index=[self.name])
+
+
+@dataclass
+class FeatureEvaluation:
+    """
+    Object to store the evaluation results for one feature type
+    (for example, metrics associated with different types of findings)
+
+    Args:
+        name (str): Name of the current feature
+        name (str): Label for the current feature
+        levels (dict[str, LevelEvaluation]): Levels of the current feature
+        (defaults to an empty dict)
+    """
+
+    name: str
+    label: str
+    levels: dict[str, LevelEvaluation] = field(default_factory=dict)
+
+    def update(
+        self, metric_name: str, metric_label: str, data: dict[str, float]
+    ) -> None:
+        # expects a dict for one metric type: {'levelA': 0.5, 'levelB': 0.5}
+        # and maps them to child level metric dicts
+        for level_name, level_metric in data.items():
+            # try to get the level item and instantiate a new one if it doesn't exist yet
+            level_eval: LevelEvaluation = self.levels.get(
+                level_name, LevelEvaluation(name=level_name)
+            )
+            # update the metrics for that level eval object and save it back to the dict
+            level_eval.update(
+                metric_name=metric_name,
+                metric_label=metric_label,
+                metric_score=level_metric,
+            )
+            self.levels[level_name] = level_eval
+
+    def update_intervals(
+        self, level_name: str, metric_intervals: dict[str, tuple[float, float]]
+    ):
+        self.levels[level_name].update_intervals(metric_intervals=metric_intervals)
+
+    def to_dataframe(
+        self, n_decimals: int = 3, add_index: bool = False, metric_labels: bool = False
+    ) -> pd.DataFrame:
+        data: list[pd.DataFrame] = []
+        for level_data in self.levels.values():
+            data.append(level_data.to_dataframe(n_decimals=n_decimals, metric_labels=metric_labels))
+
+        if add_index:
+            return pd.concat({self.label: pd.concat(data, axis=0)})
+        else:
+            return pd.concat(data, axis=0)
+
+
+@dataclass
+class ScoreEvaluation:
+    name: str
+    label: str
+    features: dict[str, FeatureEvaluation] = field(default_factory=dict)
+
+    def to_dataframe(
+        self, n_decimals: int = 3, add_index: bool = False, metric_labels: bool = False
+    ) -> pd.DataFrame:
+        data: list[pd.DataFrame] = []
+        for feature_data in self.features.values():
+            data.append(
+                feature_data.to_dataframe(n_decimals=n_decimals, add_index=True, metric_labels=metric_labels)
+            )
+
+        if add_index:
+            return pd.concat({self.label: pd.concat(data, axis=0)})
+        else:
+            return pd.concat(data, axis=0)
+
+
+@dataclass
+class AuditorFeature:
+    name: str
+    label: Optional[str] = None
+    levels: Optional[list[any]] = None
+
+
+@dataclass
+class AuditorScore:
+    name: str
+    label: Optional[str] = None
+    threshold: Optional[float] = None
+
+
+@dataclass
+class AuditorOutcome:
+    name: str
+    mapping: Optional[dict[any, int]] = None

model_auditor-0.1.0/model_auditor/utils.py

@@ -0,0 +1,27 @@
+import importlib
+import inspect
+from typing import Type
+from model_auditor.metric_inputs import AuditorMetricInput
+
+
+def is_metric_input_valid(cls: type) -> bool:
+    return (
+        inspect.isclass(cls)
+        and hasattr(cls, "name")
+        and hasattr(cls, "label")
+        and hasattr(cls, "inputs")
+        and callable(getattr(cls, "row_call", None))
+        and callable(getattr(cls, "data_transform", None))
+    )
+
+
+def collect_metric_inputs() -> dict[str, Type[AuditorMetricInput]]:
+    module = importlib.import_module("model_auditor.metric_inputs")
+
+    input_classes = {
+        cls.name: cls
+        for _, cls in inspect.getmembers(module, inspect.isclass)
+        if is_metric_input_valid(cls) and cls is not AuditorMetricInput
+    }
+
+    return input_classes

model_auditor-0.1.0/model_auditor.egg-info/PKG-INFO

@@ -0,0 +1,6 @@
+Metadata-Version: 2.4
+Name: model-auditor
+Version: 0.1.0
+Requires-Dist: pandas>=2.2
+Requires-Dist: numpy>=2.1
+Requires-Dist: scikit-learn>=1.5

model_auditor-0.1.0/model_auditor.egg-info/SOURCES.txt

@@ -0,0 +1,18 @@
+.gitignore
+README.md
+pyproject.toml
+.github/workflows/publish.yml
+model_auditor/__init__.py
+model_auditor/core.py
+model_auditor/metric_inputs.py
+model_auditor/metrics.py
+model_auditor/schemas.py
+model_auditor/utils.py
+model_auditor.egg-info/PKG-INFO
+model_auditor.egg-info/SOURCES.txt
+model_auditor.egg-info/dependency_links.txt
+model_auditor.egg-info/requires.txt
+model_auditor.egg-info/top_level.txt
+model_auditor/plotting/__init__.py
+model_auditor/plotting/plotters.py
+model_auditor/plotting/schemas.py

model_auditor-0.1.0/model_auditor.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

model_auditor-0.1.0/model_auditor.egg-info/requires.txt

@@ -0,0 +1,3 @@
+pandas>=2.2
+numpy>=2.1
+scikit-learn>=1.5

model_auditor-0.1.0/model_auditor.egg-info/top_level.txt

@@ -0,0 +1 @@
+model_auditor

model_auditor-0.1.0/pyproject.toml

@@ -0,0 +1,16 @@
+# allows the package to be installed in editable mode
+[build-system]
+requires = ["setuptools>=64", "setuptools_scm>=8", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "model-auditor"
+dynamic = ["version"]
+
+dependencies = [
+    "pandas >= 2.2",
+    "numpy >= 2.1",
+    "scikit-learn >= 1.5",
+]
+
+[tool.setuptools_scm]

model_auditor-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+