concordiahub 0.2.1__py3-none-any.whl

concordiahub/__init__.py

@@ -0,0 +1,6 @@
+from concordiahub.registry.autodiscover import autodiscover_adapters
+from concordiahub.entities.context import ModelContext, ModelData, DataSchema, ResponseSchema
+from concordiahub.runner import run
+
+# Register on package import
+autodiscover_adapters()

concordiahub/_version.py

@@ -0,0 +1,2 @@
+"""Package version identifier."""
+__version__ = "0.2.1"

concordiahub/adapters/explainers/__init__.py

@@ -0,0 +1 @@
+

concordiahub/adapters/explainers/coefficients_adapter.py

@@ -0,0 +1,90 @@
+"""
+CoefficientsAdapter class - for model with native interpretability
+"""
+
+import pandas as pd
+import numpy as np
+
+from typing import Any, Optional, Protocol
+
+from concordiahub.exceptions import MissingDataError
+from concordiahub.registry.xai_registry import register_xai
+from concordiahub.adapters.explainers.xaimethod_base import XAIMethodAdapter
+
+from concordiahub import strings, config
+from concordiahub.entities.context import ModelContext
+
+class HasCoefficients(Protocol):
+    """
+    Helper class for type hinting - the model used for the CoefficientsAdapter
+    must have a "coef_" attribute
+    """
+    coef_: np.ndarray
+
+@register_xai("coefficients")
+class CoefficientsAdapter(XAIMethodAdapter):
+    """For models with native interpretability"""
+
+    display_name = strings.COEFFICIENTS_DISPLAY_NAME
+    display_notes = strings.COEFFICIENTS_NOTES
+    default_visualization = config.COEFFS_DEFAULT_VIZ
+    compatible_visualizations = config.COEFFS_COMPATIBLE_VIZ
+
+
+    def explain_local_df(self, ctx: ModelContext, model: Optional[HasCoefficients] = None ) -> pd.DataFrame:
+        """Computes the local explanation from the coefficients of the model
+
+        Args:
+            ctx (ModelContext) : ModelContext object
+            model (Optional[HasCoefficients]): model exposing a coef_ attribute
+
+        Returns:
+            pd.DataFrame: dataframe containing local explanations
+
+        Raises:
+            MissingDataError: if data.x is not provided
+        """
+        if ctx.data.x_test is None:
+            raise MissingDataError(strings.ERROR_MISSING_MODELDATA.format(method="cls.key", data="x_test"))
+        coef = model.coef_[0]
+        instance_id = ctx.data.data_schema.instance_id_column
+        x_test = ctx.data.x_test.drop(columns=instance_id)
+        local_explanations = x_test * coef
+        local_explanations.insert(0, instance_id, ctx.data.x_test[instance_id].values)
+        return local_explanations
+
+    def explain_global_df(self, ctx: ModelContext, model: Optional[HasCoefficients] = None) -> pd.DataFrame:
+        """
+        Computes the global explanation from the coefficients of the model
+
+        Args:
+            ctx (ModelContext): ModelContext object
+            model (Optional[HasCoefficients]): model exposing a coef_ attribute
+
+        Returns:
+            pd.DataFrame: dataframe containing global explanation, with two columns, feature and importance
+        """
+
+        coef = model.coef_[0]
+        feature_names = ctx.data.x_test.drop(columns=ctx.data.data_schema.instance_id_column).columns
+        importance = pd.DataFrame({
+            "feature": feature_names,
+            "importance": coef,
+        })
+        return importance
+
+
+    def explain_local_json(self, ctx: ModelContext) -> dict[str, Any]:
+        """
+        Not implemented for CoefficientsAdapter
+
+        Args:
+            ctx (ModelContext): ModelContext object
+
+        Returns:
+            dict[str, Any]:
+
+        Raises:
+            NotImplementedError:
+        """
+        raise NotImplementedError

concordiahub/adapters/explainers/llm_reasoning_adapter.py

@@ -0,0 +1,325 @@
+"""
+LLMReasoningAdapter class - for processing explanations from LLMs
+"""
+
+import json
+
+import pandas as pd
+from typing import Any, Optional
+from dataclasses import dataclass
+
+from concordiahub.entities.context import ModelContext, ResponseSchema
+from concordiahub.exceptions import MissingDataError
+from concordiahub.registry.xai_registry import register_xai
+from concordiahub.adapters.explainers.xaimethod_base import XAIMethodAdapter
+from concordiahub import strings, config
+from concordiahub.entities.model import Model
+from concordiahub.config import TreeModel
+
+@dataclass
+class ContribScore:
+    """
+    Dataclass for a score and reasoning contribution
+    """
+    score: int
+    reasoning: str
+
+@dataclass
+class LLMResponse:
+    """
+    Dataclass for a response from LLMs
+    """
+    reasoning: str
+    prediction: int | str
+    confidence: float
+    contributions: dict[str, ContribScore]
+    raw: dict
+
+    def to_dict(self, instance_id_col: str, instance_id: str) -> dict:
+        """
+        Formats the response as a dictionary
+
+        Args:
+            instance_id_col (str): name of the instance id column
+            instance_id (str): the instance id
+
+        Returns:
+            dict: formatted to a common structure
+        """
+        return {
+            instance_id_col: instance_id,
+            "reasoning": self.reasoning,
+            "prediction": self.prediction,
+            "confidence": self.confidence,
+            "contributions": {
+                feature: {"score": c.score, "reasoning": c.reasoning}
+                for feature, c in self.contributions.items()
+            }
+        }
+
+@register_xai("llm_reasoning")
+class LLMReasoningAdapter(XAIMethodAdapter):
+    """
+    LLMReasoningAdapter class - for processing local and global explanations
+    from LLMs
+    """
+
+    display_name = strings.LLM_REASONING_DISPLAY_NAME
+    display_notes = strings.LLM_REASONING_NOTES
+    default_visualization = config.LLM_REASONING_DEFAULT_VIZ
+    compatible_visualizations = config.LLM_REASONING_COMPATIBLE_VIZ
+
+    # uses a json file for explanation storage
+    explanation_file_type: str = "json"
+
+    @classmethod
+    def load_global_explanation(cls, model: Model) -> pd.DataFrame:
+        """
+        Returns the global explanation for the specified model
+
+        Args:
+            model (Model): a Model object to get the explanation for
+
+        Returns:
+            pd.DataFrame: dataframe containing the global explanation, with the
+                following columns: feature_name, score, score_normalized_max
+        """
+        df = model.store.load_dataframe(f"exp_global_{cls.key}")
+        abs_importance = df["importance"].abs()
+        abs_max = abs_importance.max()
+        return pd.DataFrame({
+            "feature_name": df["feature"],
+            "score": df["importance"],
+            "score_normalized_max": abs_importance / abs_max if abs_max != 0 else abs_importance,
+        })
+
+    @classmethod
+    def load_local_explanation(cls,
+                               model: Model,
+                               instance_id: str) -> pd.DataFrame:
+        """
+        Returns the local explanation for a given instance
+
+        Args:
+            model (Model): a Model object
+            instance_id (str): identifier for the instance to get the local
+                explanation for
+
+        Returns:
+            pd.DataFrame: dataframe containing the local explanation, with the
+                following columns: feature_name, score, score_normalized_sum,
+                score_normalized_max, reasoning, overall_reasoning,
+                overall_confidence
+
+        Raises:
+            ValueError: if instance_id is not provided
+        """
+        records = model.store.load_json(f"exp_local_{cls.key}")
+        instance_id_col = model.get(config.DATA_SCHEMA_ID_COLUMN)
+
+        row = next((r for r in records if r[instance_id_col] == instance_id),
+                   None)
+        if row is None:
+            raise MissingDataError(strings.ERROR_INSTANCE_NOT_FOUND.format(
+                instance_id=instance_id, key=cls.key))
+
+
+        # make sure data[score] treaded as a proper float value
+        contributions = list(row["contributions"].items())
+        scores = [float(data["score"]) for _, data in contributions]
+
+        abs_max = max(abs(s) for s in scores)
+
+        overall_reasoning = row["reasoning"]
+        overall_confidence = row["confidence"]
+
+        return pd.DataFrame([
+            {
+                "feature_name": feature,
+                "score": score,
+                "score_normalized_max": score / abs_max if abs_max != 0 else score,
+                "reasoning": data["reasoning"],
+                "overall_reasoning": overall_reasoning,
+                "overall_confidence": overall_confidence
+            }
+            for (feature, data), score in zip(contributions, scores)
+        ])
+
+    def explain_global_json(self,
+                            ctx: ModelContext,
+                            _model: Optional[TreeModel] = None) -> pd.DataFrame:
+        """Generates the global explanations
+
+        Importance is the mean absolute contribution score across all
+        instances: each feature's absolute scores are summed and divided by
+        the total instance count, so features not named in an instance count
+        as zero. This matches the aggregation SHAP applies to its own values.
+        Occurrence counts are returned alongside but do not affect importance.
+
+        Args:
+            ctx (ModelContext) : ModelContext object
+            _model (Optional[TreeModel]) : Unused
+
+        Returns:
+            pd.DataFrame: dataframe containing global explanations, with the
+                following columns: feature, importance, occurrence_count,
+                occurrence_rate, n_instances
+
+        Raises:
+            MissingDataError: if data.y_exp is not provided
+        """
+        if ctx.data.y_exp is None:
+            raise MissingDataError(
+                "LLMReasoningAdapter requires y_exp in ModelData")
+        if ctx.response_schema is None:
+            raise MissingDataError(
+                "LLMReasoningAdapter requires a ResponseSchema in ModelContext")
+
+        results = {}
+        n_instances = 0
+
+        for _, row in ctx.data.y_exp.iterrows():
+            raw = json.loads(row["explanation"])
+            n_instances += 1
+
+            response = self._reformat(raw, ctx.response_schema, prediction=None)
+
+            for feature, contrib in response.contributions.items():
+                if feature not in results:
+                    results[feature] = {"sum_abs_score": 0,
+                                        "occurrence_count": 0}
+                results[feature]["sum_abs_score"] += abs(float(contrib.score))
+                results[feature]["occurrence_count"] += 1
+
+        rows = []
+        for feature, vals in results.items():
+            rows.append({
+                "feature": feature,
+                "importance": vals["sum_abs_score"] / n_instances,
+                "occurrence_count": vals["occurrence_count"],
+                "occurrence_rate": vals["occurrence_count"] / n_instances,
+                "n_instances": n_instances,
+            })
+
+        return pd.DataFrame(rows)
+
+
+    def explain_global_df(self,
+                          ctx: ModelContext,
+                          model: Optional[TreeModel] = None) -> pd.DataFrame:
+        """
+        Function is not implemented for this class instance
+
+        Args:
+            ctx (ModelContext): unused
+            model (Optional[TreeModel]): unused
+
+        Returns:
+            pd.DataFrame:
+
+        Raises:
+            NotImplementedError: always returns this as not implemented
+
+        """
+        raise NotImplementedError
+
+    def explain_local_df(self,
+                         ctx: ModelContext,
+                         model: Optional[TreeModel] = None) -> pd.DataFrame:
+        """
+        Function is not implemented for this class instance
+
+        Args:
+            ctx (ModelContext): unused
+            model (Optional[TreeModel]): unused
+
+        Returns:
+            pd.DataFrame:
+
+        Raises:
+            NotImplementedError: always returns this as not implemented
+
+        """
+        raise NotImplementedError
+
+    def explain_local_json(self,
+                           ctx: ModelContext) -> list[dict[str, Any]]:
+        """
+        Processes the local explanations
+
+        Args:
+            ctx (ModelContext): ModelContext object
+
+        Returns:
+            list[dict[str, Any]]: list of dicts containing local explanations
+
+        Raises:
+            MissingDataError: if data.x is not provided
+        """
+        if ctx.data.y_exp is None or ctx.data.y_pred is None:
+            raise MissingDataError(strings.ERROR_MISSING_MODELDATA.format(
+                method="cls.key", data="y_exp, y_pred"))
+        if ctx.response_schema is None:
+            raise MissingDataError(
+                "LLMReasoningAdapter requires a ResponseSchema in ModelContext")
+
+        instance_id_col = ctx.data.data_schema.instance_id_column
+        prediction_col = ctx.data.data_schema.prediction_column
+        y_pred = ctx.data.y_pred
+        output = []
+
+        for _, row in ctx.data.y_exp.iterrows():
+            raw = json.loads(row["explanation"])
+            instance_id = row[instance_id_col]
+            match = y_pred.loc[y_pred[instance_id_col] == instance_id, prediction_col]
+            # if a prediction for particular instance is missing, then just continue
+            if match.empty:
+                continue
+            prediction = match.iloc[0].item() if hasattr(
+                match.iloc[0], 'item') else match.iloc[0]
+            response = self._reformat(raw, ctx.response_schema, prediction)
+            output.append(response.to_dict(instance_id_col, instance_id))
+        return output
+
+    @staticmethod
+    def _reformat(raw: dict,
+                  schema: ResponseSchema,
+                  prediction: str | None) -> LLMResponse:
+        """
+        Private helper function to reformat an explanation in raw dict format
+        (from JSON) into an LLMResponse object.
+        This uses the DataSchema definition to process the specific dict
+        structure into a common LLMResponse object structure.
+
+        Args:
+            raw (dict): dict
+            schema (ResponseSchema): response schema:
+            prediction (str|None): prediction, if defined
+
+        Returns:
+            LLMResponse:
+        """
+        raw_contribs = raw[schema.contributions_field]
+        if schema.contribution_key is not None:
+            contributions = {
+                item[schema.contribution_key]: ContribScore(
+                    score=item["score"],
+                    reasoning=item["reasoning"],
+                )
+                for item in raw_contribs
+            }
+        else:
+            contributions = {
+                feature: ContribScore(
+                    score=values["score"],
+                    reasoning=values.get("reasoning", ""),
+                )
+                for feature, values in raw_contribs.items()
+            }
+        return LLMResponse(
+            reasoning=raw["reasoning"],
+            prediction=prediction,
+            confidence=raw["confidence"],
+            contributions=contributions,
+            raw=raw,
+        )

concordiahub/adapters/explainers/treeshap_adapter.py

@@ -0,0 +1,155 @@
+"""
+TreeSHAPAdapter class - for processing SHAP explanations for tree based models
+"""
+
+import shap
+import pandas as pd
+from typing import Any, Optional
+
+from concordiahub.exceptions import MissingDataError
+from concordiahub.entities.context import ModelContext
+from concordiahub.registry.xai_registry import register_xai
+from concordiahub.adapters.explainers.xaimethod_base import XAIMethodAdapter
+from concordiahub import strings, config
+from concordiahub.entities.model import Model
+from concordiahub.config import TreeModel
+
+
+@register_xai("tree_shap")
+class TreeSHAPAdapter(XAIMethodAdapter):
+    """
+    Tree SHAP Explainer
+    """
+
+    display_name = strings.TREE_SHAP_DISPLAY_NAME
+    display_notes = strings.TREE_SHAP_NOTES
+    default_visualization = config.TREE_SHAP_DEFAULT_VIZ
+    compatible_visualizations = config.TREE_SHAP_COMPATIBLE_VIZ
+
+    # SHAP adds a "class" column, so this needs to be removed
+    EXCLUDED_CONTRIBUTION_KEYS = {"class"}
+
+    @classmethod
+    def load_local_explanation(cls, model: Model, instance_id: str) -> pd.DataFrame:
+        """
+        Loads local explanation for given instance_id
+
+        Args:
+            model (Model): Model object
+            instance_id (str): Instance to get the explanation for
+
+        Returns:
+            pd.DataFrame: dataframe containing local explanations
+        """
+
+        df = super().load_local_explanation(model, instance_id)
+        return df[~df["feature_name"].isin(cls.EXCLUDED_CONTRIBUTION_KEYS)]
+
+    def explain_global_df(self, ctx: ModelContext, model: Optional[TreeModel] = None) -> pd.DataFrame:
+        """
+        Computes the global explanation from SHAP values
+
+        Args:
+            ctx (ModelContext): ModelContext object
+            model (Optional[TreeModel]): trained tree-based model object
+
+        Returns:
+            pd.DataFrame: dataframe containing global explanations
+
+        Raises:
+            MissingDataError: if data.x_test is not provided
+            DataProcessingError: if errors in processing SHAP values
+        """
+        if not hasattr(ctx.data, "x_test") or ctx.data.x_test is None:
+            raise MissingDataError(strings.ERROR_MISSING_MODELDATA.format(
+                method="cls.key", data="x_test"))
+        instance_id = ctx.data.data_schema.instance_id_column
+        data = ctx.data.x_test.copy()
+        data = data.drop(columns=[instance_id]).select_dtypes(include=['number'])
+        explainer = shap.TreeExplainer(model)
+        shap_values = explainer.shap_values(data)
+
+        if shap_values.ndim == 2:
+            importance = pd.DataFrame(shap_values, columns=data.columns).abs().mean(axis=0)
+            result = importance.reset_index()
+            result.columns = ['feature', 'importance']
+            return result
+
+        else:
+            n_classes = shap_values.shape[2]
+            dfs = []
+            for i in range(n_classes):
+                importance = pd.DataFrame(shap_values[:, :, i], columns=data.columns).abs().mean(axis=0)
+                df = importance.reset_index()
+                df.columns = ['feature', 'importance']
+                dfs.append(df)
+            return pd.concat(dfs, ignore_index=True)
+
+
+    def explain_local_df(self, ctx: ModelContext, model: Optional[TreeModel] = None) -> pd.DataFrame:
+        """
+        Computes the local explanations from SHAP values of all the instances in the test set
+
+        Args:
+            ctx (ModelContext): ModelContext object
+            model (Optional[TreeModel]): trained tree-based model object
+
+        Returns:
+            pd.DataFrame: dataframe containing local explanations
+
+        Raises:
+            MissingDataError: if data.x_test is not provided
+            DataProcessingError: if errors in processing SHAP values
+        """
+
+        if not hasattr(ctx.data, "x_test") or ctx.data.x_test is None:
+            raise MissingDataError(strings.ERROR_MISSING_MODELDATA.format(
+                method="cls.key", data="x_test"))
+
+        instance_id = ctx.data.data_schema.instance_id_column
+        data = ctx.data.x_test.copy()
+        uuids = data[instance_id].values
+        data = data.drop(columns=[instance_id]).select_dtypes(include=['number'])
+
+        explainer = shap.TreeExplainer(model)
+        shap_values = explainer.shap_values(data)
+
+        # (n_instances, 1, n_features)
+        if shap_values.ndim == 2:
+            result = pd.DataFrame(shap_values, columns=data.columns)
+            result.insert(0, instance_id, uuids)
+            result['class'] = 0
+
+        # SHAP returns (n_instances, n_features, n_classes) for multiclass tree models
+        # slice last axis per class to get (n_instances, n_features) per DataFrame
+        else:
+            n_classes = shap_values.shape[2]
+            dfs = []
+            keys = ctx.data.data_schema.class_names if ctx.data.data_schema.class_names is not None else range(
+                n_classes)
+            for i in range(n_classes):
+                df = pd.DataFrame(shap_values[:, :, i], columns=data.columns)
+                df.insert(0, instance_id, uuids)
+                df['class'] = keys[i]
+                dfs.append(df)
+            result = pd.concat(dfs, ignore_index=True)
+
+
+        # filter by abs value
+        return result
+
+
+    def explain_local_json(self, ctx: ModelContext) -> dict[str, Any]:
+        """
+        Not implemented for TreeSHAPAdapter
+
+        Args:
+            ctx (ModelContext): ModelContext object
+
+        Returns:
+            dict[str, Any]:
+
+        Raises:
+            NotImplementedError: will always return this as not implemented
+        """
+        raise NotImplementedError