erroranalysis 0.4.2__tar.gz → 0.4.4__tar.gz

{erroranalysis-0.4.2/erroranalysis.egg-info → erroranalysis-0.4.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: erroranalysis
-Version: 0.4.2
+Version: 0.4.4
 Summary: Core error analysis APIs
 Home-page: https://github.com/microsoft/responsible-ai-widgets
 Author: Roman Lutz, Ilya Matiach, Ke Xu
@@ -9,6 +9,7 @@ Classifier: Programming Language :: Python :: 3.6
 Classifier: Programming Language :: Python :: 3.7
 Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
 Classifier: Development Status :: 3 - Alpha
@@ -18,7 +19,7 @@ License-File: LICENSE
 
 # Error Analysis SDK for Python
 
-### This package has been tested with Python 3.6, 3.7, 3.8 and 3.9
+### This package has been tested with Python 3.6, 3.7, 3.8, 3.9 and 3.10
 
 The error analysis sdk enables users to get a deeper understanding of machine learning model errors. When evaluating a machine learning model, aggregate accuracy is not sufficient and single-score evaluation may hide important conditions of inaccuracies. Use Error Analysis to identify cohorts with higher error rates and diagnose the root causes behind these errors.
 

{erroranalysis-0.4.2 → erroranalysis-0.4.4}/README.md

@@ -1,6 +1,6 @@
 # Error Analysis SDK for Python
 
-### This package has been tested with Python 3.6, 3.7, 3.8 and 3.9
+### This package has been tested with Python 3.6, 3.7, 3.8, 3.9 and 3.10
 
 The error analysis sdk enables users to get a deeper understanding of machine learning model errors. When evaluating a machine learning model, aggregate accuracy is not sufficient and single-score evaluation may hide important conditions of inaccuracies. Use Error Analysis to identify cohorts with higher error rates and diagnose the root causes behind these errors.
 

{erroranalysis-0.4.2 → erroranalysis-0.4.4}/erroranalysis/_internal/constants.py

@@ -96,6 +96,9 @@ class Metrics(str, Enum):
     FALSE_POSITIVE_RATE = 'false_positive_rate'
     FALSE_NEGATIVE_RATE = 'false_negative_rate'
     SELECTION_RATE = 'selection_rate'
+    MEAN_AVERAGE_PRECISION = 'mean_average_precision'
+    AVERAGE_PRECISION = 'average_precision'
+    AVERAGE_RECALL = 'average_recall'
 
 
 class MetricKeys(str, Enum):
@@ -121,6 +124,16 @@ class TreeNode(str, Enum):
     METRIC_VALUE = MetricKeys.METRIC_VALUE.value
 
 
+class ErrorCorrelationMethods(str, Enum):
+    """Provide the supported error correlation methods.
+
+    The supported methods are 'mutual_info', 'ebm' and 'gbm_shap'.
+    """
+    MUTUAL_INFO = 'mutual_info'
+    EBM = 'ebm'
+    GBM_SHAP = 'gbm_shap'
+
+
 metric_to_display_name = {
     Metrics.ACCURACY_SCORE: 'Accuracy score',
     Metrics.MEAN_PREDICTION: 'Mean prediction',

{erroranalysis-0.4.2 → erroranalysis-0.4.4}/erroranalysis/_internal/surrogate_error_tree.py

@@ -292,7 +292,6 @@ def get_surrogate_booster_local(filtered_df, analyzer, is_model_analyzer,
         scored dataset.
     :rtype: (Booster, pandas.DataFrame, (list[str], list[int]))
     """
-    row_index = filtered_df[ROW_INDEX]
     true_y = filtered_df[TRUE_Y]
     dropped_cols = [TRUE_Y, ROW_INDEX]
     if not is_model_analyzer:
@@ -303,7 +302,7 @@ def get_surrogate_booster_local(filtered_df, analyzer, is_model_analyzer,
     if is_pandas:
         true_y = true_y.to_numpy()
     else:
-        input_data = input_data.to_numpy()
+        input_data = input_data.to_numpy(copy=True)
     if is_model_analyzer:
         pred_y = analyzer.model.predict(input_data)
     if analyzer.model_task == ModelTask.CLASSIFICATION:
@@ -317,7 +316,7 @@ def get_surrogate_booster_local(filtered_df, analyzer, is_model_analyzer,
     if not isinstance(true_y, np.ndarray):
         true_y = np.array(true_y)
     if is_pandas:
-        input_data = input_data.to_numpy()
+        input_data = input_data.to_numpy(copy=True)
 
     if analyzer.categorical_features:
         # Inplace replacement of columns
@@ -330,7 +329,7 @@ def get_surrogate_booster_local(filtered_df, analyzer, is_model_analyzer,
         else:
             string_indexed_data = analyzer.string_indexed_data
         for idx, c_i in enumerate(analyzer.categorical_indexes):
-            input_data[:, c_i] = string_indexed_data[row_index, idx]
+            input_data[:, c_i] = string_indexed_data[:, idx]
     dataset_sub_features = input_data[:, indexes]
 
     categorical_info = get_categorical_info(analyzer,
@@ -605,7 +604,7 @@ def filter_to_used_features(df, tree):
     """
     features = tree[CACHED_SUBTREE_FEATURES]
     features = features.union({PRED_Y, TRUE_Y, DIFF})
-    return df[features]
+    return df[list(features)]
 
 
 def cache_subtree_features(tree, feature_names, parent=None):

{erroranalysis-0.4.2 → erroranalysis-0.4.4}/erroranalysis/analyzer/error_analyzer.py

@@ -10,7 +10,8 @@ import pandas as pd
 from sklearn.feature_selection import (mutual_info_classif,
                                        mutual_info_regression)
 
-from erroranalysis._internal.constants import (MatrixParams, Metrics,
+from erroranalysis._internal.constants import (ErrorCorrelationMethods,
+                                               MatrixParams, Metrics,
                                                ModelTask, RootKeys,
                                                metric_to_display_name)
 from erroranalysis._internal.matrix_filter import \
@@ -25,11 +26,14 @@ from erroranalysis._internal.surrogate_error_tree import \
     compute_error_tree_on_dataset as _compute_error_tree_on_dataset
 from erroranalysis._internal.utils import generate_random_unique_indexes
 from erroranalysis._internal.version_checker import check_pandas_version
+from erroranalysis.error_correlation_methods import (
+    compute_ebm_global_importance, compute_gbm_global_importance)
 from erroranalysis.report import ErrorReport
 
 BIN_THRESHOLD = MatrixParams.BIN_THRESHOLD
 IMPORTANCES_THRESHOLD = 50000
 ROOT_COVERAGE = 100
+MUTUAL_INFO = ErrorCorrelationMethods.MUTUAL_INFO.value
 
 
 class BaseAnalyzer(ABC):
@@ -372,7 +376,8 @@ class BaseAnalyzer(ABC):
                             num_leaves=None,
                             min_child_samples=None,
                             compute_importances=False,
-                            compute_root_stats=False):
+                            compute_root_stats=False,
+                            error_correlation_method=MUTUAL_INFO):
         """Creates the error analysis ErrorReport.
 
         The ErrorReport contains the importances, heatmap and tree view json.
@@ -409,7 +414,7 @@ class BaseAnalyzer(ABC):
                                          None)
         importances = None
         if compute_importances:
-            importances = self.compute_importances()
+            importances = self.compute_importances(error_correlation_method)
         root_stats = None
         if compute_root_stats:
             root_stats = self.compute_root_stats()
@@ -419,7 +424,7 @@ class BaseAnalyzer(ABC):
                            importances=importances,
                            root_stats=root_stats)
 
-    def compute_importances(self):
+    def compute_importances(self, error_correlation_method=MUTUAL_INFO):
         """Compute the importances or correlation between features and error.
 
         Computes the feature importances or the correlation between
@@ -428,6 +433,8 @@ class BaseAnalyzer(ABC):
         uses the scikit-learn methods mutual_info_classif for classification
         and mutual_info_regression for regression tasks.
 
+        :param error_correlation_method: Method to compute error correlation.
+        :type error_correlation_method: str
         :return: The computed importances or correlation between the
             features and error.
         :rtype: list[float]
@@ -435,7 +442,7 @@ class BaseAnalyzer(ABC):
         input_data = self.dataset
         diff = self.get_diff()
         if isinstance(self.dataset, pd.DataFrame):
-            input_data = input_data.to_numpy()
+            input_data = input_data.to_numpy(copy=True)
         if self.categorical_features:
             # Inplace replacement of columns
             indexes = self.categorical_indexes
@@ -451,24 +458,29 @@ class BaseAnalyzer(ABC):
             input_data = input_data[indexes]
             diff = diff[indexes]
         try:
-            importances = self._compute_mutual_info(input_data, diff)
+            importances = self._compute_error_correlation(
+                input_data, diff, error_correlation_method)
         except ValueError:
             # Impute input_data if it contains NaNs, infinity or a value too
             # large for dtype('float64')
             input_data = np.nan_to_num(input_data)
-            importances = self._compute_mutual_info(input_data, diff)
+            importances = self._compute_error_correlation(
+                input_data, diff, error_correlation_method)
         return importances
 
-    def _compute_mutual_info(self, input_data, diff):
-        """Compute the mutual information between the features and error.
+    def _compute_error_correlation(self, input_data, diff,
+                                   error_correlation_method):
+        """Compute the correlation between the features and error.
 
-        :param input_data: The input data to compute the mutual information
+        :param input_data: The input data to compute the error correlation
             on.
         :type input_data: numpy.ndarray
         :param diff: The difference between the label and prediction
             columns.
         :type diff: numpy.ndarray
-        :return: The computed mutual information between the features and
+        :param error_correlation_method: Method to compute error correlation.
+        :type error_correlation_method: str
+        :return: The computed error correlation between the features and
             error.
         :rtype: list[float]
         """
@@ -476,13 +488,20 @@ class BaseAnalyzer(ABC):
         if input_data.shape[0] == 1:
             input_data = np.concatenate((input_data, input_data))
             diff = np.concatenate((diff, diff))
-        n_neighbors = min(3, input_data.shape[0] - 1)
-        if self._model_task == ModelTask.CLASSIFICATION:
-            return mutual_info_classif(
-                input_data, diff, n_neighbors=n_neighbors).tolist()
+        if error_correlation_method == ErrorCorrelationMethods.MUTUAL_INFO:
+            n_neighbors = min(3, input_data.shape[0] - 1)
+            if self._model_task == ModelTask.CLASSIFICATION:
+                return mutual_info_classif(
+                    input_data, diff, n_neighbors=n_neighbors).tolist()
+            else:
+                return mutual_info_regression(
+                    input_data, diff, n_neighbors=n_neighbors).tolist()
+        elif error_correlation_method == ErrorCorrelationMethods.EBM:
+            return compute_ebm_global_importance(
+                input_data, diff, self._model_task)
         else:
-            return mutual_info_regression(
-                input_data, diff, n_neighbors=n_neighbors).tolist()
+            return compute_gbm_global_importance(
+                input_data, diff, self._model_task, self._categorical_indexes)
 
     def compute_root_stats(self):
         """Compute the root all data statistics.

erroranalysis-0.4.4/erroranalysis/error_correlation_methods/__init__.py

@@ -0,0 +1,9 @@
+# Copyright (c) Microsoft Corporation
+# Licensed under the MIT License.
+
+"""Module for defining error correlation methods."""
+
+from .ebm import compute_ebm_global_importance
+from .gbm import compute_gbm_global_importance
+
+__all__ = ["compute_ebm_global_importance", "compute_gbm_global_importance"]

erroranalysis-0.4.4/erroranalysis/error_correlation_methods/ebm.py

@@ -0,0 +1,39 @@
+# Copyright (c) Microsoft Corporation
+# Licensed under the MIT License.
+
+"""Defines the error correlation computation using an EBM."""
+
+try:
+    from interpret.glassbox import (ExplainableBoostingClassifier,
+                                    ExplainableBoostingRegressor)
+    ebm_installed = True
+except ImportError:
+    ebm_installed = False
+
+from erroranalysis._internal.constants import ModelTask
+
+
+def compute_ebm_global_importance(input_data, diff, model_task):
+    """Compute global importance score for EBM between the features and error.
+    :param input_data: The input data to compute the EBM global importance
+        score on.
+    :type input_data: numpy.ndarray
+    :param diff: The difference between the label and prediction
+        columns.
+    :type diff: numpy.ndarray
+    :param model_task: The model task.
+    :type model_task: str
+    :return: The computed EBM global importance score between the features and
+        error.
+    :rtype: list[float]
+    """
+    if not ebm_installed:
+        raise ImportError('EBM is not installed. Please install it by running '
+                          '"pip install interpret"')
+    if model_task == ModelTask.CLASSIFICATION:
+        model = ExplainableBoostingClassifier(interactions=0)
+    else:
+        model = ExplainableBoostingRegressor(interactions=0)
+    model.fit(input_data, diff)
+    ebm_importances = model.term_importances()
+    return ebm_importances.tolist()

erroranalysis-0.4.4/erroranalysis/error_correlation_methods/gbm.py

@@ -0,0 +1,39 @@
+# Copyright (c) Microsoft Corporation
+# Licensed under the MIT License.
+
+"""Defines the error correlation computation using LightGBM with TreeShap."""
+
+import numpy as np
+import shap
+from lightgbm import LGBMClassifier, LGBMRegressor
+
+from erroranalysis._internal.constants import ModelTask
+
+
+def compute_gbm_global_importance(input_data, diff, model_task,
+                                  categorical_indexes):
+    """Compute global importance score for EBM between the features and error.
+    :param input_data: The input data to compute the EBM global importance
+        score on.
+    :type input_data: numpy.ndarray
+    :param diff: The difference between the label and prediction
+        columns.
+    :type diff: numpy.ndarray
+    :param model_task: The model task.
+    :type model_task: str
+    :return: The computed EBM global importance score between the features and
+        error.
+    :rtype: list[float]
+    """
+    is_classification = model_task == ModelTask.CLASSIFICATION
+    if is_classification:
+        model = LGBMClassifier()
+    else:
+        model = LGBMRegressor()
+    model.fit(input_data, diff, categorical_feature=categorical_indexes)
+    explainer = shap.TreeExplainer(model)
+    shap_values = explainer.shap_values(input_data)
+    shap_mean_abs = np.abs(shap_values).mean(axis=0)
+    if is_classification:
+        shap_mean_abs = shap_mean_abs.mean(axis=0)
+    return shap_mean_abs.tolist()

{erroranalysis-0.4.2 → erroranalysis-0.4.4}/erroranalysis/version.py

@@ -4,5 +4,5 @@
 name = 'erroranalysis'
 _major = '0'
 _minor = '4'
-_patch = '2'
+_patch = '4'
 version = '{}.{}.{}'.format(_major, _minor, _patch)

{erroranalysis-0.4.2 → erroranalysis-0.4.4/erroranalysis.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: erroranalysis
-Version: 0.4.2
+Version: 0.4.4
 Summary: Core error analysis APIs
 Home-page: https://github.com/microsoft/responsible-ai-widgets
 Author: Roman Lutz, Ilya Matiach, Ke Xu
@@ -9,6 +9,7 @@ Classifier: Programming Language :: Python :: 3.6
 Classifier: Programming Language :: Python :: 3.7
 Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
 Classifier: Development Status :: 3 - Alpha
@@ -18,7 +19,7 @@ License-File: LICENSE
 
 # Error Analysis SDK for Python
 
-### This package has been tested with Python 3.6, 3.7, 3.8 and 3.9
+### This package has been tested with Python 3.6, 3.7, 3.8, 3.9 and 3.10
 
 The error analysis sdk enables users to get a deeper understanding of machine learning model errors. When evaluating a machine learning model, aggregate accuracy is not sufficient and single-score evaluation may hide important conditions of inaccuracies. Use Error Analysis to identify cohorts with higher error rates and diagnose the root causes behind these errors.
 

{erroranalysis-0.4.2 → erroranalysis-0.4.4}/erroranalysis.egg-info/SOURCES.txt

@@ -23,6 +23,9 @@ erroranalysis/_internal/error_analyzer/__init__.py
 erroranalysis/_internal/error_report/__init__.py
 erroranalysis/analyzer/__init__.py
 erroranalysis/analyzer/error_analyzer.py
+erroranalysis/error_correlation_methods/__init__.py
+erroranalysis/error_correlation_methods/ebm.py
+erroranalysis/error_correlation_methods/gbm.py
 erroranalysis/report/__init__.py
 erroranalysis/report/error_report.py
 tests/test_cohort_filter.py

{erroranalysis-0.4.2 → erroranalysis-0.4.4}/erroranalysis.egg-info/requires.txt

@@ -1,5 +1,5 @@
 numpy>=1.17.2
-pandas>=0.25.1
+pandas<2.0.0,>=0.25.1
 scipy>=1.4.1
 scikit-learn>=0.22.1
 lightgbm>=2.0.11

{erroranalysis-0.4.2 → erroranalysis-0.4.4}/requirements.txt

@@ -1,5 +1,5 @@
 numpy>=1.17.2
-pandas>=0.25.1
+pandas>=0.25.1,<2.0.0
 scipy>=1.4.1
 scikit-learn>=0.22.1
 lightgbm>=2.0.11

{erroranalysis-0.4.2 → erroranalysis-0.4.4}/setup.py

@@ -32,6 +32,7 @@ setuptools.setup(
         "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
         "License :: OSI Approved :: MIT License",
         "Operating System :: OS Independent",
         "Development Status :: 3 - Alpha"

{erroranalysis-0.4.2 → erroranalysis-0.4.4}/tests/test_cohort_filter.py

@@ -6,8 +6,8 @@ import pandas as pd
 import pytest
 
 from erroranalysis._internal.cohort_filter import filter_from_cohort
-from erroranalysis._internal.constants import (PRED_Y, ROW_INDEX, TRUE_Y,
-                                               ModelTask)
+from erroranalysis._internal.constants import (ARG, COLUMN, METHOD, PRED_Y,
+                                               ROW_INDEX, TRUE_Y, ModelTask)
 from erroranalysis._internal.error_analyzer import ModelAnalyzer
 from rai_test_utils.datasets.tabular import (create_diabetes_data,
                                              create_iris_data,
@@ -15,6 +15,7 @@ from rai_test_utils.datasets.tabular import (create_diabetes_data,
 from rai_test_utils.models.sklearn import (
     create_sklearn_random_forest_regressor, create_sklearn_svm_classifier,
     create_titanic_pipeline)
+from raiutils.cohort import CohortFilterMethods
 
 TOL = 1e-10
 SEPAL_WIDTH = 'sepal width'
@@ -28,9 +29,9 @@ class TestCohortFilter(object):
     def test_cohort_filter_equal(self):
         X_train, X_test, y_train, y_test, feature_names, _ = \
             create_iris_pandas()
-        filters = [{'arg': [2.8],
-                    'column': SEPAL_WIDTH,
-                    'method': 'equal'}]
+        filters = [{ARG: [2.8],
+                    COLUMN: SEPAL_WIDTH,
+                    METHOD: CohortFilterMethods.METHOD_EQUAL}]
         validation_data = create_validation_data(X_test, y_test)
         validation_data = validation_data.loc[X_test[SEPAL_WIDTH] == 2.8]
         model_task = ModelTask.CLASSIFICATION
@@ -53,9 +54,9 @@ class TestCohortFilter(object):
                         "in Predicted Y cohort filtering")
         X_train, X_test, y_train, y_test, feature_names, classes = \
             create_iris_pandas(use_str_labels)
-        filters = [{'arg': [2],
-                    'column': target_type,
-                    'method': 'includes'}]
+        filters = [{ARG: [2],
+                    COLUMN: target_type,
+                    METHOD: CohortFilterMethods.METHOD_INCLUDES}]
         validation_data = create_validation_data(X_test, y_test)
         if use_str_labels:
             validation_filter = y_test == classes[2]
@@ -79,9 +80,9 @@ class TestCohortFilter(object):
     def test_cohort_filter_less(self):
         X_train, X_test, y_train, y_test, feature_names, _ = \
             create_iris_pandas()
-        filters = [{'arg': [2.8],
-                    'column': SEPAL_WIDTH,
-                    'method': 'less'}]
+        filters = [{ARG: [2.8],
+                    COLUMN: SEPAL_WIDTH,
+                    METHOD: CohortFilterMethods.METHOD_LESS}]
         validation_data = create_validation_data(X_test, y_test)
         validation_data = validation_data.loc[X_test[SEPAL_WIDTH] < 2.8]
         model_task = ModelTask.CLASSIFICATION
@@ -100,9 +101,9 @@ class TestCohortFilter(object):
         X_train, X_test, y_train, y_test, feature_names, _ = \
             create_iris_pandas()
 
-        filters = [{'arg': [2.8],
-                    'column': SEPAL_WIDTH,
-                    'method': 'less and equal'}]
+        filters = [{ARG: [2.8],
+                    COLUMN: SEPAL_WIDTH,
+                    METHOD: CohortFilterMethods.METHOD_LESS_AND_EQUAL}]
         validation_data = create_validation_data(X_test, y_test)
         validation_data = validation_data.loc[X_test[SEPAL_WIDTH] <= 2.8]
         model_task = ModelTask.CLASSIFICATION
@@ -120,9 +121,9 @@ class TestCohortFilter(object):
     def test_cohort_filter_greater(self):
         X_train, X_test, y_train, y_test, feature_names, _ = \
             create_iris_pandas()
-        filters = [{'arg': [2.8],
-                    'column': SEPAL_WIDTH,
-                    'method': 'greater'}]
+        filters = [{ARG: [2.8],
+                    COLUMN: SEPAL_WIDTH,
+                    METHOD: CohortFilterMethods.METHOD_GREATER}]
         validation_data = create_validation_data(X_test, y_test)
         validation_data = validation_data.loc[X_test[SEPAL_WIDTH] > 2.8]
         model_task = ModelTask.CLASSIFICATION
@@ -140,9 +141,9 @@ class TestCohortFilter(object):
     def test_cohort_filter_greater_and_equal(self):
         X_train, X_test, y_train, y_test, feature_names, _ = \
             create_iris_pandas()
-        filters = [{'arg': [2.8],
-                    'column': SEPAL_WIDTH,
-                    'method': 'greater and equal'}]
+        filters = [{ARG: [2.8],
+                    COLUMN: SEPAL_WIDTH,
+                    METHOD: CohortFilterMethods.METHOD_GREATER_AND_EQUAL}]
         validation_data = create_validation_data(X_test, y_test)
         validation_data = validation_data.loc[X_test[SEPAL_WIDTH] >= 2.8]
         model_task = ModelTask.CLASSIFICATION
@@ -160,9 +161,9 @@ class TestCohortFilter(object):
     def test_cohort_filter_in_the_range_of(self):
         X_train, X_test, y_train, y_test, feature_names, _ = \
             create_iris_pandas()
-        filters = [{'arg': [2.8, 3.4],
-                    'column': SEPAL_WIDTH,
-                    'method': 'in the range of'}]
+        filters = [{ARG: [2.8, 3.4],
+                    COLUMN: SEPAL_WIDTH,
+                    METHOD: CohortFilterMethods.METHOD_RANGE}]
         validation_data = create_validation_data(X_test, y_test)
         validation_data = validation_data.loc[
             (X_test[SEPAL_WIDTH] <= 3.4) & (X_test[SEPAL_WIDTH] >= 2.8)]
@@ -190,9 +191,9 @@ class TestCohortFilter(object):
 
         # the index 1, corresponds to incorrect prediction
         # the index 0 correspond to correct prediction
-        filters = [{'arg': arg,
-                    'column': CLASSIFICATION_OUTCOME,
-                    'method': 'includes'}]
+        filters = [{ARG: arg,
+                    COLUMN: CLASSIFICATION_OUTCOME,
+                    METHOD: CohortFilterMethods.METHOD_INCLUDES}]
         pred_y = model.predict(X_test)
         validation_data = create_validation_data(X_test, y_test, pred_y)
         if correct_prediction:
@@ -221,9 +222,9 @@ class TestCohortFilter(object):
         clf = create_titanic_pipeline(X_train, y_train)
         categorical_features = categorical
         # the indexes 0, 2 correspond to S, C
-        filters = [{'arg': [0, 2],
-                    'column': EMBARKED,
-                    'method': 'includes'}]
+        filters = [{ARG: [0, 2],
+                    COLUMN: EMBARKED,
+                    METHOD: CohortFilterMethods.METHOD_INCLUDES}]
         validation_data = create_validation_data(X_test, y_test)
         filter_embarked = X_test[EMBARKED].isin(['S', 'C'])
         validation_data = validation_data.loc[filter_embarked]
@@ -244,9 +245,9 @@ class TestCohortFilter(object):
         clf = create_titanic_pipeline(X_train, y_train)
         categorical_features = categorical
         # the indexes other than 0, 2 correspond to Q
-        filters = [{'arg': [0, 2],
-                    'column': EMBARKED,
-                    'method': 'excludes'}]
+        filters = [{ARG: [0, 2],
+                    COLUMN: EMBARKED,
+                    METHOD: CohortFilterMethods.METHOD_EXCLUDES}]
         validation_data = create_validation_data(X_test, y_test)
         filter_embarked = X_test[EMBARKED].isin(['Q'])
         validation_data = validation_data.loc[filter_embarked]
@@ -269,9 +270,9 @@ class TestCohortFilter(object):
         categorical_features = categorical
         # the indexes 1, 2 correspond to false positives and false negatives
         # the indexes 0, 3 correspond to true positives and true negatives
-        filters = [{'arg': arg,
-                    'column': CLASSIFICATION_OUTCOME,
-                    'method': 'includes'}]
+        filters = [{ARG: arg,
+                    COLUMN: CLASSIFICATION_OUTCOME,
+                    METHOD: CohortFilterMethods.METHOD_INCLUDES}]
         pred_y = clf.predict(X_test)
         validation_data = create_validation_data(X_test, y_test, pred_y)
         if not outcome:
@@ -296,9 +297,9 @@ class TestCohortFilter(object):
         X_train, X_test, y_train, y_test, feature_names, _ = \
             create_iris_pandas()
         # filter on index, which can be done from the RAI dashboard
-        filters = [{'arg': [40],
-                    'column': ROW_INDEX,
-                    'method': 'less and equal'}]
+        filters = [{ARG: [40],
+                    COLUMN: ROW_INDEX,
+                    METHOD: CohortFilterMethods.METHOD_LESS_AND_EQUAL}]
         validation_data = create_validation_data(X_test, y_test)
         validation_data = validation_data.loc[validation_data[ROW_INDEX] <= 40]
         model_task = ModelTask.CLASSIFICATION
@@ -321,9 +322,9 @@ class TestCohortFilter(object):
 
         # filter on regression error, which can be done from the
         # RAI dashboard
-        filters = [{'arg': [40],
-                    'column': REGRESSION_ERROR,
-                    'method': 'less and equal'}]
+        filters = [{ARG: [40],
+                    COLUMN: REGRESSION_ERROR,
+                    METHOD: CohortFilterMethods.METHOD_LESS_AND_EQUAL}]
 
         model = create_sklearn_random_forest_regressor(X_train, y_train)
         pred_y = model.predict(X_test)

{erroranalysis-0.4.2 → erroranalysis-0.4.4}/tests/test_error_report.py

@@ -1,8 +1,6 @@
 # Copyright (c) Microsoft Corporation
 # Licensed under the MIT License.
 
-import uuid
-
 import numpy as np
 import pandas as pd
 import pytest
@@ -11,9 +9,13 @@ from erroranalysis._internal.error_analyzer import ModelAnalyzer
 from erroranalysis._internal.error_report import ErrorReport
 from rai_test_utils.datasets.tabular import (create_cancer_data,
                                              create_housing_data,
-                                             create_iris_data)
+                                             create_iris_data,
+                                             create_simple_titanic_data)
 from rai_test_utils.models.model_utils import (create_models_classification,
                                                create_models_regression)
+from rai_test_utils.models.sklearn import \
+    create_complex_classification_pipeline
+from rai_test_utils.utilities import is_valid_uuid
 
 
 class TestErrorReport(object):
@@ -79,13 +81,21 @@ class TestErrorReport(object):
                                categorical_features,
                                filter_features=filter_features)
 
+    def test_error_report_titanic(self):
+        X_train, X_test, y_train, y_test, numeric, categorical = \
+            create_simple_titanic_data()
+
+        # Drop all numeric features
+        X_train = X_train.drop(['pclass', 'age', 'fare'], axis=1)
+        X_test = X_test.drop(['pclass', 'age', 'fare'], axis=1)
+
+        clf = create_complex_classification_pipeline(
+            X_train, y_train, [], ['embarked', 'sex'])
 
-def is_valid_uuid(id):
-    try:
-        uuid.UUID(str(id))
-        return True
-    except ValueError:
-        return False
+        # Pass the remaining categorical features
+        run_error_analyzer(clf, X_test, y_test, ['embarked', 'sex'],
+                           ['embarked', 'sex'],
+                           filter_features=[])
 
 
 def run_error_analyzer(model, X_test, y_test, feature_names,

{erroranalysis-0.4.2 → erroranalysis-0.4.4}/tests/test_importances.py

@@ -7,7 +7,8 @@ import numpy as np
 import pytest
 from common_utils import replicate_dataset
 
-from erroranalysis._internal.constants import ModelTask
+from erroranalysis._internal.constants import (ErrorCorrelationMethods,
+                                               ModelTask)
 from erroranalysis._internal.error_analyzer import ModelAnalyzer
 from rai_test_utils.datasets.tabular import (
     create_binary_classification_dataset, create_cancer_data,
@@ -22,11 +23,17 @@ from rai_test_utils.models.sklearn import (
 TOL = 1e-10
 NUM_SAMPLE_ROWS = 100
 DEFAULT_SAMPLE_COLS = 20
+MUTUAL_INFO = ErrorCorrelationMethods.MUTUAL_INFO
+EBM = ErrorCorrelationMethods.EBM
+GBM_SHAP = ErrorCorrelationMethods.GBM_SHAP
+GBM_ERROR_SAMPLES_TOL = 6
 
 
 class TestImportances(object):
 
-    def test_importances_iris(self):
+    @pytest.mark.parametrize('error_correlation_method',
+                             [MUTUAL_INFO, EBM, GBM_SHAP])
+    def test_importances_iris(self, error_correlation_method):
         X_train, X_test, y_train, y_test, feature_names, _ = create_iris_data()
 
         models = create_models_classification(X_train, y_train)
@@ -34,9 +41,11 @@ class TestImportances(object):
         for model in models:
             categorical_features = []
             run_error_analyzer(model, X_test, y_test, feature_names,
-                               categorical_features)
+                               categorical_features, error_correlation_method)
 
-    def test_importances_cancer(self):
+    @pytest.mark.parametrize('error_correlation_method',
+                             [MUTUAL_INFO, EBM, GBM_SHAP])
+    def test_importances_cancer(self, error_correlation_method):
         X_train, X_test, y_train, y_test, feature_names, _ = \
             create_cancer_data()
 
@@ -45,9 +54,12 @@ class TestImportances(object):
         for model in models:
             categorical_features = []
             run_error_analyzer(model, X_test, y_test, feature_names,
-                               categorical_features)
+                               categorical_features, error_correlation_method)
 
-    def test_importances_binary_classification(self):
+    @pytest.mark.parametrize('error_correlation_method',
+                             [MUTUAL_INFO, EBM, GBM_SHAP])
+    def test_importances_binary_classification(self,
+                                               error_correlation_method):
         X_train, y_train, X_test, y_test, _ = \
             create_binary_classification_dataset()
         feature_names = list(X_train.columns)
@@ -56,18 +68,22 @@ class TestImportances(object):
         for model in models:
             categorical_features = []
             run_error_analyzer(model, X_test, y_test, feature_names,
-                               categorical_features)
+                               categorical_features, error_correlation_method)
 
-    def test_importances_titanic(self):
+    @pytest.mark.parametrize('error_correlation_method',
+                             [MUTUAL_INFO, EBM, GBM_SHAP])
+    def test_importances_titanic(self, error_correlation_method):
         X_train, X_test, y_train, y_test, numeric, categorical = \
             create_simple_titanic_data()
         feature_names = categorical + numeric
         clf = create_titanic_pipeline(X_train, y_train)
         categorical_features = categorical
         run_error_analyzer(clf, X_test, y_test, feature_names,
-                           categorical_features)
+                           categorical_features, error_correlation_method)
 
-    def test_importances_housing(self):
+    @pytest.mark.parametrize('error_correlation_method',
+                             [MUTUAL_INFO, EBM, GBM_SHAP])
+    def test_importances_housing(self, error_correlation_method):
         X_train, X_test, y_train, y_test, feature_names = \
             create_housing_data()
         models = create_models_regression(X_train, y_train)
@@ -75,9 +91,11 @@ class TestImportances(object):
         for model in models:
             categorical_features = []
             run_error_analyzer(model, X_test, y_test, feature_names,
-                               categorical_features)
+                               categorical_features, error_correlation_method)
 
-    def test_large_data_importances(self):
+    @pytest.mark.parametrize('error_correlation_method',
+                             [MUTUAL_INFO, EBM, GBM_SHAP])
+    def test_large_data_importances(self, error_correlation_method):
         # mutual information can be very costly for large number of rows
         # hence, assert we downsample to compute importances for large data
         X_train, y_train, X_test, y_test, _ = \
@@ -91,16 +109,22 @@ class TestImportances(object):
         model_analyzer = ModelAnalyzer(model, X_test, y_test,
                                        feature_names,
                                        categorical_features)
-        model_analyzer.compute_importances()
+        model_analyzer.compute_importances(error_correlation_method)
         t1 = time.time()
         execution_time = t1 - t0
         print(execution_time)
         # assert we don't take too long and downsample the dataset
         # note execution time is in seconds
-        assert execution_time < 20
+        # note GBM_SHAP is slower than the other methods
+        if error_correlation_method == GBM_SHAP:
+            assert execution_time < 35
+        else:
+            assert execution_time < 25
 
     @pytest.mark.parametrize('num_rows', [1, 2, 3, 4])
-    def test_small_data_importances(self, num_rows):
+    @pytest.mark.parametrize('error_correlation_method',
+                             [MUTUAL_INFO, EBM, GBM_SHAP])
+    def test_small_data_importances(self, num_rows, error_correlation_method):
         # validate we can run on very few rows
         X_train, y_train, X_test, y_test, _ = \
             create_binary_classification_dataset(NUM_SAMPLE_ROWS)
@@ -112,10 +136,12 @@ class TestImportances(object):
         model_analyzer = ModelAnalyzer(model, X_test, y_test,
                                        feature_names,
                                        categorical_features)
-        scores = model_analyzer.compute_importances()
+        scores = model_analyzer.compute_importances(error_correlation_method)
         assert len(scores) == DEFAULT_SAMPLE_COLS
 
-    def test_importances_missings(self):
+    @pytest.mark.parametrize('error_correlation_method',
+                             [MUTUAL_INFO, EBM, GBM_SHAP])
+    def test_importances_missings(self, error_correlation_method):
         X_train, X_test, y_train, y_test, feature_names, _ = create_iris_data()
 
         # add missing values to X_test
@@ -127,23 +153,29 @@ class TestImportances(object):
 
         categorical_features = []
         run_error_analyzer(model, X_test, y_test, feature_names,
-                           categorical_features)
+                           categorical_features, error_correlation_method)
 
 
 def run_error_analyzer(model, X_test, y_test, feature_names,
-                       categorical_features):
+                       categorical_features,
+                       error_correlation_method):
     model_analyzer = ModelAnalyzer(model, X_test, y_test,
                                    feature_names,
                                    categorical_features)
-    scores = model_analyzer.compute_importances()
+    scores = model_analyzer.compute_importances(error_correlation_method)
     if model_analyzer.model_task == ModelTask.CLASSIFICATION:
         diff = model.predict(model_analyzer.dataset) != model_analyzer.true_y
     else:
         diff = model.predict(model_analyzer.dataset) - model_analyzer.true_y
     assert isinstance(scores, list)
     assert len(scores) == len(feature_names)
+    # GBM methods seems to predict all zeros
+    # (and sometimes otherwise) even if very few instances non-zero
+    is_gbm = error_correlation_method == GBM_SHAP
+    sum_diff = sum(diff)
+    noisy_gbm = sum_diff < GBM_ERROR_SAMPLES_TOL and sum_diff > 0 and is_gbm
     # If model predicted perfectly, assert all scores are zeros
     if not any(diff):
         assert all(abs(score - 0) < TOL for score in scores)
-    else:
+    elif not noisy_gbm:
         assert any(score != 0 for score in scores)

{erroranalysis-0.4.2 → erroranalysis-0.4.4}/tests/test_matrix_filter.py

@@ -6,8 +6,9 @@ import pandas as pd
 import pytest
 
 from erroranalysis._internal.cohort_filter import filter_from_cohort
-from erroranalysis._internal.constants import (ROW_INDEX, TRUE_Y, MatrixParams,
-                                               Metrics, ModelTask, f1_metrics,
+from erroranalysis._internal.constants import (ARG, COLUMN, METHOD, ROW_INDEX,
+                                               TRUE_Y, MatrixParams, Metrics,
+                                               ModelTask, f1_metrics,
                                                metric_to_display_name,
                                                precision_metrics,
                                                recall_metrics)
@@ -50,9 +51,9 @@ class TestMatrixFilter(object):
         # cohort was chosen in tree view
         X_train, X_test, y_train, y_test, feature_names, _ = create_iris_data()
 
-        filters = [{'arg': [2.85],
-                    'column': feature_names[1],
-                    'method': 'less and equal'}]
+        filters = [{ARG: [2.85],
+                    COLUMN: feature_names[1],
+                    METHOD: 'less and equal'}]
 
         model_task = ModelTask.CLASSIFICATION
         run_error_analyzer_on_models(X_train, y_train, X_test,
@@ -67,9 +68,9 @@ class TestMatrixFilter(object):
         X_train = pd.DataFrame(X_train, columns=feature_names)
         X_test = pd.DataFrame(X_test, columns=feature_names)
 
-        filters = [{'arg': [2.85],
-                    'column': feature_names[1],
-                    'method': 'less and equal'}]
+        filters = [{ARG: [2.85],
+                    COLUMN: feature_names[1],
+                    METHOD: 'less and equal'}]
 
         model_task = ModelTask.CLASSIFICATION
         run_error_analyzer_on_models(X_train, y_train, X_test,
@@ -222,24 +223,24 @@ class TestMatrixFilter(object):
 
         composite_filters = [{'compositeFilters':
                              [{'compositeFilters':
-                              [{'arg': [11.364, 13.182],
-                                'column': 'mean radius',
-                                'method': 'in the range of'}],
+                              [{ARG: [11.364, 13.182],
+                                COLUMN: 'mean radius',
+                                METHOD: 'in the range of'}],
                                'operation': 'and'},
                               {'compositeFilters':
-                               [{'arg': [13.182, 15],
-                                 'column': 'mean radius',
-                                 'method': 'in the range of'}],
+                               [{ARG: [13.182, 15],
+                                 COLUMN: 'mean radius',
+                                 METHOD: 'in the range of'}],
                                'operation': 'and'},
                               {'compositeFilters':
-                               [{'arg': [15, 16.817],
-                                 'column': 'mean radius',
-                                 'method': 'in the range of'}],
+                               [{ARG: [15, 16.817],
+                                 COLUMN: 'mean radius',
+                                 METHOD: 'in the range of'}],
                                'operation': 'and'},
                               {'compositeFilters':
-                               [{'arg': [16.817, 18.635],
-                                 'column': 'mean radius',
-                                 'method': 'in the range of'}],
+                               [{ARG: [16.817, 18.635],
+                                 COLUMN: 'mean radius',
+                                 METHOD: 'in the range of'}],
                                'operation': 'and'}],
                              'operation': 'or'}]
 
@@ -293,12 +294,12 @@ class TestMatrixFilter(object):
     def test_matrix_filter_housing_filters(self):
         X_train, X_test, y_train, y_test, feature_names = create_housing_data()
 
-        filters = [{'arg': [600],
-                    'column': 'Population',
-                    'method': 'less and equal'},
-                   {'arg': [6],
-                    'column': 'AveRooms',
-                    'method': 'greater'}]
+        filters = [{ARG: [600],
+                    COLUMN: 'Population',
+                    METHOD: 'less and equal'},
+                   {ARG: [6],
+                    COLUMN: 'AveRooms',
+                    METHOD: 'greater'}]
 
         model_task = ModelTask.REGRESSION
         run_error_analyzer_on_models(X_train, y_train, X_test,

{erroranalysis-0.4.2 → erroranalysis-0.4.4}/tests/test_surrogate_error_tree.py

@@ -66,12 +66,24 @@ class TestSurrogateErrorTree(object):
     def test_surrogate_error_tree_int_categorical(self, analyzer_type):
         X_train, X_test, y_train, y_test, categorical_features = \
             create_adult_census_data()
-
         model = create_kneighbors_classifier(X_train, y_train)
 
         run_error_analyzer(model, X_test, y_test, list(X_train.columns),
                            analyzer_type, categorical_features)
 
+    @pytest.mark.parametrize('analyzer_type', [AnalyzerType.MODEL,
+                                               AnalyzerType.PREDICTIONS])
+    def test_surrogate_error_tree_categorical_filtered(self, analyzer_type):
+        X_train, X_test, y_train, y_test, categorical_features = \
+            create_adult_census_data()
+        model = create_kneighbors_classifier(X_train, y_train)
+        filters = [{ARG: [40],
+                    COLUMN: 'Age',
+                    METHOD: 'less and equal'}]
+        run_error_analyzer(model, X_test, y_test, list(X_train.columns),
+                           analyzer_type, categorical_features,
+                           filters=filters)
+
     def test_large_data_surrogate_error_tree(self):
         # validate tree trains quickly for large data
         X_train, y_train, X_test, y_test, _ = \