erroranalysis 0.4.3__tar.gz → 0.4.4__tar.gz

{erroranalysis-0.4.3/erroranalysis.egg-info → erroranalysis-0.4.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: erroranalysis
-Version: 0.4.3
+Version: 0.4.4
 Summary: Core error analysis APIs
 Home-page: https://github.com/microsoft/responsible-ai-widgets
 Author: Roman Lutz, Ilya Matiach, Ke Xu

{erroranalysis-0.4.3 → erroranalysis-0.4.4}/erroranalysis/_internal/surrogate_error_tree.py

@@ -302,7 +302,7 @@ def get_surrogate_booster_local(filtered_df, analyzer, is_model_analyzer,
     if is_pandas:
         true_y = true_y.to_numpy()
     else:
-        input_data = input_data.to_numpy()
+        input_data = input_data.to_numpy(copy=True)
     if is_model_analyzer:
         pred_y = analyzer.model.predict(input_data)
     if analyzer.model_task == ModelTask.CLASSIFICATION:
@@ -316,7 +316,7 @@ def get_surrogate_booster_local(filtered_df, analyzer, is_model_analyzer,
     if not isinstance(true_y, np.ndarray):
         true_y = np.array(true_y)
     if is_pandas:
-        input_data = input_data.to_numpy()
+        input_data = input_data.to_numpy(copy=True)
 
     if analyzer.categorical_features:
         # Inplace replacement of columns

{erroranalysis-0.4.3 → erroranalysis-0.4.4}/erroranalysis/analyzer/error_analyzer.py

@@ -442,7 +442,7 @@ class BaseAnalyzer(ABC):
         input_data = self.dataset
         diff = self.get_diff()
         if isinstance(self.dataset, pd.DataFrame):
-            input_data = input_data.to_numpy()
+            input_data = input_data.to_numpy(copy=True)
         if self.categorical_features:
             # Inplace replacement of columns
             indexes = self.categorical_indexes

{erroranalysis-0.4.3 → erroranalysis-0.4.4}/erroranalysis/version.py

@@ -4,5 +4,5 @@
 name = 'erroranalysis'
 _major = '0'
 _minor = '4'
-_patch = '3'
+_patch = '4'
 version = '{}.{}.{}'.format(_major, _minor, _patch)

{erroranalysis-0.4.3 → erroranalysis-0.4.4/erroranalysis.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: erroranalysis
-Version: 0.4.3
+Version: 0.4.4
 Summary: Core error analysis APIs
 Home-page: https://github.com/microsoft/responsible-ai-widgets
 Author: Roman Lutz, Ilya Matiach, Ke Xu

{erroranalysis-0.4.3 → erroranalysis-0.4.4}/tests/test_cohort_filter.py

@@ -6,8 +6,8 @@ import pandas as pd
 import pytest
 
 from erroranalysis._internal.cohort_filter import filter_from_cohort
-from erroranalysis._internal.constants import (PRED_Y, ROW_INDEX, TRUE_Y,
-                                               ModelTask)
+from erroranalysis._internal.constants import (ARG, COLUMN, METHOD, PRED_Y,
+                                               ROW_INDEX, TRUE_Y, ModelTask)
 from erroranalysis._internal.error_analyzer import ModelAnalyzer
 from rai_test_utils.datasets.tabular import (create_diabetes_data,
                                              create_iris_data,
@@ -15,6 +15,7 @@ from rai_test_utils.datasets.tabular import (create_diabetes_data,
 from rai_test_utils.models.sklearn import (
     create_sklearn_random_forest_regressor, create_sklearn_svm_classifier,
     create_titanic_pipeline)
+from raiutils.cohort import CohortFilterMethods
 
 TOL = 1e-10
 SEPAL_WIDTH = 'sepal width'
@@ -28,9 +29,9 @@ class TestCohortFilter(object):
     def test_cohort_filter_equal(self):
         X_train, X_test, y_train, y_test, feature_names, _ = \
             create_iris_pandas()
-        filters = [{'arg': [2.8],
-                    'column': SEPAL_WIDTH,
-                    'method': 'equal'}]
+        filters = [{ARG: [2.8],
+                    COLUMN: SEPAL_WIDTH,
+                    METHOD: CohortFilterMethods.METHOD_EQUAL}]
         validation_data = create_validation_data(X_test, y_test)
         validation_data = validation_data.loc[X_test[SEPAL_WIDTH] == 2.8]
         model_task = ModelTask.CLASSIFICATION
@@ -53,9 +54,9 @@ class TestCohortFilter(object):
                         "in Predicted Y cohort filtering")
         X_train, X_test, y_train, y_test, feature_names, classes = \
             create_iris_pandas(use_str_labels)
-        filters = [{'arg': [2],
-                    'column': target_type,
-                    'method': 'includes'}]
+        filters = [{ARG: [2],
+                    COLUMN: target_type,
+                    METHOD: CohortFilterMethods.METHOD_INCLUDES}]
         validation_data = create_validation_data(X_test, y_test)
         if use_str_labels:
             validation_filter = y_test == classes[2]
@@ -79,9 +80,9 @@ class TestCohortFilter(object):
     def test_cohort_filter_less(self):
         X_train, X_test, y_train, y_test, feature_names, _ = \
             create_iris_pandas()
-        filters = [{'arg': [2.8],
-                    'column': SEPAL_WIDTH,
-                    'method': 'less'}]
+        filters = [{ARG: [2.8],
+                    COLUMN: SEPAL_WIDTH,
+                    METHOD: CohortFilterMethods.METHOD_LESS}]
         validation_data = create_validation_data(X_test, y_test)
         validation_data = validation_data.loc[X_test[SEPAL_WIDTH] < 2.8]
         model_task = ModelTask.CLASSIFICATION
@@ -100,9 +101,9 @@ class TestCohortFilter(object):
         X_train, X_test, y_train, y_test, feature_names, _ = \
             create_iris_pandas()
 
-        filters = [{'arg': [2.8],
-                    'column': SEPAL_WIDTH,
-                    'method': 'less and equal'}]
+        filters = [{ARG: [2.8],
+                    COLUMN: SEPAL_WIDTH,
+                    METHOD: CohortFilterMethods.METHOD_LESS_AND_EQUAL}]
         validation_data = create_validation_data(X_test, y_test)
         validation_data = validation_data.loc[X_test[SEPAL_WIDTH] <= 2.8]
         model_task = ModelTask.CLASSIFICATION
@@ -120,9 +121,9 @@ class TestCohortFilter(object):
     def test_cohort_filter_greater(self):
         X_train, X_test, y_train, y_test, feature_names, _ = \
             create_iris_pandas()
-        filters = [{'arg': [2.8],
-                    'column': SEPAL_WIDTH,
-                    'method': 'greater'}]
+        filters = [{ARG: [2.8],
+                    COLUMN: SEPAL_WIDTH,
+                    METHOD: CohortFilterMethods.METHOD_GREATER}]
         validation_data = create_validation_data(X_test, y_test)
         validation_data = validation_data.loc[X_test[SEPAL_WIDTH] > 2.8]
         model_task = ModelTask.CLASSIFICATION
@@ -140,9 +141,9 @@ class TestCohortFilter(object):
     def test_cohort_filter_greater_and_equal(self):
         X_train, X_test, y_train, y_test, feature_names, _ = \
             create_iris_pandas()
-        filters = [{'arg': [2.8],
-                    'column': SEPAL_WIDTH,
-                    'method': 'greater and equal'}]
+        filters = [{ARG: [2.8],
+                    COLUMN: SEPAL_WIDTH,
+                    METHOD: CohortFilterMethods.METHOD_GREATER_AND_EQUAL}]
         validation_data = create_validation_data(X_test, y_test)
         validation_data = validation_data.loc[X_test[SEPAL_WIDTH] >= 2.8]
         model_task = ModelTask.CLASSIFICATION
@@ -160,9 +161,9 @@ class TestCohortFilter(object):
     def test_cohort_filter_in_the_range_of(self):
         X_train, X_test, y_train, y_test, feature_names, _ = \
             create_iris_pandas()
-        filters = [{'arg': [2.8, 3.4],
-                    'column': SEPAL_WIDTH,
-                    'method': 'in the range of'}]
+        filters = [{ARG: [2.8, 3.4],
+                    COLUMN: SEPAL_WIDTH,
+                    METHOD: CohortFilterMethods.METHOD_RANGE}]
         validation_data = create_validation_data(X_test, y_test)
         validation_data = validation_data.loc[
             (X_test[SEPAL_WIDTH] <= 3.4) & (X_test[SEPAL_WIDTH] >= 2.8)]
@@ -190,9 +191,9 @@ class TestCohortFilter(object):
 
         # the index 1, corresponds to incorrect prediction
         # the index 0 correspond to correct prediction
-        filters = [{'arg': arg,
-                    'column': CLASSIFICATION_OUTCOME,
-                    'method': 'includes'}]
+        filters = [{ARG: arg,
+                    COLUMN: CLASSIFICATION_OUTCOME,
+                    METHOD: CohortFilterMethods.METHOD_INCLUDES}]
         pred_y = model.predict(X_test)
         validation_data = create_validation_data(X_test, y_test, pred_y)
         if correct_prediction:
@@ -221,9 +222,9 @@ class TestCohortFilter(object):
         clf = create_titanic_pipeline(X_train, y_train)
         categorical_features = categorical
         # the indexes 0, 2 correspond to S, C
-        filters = [{'arg': [0, 2],
-                    'column': EMBARKED,
-                    'method': 'includes'}]
+        filters = [{ARG: [0, 2],
+                    COLUMN: EMBARKED,
+                    METHOD: CohortFilterMethods.METHOD_INCLUDES}]
         validation_data = create_validation_data(X_test, y_test)
         filter_embarked = X_test[EMBARKED].isin(['S', 'C'])
         validation_data = validation_data.loc[filter_embarked]
@@ -244,9 +245,9 @@ class TestCohortFilter(object):
         clf = create_titanic_pipeline(X_train, y_train)
         categorical_features = categorical
         # the indexes other than 0, 2 correspond to Q
-        filters = [{'arg': [0, 2],
-                    'column': EMBARKED,
-                    'method': 'excludes'}]
+        filters = [{ARG: [0, 2],
+                    COLUMN: EMBARKED,
+                    METHOD: CohortFilterMethods.METHOD_EXCLUDES}]
         validation_data = create_validation_data(X_test, y_test)
         filter_embarked = X_test[EMBARKED].isin(['Q'])
         validation_data = validation_data.loc[filter_embarked]
@@ -269,9 +270,9 @@ class TestCohortFilter(object):
         categorical_features = categorical
         # the indexes 1, 2 correspond to false positives and false negatives
         # the indexes 0, 3 correspond to true positives and true negatives
-        filters = [{'arg': arg,
-                    'column': CLASSIFICATION_OUTCOME,
-                    'method': 'includes'}]
+        filters = [{ARG: arg,
+                    COLUMN: CLASSIFICATION_OUTCOME,
+                    METHOD: CohortFilterMethods.METHOD_INCLUDES}]
         pred_y = clf.predict(X_test)
         validation_data = create_validation_data(X_test, y_test, pred_y)
         if not outcome:
@@ -296,9 +297,9 @@ class TestCohortFilter(object):
         X_train, X_test, y_train, y_test, feature_names, _ = \
             create_iris_pandas()
         # filter on index, which can be done from the RAI dashboard
-        filters = [{'arg': [40],
-                    'column': ROW_INDEX,
-                    'method': 'less and equal'}]
+        filters = [{ARG: [40],
+                    COLUMN: ROW_INDEX,
+                    METHOD: CohortFilterMethods.METHOD_LESS_AND_EQUAL}]
         validation_data = create_validation_data(X_test, y_test)
         validation_data = validation_data.loc[validation_data[ROW_INDEX] <= 40]
         model_task = ModelTask.CLASSIFICATION
@@ -321,9 +322,9 @@ class TestCohortFilter(object):
 
         # filter on regression error, which can be done from the
         # RAI dashboard
-        filters = [{'arg': [40],
-                    'column': REGRESSION_ERROR,
-                    'method': 'less and equal'}]
+        filters = [{ARG: [40],
+                    COLUMN: REGRESSION_ERROR,
+                    METHOD: CohortFilterMethods.METHOD_LESS_AND_EQUAL}]
 
         model = create_sklearn_random_forest_regressor(X_train, y_train)
         pred_y = model.predict(X_test)

{erroranalysis-0.4.3 → erroranalysis-0.4.4}/tests/test_error_report.py

@@ -1,8 +1,6 @@
 # Copyright (c) Microsoft Corporation
 # Licensed under the MIT License.
 
-import uuid
-
 import numpy as np
 import pandas as pd
 import pytest
@@ -11,9 +9,13 @@ from erroranalysis._internal.error_analyzer import ModelAnalyzer
 from erroranalysis._internal.error_report import ErrorReport
 from rai_test_utils.datasets.tabular import (create_cancer_data,
                                              create_housing_data,
-                                             create_iris_data)
+                                             create_iris_data,
+                                             create_simple_titanic_data)
 from rai_test_utils.models.model_utils import (create_models_classification,
                                                create_models_regression)
+from rai_test_utils.models.sklearn import \
+    create_complex_classification_pipeline
+from rai_test_utils.utilities import is_valid_uuid
 
 
 class TestErrorReport(object):
@@ -79,13 +81,21 @@ class TestErrorReport(object):
                                categorical_features,
                                filter_features=filter_features)
 
+    def test_error_report_titanic(self):
+        X_train, X_test, y_train, y_test, numeric, categorical = \
+            create_simple_titanic_data()
+
+        # Drop all numeric features
+        X_train = X_train.drop(['pclass', 'age', 'fare'], axis=1)
+        X_test = X_test.drop(['pclass', 'age', 'fare'], axis=1)
+
+        clf = create_complex_classification_pipeline(
+            X_train, y_train, [], ['embarked', 'sex'])
 
-def is_valid_uuid(id):
-    try:
-        uuid.UUID(str(id))
-        return True
-    except ValueError:
-        return False
+        # Pass the remaining categorical features
+        run_error_analyzer(clf, X_test, y_test, ['embarked', 'sex'],
+                           ['embarked', 'sex'],
+                           filter_features=[])
 
 
 def run_error_analyzer(model, X_test, y_test, feature_names,

{erroranalysis-0.4.3 → erroranalysis-0.4.4}/tests/test_importances.py

@@ -115,7 +115,11 @@ class TestImportances(object):
         print(execution_time)
         # assert we don't take too long and downsample the dataset
         # note execution time is in seconds
-        assert execution_time < 20
+        # note GBM_SHAP is slower than the other methods
+        if error_correlation_method == GBM_SHAP:
+            assert execution_time < 35
+        else:
+            assert execution_time < 25
 
     @pytest.mark.parametrize('num_rows', [1, 2, 3, 4])
     @pytest.mark.parametrize('error_correlation_method',

{erroranalysis-0.4.3 → erroranalysis-0.4.4}/tests/test_matrix_filter.py

@@ -6,8 +6,9 @@ import pandas as pd
 import pytest
 
 from erroranalysis._internal.cohort_filter import filter_from_cohort
-from erroranalysis._internal.constants import (ROW_INDEX, TRUE_Y, MatrixParams,
-                                               Metrics, ModelTask, f1_metrics,
+from erroranalysis._internal.constants import (ARG, COLUMN, METHOD, ROW_INDEX,
+                                               TRUE_Y, MatrixParams, Metrics,
+                                               ModelTask, f1_metrics,
                                                metric_to_display_name,
                                                precision_metrics,
                                                recall_metrics)
@@ -50,9 +51,9 @@ class TestMatrixFilter(object):
         # cohort was chosen in tree view
         X_train, X_test, y_train, y_test, feature_names, _ = create_iris_data()
 
-        filters = [{'arg': [2.85],
-                    'column': feature_names[1],
-                    'method': 'less and equal'}]
+        filters = [{ARG: [2.85],
+                    COLUMN: feature_names[1],
+                    METHOD: 'less and equal'}]
 
         model_task = ModelTask.CLASSIFICATION
         run_error_analyzer_on_models(X_train, y_train, X_test,
@@ -67,9 +68,9 @@ class TestMatrixFilter(object):
         X_train = pd.DataFrame(X_train, columns=feature_names)
         X_test = pd.DataFrame(X_test, columns=feature_names)
 
-        filters = [{'arg': [2.85],
-                    'column': feature_names[1],
-                    'method': 'less and equal'}]
+        filters = [{ARG: [2.85],
+                    COLUMN: feature_names[1],
+                    METHOD: 'less and equal'}]
 
         model_task = ModelTask.CLASSIFICATION
         run_error_analyzer_on_models(X_train, y_train, X_test,
@@ -222,24 +223,24 @@ class TestMatrixFilter(object):
 
         composite_filters = [{'compositeFilters':
                              [{'compositeFilters':
-                              [{'arg': [11.364, 13.182],
-                                'column': 'mean radius',
-                                'method': 'in the range of'}],
+                              [{ARG: [11.364, 13.182],
+                                COLUMN: 'mean radius',
+                                METHOD: 'in the range of'}],
                                'operation': 'and'},
                               {'compositeFilters':
-                               [{'arg': [13.182, 15],
-                                 'column': 'mean radius',
-                                 'method': 'in the range of'}],
+                               [{ARG: [13.182, 15],
+                                 COLUMN: 'mean radius',
+                                 METHOD: 'in the range of'}],
                                'operation': 'and'},
                               {'compositeFilters':
-                               [{'arg': [15, 16.817],
-                                 'column': 'mean radius',
-                                 'method': 'in the range of'}],
+                               [{ARG: [15, 16.817],
+                                 COLUMN: 'mean radius',
+                                 METHOD: 'in the range of'}],
                                'operation': 'and'},
                               {'compositeFilters':
-                               [{'arg': [16.817, 18.635],
-                                 'column': 'mean radius',
-                                 'method': 'in the range of'}],
+                               [{ARG: [16.817, 18.635],
+                                 COLUMN: 'mean radius',
+                                 METHOD: 'in the range of'}],
                                'operation': 'and'}],
                              'operation': 'or'}]
 
@@ -293,12 +294,12 @@ class TestMatrixFilter(object):
     def test_matrix_filter_housing_filters(self):
         X_train, X_test, y_train, y_test, feature_names = create_housing_data()
 
-        filters = [{'arg': [600],
-                    'column': 'Population',
-                    'method': 'less and equal'},
-                   {'arg': [6],
-                    'column': 'AveRooms',
-                    'method': 'greater'}]
+        filters = [{ARG: [600],
+                    COLUMN: 'Population',
+                    METHOD: 'less and equal'},
+                   {ARG: [6],
+                    COLUMN: 'AveRooms',
+                    METHOD: 'greater'}]
 
         model_task = ModelTask.REGRESSION
         run_error_analyzer_on_models(X_train, y_train, X_test,

{erroranalysis-0.4.3 → erroranalysis-0.4.4}/tests/test_surrogate_error_tree.py

@@ -77,9 +77,9 @@ class TestSurrogateErrorTree(object):
         X_train, X_test, y_train, y_test, categorical_features = \
             create_adult_census_data()
         model = create_kneighbors_classifier(X_train, y_train)
-        filters = [{'arg': [40],
-                    'column': 'Age',
-                    'method': 'less and equal'}]
+        filters = [{ARG: [40],
+                    COLUMN: 'Age',
+                    METHOD: 'less and equal'}]
         run_error_analyzer(model, X_test, y_test, list(X_train.columns),
                            analyzer_type, categorical_features,
                            filters=filters)