erroranalysis 0.4.2__py3-none-any.whl → 0.4.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -96,6 +96,9 @@ class Metrics(str, Enum):
96
96
  FALSE_POSITIVE_RATE = 'false_positive_rate'
97
97
  FALSE_NEGATIVE_RATE = 'false_negative_rate'
98
98
  SELECTION_RATE = 'selection_rate'
99
+ MEAN_AVERAGE_PRECISION = 'mean_average_precision'
100
+ AVERAGE_PRECISION = 'average_precision'
101
+ AVERAGE_RECALL = 'average_recall'
99
102
 
100
103
 
101
104
  class MetricKeys(str, Enum):
@@ -121,6 +124,16 @@ class TreeNode(str, Enum):
121
124
  METRIC_VALUE = MetricKeys.METRIC_VALUE.value
122
125
 
123
126
 
127
+ class ErrorCorrelationMethods(str, Enum):
128
+ """Provide the supported error correlation methods.
129
+
130
+ The supported methods are 'mutual_info', 'ebm' and 'gbm_shap'.
131
+ """
132
+ MUTUAL_INFO = 'mutual_info'
133
+ EBM = 'ebm'
134
+ GBM_SHAP = 'gbm_shap'
135
+
136
+
124
137
  metric_to_display_name = {
125
138
  Metrics.ACCURACY_SCORE: 'Accuracy score',
126
139
  Metrics.MEAN_PREDICTION: 'Mean prediction',
@@ -292,7 +292,6 @@ def get_surrogate_booster_local(filtered_df, analyzer, is_model_analyzer,
292
292
  scored dataset.
293
293
  :rtype: (Booster, pandas.DataFrame, (list[str], list[int]))
294
294
  """
295
- row_index = filtered_df[ROW_INDEX]
296
295
  true_y = filtered_df[TRUE_Y]
297
296
  dropped_cols = [TRUE_Y, ROW_INDEX]
298
297
  if not is_model_analyzer:
@@ -303,7 +302,7 @@ def get_surrogate_booster_local(filtered_df, analyzer, is_model_analyzer,
303
302
  if is_pandas:
304
303
  true_y = true_y.to_numpy()
305
304
  else:
306
- input_data = input_data.to_numpy()
305
+ input_data = input_data.to_numpy(copy=True)
307
306
  if is_model_analyzer:
308
307
  pred_y = analyzer.model.predict(input_data)
309
308
  if analyzer.model_task == ModelTask.CLASSIFICATION:
@@ -317,7 +316,7 @@ def get_surrogate_booster_local(filtered_df, analyzer, is_model_analyzer,
317
316
  if not isinstance(true_y, np.ndarray):
318
317
  true_y = np.array(true_y)
319
318
  if is_pandas:
320
- input_data = input_data.to_numpy()
319
+ input_data = input_data.to_numpy(copy=True)
321
320
 
322
321
  if analyzer.categorical_features:
323
322
  # Inplace replacement of columns
@@ -330,7 +329,7 @@ def get_surrogate_booster_local(filtered_df, analyzer, is_model_analyzer,
330
329
  else:
331
330
  string_indexed_data = analyzer.string_indexed_data
332
331
  for idx, c_i in enumerate(analyzer.categorical_indexes):
333
- input_data[:, c_i] = string_indexed_data[row_index, idx]
332
+ input_data[:, c_i] = string_indexed_data[:, idx]
334
333
  dataset_sub_features = input_data[:, indexes]
335
334
 
336
335
  categorical_info = get_categorical_info(analyzer,
@@ -605,7 +604,7 @@ def filter_to_used_features(df, tree):
605
604
  """
606
605
  features = tree[CACHED_SUBTREE_FEATURES]
607
606
  features = features.union({PRED_Y, TRUE_Y, DIFF})
608
- return df[features]
607
+ return df[list(features)]
609
608
 
610
609
 
611
610
  def cache_subtree_features(tree, feature_names, parent=None):
@@ -10,7 +10,8 @@ import pandas as pd
10
10
  from sklearn.feature_selection import (mutual_info_classif,
11
11
  mutual_info_regression)
12
12
 
13
- from erroranalysis._internal.constants import (MatrixParams, Metrics,
13
+ from erroranalysis._internal.constants import (ErrorCorrelationMethods,
14
+ MatrixParams, Metrics,
14
15
  ModelTask, RootKeys,
15
16
  metric_to_display_name)
16
17
  from erroranalysis._internal.matrix_filter import \
@@ -25,11 +26,14 @@ from erroranalysis._internal.surrogate_error_tree import \
25
26
  compute_error_tree_on_dataset as _compute_error_tree_on_dataset
26
27
  from erroranalysis._internal.utils import generate_random_unique_indexes
27
28
  from erroranalysis._internal.version_checker import check_pandas_version
29
+ from erroranalysis.error_correlation_methods import (
30
+ compute_ebm_global_importance, compute_gbm_global_importance)
28
31
  from erroranalysis.report import ErrorReport
29
32
 
30
33
  BIN_THRESHOLD = MatrixParams.BIN_THRESHOLD
31
34
  IMPORTANCES_THRESHOLD = 50000
32
35
  ROOT_COVERAGE = 100
36
+ MUTUAL_INFO = ErrorCorrelationMethods.MUTUAL_INFO.value
33
37
 
34
38
 
35
39
  class BaseAnalyzer(ABC):
@@ -372,7 +376,8 @@ class BaseAnalyzer(ABC):
372
376
  num_leaves=None,
373
377
  min_child_samples=None,
374
378
  compute_importances=False,
375
- compute_root_stats=False):
379
+ compute_root_stats=False,
380
+ error_correlation_method=MUTUAL_INFO):
376
381
  """Creates the error analysis ErrorReport.
377
382
 
378
383
  The ErrorReport contains the importances, heatmap and tree view json.
@@ -409,7 +414,7 @@ class BaseAnalyzer(ABC):
409
414
  None)
410
415
  importances = None
411
416
  if compute_importances:
412
- importances = self.compute_importances()
417
+ importances = self.compute_importances(error_correlation_method)
413
418
  root_stats = None
414
419
  if compute_root_stats:
415
420
  root_stats = self.compute_root_stats()
@@ -419,7 +424,7 @@ class BaseAnalyzer(ABC):
419
424
  importances=importances,
420
425
  root_stats=root_stats)
421
426
 
422
- def compute_importances(self):
427
+ def compute_importances(self, error_correlation_method=MUTUAL_INFO):
423
428
  """Compute the importances or correlation between features and error.
424
429
 
425
430
  Computes the feature importances or the correlation between
@@ -428,6 +433,8 @@ class BaseAnalyzer(ABC):
428
433
  uses the scikit-learn methods mutual_info_classif for classification
429
434
  and mutual_info_regression for regression tasks.
430
435
 
436
+ :param error_correlation_method: Method to compute error correlation.
437
+ :type error_correlation_method: str
431
438
  :return: The computed importances or correlation between the
432
439
  features and error.
433
440
  :rtype: list[float]
@@ -435,7 +442,7 @@ class BaseAnalyzer(ABC):
435
442
  input_data = self.dataset
436
443
  diff = self.get_diff()
437
444
  if isinstance(self.dataset, pd.DataFrame):
438
- input_data = input_data.to_numpy()
445
+ input_data = input_data.to_numpy(copy=True)
439
446
  if self.categorical_features:
440
447
  # Inplace replacement of columns
441
448
  indexes = self.categorical_indexes
@@ -451,24 +458,29 @@ class BaseAnalyzer(ABC):
451
458
  input_data = input_data[indexes]
452
459
  diff = diff[indexes]
453
460
  try:
454
- importances = self._compute_mutual_info(input_data, diff)
461
+ importances = self._compute_error_correlation(
462
+ input_data, diff, error_correlation_method)
455
463
  except ValueError:
456
464
  # Impute input_data if it contains NaNs, infinity or a value too
457
465
  # large for dtype('float64')
458
466
  input_data = np.nan_to_num(input_data)
459
- importances = self._compute_mutual_info(input_data, diff)
467
+ importances = self._compute_error_correlation(
468
+ input_data, diff, error_correlation_method)
460
469
  return importances
461
470
 
462
- def _compute_mutual_info(self, input_data, diff):
463
- """Compute the mutual information between the features and error.
471
+ def _compute_error_correlation(self, input_data, diff,
472
+ error_correlation_method):
473
+ """Compute the correlation between the features and error.
464
474
 
465
- :param input_data: The input data to compute the mutual information
475
+ :param input_data: The input data to compute the error correlation
466
476
  on.
467
477
  :type input_data: numpy.ndarray
468
478
  :param diff: The difference between the label and prediction
469
479
  columns.
470
480
  :type diff: numpy.ndarray
471
- :return: The computed mutual information between the features and
481
+ :param error_correlation_method: Method to compute error correlation.
482
+ :type error_correlation_method: str
483
+ :return: The computed error correlation between the features and
472
484
  error.
473
485
  :rtype: list[float]
474
486
  """
@@ -476,13 +488,20 @@ class BaseAnalyzer(ABC):
476
488
  if input_data.shape[0] == 1:
477
489
  input_data = np.concatenate((input_data, input_data))
478
490
  diff = np.concatenate((diff, diff))
479
- n_neighbors = min(3, input_data.shape[0] - 1)
480
- if self._model_task == ModelTask.CLASSIFICATION:
481
- return mutual_info_classif(
482
- input_data, diff, n_neighbors=n_neighbors).tolist()
491
+ if error_correlation_method == ErrorCorrelationMethods.MUTUAL_INFO:
492
+ n_neighbors = min(3, input_data.shape[0] - 1)
493
+ if self._model_task == ModelTask.CLASSIFICATION:
494
+ return mutual_info_classif(
495
+ input_data, diff, n_neighbors=n_neighbors).tolist()
496
+ else:
497
+ return mutual_info_regression(
498
+ input_data, diff, n_neighbors=n_neighbors).tolist()
499
+ elif error_correlation_method == ErrorCorrelationMethods.EBM:
500
+ return compute_ebm_global_importance(
501
+ input_data, diff, self._model_task)
483
502
  else:
484
- return mutual_info_regression(
485
- input_data, diff, n_neighbors=n_neighbors).tolist()
503
+ return compute_gbm_global_importance(
504
+ input_data, diff, self._model_task, self._categorical_indexes)
486
505
 
487
506
  def compute_root_stats(self):
488
507
  """Compute the root all data statistics.
@@ -0,0 +1,9 @@
1
+ # Copyright (c) Microsoft Corporation
2
+ # Licensed under the MIT License.
3
+
4
+ """Module for defining error correlation methods."""
5
+
6
+ from .ebm import compute_ebm_global_importance
7
+ from .gbm import compute_gbm_global_importance
8
+
9
+ __all__ = ["compute_ebm_global_importance", "compute_gbm_global_importance"]
@@ -0,0 +1,39 @@
1
+ # Copyright (c) Microsoft Corporation
2
+ # Licensed under the MIT License.
3
+
4
+ """Defines the error correlation computation using an EBM."""
5
+
6
+ try:
7
+ from interpret.glassbox import (ExplainableBoostingClassifier,
8
+ ExplainableBoostingRegressor)
9
+ ebm_installed = True
10
+ except ImportError:
11
+ ebm_installed = False
12
+
13
+ from erroranalysis._internal.constants import ModelTask
14
+
15
+
16
+ def compute_ebm_global_importance(input_data, diff, model_task):
17
+ """Compute global importance score for EBM between the features and error.
18
+ :param input_data: The input data to compute the EBM global importance
19
+ score on.
20
+ :type input_data: numpy.ndarray
21
+ :param diff: The difference between the label and prediction
22
+ columns.
23
+ :type diff: numpy.ndarray
24
+ :param model_task: The model task.
25
+ :type model_task: str
26
+ :return: The computed EBM global importance score between the features and
27
+ error.
28
+ :rtype: list[float]
29
+ """
30
+ if not ebm_installed:
31
+ raise ImportError('EBM is not installed. Please install it by running '
32
+ '"pip install interpret"')
33
+ if model_task == ModelTask.CLASSIFICATION:
34
+ model = ExplainableBoostingClassifier(interactions=0)
35
+ else:
36
+ model = ExplainableBoostingRegressor(interactions=0)
37
+ model.fit(input_data, diff)
38
+ ebm_importances = model.term_importances()
39
+ return ebm_importances.tolist()
@@ -0,0 +1,39 @@
1
+ # Copyright (c) Microsoft Corporation
2
+ # Licensed under the MIT License.
3
+
4
+ """Defines the error correlation computation using LightGBM with TreeShap."""
5
+
6
+ import numpy as np
7
+ import shap
8
+ from lightgbm import LGBMClassifier, LGBMRegressor
9
+
10
+ from erroranalysis._internal.constants import ModelTask
11
+
12
+
13
+ def compute_gbm_global_importance(input_data, diff, model_task,
14
+ categorical_indexes):
15
+ """Compute global importance score for EBM between the features and error.
16
+ :param input_data: The input data to compute the EBM global importance
17
+ score on.
18
+ :type input_data: numpy.ndarray
19
+ :param diff: The difference between the label and prediction
20
+ columns.
21
+ :type diff: numpy.ndarray
22
+ :param model_task: The model task.
23
+ :type model_task: str
24
+ :return: The computed EBM global importance score between the features and
25
+ error.
26
+ :rtype: list[float]
27
+ """
28
+ is_classification = model_task == ModelTask.CLASSIFICATION
29
+ if is_classification:
30
+ model = LGBMClassifier()
31
+ else:
32
+ model = LGBMRegressor()
33
+ model.fit(input_data, diff, categorical_feature=categorical_indexes)
34
+ explainer = shap.TreeExplainer(model)
35
+ shap_values = explainer.shap_values(input_data)
36
+ shap_mean_abs = np.abs(shap_values).mean(axis=0)
37
+ if is_classification:
38
+ shap_mean_abs = shap_mean_abs.mean(axis=0)
39
+ return shap_mean_abs.tolist()
erroranalysis/version.py CHANGED
@@ -4,5 +4,5 @@
4
4
  name = 'erroranalysis'
5
5
  _major = '0'
6
6
  _minor = '4'
7
- _patch = '2'
7
+ _patch = '4'
8
8
  version = '{}.{}.{}'.format(_major, _minor, _patch)
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: erroranalysis
3
- Version: 0.4.2
3
+ Version: 0.4.4
4
4
  Summary: Core error analysis APIs
5
5
  Home-page: https://github.com/microsoft/responsible-ai-widgets
6
6
  Author: Roman Lutz, Ilya Matiach, Ke Xu
@@ -9,6 +9,7 @@ Classifier: Programming Language :: Python :: 3.6
9
9
  Classifier: Programming Language :: Python :: 3.7
10
10
  Classifier: Programming Language :: Python :: 3.8
11
11
  Classifier: Programming Language :: Python :: 3.9
12
+ Classifier: Programming Language :: Python :: 3.10
12
13
  Classifier: License :: OSI Approved :: MIT License
13
14
  Classifier: Operating System :: OS Independent
14
15
  Classifier: Development Status :: 3 - Alpha
@@ -16,7 +17,7 @@ Requires-Python: >=3.6
16
17
  Description-Content-Type: text/markdown
17
18
  License-File: LICENSE
18
19
  Requires-Dist: numpy (>=1.17.2)
19
- Requires-Dist: pandas (>=0.25.1)
20
+ Requires-Dist: pandas (<2.0.0,>=0.25.1)
20
21
  Requires-Dist: scipy (>=1.4.1)
21
22
  Requires-Dist: scikit-learn (>=0.22.1)
22
23
  Requires-Dist: lightgbm (>=2.0.11)
@@ -24,7 +25,7 @@ Requires-Dist: raiutils (>=0.4.0)
24
25
 
25
26
  # Error Analysis SDK for Python
26
27
 
27
- ### This package has been tested with Python 3.6, 3.7, 3.8 and 3.9
28
+ ### This package has been tested with Python 3.6, 3.7, 3.8, 3.9 and 3.10
28
29
 
29
30
  The error analysis sdk enables users to get a deeper understanding of machine learning model errors. When evaluating a machine learning model, aggregate accuracy is not sufficient and single-score evaluation may hide important conditions of inaccuracies. Use Error Analysis to identify cohorts with higher error rates and diagnose the root causes behind these errors.
30
31
 
@@ -1,22 +1,25 @@
1
1
  erroranalysis/__init__.py,sha256=85U8EP6x4cDHR6iUXzaQP7Octc6rR_yKCEMhLI5xK5E,182
2
- erroranalysis/version.py,sha256=bczENlZp92415SC8Ec0hdXlrGFVoPlaVUaQH6KbkdtM,187
2
+ erroranalysis/version.py,sha256=13dXrrVrIzWsCmPWGfhywPuipR09dV1y6IvZjD6zFH0,187
3
3
  erroranalysis/_internal/__init__.py,sha256=PVrm5KH4D_BqHrsMRTA-2WfYykArKOQOOMcdCKIBIZU,115
4
4
  erroranalysis/_internal/cohort_filter.py,sha256=9VMLaRwwj2AJK5KRpH3Dw4BnrP5fdTkaH0dKJixNNRo,20158
5
- erroranalysis/_internal/constants.py,sha256=ybxtQH2pNGMRtKKatNk3r9c8uwcLMyHqt3KLManLAkM,5838
5
+ erroranalysis/_internal/constants.py,sha256=mcAImyciOl3XQKOuF3KV5Yb90kGiIhC7ltXTpYJL9kU,6224
6
6
  erroranalysis/_internal/matrix_filter.py,sha256=1UBXD7v_sX0hqDssvFUwV0nm30F-RSFdzTpazPe4OUA,34348
7
7
  erroranalysis/_internal/metrics.py,sha256=3GSDDE8hZknebT0mnQCdgYUfHOvyMSM8mpbU-nz-5Ps,8917
8
8
  erroranalysis/_internal/process_categoricals.py,sha256=PimJQ3g00qOkHCJM_oR_XDAAsiKaVEZf99EXmBbDXmc,2175
9
- erroranalysis/_internal/surrogate_error_tree.py,sha256=DLebQabTL4-NY2Ax9AbF9rcNatwMQrj8hUWS-W3aQiI,41699
9
+ erroranalysis/_internal/surrogate_error_tree.py,sha256=kgrCufRmMJ3U0Bqz-2HQBUNRC_6Zefnq0t7xK1NW06c,41676
10
10
  erroranalysis/_internal/utils.py,sha256=ltomW3ry6Ch4KWgcEhlv1gpub3Pdth-jSQ8ByFuaVNI,1443
11
11
  erroranalysis/_internal/version_checker.py,sha256=jdI9j3EOIPKaHQGsSK-VslINUGgRzVmzxnxiogm0uUU,728
12
12
  erroranalysis/_internal/error_analyzer/__init__.py,sha256=vBt8WuwJDK7y_AexiFJylf4ajr_Uiq7tg52LcGRAdjU,246
13
13
  erroranalysis/_internal/error_report/__init__.py,sha256=KMARa8kO4ohRKlq_2DUBowq7DVs6KTls38RifQYj-gc,288
14
14
  erroranalysis/analyzer/__init__.py,sha256=NnPENfqCLKIIxlD_Lf3Ad96_ZEPff-zx7vGUwkB82Cg,229
15
- erroranalysis/analyzer/error_analyzer.py,sha256=HaelOMeAqklQ_rFZPqewOwaEzNLtGVhgRWEP4bQDSko,30887
15
+ erroranalysis/analyzer/error_analyzer.py,sha256=Jcad3838KwDoH2eAPrxXY7YEdul2uVHqTjC2SlkbC7c,32091
16
+ erroranalysis/error_correlation_methods/__init__.py,sha256=WbrZbWGlf2jBGIPWDeLBE9sqXye9Qmvdqbw9QDTX4UA,299
17
+ erroranalysis/error_correlation_methods/ebm.py,sha256=i_IyLxSfbACwDkpma71R0Nl4HJ0Ui8XCVhk584MwLnE,1434
18
+ erroranalysis/error_correlation_methods/gbm.py,sha256=lCx7AlepaFcqfLHEErJHJmXpQcHIz79k6fGgIJDpd5U,1419
16
19
  erroranalysis/report/__init__.py,sha256=sQgHMK55FJTMLrA2BBavYmViwgmQmb11zXldRf0KITI,310
17
20
  erroranalysis/report/error_report.py,sha256=Bu9fK0JPHfxYNAGWGDf9rJo6VaMX5WZin1v-rGXUvGM,7895
18
- erroranalysis-0.4.2.dist-info/LICENSE,sha256=ws_MuBL-SCEBqPBFl9_FqZkaaydIJmxHrJG2parhU4M,1141
19
- erroranalysis-0.4.2.dist-info/METADATA,sha256=aOYIBUD4v7R45gNbOYI6N0sgtIzjDHdLa4fwDDqmegA,1979
20
- erroranalysis-0.4.2.dist-info/WHEEL,sha256=2wepM1nk4DS4eFpYrW1TTqPcoGNfHhhO_i5m4cOimbo,92
21
- erroranalysis-0.4.2.dist-info/top_level.txt,sha256=GujRja2OTBpoRF4KpwnEsDskyvHe3qzqdooJthylbCk,14
22
- erroranalysis-0.4.2.dist-info/RECORD,,
21
+ erroranalysis-0.4.4.dist-info/LICENSE,sha256=ws_MuBL-SCEBqPBFl9_FqZkaaydIJmxHrJG2parhU4M,1141
22
+ erroranalysis-0.4.4.dist-info/METADATA,sha256=aI1_GHkTy8cVUjjB1TvJu1ve9E5q0Z2jdsELrt-Ip64,2043
23
+ erroranalysis-0.4.4.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
24
+ erroranalysis-0.4.4.dist-info/top_level.txt,sha256=GujRja2OTBpoRF4KpwnEsDskyvHe3qzqdooJthylbCk,14
25
+ erroranalysis-0.4.4.dist-info/RECORD,,
@@ -1,5 +1,5 @@
1
1
  Wheel-Version: 1.0
2
- Generator: bdist_wheel (0.38.4)
2
+ Generator: bdist_wheel (0.40.0)
3
3
  Root-Is-Purelib: true
4
4
  Tag: py3-none-any
5
5