erroranalysis 0.5.4__tar.gz → 0.6.0__tar.gz

{erroranalysis-0.5.4/erroranalysis.egg-info → erroranalysis-0.6.0}/PKG-INFO

@@ -1,33 +1,42 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: erroranalysis
-Version: 0.5.4
+Version: 0.6.0
 Summary: Core error analysis APIs
 Home-page: https://github.com/microsoft/responsible-ai-widgets
 Author: Roman Lutz, Ilya Matiach, Ke Xu
 Author-email: raiwidgets-maintain@microsoft.com
-Classifier: Programming Language :: Python :: 3.7
-Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
 Classifier: Development Status :: 3 - Alpha
-Requires-Python: >=3.7
+Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: numpy>=1.17.2
-Requires-Dist: pandas<2.0.0,>=0.25.1
+Requires-Dist: numpy<3,>=1.17.2
+Requires-Dist: pandas<3,>=0.25.1
 Requires-Dist: scipy>=1.4.1
 Requires-Dist: scikit-learn>=0.22.1
 Requires-Dist: lightgbm>=2.0.11
 Requires-Dist: raiutils>=0.4.2
 Provides-Extra: object-detection
 Requires-Dist: vision_explanation_methods; extra == "object-detection"
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license-file
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
 
 # Error Analysis SDK for Python
 
-### This package has been tested with Python 3.7, 3.8, 3.9, 3.10 and 3.11
+### This package has been tested with Python 3.9, 3.10 and 3.11
 
 The error analysis sdk enables users to get a deeper understanding of machine learning model errors. When evaluating a machine learning model, aggregate accuracy is not sufficient and single-score evaluation may hide important conditions of inaccuracies. Use Error Analysis to identify cohorts with higher error rates and diagnose the root causes behind these errors.
 

{erroranalysis-0.5.4 → erroranalysis-0.6.0}/README.md

@@ -1,6 +1,6 @@
 # Error Analysis SDK for Python
 
-### This package has been tested with Python 3.7, 3.8, 3.9, 3.10 and 3.11
+### This package has been tested with Python 3.9, 3.10 and 3.11
 
 The error analysis sdk enables users to get a deeper understanding of machine learning model errors. When evaluating a machine learning model, aggregate accuracy is not sufficient and single-score evaluation may hide important conditions of inaccuracies. Use Error Analysis to identify cohorts with higher error rates and diagnose the root causes behind these errors.
 

{erroranalysis-0.5.4 → erroranalysis-0.6.0}/erroranalysis/_internal/cohort_filter.py

@@ -11,14 +11,60 @@ from erroranalysis._internal.constants import (ARG, COLUMN, COMPOSITE_FILTERS,
                                                ROW_INDEX, TRUE_Y,
                                                CohortFilterMethods,
                                                CohortFilterOps, ModelTask)
-from erroranalysis._internal.metrics import get_ordered_classes
-from erroranalysis._internal.utils import is_spark
+from erroranalysis._internal.metrics import (encode_classification_labels,
+                                             get_ordered_classes,
+                                             validate_ordered_classes)
+from erroranalysis._internal.utils import (compute_classification_diff,
+                                           compute_regression_residuals,
+                                           is_spark,
+                                           normalize_multioutput_array,
+                                           normalize_single_output_array,
+                                           object_label_series)
 
 MODEL = 'model'
 CLASSIFICATION_OUTCOME = 'Classification outcome'
 REGRESSION_ERROR = 'Regression error'
 
 
+def _get_object_detection_values(values):
+    """Return one object cell per sample for nested labels."""
+    if isinstance(values, pd.DataFrame):
+        if values.shape[1] != 1:
+            raise ValueError(
+                "Object detection labels must have exactly one column")
+        values = values.iloc[:, 0]
+    elif isinstance(values, np.ndarray) and \
+            values.ndim == 2 and values.shape[1] == 1:
+        values = values[:, 0]
+    result = np.empty(len(values), dtype=object)
+    result[:] = list(values)
+    return result
+
+
+def _get_cohort_values(values):
+    """Return scalar labels or one object cell per multi-output row."""
+    multioutput = normalize_multioutput_array(values)
+    if multioutput is not None:
+        result = np.empty(len(multioutput), dtype=object)
+        result[:] = list(multioutput)
+        return result
+    normalized = normalize_single_output_array(values)
+    if normalized.ndim == 2 and normalized.shape[1] > 1:
+        result = np.empty(len(normalized), dtype=object)
+        result[:] = list(normalized)
+        return result
+    return normalized
+
+
+def _assign_label_column(df, column, values):
+    """Assign object labels without pandas temporal or numeric inference."""
+    if (isinstance(values, np.ndarray) and
+            values.dtype.kind in ("M", "O", "m")):
+        df[column] = object_label_series(values, index=df.index)
+    else:
+        df[column] = values
+
+
 def filter_from_cohort(analyzer, filters, composite_filters,
                        include_original_columns_only=False):
     """Filters the dataset on the analyzer based on the specified filters.
@@ -124,7 +170,7 @@ class FilterDataWithCohortFilters:
                 # Note: we make a non-deep copy of the input DataFrame since
                 # we will add columns below
                 df = df.copy()
-        self._add_filter_cols(df, filters)
+        self._add_filter_cols(df, (filters or []) + (composite_filters or []))
         df = self._apply_recursive_filter(
             df, filters, self.categorical_features, self.categories)
         df = self._apply_recursive_filter(
@@ -148,11 +194,7 @@ class FilterDataWithCohortFilters:
         if (model_task == ModelTask.CLASSIFICATION or
             model_task == ModelTask.OBJECT_DETECTION) \
                 and filters:
-            for filter in filters:
-                if COLUMN in filter:
-                    column = filter[COLUMN]
-                    if column == CLASSIFICATION_OUTCOME:
-                        return True
+            return self._filters_has_column(filters, CLASSIFICATION_OUTCOME)
         return False
 
     def _filters_has_regression_error(self, filters):
@@ -169,11 +211,17 @@ class FilterDataWithCohortFilters:
         # is included in filters, and if it is then
         # compute the necessary column data on the fly
         if model_task == ModelTask.REGRESSION and filters:
-            for filter in filters:
-                if COLUMN in filter:
-                    column = filter[COLUMN]
-                    if column == REGRESSION_ERROR:
-                        return True
+            return self._filters_has_column(filters, REGRESSION_ERROR)
+        return False
+
+    def _filters_has_column(self, filters, column):
+        """Find a column in flat filters or nested composite filters."""
+        for filter in filters:
+            if filter.get(COLUMN) == column:
+                return True
+            if self._filters_has_column(
+                    filter.get(COMPOSITE_FILTERS, []), column):
+                return True
         return False
 
     def _compute_binary_classification_outcome_data(self, true_y, pred_y,
@@ -181,6 +229,10 @@ class FilterDataWithCohortFilters:
         """Creates the classification outcome data for binary
         classification scenario.
 
+        The second class is positive. Outcome codes match the dashboard:
+        0 is true negative, 1 false positive, 2 false negative,
+        and 3 true positive.
+
         :param true_y: The true labels.
         :type true_y: list or numpy.ndarray
         :param pred_y: The predicted labels.
@@ -190,29 +242,15 @@ class FilterDataWithCohortFilters:
         :return: The classification outcome data.
         :rtype: list
         """
-        classification_outcome = []
-        if not isinstance(pred_y, np.ndarray):
-            pred_y = np.array(pred_y)
-
-        if not isinstance(true_y, np.ndarray):
-            true_y = np.array(true_y)
-
-        for i in range(len(true_y)):
-            if true_y[i] == pred_y[i]:
-                if true_y[i] == classes[0]:
-                    # True positive == 0
-                    classification_outcome.append(0)
-                else:
-                    # True negative == 3
-                    classification_outcome.append(3)
-            else:
-                if true_y[i] == classes[0]:
-                    # False negative == 2
-                    classification_outcome.append(2)
-                else:
-                    # False positive == 1
-                    classification_outcome.append(1)
-        return classification_outcome
+        true_codes = encode_classification_labels(
+            classes, true_y, allow_unknown=True)[0]
+        matches = ~compute_classification_diff(pred_y, true_y)
+        true_negative_class = true_codes == 0
+        return np.where(
+            matches,
+            np.where(true_negative_class, 0, 3),
+            np.where(true_negative_class, 1, 2),
+        ).tolist()
 
     def _compute_multiclass_classification_outcome_data(
             self, true_y, pred_y):
@@ -226,22 +264,7 @@ class FilterDataWithCohortFilters:
         :return: The classification outcome data.
         :rtype: list
         """
-        classification_outcome = []
-        if not isinstance(pred_y, np.ndarray):
-            pred_y = np.array(pred_y)
-
-        if not isinstance(true_y, np.ndarray):
-            true_y = np.array(true_y)
-
-        for i in range(len(true_y)):
-            if true_y[i] == pred_y[i]:
-                # Correct prediction == 0
-                classification_outcome.append(0)
-            else:
-                # Incorrect prediction == 1
-                classification_outcome.append(1)
-
-        return classification_outcome
+        return compute_classification_diff(pred_y, true_y).astype(int).tolist()
 
     def _compute_regression_error_data(self, true_y, pred_y):
         """Computes the per instance absolute difference
@@ -254,16 +277,7 @@ class FilterDataWithCohortFilters:
         :return: The regression error data.
         :rtype: list
         """
-        regression_error = []
-        if not isinstance(pred_y, np.ndarray):
-            pred_y = np.array(pred_y)
-
-        if not isinstance(true_y, np.ndarray):
-            true_y = np.array(true_y)
-
-        for i in range(len(true_y)):
-            regression_error.append(abs(true_y[i] - pred_y[i]))
-        return regression_error
+        return np.abs(compute_regression_residuals(pred_y, true_y)).tolist()
 
     def _add_filter_cols(self, df, filters):
         """Adds special columns to the dataset for filtering
@@ -277,17 +291,39 @@ class FilterDataWithCohortFilters:
         has_classification_outcome = self._filters_has_classification_outcome(
             filters)
         has_regression_error = self._filters_has_regression_error(filters)
+        if self.model is None and self.pred_y is None and (
+                has_classification_outcome or has_regression_error):
+            raise ValueError(
+                "A model or predicted labels are required to filter by "
+                "classification outcome or regression error.")
 
         if isinstance(self.true_y, str):
             df.rename(columns={self.true_y: TRUE_Y})
+        elif self.model_task in (
+                ModelTask.OBJECT_DETECTION,
+                ModelTask.MULTILABEL_IMAGE_CLASSIFICATION):
+            _assign_label_column(
+                df, TRUE_Y, _get_object_detection_values(self.true_y))
         else:
-            df[TRUE_Y] = self.true_y
+            _assign_label_column(
+                df, TRUE_Y, _get_cohort_values(self.true_y))
 
         if self.model is None:
-            df[PRED_Y] = self.pred_y
+            if self.pred_y is not None and self.model_task in (
+                    ModelTask.OBJECT_DETECTION,
+                    ModelTask.MULTILABEL_IMAGE_CLASSIFICATION):
+                _assign_label_column(
+                    df, PRED_Y,
+                    _get_object_detection_values(self.pred_y))
+            elif self.pred_y is not None:
+                _assign_label_column(
+                    df, PRED_Y, _get_cohort_values(self.pred_y))
 
         if not is_spark(df):
-            df[ROW_INDEX] = np.arange(0, len(self.true_y))
+            row_count = (self.true_y.shape[0]
+                         if hasattr(self.true_y, "shape")
+                         else len(self.true_y))
+            df[ROW_INDEX] = np.arange(0, row_count)
         if has_classification_outcome:
             if PRED_Y in df:
                 pred_y = df[PRED_Y]
@@ -296,11 +332,51 @@ class FilterDataWithCohortFilters:
                 pred_y = self.model.predict(
                     df.drop(columns=[TRUE_Y, ROW_INDEX]))
 
+            multi_true_y = normalize_multioutput_array(self.true_y)
+            multi_pred_y = normalize_multioutput_array(pred_y)
+            if multi_true_y is not None and multi_pred_y is not None:
+                df[CLASSIFICATION_OUTCOME] = \
+                    self._compute_multiclass_classification_outcome_data(
+                        multi_true_y, multi_pred_y)
+                return
+
+            model_classes = getattr(self.model, "classes_", None)
+            declared_classes = self.classes
+            if declared_classes is None and model_classes is not None:
+                model_order = get_ordered_classes(
+                    model_classes, self.true_y, pred_y)
+                if model_order is model_classes and len(model_classes) == 2:
+                    declared_classes = model_classes
+            true_labels = normalize_single_output_array(self.true_y).ravel()
+            nonmissing_true_labels = [
+                label for label in true_labels
+                if not (
+                    np.asarray(pd.isna(label)).ndim == 0 and
+                    bool(pd.isna(label)))
+            ]
             classes = get_ordered_classes(
-                self.classes, self.true_y, pred_y)
+                declared_classes, nonmissing_true_labels,
+                nonmissing_true_labels)
+            if declared_classes is None or classes is not declared_classes:
+                if model_classes is not None and declared_classes is not None:
+                    model_order = get_ordered_classes(
+                        model_classes, self.true_y, pred_y)
+                    if model_order is model_classes and \
+                            len(model_classes) == 2:
+                        classes = model_classes
+                    else:
+                        classes = get_ordered_classes(
+                            None, self.true_y, pred_y)
+                else:
+                    classes = get_ordered_classes(
+                        None, self.true_y, pred_y)
 
             # calculate classification outcome and add to df
             if len(classes) == 2:
+                validate_ordered_classes(
+                    classes, nonmissing_true_labels,
+                    nonmissing_true_labels,
+                    require_two_classes=True)
                 df[CLASSIFICATION_OUTCOME] = \
                     self._compute_binary_classification_outcome_data(
                         self.true_y, pred_y, classes)