dataeval 0.75.0__py3-none-any.whl → 0.76.1__py3-none-any.whl

dataeval/utils/metadata.py

@@ -1,14 +1,15 @@
 """
-Metadata related utility functions that help organize raw metadata into :class:`Metadata` objects
-for use within `DataEval`.
+Metadata related utility functions that help organize raw metadata into \
+:class:`Metadata` objects for use within `DataEval`.
 """
 
 from __future__ import annotations
 
-__all__ = ["Metadata", "preprocess", "merge"]
+__all__ = ["Metadata", "preprocess", "merge", "flatten"]
 
 import warnings
 from dataclasses import dataclass
+from enum import Enum
 from typing import Any, Iterable, Literal, Mapping, TypeVar, overload
 
 import numpy as np
@@ -18,9 +19,15 @@ from scipy.stats import wasserstein_distance as wd
 from dataeval.interop import as_numpy, to_numpy
 from dataeval.output import Output, set_metadata
 
-TNum = TypeVar("TNum", int, float)
 DISCRETE_MIN_WD = 0.054
 CONTINUOUS_MIN_SAMPLE_SIZE = 20
+DEFAULT_IMAGE_INDEX_KEY = "_image_index"
+
+
+class DropReason(Enum):
+    INCONSISTENT_KEY = "inconsistent_key"
+    INCONSISTENT_SIZE = "inconsistent_size"
+    NESTED_LIST = "nested_list"
 
 
 T = TypeVar("T")
@@ -42,8 +49,8 @@ def _convert_type(data: str) -> int | float | str: ...
 
 def _convert_type(data: list[str] | str) -> list[int] | list[float] | list[str] | int | float | str:
     """
-    Converts a value or a list of values to the simplest form possible, in preferred order of `int`,
-    `float`, or `string`.
+    Converts a value or a list of values to the simplest form possible,
+    in preferred order of `int`, `float`, or `string`.
 
     Parameters
     ----------
@@ -100,8 +107,16 @@ def _get_key_indices(keys: Iterable[tuple[str, ...]]) -> dict[tuple[str, ...], i
     return indices
 
 
+def _sorted_drop_reasons(d: dict[str, set[DropReason]]) -> dict[str, list[str]]:
+    return {k: sorted({vv.value for vv in v}) for k, v in sorted(d.items(), key=lambda item: item[1])}
+
+
 def _flatten_dict_inner(
-    d: Mapping[str, Any], parent_keys: tuple[str, ...], size: int | None = None, nested: bool = False
+    d: Mapping[str, Any],
+    dropped: dict[tuple[str, ...], set[DropReason]],
+    parent_keys: tuple[str, ...],
+    size: int | None = None,
+    nested: bool = False,
 ) -> tuple[dict[tuple[str, ...], Any], int | None]:
     """
     Recursive internal function for flattening a dictionary.
@@ -110,6 +125,8 @@ def _flatten_dict_inner(
     ----------
     d : dict[str, Any]
         Dictionary to flatten
+    dropped: set[tuple[str, ...]]
+        Reference to set of dropped keys from the dictionary
     parent_keys : tuple[str, ...]
         Parent keys to the current dictionary being flattened
     size : int or None, default None
@@ -120,35 +137,62 @@ def _flatten_dict_inner(
     Returns
     -------
     tuple[dict[tuple[str, ...], Any], int | None]
-        - [0]: Dictionary of flattened values with the keys reformatted as a hierarchical tuple of strings
+        - [0]: Dictionary of flattened values with the keys reformatted as a
+               hierarchical tuple of strings
         - [1]: Size, if any, of the current list of values
     """
     items: dict[tuple[str, ...], Any] = {}
     for k, v in d.items():
         new_keys: tuple[str, ...] = parent_keys + (k,)
         if isinstance(v, dict):
-            fd, size = _flatten_dict_inner(v, new_keys, size=size, nested=nested)
+            fd, size = _flatten_dict_inner(v, dropped, new_keys, size=size, nested=nested)
             items.update(fd)
         elif isinstance(v, (list, tuple)):
-            if not nested and (size is None or size == len(v)):
+            if nested:
+                dropped.setdefault(parent_keys + (k,), set()).add(DropReason.NESTED_LIST)
+            elif size is not None and size != len(v):
+                dropped.setdefault(parent_keys + (k,), set()).add(DropReason.INCONSISTENT_SIZE)
+            else:
                 size = len(v)
                 if all(isinstance(i, dict) for i in v):
                     for sub_dict in v:
-                        fd, size = _flatten_dict_inner(sub_dict, new_keys, size=size, nested=True)
+                        fd, size = _flatten_dict_inner(sub_dict, dropped, new_keys, size=size, nested=True)
                         for fk, fv in fd.items():
                             items.setdefault(fk, []).append(fv)
                 else:
                     items[new_keys] = v
-            else:
-                warnings.warn(f"Dropping nested list found in '{parent_keys + (k, )}'.")
         else:
             items[new_keys] = v
     return items, size
 
 
-def _flatten_dict(
-    d: Mapping[str, Any], sep: str, ignore_lists: bool, fully_qualified: bool
-) -> tuple[dict[str, Any], int]:
+@overload
+def flatten(
+    d: Mapping[str, Any],
+    return_dropped: Literal[True],
+    sep: str = "_",
+    ignore_lists: bool = False,
+    fully_qualified: bool = False,
+) -> tuple[dict[str, Any], int, dict[str, list[str]]]: ...
+
+
+@overload
+def flatten(
+    d: Mapping[str, Any],
+    return_dropped: Literal[False] = False,
+    sep: str = "_",
+    ignore_lists: bool = False,
+    fully_qualified: bool = False,
+) -> tuple[dict[str, Any], int]: ...
+
+
+def flatten(
+    d: Mapping[str, Any],
+    return_dropped: bool = False,
+    sep: str = "_",
+    ignore_lists: bool = False,
+    fully_qualified: bool = False,
+):
     """
     Flattens a dictionary and converts values to numeric values when possible.
 
@@ -156,33 +200,53 @@ def _flatten_dict(
     ----------
     d : dict[str, Any]
         Dictionary to flatten
-    sep : str
+    return_dropped: bool, default False
+        Option to return a dictionary of dropped keys and the reason(s) for dropping
+    sep : str, default "_"
         String separator to use when concatenating key names
-    ignore_lists : bool
+    ignore_lists : bool, default False
         Option to skip expanding lists within metadata
-    fully_qualified : bool
-        Option to return dictionary keys full qualified instead of minimized
+    fully_qualified : bool, default False
+        Option to return dictionary keys fully qualified instead of reduced
 
     Returns
     -------
     dict[str, Any]
-        A flattened dictionary
+        Dictionary of flattened values with the keys reformatted as a hierarchical tuple of strings
+    int
+        Size of the values in the flattened dictionary
+    dict[str, list[str]], Optional
+        Dictionary containing dropped keys and reason(s) for dropping
     """
-    expanded, size = _flatten_dict_inner(d, parent_keys=(), nested=ignore_lists)
+    dropped_inner: dict[tuple[str, ...], set[DropReason]] = {}
+    expanded, size = _flatten_dict_inner(d, dropped=dropped_inner, parent_keys=(), nested=ignore_lists)
 
     output = {}
-    if fully_qualified:
-        expanded = {sep.join(k): v for k, v in expanded.items()}
-    else:
-        keys = _get_key_indices(expanded)
-        expanded = {sep.join(k[keys[k] :]): v for k, v in expanded.items()}
     for k, v in expanded.items():
         cv = _convert_type(v)
-        if isinstance(cv, list) and len(cv) == size:
-            output[k] = cv
+        if isinstance(cv, list):
+            if len(cv) == size:
+                output[k] = cv
+            else:
+                dropped_inner.setdefault(k, set()).add(DropReason.INCONSISTENT_KEY)
         elif not isinstance(cv, list):
             output[k] = cv if not size else [cv] * size
-    return output, size if size is not None else 1
+
+    if fully_qualified:
+        output = {sep.join(k): v for k, v in output.items()}
+    else:
+        keys = _get_key_indices(output)
+        output = {sep.join(k[keys[k] :]): v for k, v in output.items()}
+
+    size = size if size is not None else 1
+    dropped = {sep.join(k): v for k, v in dropped_inner.items()}
+
+    if return_dropped:
+        return output, size, _sorted_drop_reasons(dropped)
+    else:
+        if dropped:
+            warnings.warn(f"Metadata keys {list(dropped)} were dropped.")
+        return output, size
 
 
 def _is_metadata_dict_of_dicts(metadata: Mapping) -> bool:
@@ -200,48 +264,75 @@ def _is_metadata_dict_of_dicts(metadata: Mapping) -> bool:
     return set(metadata[keys[0]]) == set(metadata[keys[1]])
 
 
+@overload
+def merge(
+    metadata: Iterable[Mapping[str, Any]],
+    return_dropped: Literal[True],
+    ignore_lists: bool = False,
+    fully_qualified: bool = False,
+    return_numpy: bool = False,
+) -> tuple[dict[str, list[Any]] | dict[str, NDArray[Any]], dict[str, list[str]]]: ...
+
+
+@overload
+def merge(
+    metadata: Iterable[Mapping[str, Any]],
+    return_dropped: Literal[False] = False,
+    ignore_lists: bool = False,
+    fully_qualified: bool = False,
+    return_numpy: bool = False,
+) -> dict[str, list[Any]] | dict[str, NDArray[Any]]: ...
+
+
 def merge(
     metadata: Iterable[Mapping[str, Any]],
+    return_dropped: bool = False,
     ignore_lists: bool = False,
     fully_qualified: bool = False,
-    as_numpy: bool = False,
-) -> tuple[dict[str, list[Any]] | dict[str, NDArray[Any]], NDArray[np.int_]]:
+    return_numpy: bool = False,
+):
     """
-    Merges a collection of metadata dictionaries into a single flattened dictionary of keys and values.
+    Merges a collection of metadata dictionaries into a single flattened
+    dictionary of keys and values.
 
-    Nested dictionaries are flattened, and lists are expanded. Nested lists are dropped as the
-    expanding into multiple hierarchical trees is not supported.
+    Nested dictionaries are flattened, and lists are expanded. Nested lists are
+    dropped as the expanding into multiple hierarchical trees is not supported.
+    The function adds an internal "_image_index" key to the metadata dictionary
+    for consumption by the preprocess function.
 
     Parameters
     ----------
     metadata : Iterable[Mapping[str, Any]]
         Iterable collection of metadata dictionaries to flatten and merge
+    return_dropped: bool, default False
+        Option to return a dictionary of dropped keys and the reason(s) for dropping
     ignore_lists : bool, default False
         Option to skip expanding lists within metadata
     fully_qualified : bool, default False
         Option to return dictionary keys full qualified instead of minimized
-    as_numpy : bool, default False
+    return_numpy : bool, default False
         Option to return results as lists or NumPy arrays
 
     Returns
     -------
-    dict[str, list[Any]] or dict[str, NDArray[Any]]
+    dict[str, list[Any]] | dict[str, NDArray[Any]]
         A single dictionary containing the flattened data as lists or NumPy arrays
-    NDArray[np.int_]
-        Array defining where individual images start, helpful when working with object detection metadata
+    dict[str, list[str]], Optional
+        Dictionary containing dropped keys and reason(s) for dropping
 
     Note
     ----
-    Nested lists of values and inconsistent keys are dropped in the merged metadata dictionary
+    Nested lists of values and inconsistent keys are dropped in the merged
+    metadata dictionary
 
     Example
     -------
     >>> list_metadata = [{"common": 1, "target": [{"a": 1, "b": 3, "c": 5}, {"a": 2, "b": 4}], "source": "example"}]
-    >>> reorganized_metadata, image_indicies = merge(list_metadata)
+    >>> reorganized_metadata, dropped_keys = merge(list_metadata, return_dropped=True)
     >>> reorganized_metadata
-    {'common': [1, 1], 'a': [1, 2], 'b': [3, 4], 'source': ['example', 'example']}
-    >>> image_indicies
-    array([0])
+    {'common': [1, 1], 'a': [1, 2], 'b': [3, 4], 'source': ['example', 'example'], '_image_index': [0, 0]}
+    >>> dropped_keys
+    {'target_c': ['inconsistent_key']}
     """
     merged: dict[str, list[Any]] = {}
     isect: set[str] = set()
@@ -258,56 +349,71 @@ def merge(
     else:
         dicts = list(metadata)
 
-    image_repeats = np.zeros(len(dicts))
+    image_repeats = np.zeros(len(dicts), dtype=np.int_)
+    dropped: dict[str, set[DropReason]] = {}
     for i, d in enumerate(dicts):
-        flattened, image_repeats[i] = _flatten_dict(
-            d, sep="_", ignore_lists=ignore_lists, fully_qualified=fully_qualified
+        flattened, image_repeats[i], dropped_inner = flatten(
+            d,
+            return_dropped=True,
+            ignore_lists=ignore_lists,
+            fully_qualified=fully_qualified,
         )
         isect = isect.intersection(flattened.keys()) if isect else set(flattened.keys())
-        union = union.union(flattened.keys())
+        union.update(flattened.keys())
+        for k, v in dropped_inner.items():
+            dropped.setdefault(k, set()).update({DropReason(vv) for vv in v})
         for k, v in flattened.items():
             merged.setdefault(k, []).extend(flattened[k]) if isinstance(v, list) else merged.setdefault(k, []).append(v)
 
-    if len(union) > len(isect):
-        warnings.warn(f"Inconsistent metadata keys found. Dropping {union - isect} from metadata.")
-
-    output: dict[str, Any] = {}
+    for k in union - isect:
+        dropped.setdefault(k, set()).add(DropReason.INCONSISTENT_KEY)
 
     if image_repeats.sum() == image_repeats.size:
-        image_indicies = np.arange(image_repeats.size)
+        image_indices = np.arange(image_repeats.size)
     else:
         image_ids = np.arange(image_repeats.size)
         image_data = np.concatenate(
             [np.repeat(image_ids[i], image_repeats[i]) for i in range(image_ids.size)], dtype=np.int_
         )
-        _, image_unsorted = np.unique(image_data, return_index=True)
-        image_indicies = np.sort(image_unsorted)
+        _, image_unsorted = np.unique(image_data, return_inverse=True)
+        image_indices = np.sort(image_unsorted)
+
+    output: dict[str, Any] = {}
 
     if keys:
-        output["keys"] = np.array(keys) if as_numpy else keys
+        output["keys"] = np.array(keys) if return_numpy else keys
 
     for k in (key for key in merged if key in isect):
         cv = _convert_type(merged[k])
-        output[k] = np.array(cv) if as_numpy else cv
+        output[k] = np.array(cv) if return_numpy else cv
+    output[DEFAULT_IMAGE_INDEX_KEY] = np.array(image_indices) if return_numpy else list(image_indices)
 
-    return output, image_indicies
+    if return_dropped:
+        return output, _sorted_drop_reasons(dropped)
+    else:
+        if dropped:
+            warnings.warn(f"Metadata keys {list(dropped)} were dropped.")
+        return output
 
 
 @dataclass(frozen=True)
 class Metadata(Output):
     """
-    Dataclass containing binned metadata from the :func:`preprocess` function
+    Dataclass containing binned metadata from the :func:`preprocess` function.
 
     Attributes
     ----------
     discrete_factor_names : list[str]
-        List containing factor names for the original data that was discrete and the binned continuous data
+        List containing factor names for the original data that was discrete and
+        the binned continuous data
     discrete_data : NDArray[np.int]
-        Array containing values for the original data that was discrete and the binned continuous data
+        Array containing values for the original data that was discrete and the
+        binned continuous data
     continuous_factor_names : list[str]
         List containing factor names for the original continuous data
     continuous_data : NDArray[np.int or np.double] | None
-        Array containing values for the original continuous data or None if there was no continuous data
+        Array containing values for the original continuous data or None if there
+        was no continuous data
     class_labels : NDArray[np.int]
         Numerical class labels for the images/objects
     class_names : NDArray[Any]
@@ -327,11 +433,12 @@ class Metadata(Output):
 
 @set_metadata
 def preprocess(
-    raw_metadata: Iterable[Mapping[str, Any]],
+    metadata: dict[str, list[Any]] | dict[str, NDArray[Any]],
     class_labels: ArrayLike | str,
-    continuous_factor_bins: Mapping[str, int | list[tuple[TNum, TNum]]] | None = None,
+    continuous_factor_bins: Mapping[str, int | Iterable[float]] | None = None,
     auto_bin_method: Literal["uniform_width", "uniform_count", "clusters"] = "uniform_width",
     exclude: Iterable[str] | None = None,
+    image_index_key: str = "_image_index",
 ) -> Metadata:
     """
     Restructures the metadata to be in the correct format for the bias functions.
@@ -343,46 +450,84 @@ def preprocess(
 
     Parameters
     ----------
-    raw_metadata : Iterable[Mapping[str, Any]]
-        Iterable collection of metadata dictionaries to flatten and merge.
+    metadata : dict[str, list[Any] | NDArray[Any]]
+        A flat dictionary which contains all of the metadata on a per image (classification)
+        or per object (object detection) basis. Length of lists/array should match the length
+        of the label list/array.
     class_labels : ArrayLike or string
-        If arraylike, expects the labels for each image (image classification) or each object (object detection).
-        If the labels are included in the metadata dictionary, pass in the key value.
-    continuous_factor_bins : Mapping[str, int] or Mapping[str, list[tuple[TNum, TNum]]] or None, default None
-        User provided dictionary specifying how to bin the continuous metadata factors
+        If arraylike, expects the labels for each image (image classification)
+        or each object (object detection). If the labels are included in the
+        metadata dictionary, pass in the key value.
+    continuous_factor_bins : Mapping[str, int or Iterable[float]] or None, default None
+        User provided dictionary specifying how to bin the continuous metadata
+        factors where the value is either an int to represent the number of bins,
+        or a list of floats representing the edges for each bin.
     auto_bin_method : "uniform_width" or "uniform_count" or "clusters", default "uniform_width"
-        Method by which the function will automatically bin continuous metadata factors. It is recommended
-        that the user provide the bins through the `continuous_factor_bins`.
+        Method by which the function will automatically bin continuous metadata factors.
+        It is recommended that the user provide the bins through the `continuous_factor_bins`.
     exclude : Iterable[str] or None, default None
         User provided collection of metadata keys to exclude when processing metadata.
+    image_index_key : str, default "_image_index"
+        User provided metadata key which maps the metadata entry to the source image.
 
     Returns
     -------
     Metadata
         Output class containing the binned metadata
+
+    See Also
+    --------
+    merge
     """
-    # Transform metadata into single, flattened dictionary
-    metadata, image_repeats = merge(raw_metadata)
+    # Check that metadata is a single, flattened dictionary with uniform array lengths
+    check_length = -1
+    for k, v in metadata.items():
+        if not isinstance(v, (list, tuple, np.ndarray)):
+            raise TypeError(
+                "Metadata dictionary needs to be a single dictionary whose values "
+                "are arraylike containing the metadata on a per image or per object basis."
+            )
+        else:
+            if check_length == -1:
+                check_length = len(v)
+            else:
+                if check_length != len(v):
+                    raise ValueError(
+                        "The lists/arrays in the metadata dict have varying lengths. "
+                        "Preprocess needs them to be uniform in length."
+                    )
+
+    # Grab continuous factors if supplied
+    continuous_factor_bins = dict(continuous_factor_bins) if continuous_factor_bins else None
 
     # Drop any excluded metadata keys
-    if exclude:
-        for k in list(metadata):
-            if k in exclude:
-                metadata.pop(k)
+    for k in exclude or ():
+        metadata.pop(k, None)
+        if continuous_factor_bins:
+            continuous_factor_bins.pop(k, None)
 
-    # Get the class label array in numeric form
+    # Get the class label array in numeric form and check its dimensions
     class_array = as_numpy(metadata.pop(class_labels)) if isinstance(class_labels, str) else as_numpy(class_labels)
     if class_array.ndim > 1:
         raise ValueError(
             f"Got class labels with {class_array.ndim}-dimensional "
             f"shape {class_array.shape}, but expected a 1-dimensional array."
         )
+    # Check if the label array is the same length as the metadata arrays
+    elif len(class_array) != check_length:
+        raise ValueError(
+            f"The length of the label array {len(class_array)} is not the same as "
+            f"the length of the metadata arrays {check_length}."
+        )
     if not np.issubdtype(class_array.dtype, np.int_):
         unique_classes, numerical_labels = np.unique(class_array, return_inverse=True)
     else:
         numerical_labels = class_array
         unique_classes = np.unique(class_array)
 
+    # Determine if _image_index is given
+    image_indices = as_numpy(metadata[image_index_key]) if image_index_key in metadata else np.arange(check_length)
+
     # Bin according to user supplied bins
     continuous_metadata = {}
     discrete_metadata = {}
@@ -394,8 +539,8 @@ def preprocess(
                 "but are not keys in the `metadata` dictionary. Delete these keys from `continuous_factor_bins` "
                 "or add corresponding entries to the `metadata` dictionary."
             )
-        for factor, grouping in continuous_factor_bins.items():
-            discrete_metadata[factor] = _user_defined_bin(metadata[factor], grouping)
+        for factor, bins in continuous_factor_bins.items():
+            discrete_metadata[factor] = _digitize_data(metadata[factor], bins)
             continuous_metadata[factor] = metadata[factor]
 
     # Determine category of the rest of the keys
@@ -403,7 +548,7 @@ def preprocess(
     for key in remaining_keys:
         data = to_numpy(metadata[key])
         if np.issubdtype(data.dtype, np.number):
-            result = _is_continuous(data, image_repeats)
+            result = _is_continuous(data, image_indices)
             if result:
                 continuous_metadata[key] = data
             unique_samples, ordinal_data = np.unique(data, return_inverse=True)
@@ -417,11 +562,11 @@ def preprocess(
                     "bins using the continuous_factor_bins parameter.",
                     UserWarning,
                 )
-                discrete_metadata[key] = _binning_function(data, auto_bin_method)
+                discrete_metadata[key] = _bin_data(data, auto_bin_method)
         else:
             _, discrete_metadata[key] = np.unique(data, return_inverse=True)
 
-    # splitting out the dictionaries into the keys and values
+    # Split out the dictionaries into the keys and values
     discrete_factor_names = list(discrete_metadata.keys())
     discrete_data = np.stack(list(discrete_metadata.values()), axis=-1)
     continuous_factor_names = list(continuous_metadata.keys())
@@ -439,7 +584,7 @@ def preprocess(
     )
 
 
-def _user_defined_bin(data: list[Any] | NDArray[Any], binning: int | list[tuple[TNum, TNum]]) -> NDArray[np.intp]:
+def _digitize_data(data: list[Any] | NDArray[Any], bins: int | Iterable[float]) -> NDArray[np.intp]:
     """
     Digitizes a list of values into a given number of bins.
 
@@ -447,8 +592,8 @@ def _user_defined_bin(data: list[Any] | NDArray[Any], binning: int | list[tuple[
     ----------
     data : list | NDArray
         The values to be digitized.
-    binning :  int | list[tuple[TNum, TNum]]
-        The number of bins for the discrete values that data will be digitized into.
+    bins : int | Iterable[float]
+        The number of bins or list of bin edges for the discrete values that data will be digitized into.
 
     Returns
     -------
@@ -461,16 +606,16 @@ def _user_defined_bin(data: list[Any] | NDArray[Any], binning: int | list[tuple[
             "Encountered a data value with non-numeric type when digitizing a factor. "
             "Ensure all occurrences of continuous factors are numeric types."
         )
-    if type(binning) is int:
-        _, bin_edges = np.histogram(data, bins=binning)
+    if isinstance(bins, int):
+        _, bin_edges = np.histogram(data, bins=bins)
         bin_edges[-1] = np.inf
         bin_edges[0] = -np.inf
     else:
-        bin_edges = binning
+        bin_edges = list(bins)
     return np.digitize(data, bin_edges)
 
 
-def _binning_function(data: NDArray[Any], bin_method: str) -> NDArray[np.int_]:
+def _bin_data(data: NDArray[Any], bin_method: str) -> NDArray[np.int_]:
     """
     Bins continuous data through either equal width bins, equal amounts in each bin, or by clusters.
     """
@@ -482,26 +627,26 @@ def _binning_function(data: NDArray[Any], bin_method: str) -> NDArray[np.int_]:
         )
         bin_method = "uniform_width"
 
-    if bin_method != "clusters":
-        counts, bin_edges = np.histogram(data, bins="auto")
-        n_bins = counts.size
-        if counts[counts > 0].min() < 10:
-            for _ in range(20):
-                n_bins -= 1
-                counts, bin_edges = np.histogram(data, bins=n_bins)
-                if counts[counts > 0].min() >= 10 or n_bins < 2:
-                    break
+    # if bin_method != "clusters":  # restore this when clusters bin_method is available
+    counts, bin_edges = np.histogram(data, bins="auto")
+    n_bins = counts.size
+    if counts[counts > 0].min() < 10:
+        counter = 20
+        while counts[counts > 0].min() < 10 and n_bins >= 2 and counter > 0:
+            counter -= 1
+            n_bins -= 1
+            counts, bin_edges = np.histogram(data, bins=n_bins)
 
-        if bin_method == "uniform_count":
-            quantiles = np.linspace(0, 100, n_bins + 1)
-            bin_edges = np.asarray(np.percentile(data, quantiles))
+    if bin_method == "uniform_count":
+        quantiles = np.linspace(0, 100, n_bins + 1)
+        bin_edges = np.asarray(np.percentile(data, quantiles))
 
     bin_edges[0] = -np.inf  # type: ignore # until the clusters speed up is merged
     bin_edges[-1] = np.inf  # type: ignore # and the _binning_by_clusters can be uncommented
     return np.digitize(data, bin_edges)  # type: ignore
 
 
-def _is_continuous(data: NDArray[np.number], image_indicies: NDArray[np.number]) -> bool:
+def _is_continuous(data: NDArray[np.number], image_indices: NDArray[np.number]) -> bool:
     """
     Determines whether the data is continuous or discrete using the Wasserstein distance.
 
@@ -520,11 +665,11 @@ def _is_continuous(data: NDArray[np.number], image_indicies: NDArray[np.number])
     measured from a uniform distribution is greater or less than 0.054, respectively.
     """
     # Check if the metadata is image specific
-    _, data_indicies_unsorted = np.unique(data, return_index=True)
-    if data_indicies_unsorted.size == image_indicies.size:
-        data_indicies = np.sort(data_indicies_unsorted)
-        if (data_indicies == image_indicies).all():
-            data = data[data_indicies]
+    _, data_indices_unsorted = np.unique(data, return_index=True)
+    if data_indices_unsorted.size == image_indices.size:
+        data_indices = np.sort(data_indices_unsorted)
+        if (data_indices == image_indices).all():
+            data = data[data_indices]
 
     # OLD METHOD
     # uvals = np.unique(data)
@@ -572,7 +717,7 @@ def get_counts(data: NDArray[np.int_], min_num_bins: int | None = None) -> NDArr
 
     Returns
     -------
-    NDArray[np.int_]
+    NDArray[np.int]
         Bin counts per column of data.
     """
     max_value = data.max() + 1 if min_num_bins is None else min_num_bins