dataeval 0.64.0__py3-none-any.whl → 0.65.0__py3-none-any.whl

dataeval/_internal/metrics/utils.py

@@ -2,6 +2,7 @@ from typing import Any, Callable, Dict, List, Literal, NamedTuple, Optional, Seq
 
 import numpy as np
 import xxhash as xxh
+from numpy.typing import NDArray
 from PIL import Image
 from scipy.fftpack import dct
 from scipy.signal import convolve2d
@@ -25,7 +26,7 @@ def get_method(method_map: Dict[str, Callable], method: str) -> Callable:
 
 
 def get_counts(
-    data: np.ndarray, names: List[str], is_categorical: List[bool], subset_mask: Optional[np.ndarray] = None
+    data: NDArray, names: List[str], is_categorical: List[bool], subset_mask: Optional[NDArray[np.bool_]] = None
 ) -> tuple[Dict, Dict]:
     """
     Initialize dictionary of histogram counts --- treat categorical values
@@ -33,7 +34,7 @@ def get_counts(
 
     Parameters
     ----------
-    subset_mask: Optional[np.ndarray[bool]]
+    subset_mask: Optional[NDArray[np.bool_]]
         Boolean mask of samples to bin (e.g. when computing per class).  True -> include in histogram counts
 
     Returns
@@ -66,24 +67,24 @@ def get_counts(
 
 
 def entropy(
-    data: np.ndarray,
+    data: NDArray,
     names: List[str],
     is_categorical: List[bool],
     normalized: bool = False,
-    subset_mask: Optional[np.ndarray] = None,
-) -> np.ndarray:
+    subset_mask: Optional[NDArray[np.bool_]] = None,
+) -> NDArray[np.float64]:
     """
     Meant for use with Bias metrics, Balance, Diversity, ClasswiseBalance,
     and Classwise Diversity.
 
-    Compute entropy for discrete/categorical variables and, through standard
-    histogram binning, for continuous variables.
+    Compute entropy for discrete/categorical variables and for continuous variables through standard
+    histogram binning.
 
     Parameters
     ----------
     normalized: bool
         Flag that determines whether or not to normalize entropy by log(num_bins)
-    subset_mask: Optional[np.ndarray[bool]]
+    subset_mask: Optional[NDArray[np.bool_]]
         Boolean mask of samples to bin (e.g. when computing per class).  True -> include in histogram counts
 
     Notes
@@ -93,7 +94,7 @@ def entropy(
 
     Returns
     -------
-    ent: np.ndarray[float]
+    ent: NDArray[np.float64]
         Entropy estimate per column of X
 
     See Also
@@ -119,16 +120,20 @@ def entropy(
 
 
 def get_num_bins(
-    data: np.ndarray, names: List[str], is_categorical: List[bool], subset_mask: Optional[np.ndarray] = None
-) -> np.ndarray:
+    data: NDArray, names: List[str], is_categorical: List[bool], subset_mask: Optional[NDArray[np.bool_]] = None
+) -> NDArray[np.float64]:
     """
     Number of bins or unique values for each metadata factor, used to
     normalize entropy/diversity.
 
     Parameters
     ----------
-    subset_mask: Optional[np.ndarray[bool]]
+    subset_mask: Optional[NDArray[np.bool_]]
         Boolean mask of samples to bin (e.g. when computing per class).  True -> include in histogram counts
+
+    Returns
+    -------
+    NDArray[np.float64]
     """
     # likely cached
     hist_counts, _ = get_counts(data, names, is_categorical, subset_mask)
@@ -139,7 +144,7 @@ def get_num_bins(
     return num_bins
 
 
-def infer_categorical(X: np.ndarray, threshold: float = 0.5) -> np.ndarray:
+def infer_categorical(X: NDArray, threshold: float = 0.5) -> NDArray:
     """
     Compute fraction of feature values that are unique --- intended to be used
     for inferring whether variables are categorical.
@@ -154,9 +159,11 @@ def infer_categorical(X: np.ndarray, threshold: float = 0.5) -> np.ndarray:
     return pct_unique < threshold
 
 
-def preprocess_metadata(class_labels: Sequence[int], metadata: List[Dict]) -> Tuple[np.ndarray, List[str], List[bool]]:
+def preprocess_metadata(
+    class_labels: Sequence[int], metadata: List[Dict], cat_thresh: float = 0.2
+) -> Tuple[NDArray, List[str], List[bool]]:
     # convert class_labels and list of metadata dicts to dict of ndarrays
-    metadata_dict: Dict[str, np.ndarray] = {
+    metadata_dict: Dict[str, NDArray] = {
         "class_label": np.asarray(class_labels, dtype=int),
         **{k: np.array([d[k] for d in metadata]) for k in metadata[0]},
     }
@@ -172,18 +179,35 @@ def preprocess_metadata(class_labels: Sequence[int], metadata: List[Dict]) -> Tu
 
     data = np.stack(list(metadata_dict.values()), axis=-1)
     names = list(metadata_dict.keys())
-    is_categorical = [infer_categorical(metadata_dict[var], 0.25)[0] for var in names]
+    is_categorical = [infer_categorical(metadata_dict[var], cat_thresh)[0] for var in names]
 
     return data, names, is_categorical
 
 
-def minimum_spanning_tree(X: np.ndarray) -> Any:
+def flatten(X: NDArray):
+    """
+    Flattens input array from (N, ... ) to (N, -1) where all samples N have all data in their last dimension
+
+    Parameters
+    ----------
+    X : NDArray, shape - (N, ... )
+        Input array
+
+    Returns
+    -------
+    NDArray, shape - (N, -1)
+    """
+
+    return X.reshape((X.shape[0], -1))
+
+
+def minimum_spanning_tree(X: NDArray) -> Any:
     """
     Returns the minimum spanning tree from a NumPy image array.
 
     Parameters
     ----------
-    X: np.ndarray
+    X : NDArray
         Numpy image array
 
     Returns
@@ -191,7 +215,7 @@ def minimum_spanning_tree(X: np.ndarray) -> Any:
         Data representing the minimum spanning tree
     """
     # All features belong on second dimension
-    X = X.reshape((X.shape[0], -1))
+    X = flatten(X)
     # We add a small constant to the distance matrix to ensure scipy interprets
     # the input graph as fully-connected.
     dense_eudist = squareform(pdist(X)) + EPSILON
@@ -199,13 +223,13 @@ def minimum_spanning_tree(X: np.ndarray) -> Any:
     return mst(eudist_csr)
 
 
-def get_classes_counts(labels: np.ndarray) -> Tuple[int, int]:
+def get_classes_counts(labels: NDArray) -> Tuple[int, int]:
     """
     Returns the classes and counts of from an array of labels
 
     Parameters
     ----------
-    label: np.ndarray
+    label : NDArray
         Numpy labels array
 
     Returns
@@ -226,17 +250,17 @@ def get_classes_counts(labels: np.ndarray) -> Tuple[int, int]:
 
 
 def compute_neighbors(
-    A: np.ndarray,
-    B: np.ndarray,
+    A: NDArray,
+    B: NDArray,
     k: int = 1,
     algorithm: Literal["auto", "ball_tree", "kd_tree"] = "auto",
-) -> np.ndarray:
+) -> NDArray:
     """
     For each sample in A, compute the nearest neighbor in B
 
     Parameters
     ----------
-    A, B : np.ndarray
+    A, B : NDArray
         The n_samples and n_features respectively
     k : int
         The number of neighbors to find
@@ -252,11 +276,24 @@ def compute_neighbors(
     List:
         Closest points to each point in A and B
 
+    Raises
+    ------
+    ValueError
+        If algorithm is not "auto", "ball_tree", or "kd_tree"
+
     See Also
     --------
     sklearn.neighbors.NearestNeighbors
     """
 
+    if k < 1:
+        raise ValueError("k must be >= 1")
+    if algorithm not in ["auto", "ball_tree", "kd_tree"]:
+        raise ValueError("Algorithm must be 'auto', 'ball_tree', or 'kd_tree'")
+
+    A = flatten(A)
+    B = flatten(B)
+
     nbrs = NearestNeighbors(n_neighbors=k + 1, algorithm=algorithm).fit(B)
     nns = nbrs.kneighbors(A)[1]
     nns = nns[:, 1:].squeeze()
@@ -270,7 +307,7 @@ class BitDepth(NamedTuple):
     pmax: Union[float, int]
 
 
-def get_bitdepth(image: np.ndarray) -> BitDepth:
+def get_bitdepth(image: NDArray) -> BitDepth:
     """
     Approximates the bit depth of the image using the
     min and max pixel values.
@@ -283,7 +320,7 @@ def get_bitdepth(image: np.ndarray) -> BitDepth:
         return BitDepth(depth, 0, 2**depth - 1)
 
 
-def rescale(image: np.ndarray, depth: int = 1) -> np.ndarray:
+def rescale(image: NDArray, depth: int = 1) -> NDArray:
     """
     Rescales the image using the bit depth provided.
     """
@@ -295,7 +332,7 @@ def rescale(image: np.ndarray, depth: int = 1) -> np.ndarray:
         return normalized * (2**depth - 1)
 
 
-def normalize_image_shape(image: np.ndarray) -> np.ndarray:
+def normalize_image_shape(image: NDArray) -> NDArray:
     """
     Normalizes the image shape into (C,H,W).
     """
@@ -311,7 +348,7 @@ def normalize_image_shape(image: np.ndarray) -> np.ndarray:
         raise ValueError("Images must have 2 or more dimensions.")
 
 
-def edge_filter(image: np.ndarray, offset: float = 0.5) -> np.ndarray:
+def edge_filter(image: NDArray, offset: float = 0.5) -> NDArray:
     """
     Returns the image filtered using a 3x3 edge detection kernel:
     [[ -1, -1, -1 ],
@@ -323,7 +360,7 @@ def edge_filter(image: np.ndarray, offset: float = 0.5) -> np.ndarray:
     return edges
 
 
-def pchash(image: np.ndarray) -> str:
+def pchash(image: NDArray) -> str:
     """
     Performs a perceptual hash on an image by resizing to a square NxN image
     using the Lanczos algorithm where N is 32x32 or the largest multiple of
@@ -334,7 +371,7 @@ def pchash(image: np.ndarray) -> str:
 
     Parameters
     ----------
-    image : np.ndarray
+    image : NDArray
         An image as a numpy array in CxHxW format
 
     Returns
@@ -374,7 +411,7 @@ def pchash(image: np.ndarray) -> str:
     return hash_hex if hash_hex else "0"
 
 
-def xxhash(image: np.ndarray) -> str:
+def xxhash(image: NDArray) -> str:
     """
     Performs a fast non-cryptographic hash using the xxhash algorithm
     (xxhash.com) against the image as a flattened bytearray.  The hash
@@ -382,7 +419,7 @@ def xxhash(image: np.ndarray) -> str:
 
     Parameters
     ----------
-    image : np.ndarray
+    image : NDArray
         An image as a numpy array
 
     Returns

dataeval/_internal/models/tensorflow/losses.py

@@ -8,9 +8,9 @@ Licensed under Apache Software License (Apache 2.0)
 
 from typing import Literal, Optional, Union, cast
 
-import numpy as np
 import tensorflow as tf
 from keras.layers import Flatten
+from numpy.typing import NDArray
 from tensorflow_probability.python.distributions.mvn_diag import MultivariateNormalDiag
 from tensorflow_probability.python.distributions.mvn_tril import MultivariateNormalTriL
 from tensorflow_probability.python.stats import covariance
@@ -35,12 +35,12 @@ class Elbo:
     def __init__(
         self,
         cov_type: Union[Literal["cov_full", "cov_diag"], float] = 1.0,
-        x: Optional[Union[tf.Tensor, np.ndarray]] = None,
+        x: Optional[Union[tf.Tensor, NDArray]] = None,
     ):
         if isinstance(cov_type, float):
             self.cov = ("sim", cov_type)
         elif cov_type in ["cov_full", "cov_diag"]:
-            x_np: np.ndarray = x.numpy() if tf.is_tensor(x) else x  # type: ignore
+            x_np: NDArray = x.numpy() if tf.is_tensor(x) else x  # type: ignore
             cov = covariance(x_np.reshape(x_np.shape[0], -1))  # type: ignore py38
             if cov_type == "cov_diag":  # infer standard deviation from covariance matrix
                 cov = tf.math.sqrt(tf.linalg.diag_part(cov))

dataeval/_internal/models/tensorflow/trainer.py

@@ -11,12 +11,13 @@ from typing import Callable, Iterable, Optional, Tuple, cast
 import keras
 import numpy as np
 import tensorflow as tf
+from numpy.typing import NDArray
 
 
 def trainer(
     model: keras.Model,
-    x_train: np.ndarray,
-    y_train: Optional[np.ndarray] = None,
+    x_train: NDArray,
+    y_train: Optional[NDArray] = None,
     loss_fn: Optional[Callable[..., tf.Tensor]] = None,
     optimizer: keras.optimizers.Optimizer = keras.optimizers.Adam,
     preprocess_fn: Optional[Callable[[tf.Tensor], tf.Tensor]] = None,

dataeval/_internal/models/tensorflow/utils.py

@@ -21,6 +21,7 @@ from keras.layers import (
     InputLayer,
     Reshape,
 )
+from numpy.typing import NDArray
 from tensorflow._api.v2.nn import relu, softmax, tanh
 
 from dataeval._internal.models.tensorflow.autoencoder import AE, AEGMM, VAE, VAEGMM
@@ -28,12 +29,12 @@ from dataeval._internal.models.tensorflow.pixelcnn import PixelCNN
 
 
 def predict_batch(
-    x: Union[list, np.ndarray, tf.Tensor],
+    x: Union[list, NDArray, tf.Tensor],
     model: Union[Callable, keras.Model],
     batch_size: int = int(1e10),
     preprocess_fn: Optional[Callable] = None,
     dtype: Union[Type[np.generic], tf.DType] = np.float32,
-) -> Union[np.ndarray, tf.Tensor, tuple, list]:
+) -> Union[NDArray, tf.Tensor, tuple, list]:
     """
     Make batch predictions on a model.
 
@@ -80,7 +81,7 @@ def predict_batch(
         else:
             raise TypeError(
                 f"Model output type {type(preds_tmp)} not supported. The model output "
-                f"type needs to be one of list, tuple, np.ndarray or tf.Tensor."
+                f"type needs to be one of list, tuple, NDArray or tf.Tensor."
             )
     concat = np.concatenate if return_np else tf.concat
     out = cast(

dataeval/_internal/output.py

@@ -0,0 +1,82 @@
+import inspect
+from datetime import datetime, timezone
+from functools import wraps
+from typing import Dict, List, Optional
+
+import numpy as np
+
+from dataeval import __version__
+
+
+class OutputMetadata:
+    _name: str
+    _execution_time: str
+    _execution_duration: float
+    _arguments: Dict[str, str]
+    _state: Dict[str, str]
+    _version: str
+
+    def dict(self) -> Dict:
+        return {k: v for k, v in self.__dict__.items() if not k.startswith("_")}
+
+    def meta(self) -> Dict:
+        return {k.removeprefix("_"): v for k, v in self.__dict__.items() if k.startswith("_")}
+
+
+def set_metadata(module_name: str = "", state_attr: Optional[List[str]] = None):
+    def decorator(fn):
+        @wraps(fn)
+        def wrapper(*args, **kwargs):
+            def fmt(v):
+                if np.isscalar(v):
+                    return v
+                if hasattr(v, "shape"):
+                    return f"{v.__class__.__name__}: shape={getattr(v, 'shape')}"
+                if hasattr(v, "__len__"):
+                    return f"{v.__class__.__name__}: len={len(v)}"
+                return f"{v.__class__.__name__}"
+
+            time = datetime.now(timezone.utc)
+            result = fn(*args, **kwargs)
+            duration = (datetime.now(timezone.utc) - time).total_seconds()
+            fn_params = inspect.signature(fn).parameters
+            # set all params with defaults then update params with mapped arguments and explicit keyword args
+            arguments = {k: None if v.default is inspect.Parameter.empty else v.default for k, v in fn_params.items()}
+            arguments.update(zip(fn_params, args))
+            arguments.update(kwargs)
+            arguments = {k: fmt(v) for k, v in arguments.items()}
+            state = (
+                {k: fmt(getattr(args[0], k)) for k in state_attr if "self" in arguments}
+                if "self" in arguments and state_attr
+                else {}
+            )
+            name = args[0].__class__.__name__ if "self" in arguments else fn.__name__
+            metadata = {
+                "_name": f"{module_name}.{name}",
+                "_execution_time": time,
+                "_execution_duration": duration,
+                "_arguments": {k: v for k, v in arguments.items() if k != "self"},
+                "_state": state,
+                "_version": __version__,
+            }
+            for k, v in metadata.items():
+                object.__setattr__(result, k, v)
+            return result
+
+        return wrapper
+
+    return decorator
+
+
+def populate_defaults(d: dict, c: type) -> dict:
+    def default(t):
+        name = t._name if hasattr(t, "_name") else t.__name__  # py3.9 : _name, py3.10 : __name__
+        if name == "Dict":
+            return {}
+        if name == "List":
+            return []
+        if name == "ndarray":
+            return np.array([])
+        raise TypeError("Unrecognized annotation type")
+
+    return {k: d[k] if k in d else default(t) for k, t in c.__annotations__.items()}