req-metrics 0.1.0__py3-none-any.whl

req_metrics/__init__.py

@@ -0,0 +1,180 @@
+"""Label-free representation-quality metrics for layer-wise analysis of pretrained models and for monitoring during training.
+
+Estimators are pure functions on tensors that return a MetricResult. The
+registry records, for each metric, its input contract, canonical preprocessing
+and source citation. Levels (sequence, sample, population) and I/O belong to the
+pipeline, not to the estimators.
+"""
+
+__version__ = "0.1.0"
+
+from req_metrics import protocols
+from req_metrics._types import InputKind, MetricResult, Preprocess
+from req_metrics.calibration import (
+    Calibrated,
+    CalibratedGrid,
+    calibrate,
+    calibrate_grid,
+    calibrate_layers,
+    group_permutation,
+    jackknife,
+)
+from req_metrics.layouts import (
+    frame_tokens_to_pooled,
+    grid_to_pooled,
+    grid_to_tokens,
+    grid_to_trajectory,
+    stack_samples,
+    strip_prefix_tokens,
+)
+from req_metrics.metrics.clustering import cluster_quality, dense_structure, recombine_dense_structure
+from req_metrics.metrics.compare import (
+    cka,
+    cycle_knn,
+    information_imbalance,
+    neighborhood_overlap,
+    procrustes,
+    ridge_predictivity,
+    rsa,
+    svcca,
+)
+from req_metrics.metrics.dimension import gride, mle, mlid, mst_dimension, twonn
+from req_metrics.metrics.distribution import embedding_norm, gaussianity, sparsity
+from req_metrics.metrics.equivariance import pte
+from req_metrics.metrics.functional import jacobian_effective_rank
+from req_metrics.metrics.relational import anisotropy_cosine, normalized_std, self_clustering, uniformity
+from req_metrics.metrics.spectral import (
+    alpha_req,
+    anisotropy_spectral,
+    effective_rank,
+    eigenvalue_early_enrichment,
+    matrix_entropy,
+    participation_ratio,
+)
+from req_metrics.metrics.tokens import cls_patch_cosine, token_cosine, token_gram_drift, token_norm_outliers
+from req_metrics.metrics.trajectory import trajectory_curvature
+from req_metrics.metrics.views import alignment, dime, infonce, lidar
+from req_metrics.monitor import (
+    LayerMonitor,
+    OnlineBuffer,
+    csv_sink,
+    jacobian_products,
+    json_sink,
+    layer_scalars,
+    make_pooler,
+    metric_key,
+    metric_key_prefix,
+    monitor_loader,
+    resolve_layers,
+    tensorboard_sink,
+    wandb_sink,
+)
+from req_metrics.neighbors import Neighbors
+from req_metrics.pipeline import choose_indices, compute, compute_pairs
+from req_metrics.preprocess import apply_preprocess, center, l2_normalize, standardize
+from req_metrics.records import Record, Records
+from req_metrics.registry import MetricSpec, describe, get_metric, list_metrics, register_metric
+from req_metrics.selection import rank_runs, top_layers, top_local_maxima, value_at
+from req_metrics.spectrum import Spectrum
+from req_metrics.stability import Convergence, convergence
+from req_metrics.view_construction import ShiftSpec, ViewSpec, make_shifted, make_views, stack_views
+
+__all__ = [
+    "InputKind",
+    "MetricResult",
+    "Preprocess",
+    "Spectrum",
+    "Neighbors",
+    "MetricSpec",
+    "register_metric",
+    "describe",
+    "get_metric",
+    "list_metrics",
+    "apply_preprocess",
+    "center",
+    "l2_normalize",
+    "standardize",
+    "effective_rank",
+    "matrix_entropy",
+    "alpha_req",
+    "anisotropy_spectral",
+    "anisotropy_cosine",
+    "participation_ratio",
+    "eigenvalue_early_enrichment",
+    "gaussianity",
+    "sparsity",
+    "twonn",
+    "gride",
+    "mle",
+    "mlid",
+    "mst_dimension",
+    "information_imbalance",
+    "trajectory_curvature",
+    "lidar",
+    "infonce",
+    "dime",
+    "ViewSpec",
+    "make_views",
+    "stack_views",
+    "ShiftSpec",
+    "make_shifted",
+    "pte",
+    "token_norm_outliers",
+    "token_cosine",
+    "cls_patch_cosine",
+    "token_gram_drift",
+    "embedding_norm",
+    "Record",
+    "Records",
+    "compute",
+    "compute_pairs",
+    "choose_indices",
+    "protocols",
+    "LayerMonitor",
+    "OnlineBuffer",
+    "metric_key_prefix",
+    "csv_sink",
+    "json_sink",
+    "tensorboard_sink",
+    "wandb_sink",
+    "layer_scalars",
+    "metric_key",
+    "resolve_layers",
+    "make_pooler",
+    "monitor_loader",
+    "grid_to_trajectory",
+    "grid_to_tokens",
+    "grid_to_pooled",
+    "frame_tokens_to_pooled",
+    "strip_prefix_tokens",
+    "stack_samples",
+    "alignment",
+    "normalized_std",
+    "self_clustering",
+    "uniformity",
+    "convergence",
+    "Convergence",
+    "calibrate",
+    "calibrate_grid",
+    "calibrate_layers",
+    "CalibratedGrid",
+    "Calibrated",
+    "group_permutation",
+    "jackknife",
+    "neighborhood_overlap",
+    "cka",
+    "cycle_knn",
+    "procrustes",
+    "ridge_predictivity",
+    "rsa",
+    "svcca",
+    "cluster_quality",
+    "dense_structure",
+    "recombine_dense_structure",
+    "jacobian_effective_rank",
+    "jacobian_products",
+    "rank_runs",
+    "top_layers",
+    "top_local_maxima",
+    "value_at",
+]

req_metrics/_types.py

@@ -0,0 +1,44 @@
+"""Core types shared by estimators, registry and pipeline."""
+
+from dataclasses import dataclass, field
+from enum import Enum
+
+
+class InputKind(str, Enum):
+    """What an estimator consumes. Shapes use N samples, T tokens, D dims, q views."""
+
+    POINTS = "points"  # (N, D)
+    TRAJECTORY = "trajectory"  # (T, D), time-ordered tokens of one sample
+    VIEWS = "views"  # (q, N, D), q augmented views of the same N samples
+    SHIFTED = "shifted"  # (N, D) plus {semitones: (N, D)} transposed copies
+    TOKENS = "tokens"  # (T, D), the token field of one sample
+    PAIR = "pair"  # two point clouds (N, D_a), (N, D_b) describing the same N items
+    TOKEN_PAIR = "token_pair"  # (T, D) token field and a (T, D_ref) reference field of the same sample
+    JACOBIAN = "jacobian"  # (B, k, M) Jacobian sketches: B inputs, k rows per input
+
+
+@dataclass(frozen=True)
+class Preprocess:
+    """Preprocessing of a (..., N, D) tensor before an estimator.
+
+    center subtracts the feature means over the N samples, standardize also divides
+    each feature by its standard deviation, l2 scales each sample to unit norm;
+    applied in that order.
+    """
+
+    center: bool = False
+    standardize: bool = False
+    l2: bool = False
+
+    def describe(self) -> str:
+        """Short text of the preprocessing steps, as stored in records."""
+        parts = [n for n in ("center", "standardize", "l2") if getattr(self, n)]
+        return "+".join(parts) if parts else "none"
+
+
+@dataclass
+class MetricResult:
+    """One estimator output: the headline value and named secondary quantities."""
+
+    value: float
+    extras: dict[str, float] = field(default_factory=dict)