kuration-sdk 0.1.0__py3-none-any.whl

kuration/__init__.py

@@ -0,0 +1,16 @@
+"""Kuration's public package surface.
+
+Implementation lives in explicit product domains:
+
+- :mod:`kuration.core` contains the dataset-agnostic curation SDK.
+- :mod:`kuration.gameplay` contains maintained gameplay capabilities.
+- :mod:`kuration.selection` contains built-in dataset-selection recipes
+  (balanced coverage, rare-event weighting, cluster-balanced diversity).
+
+Core symbols remain available from :mod:`kuration` for a concise default API, while
+domain-specific code is imported from its domain package.
+"""
+from __future__ import annotations
+
+from .core import *  # noqa: F403
+from .core import __all__ as __all__

kuration/core/__init__.py

@@ -0,0 +1,180 @@
+"""Kuration's dataset-agnostic curation core.
+
+Full pipeline: schema-driven metric DSL, input adapters, segmentation, feature engine,
+binning, declarative strategies (base + feature-space + filter + pre-pass filters + a
+learned-scorer seam), the output layer (declared modalities, npz/json/hdf5/parquet shards,
+hours-budget fill, sub-track spans), tag-coverage, experiment + report, parallel /
+skip-and-log / resume / out-of-core execution, path-agnostic local + cloud I/O, schema
+inference, and an `input(...).operator(...)` pipeline API.
+This package contains no gameplay-specific implementation. See README.md and docs/.
+"""
+from importlib.metadata import PackageNotFoundError, version
+
+try:
+    __version__ = version("kuration-sdk")
+except PackageNotFoundError:  # source tree imported before installation
+    __version__ = "0+unknown"
+
+from .adapters import (
+    CuratedDatasetAdapter,
+    HDF5Adapter,
+    HuggingFaceAdapter,
+    InputAdapter,
+    JsonlAdapter,
+    NpzAdapter,
+    ParquetDatasetAdapter,
+    TimeseriesAdapter,
+    ZarrAdapter,
+    VideoAdapter,
+    JoinedMediaAdapter,
+    detect_adapter,
+    get_adapter,
+    register_adapter,
+    schema_from_output,
+)
+from .binning import bin_segments, quantile_bins
+from .buffer import Tick, WindowBuffer, build_window, empty_like
+from .columnar import IDENTITY_COLUMNS, SegmentBatch, iter_segment_batches
+from .checkpoints import (CheckpointIdentity, PartitionCommit, StageCheckpoint,
+                          canonical_hash)
+from .bounded import FeatureStageResult, checkpoint_feature_stage
+from .external import (external_quantile_edges, external_top_k, external_uniform_bins,
+                       external_weighted, iter_external_span_items,
+                       iter_external_hash_dedup, iter_external_top_k_priorities,
+                       iter_external_uniform_priorities,
+                       iter_external_weighted_priorities)
+from .curate import CurateResult, curate
+from .dsl import compile_metric, register_compiled, register_metric
+from .experiment import Arm, run_experiment, union_define
+from .features import FeatureEngine
+from .infer import infer_schema
+from .io import exists, local_copy, read_text, write_bytes, write_text
+from .loaders import iter_shards, read_shard_arrays, shard_entries
+from .inspection import compare_datasets, inspect_dataset
+from .importing import import_dataset
+from .html_report import render_html_report, write_html_report
+from .manifest import build_config, dump_segment_manifest, write_config
+from .methods import apply_filters, method_priorities, register_filter
+from .pipeline import Pipeline, input
+from . import builtin_operators  # noqa: F401 -- registers every built-in Method/Filter/Scorer
+                                 # as a named Pipeline operator; see builtin_operators.py
+from .plugins import load_plugins, loaded_plugin_modules, plugin_failures, plugin_provenance
+from .plugin_validation import PluginCheckResult, check_plugins
+from .progress import (ProgressCallback, ProgressSpec, cli_progress, notebook_progress,
+                       progress_callback)
+from .report import dataset_report
+from .stats import stats
+from .review import hours_selection_review, preview_dataset, selection_review
+from .quality import (QualityGateError, check_quality, evaluate_quality,
+                      validate_expectations)
+from .scorers import SegmentScorer, apply_scorers, get_scorer, register_scorer
+from .output import (
+    Container,
+    Modality,
+    OutputEncoder,
+    OutputSpec,
+    PassthroughEncoder,
+    Track,
+    fill_to_hours,
+    get_encoder,
+    register_encoder,
+    register_container,
+    output_preset,
+    tracks_from_ticks,
+    write_dataset,
+    write_track,
+)
+from .schema import FieldSpec, MediaSpec, SchemaSpec
+from .media import FrameRef, MediaProvider, PyAVMediaProvider, VideoMetadata
+from .transforms import (BatchTransform, TransformContext, apply_transforms,
+                         apply_transforms_to_windows, get_transform,
+                         register_batch_transform, register_transform)
+from .analyzers import (
+    Analyzer,
+    DatasetAnalyzer,
+    SplitLeakageAnalyzer,
+    StreamAnalyzer,
+    get_analyzer,
+    register_analyzer,
+    run_dataset_analyzer,
+    run_stream_analyzer,
+)
+from .execution import (ExecutionBackend, LocalExecutionBackend, RayExecutionBackend,
+                        ResourceSpec, get_execution_backend)
+from .planning import (OperatorResources, OperatorTraits, PhysicalOperator, PhysicalPlan,
+                       PhysicalStage, compile_physical_plan)
+from .scoring import score_sources
+from .segment import Segment, Segmenter
+from .select import assign_splits
+from .strategy import Strategy, load_strategy, segment_priorities, select
+from .registry import (ComponentDescriptor, Registry, default_registry, describe_component,
+                       list_components, register_method, register_operator, register_preset)
+from .results import (DatasetComparison, DatasetInspection, DatasetPreview, DatasetReport,
+                      DatasetWriteResult, ExecutionPlan, ExperimentResult,
+                      HoursSelectionReview, QualityGateResult, SelectionReview,
+                      StatsExamples, StatsReport, ValidationResult)
+from .versioning import (ARTIFACT_FORMAT_VERSION, JOB_SCHEMA_VERSION, migrate,
+                         migrate_config, migrate_job)
+
+__all__ = [
+    "__version__",
+    "SchemaSpec", "FieldSpec", "MediaSpec",
+    "WindowBuffer", "Tick", "build_window", "empty_like",
+    "SegmentBatch", "IDENTITY_COLUMNS", "iter_segment_batches",
+    "CheckpointIdentity", "PartitionCommit", "StageCheckpoint", "canonical_hash",
+    "FeatureStageResult", "checkpoint_feature_stage",
+    "external_quantile_edges", "external_top_k",
+    "external_uniform_bins",
+    "iter_external_top_k_priorities", "iter_external_span_items",
+    "iter_external_uniform_priorities",
+    "external_weighted", "iter_external_weighted_priorities",
+    "iter_external_hash_dedup",
+    "compile_metric", "register_metric", "register_compiled",
+    "InputAdapter", "TimeseriesAdapter", "HDF5Adapter", "NpzAdapter", "JsonlAdapter",
+    "CuratedDatasetAdapter", "ParquetDatasetAdapter", "HuggingFaceAdapter", "ZarrAdapter",
+    "schema_from_output", "register_adapter", "get_adapter", "detect_adapter",
+    "FrameRef", "VideoMetadata", "MediaProvider", "PyAVMediaProvider",
+    "VideoAdapter", "JoinedMediaAdapter",
+    "BatchTransform", "TransformContext", "register_batch_transform", "register_transform",
+    "get_transform",
+    "apply_transforms", "apply_transforms_to_windows",
+    "Analyzer", "DatasetAnalyzer", "StreamAnalyzer", "SplitLeakageAnalyzer",
+    "register_analyzer", "get_analyzer", "run_dataset_analyzer", "run_stream_analyzer",
+    "ExecutionBackend", "LocalExecutionBackend", "RayExecutionBackend", "ResourceSpec",
+    "get_execution_backend",
+    "OperatorResources", "OperatorTraits", "PhysicalOperator", "PhysicalStage",
+    "PhysicalPlan", "compile_physical_plan",
+    "Arm", "run_experiment", "union_define",
+    "Segment", "Segmenter", "FeatureEngine",
+    "bin_segments", "quantile_bins",
+    "Strategy", "load_strategy", "select", "segment_priorities", "assign_splits",
+    "apply_filters", "method_priorities", "score_sources", "dataset_report", "stats",
+    "register_method", "register_filter", "register_operator",
+    "SegmentScorer", "register_scorer", "get_scorer", "apply_scorers",
+    "curate", "CurateResult",
+    "build_config", "dump_segment_manifest", "write_config",
+    "read_text", "write_text", "write_bytes", "exists", "local_copy",
+    "OutputSpec", "Modality", "OutputEncoder", "PassthroughEncoder",
+    "register_encoder", "get_encoder", "Track", "tracks_from_ticks", "write_track",
+    "fill_to_hours", "write_dataset",
+    "Container", "register_container", "output_preset", "register_preset",
+    "iter_shards", "read_shard_arrays", "shard_entries",
+    "inspect_dataset", "DatasetInspection", "compare_datasets", "DatasetComparison",
+    "import_dataset",
+    "preview_dataset", "DatasetPreview", "selection_review", "SelectionReview",
+    "hours_selection_review", "HoursSelectionReview",
+    "evaluate_quality", "check_quality", "validate_expectations",
+    "QualityGateResult", "QualityGateError",
+    "render_html_report", "write_html_report",
+    "infer_schema", "Pipeline", "input",
+    "load_plugins", "loaded_plugin_modules", "plugin_failures", "plugin_provenance",
+    "check_plugins",
+    "PluginCheckResult", "ProgressCallback", "ProgressSpec", "progress_callback",
+    "notebook_progress", "cli_progress",
+    "Registry", "default_registry", "ComponentDescriptor", "list_components",
+    "describe_component", "ExecutionPlan", "ARTIFACT_FORMAT_VERSION",
+    "DatasetWriteResult", "ExperimentResult", "DatasetReport", "StatsReport",
+    "StatsExamples",
+    "ValidationResult",
+    "JOB_SCHEMA_VERSION", "migrate", "migrate_job", "migrate_config",
+]

kuration/core/_pipeline_support.py

@@ -0,0 +1,182 @@
+"""Input adapters and validation helpers used by the pipeline facade.
+
+Kept separate so ``pipeline.py`` can focus on pipeline plan state and terminal execution. These
+objects remain module-level because in-memory adapters must be pickleable.
+"""
+from __future__ import annotations
+
+import hashlib
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+
+from .adapters.timeseries import TimeseriesAdapter
+from .schema import SchemaSpec
+
+GLOB_CHARS = set("*?[")
+FILTER_TYPES = {"anomaly", "dedup"}
+OPTION_KEYS = {"workers", "on_error", "seed", "streaming", "resume", "sample_hz", "progress",
+               "mode", "executor", "memory_budget", "batch_size", "checkpoint_dir",
+               "spill_dir", "approximation", "max_in_flight"}
+TABULAR_KINDS = {"timeseries", "jsonl", "json-rows", "parquet-dataset"}
+SUFFIX_CONTAINER = {".hdf5": "hdf5", ".h5": "hdf5", ".npz": "npz",
+                    ".parquet": "parquet", ".json": "json"}
+
+OP_ALIASES = {
+    "<=": "<=", "le": "<=", "lte": "<=", "lessorequal": "<=", "leq": "<=",
+    ">=": ">=", "ge": ">=", "gte": ">=", "greaterorequal": ">=", "geq": ">=",
+    "<": "<", "lt": "<", "less": "<", ">": ">", "gt": ">", "greater": ">",
+    "==": "==", "eq": "==", "equal": "==", "=": "==",
+    "!=": "!=", "ne": "!=", "neq": "!=", "notequal": "!=",
+}
+
+METHOD_ALIASES = {
+    "equal": "uniform-bins", "uniform": "uniform-bins", "uniform-bins": "uniform-bins",
+    "top": "top-k", "top-k": "top-k", "topk": "top-k",
+    "target": "target-distribution", "target-distribution": "target-distribution",
+    "cluster": "cluster-balanced", "cluster-balanced": "cluster-balanced",
+    "facility": "facility-location", "facility-location": "facility-location",
+    "weighted": "weighted", "k-center": "k-center", "dsir": "dsir",
+    "rare": "rare", "typical": "typical", "filter": "filter",
+}
+
+
+def jsonable(value):
+    """Return a compact, stable, publishable provenance representation."""
+    if value is None or isinstance(value, (bool, int, float)):
+        return value
+    if isinstance(value, str):
+        looks_private = (
+            "://" in value
+            or value.startswith(("/", "\\", "file://"))
+            or (len(value) >= 3 and value[1] == ":" and value[2] in "\\/")
+        )
+        if looks_private:
+            from .provenance import publishable_source_id
+            return publishable_source_id(value)
+        return value
+    if isinstance(value, Path):
+        from .provenance import publishable_source_id
+        return publishable_source_id(value)
+    if isinstance(value, np.generic):
+        return jsonable(value.item())
+    if isinstance(value, np.ndarray):
+        return {
+            "type": "numpy.ndarray",
+            "shape": list(value.shape),
+            "dtype": str(value.dtype),
+            "content_sha256": hashlib.sha256(value.tobytes(order="C")).hexdigest(),
+        }
+    if isinstance(value, bytes):
+        return {
+            "type": "bytes",
+            "size": len(value),
+            "content_sha256": hashlib.sha256(value).hexdigest(),
+        }
+    if isinstance(value, (list, tuple)):
+        return [jsonable(item) for item in value]
+    if isinstance(value, dict):
+        return {str(jsonable(key)): jsonable(item) for key, item in value.items()}
+    to_dict = getattr(value, "to_dict", None)
+    if callable(to_dict):
+        return jsonable(to_dict())
+    from .provenance import component_identity
+    try:
+        identity = component_identity(value)
+    except Exception:
+        identity = {
+            "type": f"{type(value).__module__}.{type(value).__qualname__}",
+            "identity": "unavailable",
+        }
+    return {"component": identity}
+
+
+def describe_src(src) -> Any:
+    from .provenance import publishable_source_id
+    if isinstance(src, str):
+        return publishable_source_id(src)
+    if isinstance(src, (list, tuple)) and src and all(isinstance(item, str) for item in src):
+        return [publishable_source_id(item) for item in src]
+    return f"<in-memory {type(src).__name__}>"
+
+
+def auto_qualify(token: str, schema: SchemaSpec) -> str:
+    if "." in token:
+        return token
+    if token in schema.fields:
+        return f"{schema.entity_axis}.{token}" if schema.entity_axis else token
+    if token in schema.globals:
+        return token
+    if token in schema.event_streams:
+        raise ValueError(
+            f"{token!r} is an event stream — name a field: '{token}.<field>' "
+            f"(fields: {sorted(schema.event_streams[token]['fields'])})")
+    raise ValueError(f"unknown metric source {token!r}; schema fields: "
+                     f"{sorted(schema.fields)}, globals: {sorted(schema.globals)}")
+
+
+def default_field_map(schema: SchemaSpec, *, tabular: bool) -> dict:
+    out: dict = {}
+    for name, field in schema.fields.items():
+        out[name] = ([f"{name}.{i}" for i in range(len(field.dims))]
+                     if tabular and field.kind == "vector" else name)
+    return out
+
+
+def expand_row(row: dict) -> dict:
+    out = dict(row)
+    for key, value in row.items():
+        if isinstance(value, np.ndarray) and value.ndim == 1:
+            value = value.tolist()
+        if isinstance(value, (list, tuple)):
+            for index, item in enumerate(value):
+                out[f"{key}.{index}"] = item
+    return out
+
+
+class RowsAdapter(TimeseriesAdapter):
+    in_memory = True
+
+    def __init__(self, schema, *, data: dict, fields: dict, time_column=None,
+                 entity_column=None, globals=None):
+        super().__init__(schema, fields=fields, time_column=time_column,
+                         entity_column=entity_column, globals=globals)
+        self._data = data
+
+    def ticks(self, source_uri):
+        if source_uri not in self._data:
+            raise ValueError(f"unknown in-memory source {source_uri!r} "
+                             f"(have: {sorted(self._data)})")
+        return self._rows_to_ticks([expand_row(row) for row in self._data[source_uri]])
+
+    def source_fingerprint(self, source_uri):
+        from .provenance import _state_hash
+        return _state_hash(self._data[source_uri])
+
+
+class TickAdapter:
+    in_memory = True
+
+    def __init__(self, schema, data: dict):
+        self.schema = schema
+        self._data = data
+
+    def ticks(self, source_uri):
+        if source_uri not in self._data:
+            raise ValueError(f"unknown in-memory source {source_uri!r} "
+                             f"(have: {sorted(self._data)})")
+        return self._data[source_uri]
+
+    def source_fingerprint(self, source_uri):
+        from .provenance import _state_hash
+        return _state_hash(self._data[source_uri])
+
+
+class JsonRowsAdapter(TimeseriesAdapter):
+    def ticks(self, source_uri):
+        from .infer import load_rows
+        return self._rows_to_ticks([expand_row(row) for row in load_rows(source_uri)])
+
+    def iter_ticks(self, source_uri):
+        yield from self.ticks(source_uri)

kuration/core/_registration.py

@@ -0,0 +1,15 @@
+"""Shared collision-safe idempotence for built-in component registration."""
+from __future__ import annotations
+
+
+def needs_registration(registry, kind: str, name: str, component) -> bool:
+    """Return whether a component is absent, accepting only exact repeat registration."""
+    if name not in registry.names(kind):
+        return True
+    if registry.get(kind, name) is component:
+        return False
+    descriptor = registry.descriptor(kind, name)
+    owner = descriptor.module or descriptor.package or "an unknown implementation"
+    raise ValueError(
+        f"cannot register {kind} {name!r}: the name is already owned by {owner!r}"
+    )

kuration/core/adapters/__init__.py

@@ -0,0 +1,18 @@
+"""Input adapters: raw source -> canonical Tick stream."""
+from .base import InputAdapter, detect_adapter, get_adapter, register_adapter
+from .curated import CuratedDatasetAdapter, schema_from_output
+from .hdf5 import HDF5Adapter
+from .huggingface import HuggingFaceAdapter
+from .jsonl import JsonlAdapter
+from .npz import NpzAdapter
+from .parquet_dataset import ParquetDatasetAdapter
+from .timeseries import TimeseriesAdapter
+from .zarr_store import ZarrAdapter
+from ..media import JoinedMediaAdapter, VideoAdapter
+
+__all__ = ["InputAdapter", "register_adapter", "get_adapter", "detect_adapter",
+           "TimeseriesAdapter",
+           "HDF5Adapter", "NpzAdapter", "JsonlAdapter", "CuratedDatasetAdapter",
+           "ParquetDatasetAdapter", "HuggingFaceAdapter", "ZarrAdapter",
+           "schema_from_output"]
+__all__ += ["VideoAdapter", "JoinedMediaAdapter"]

kuration/core/adapters/base.py

@@ -0,0 +1,248 @@
+"""`InputAdapter` — the input seam. The one place domain knowledge about a raw format
+enters the SDK.
+
+An adapter's contract is deliberately tiny: turn a raw source into an ordered stream of
+canonical `Tick`s (parse + group-by-timestep). Everything after that — building the
+`(T,E,D)` buffer, segmentation, metrics, selection — is generic SDK code that never knows
+which dataset it's looking at. Streaming-capable adapters expose `iter_ticks`; the compatible
+`ticks` path materializes a list for callers that need random access.
+
+Adapters are registered by name for declarative job construction. The parallel feature pass
+pickles the configured adapter into spawned workers.
+"""
+from __future__ import annotations
+
+from typing import Callable, Iterable, Protocol, runtime_checkable
+
+import numpy as np
+
+from ..buffer import Tick
+from ..schema import SchemaSpec
+
+
+@runtime_checkable
+class InputAdapter(Protocol):
+    schema: SchemaSpec
+    def ticks(self, source_uri: str) -> Iterable[Tick]: ...
+
+
+def validate_adapter_availability(adapter, schema: SchemaSpec, *, define: dict | None = None,
+                                  tags_spec: dict | None = None, output_spec=None) -> None:
+    """Reject plans that depend on schema values a configured adapter cannot emit.
+
+    Third-party adapters that do not declare ``available_fields``/``available_globals`` retain
+    the protocol's historical promise that they can emit the full schema. Built-in configured
+    adapters expose these sets, allowing a typo or partial mapping to fail before data is read
+    instead of becoming an all-missing buffer that reduces to a plausible zero.
+    """
+    available_fields = getattr(adapter, "available_fields", None)
+    available_globals = getattr(adapter, "available_globals", None)
+    if available_fields is None and available_globals is None:
+        return
+    available_fields = set(schema.fields if available_fields is None else available_fields)
+    available_globals = set(schema.globals if available_globals is None else available_globals)
+    required_fields: set[str] = set()
+    required_globals: set[str] = set()
+    for spec in (define or {}).values():
+        if not isinstance(spec, dict):
+            continue
+        source = spec.get("source")
+        if isinstance(source, str):
+            parts = source.split(".")
+            if schema.entity_axis is not None and len(parts) >= 2 and parts[0] == schema.entity_axis:
+                required_fields.add(parts[1])
+            elif source in schema.globals:
+                required_globals.add(source)
+        filter_expr = spec.get("filter")
+        if isinstance(filter_expr, str):
+            from ..dsl.grammar import _parse_filter
+            field = _parse_filter(filter_expr)[1]
+            if field in schema.fields:
+                required_fields.add(field)
+    for source in ((tags_spec or {}).get("from") or ()):
+        if source in schema.fields:
+            required_fields.add(source)
+        elif source in schema.globals:
+            required_globals.add(source)
+    if output_spec is not None:
+        for modality in output_spec.modalities:
+            if modality.src_kind == "field":
+                required_fields.add(modality.src_arg)
+    missing_fields = sorted(required_fields - available_fields)
+    missing_globals = sorted(required_globals - available_globals)
+    if missing_fields or missing_globals:
+        details = []
+        if missing_fields:
+            details.append(f"fields {missing_fields}")
+        if missing_globals:
+            details.append(f"globals {missing_globals}")
+        raise ValueError(f"{type(adapter).__name__} does not map required " + " and ".join(details))
+
+
+def iter_arrays_to_ticks(schema: SchemaSpec, arrs: dict, gl: dict = None):
+    """Lazily build single-entity ticks from indexable per-field arrays."""
+    gl = gl or {}
+    overlap = sorted(set(arrs) & set(gl))
+    if overlap:
+        raise ValueError(f"arrays_to_ticks: names cannot be both fields and globals: {overlap}")
+    arrays = {**arrs, **gl}
+
+    def step_count(array) -> int:
+        # Zarr 3 arrays deliberately do not implement ``len``. Array stores do expose
+        # ``shape`` as metadata, while ordinary Python sequences still use ``len``.
+        shape = getattr(array, "shape", None)
+        return int(shape[0]) if shape else (0 if shape == () else len(array))
+
+    n = step_count(next(iter(arrays.values()))) if arrays else 0
+    mismatched = {k: step_count(a) for k, a in arrays.items() if step_count(a) != n}
+    if mismatched:                       # silent truncation to the first field's length is worse
+        raise ValueError(f"arrays_to_ticks: length mismatch — expected {n} steps, got "
+                         f"{mismatched}")
+    import math
+    for fn, arr in arrs.items():
+        if fn not in schema.fields:
+            raise ValueError(f"arrays_to_ticks: undeclared schema field {fn!r}")
+        shape = tuple(getattr(arr, "shape", np.asarray(arr).shape))
+        width = math.prod(shape[1:]) if shape else 0
+        expected = schema.fields[fn].width
+        if width != expected:
+            raise ValueError(f"arrays_to_ticks: field {fn!r} has per-step width {width}, "
+                             f"expected {expected} for {schema.fields[fn].kind}")
+    for gn, arr in gl.items():
+        if gn not in schema.globals:
+            raise ValueError(f"arrays_to_ticks: undeclared schema global {gn!r}")
+        shape = tuple(getattr(arr, "shape", np.asarray(arr).shape))
+        width = math.prod(shape[1:]) if shape else 0
+        if width != 1:
+            raise ValueError(f"arrays_to_ticks: global {gn!r} has per-step width {width}, "
+                             "expected 1")
+    for t in range(n):
+        ent: dict[str, object] = {}
+        for fn, arr in arrs.items():
+            fs = schema.fields[fn]
+            x = arr[t]
+            if fs.kind == "vector":
+                ent[fn] = tuple(float(v) for v in np.ravel(x))
+            elif fs.kind == "categorical":
+                ent[fn] = x.decode() if isinstance(x, bytes) else str(x)
+            else:
+                ent[fn] = float(np.ravel(x)[0]) if np.ndim(x) else float(x)
+        g = {gn: float(np.ravel(arr[t])[0]) for gn, arr in gl.items()}
+        yield Tick(time=t, entities={0: ent}, globals=g)
+
+
+def arrays_to_ticks(schema: SchemaSpec, arrs: dict, gl: dict = None) -> list:
+    """Materialized counterpart to `iter_arrays_to_ticks` for existing callers."""
+    return list(iter_arrays_to_ticks(schema, arrs, gl))
+
+
+def check_mapped_arrays(schema: SchemaSpec, fields: dict, globals_: dict, source_uri: str,
+                        shape_of: Callable) -> None:
+    """Metadata-level mirror of `arrays_to_ticks`'s run-time checks, shared by the array-store
+    adapters' `check_source` hooks. `shape_of(key)` returns the stored array's shape, or None
+    if the key doesn't exist — shapes are store metadata, so no array data is read. Checks:
+    every mapped key exists, all first dims agree (step count), and each field's per-step
+    width matches the schema (a wide array mapped to a scalar would silently truncate to its
+    first component at run time — loud beats masked)."""
+    import math
+    overlap = sorted(set(fields) & set(globals_))
+    if overlap:
+        raise ValueError(f"source {source_uri!r}: names cannot be both fields and globals: "
+                         f"{overlap}")
+    keys = {**fields, **globals_}                # mapped name -> store key
+    shapes, missing = {}, []
+    for key in keys.values():
+        shp = shape_of(key)
+        if shp is None:
+            missing.append(key)
+        else:
+            shapes[key] = tuple(shp)
+    if missing:
+        raise ValueError(f"source {source_uri!r} is missing mapped key(s) {sorted(missing)}")
+    lens = {k: (s[0] if s else 0) for k, s in shapes.items()}
+    if len(set(lens.values())) > 1:
+        raise ValueError(f"source {source_uri!r}: mapped arrays disagree on step count: {lens}")
+    for f, key in fields.items():
+        fs = schema.fields[f]
+        width = math.prod(shapes[key][1:])
+        if width != fs.width:
+            raise ValueError(f"source {source_uri!r}: array {key!r} has per-step width {width}, "
+                             f"but field {f!r} ({fs.kind}) expects {fs.width}")
+
+
+_ADAPTERS: dict[str, Callable[..., InputAdapter]] = {}
+
+
+def register_adapter(name: str, factory: Callable[..., InputAdapter], **metadata) -> None:
+    """Register an input-adapter factory under a unique name.
+
+    ``metadata`` becomes its component descriptor. Duplicate names raise ``ValueError``.
+    """
+    from ..registry import default_registry
+    default_registry.register("adapter", name, factory, **metadata)
+    _ADAPTERS[name] = factory
+
+
+def get_adapter(name: str, *, registry=None, **kwargs) -> InputAdapter:
+    """Construct a named adapter, loading plugins on demand.
+
+    ``kwargs`` are passed to the factory. An explicit ``registry`` bypasses the legacy global
+    adapter table; an unknown name raises ``ValueError``.
+    """
+    from ..registry import default_registry
+    active_registry = registry or default_registry
+    if name not in active_registry.names("adapter") and active_registry is default_registry:
+        from ..plugins import load_plugins
+        load_plugins()
+    return active_registry.get("adapter", name)(**kwargs)
+
+
+def detect_adapter(source_uri: str, *, registry=None) -> str:
+    """Return the uniquely highest-confidence registered adapter for a source.
+
+    Adapter factories may expose ``detect_source(uri) -> bool | number``. A numeric result is
+    a confidence score; ``True`` is score 1 and falsey values mean no match. Detection is
+    intentionally separate from construction because many adapters need inferred schema and
+    mapping parameters before they can be instantiated.
+    """
+    from ..registry import default_registry
+
+    active_registry = registry or default_registry
+    if active_registry is default_registry:
+        from ..plugins import load_plugins
+
+        load_plugins()
+    matches: list[tuple[float, str]] = []
+    for name in active_registry.names("adapter"):
+        factory = active_registry.get("adapter", name)
+        detector = getattr(factory, "detect_source", None)
+        if not callable(detector):
+            continue
+        confidence = detector(str(source_uri))
+        if isinstance(confidence, bool):
+            score = 1.0 if confidence else 0.0
+        elif isinstance(confidence, (int, float)) and not isinstance(confidence, bool):
+            score = float(confidence)
+        else:
+            raise ValueError(
+                f"adapter {name!r} detect_source() must return a boolean or number"
+            )
+        if not np.isfinite(score) or score < 0:
+            raise ValueError(
+                f"adapter {name!r} detect_source() returned invalid confidence {score!r}"
+            )
+        if score > 0:
+            matches.append((score, name))
+    if not matches:
+        raise ValueError(
+            f"no registered input adapter recognized {source_uri!r}; "
+            "pass adapter= explicitly or install/register a matching adapter"
+        )
+    best = max(score for score, _name in matches)
+    winners = sorted(name for score, name in matches if score == best)
+    if len(winners) > 1:
+        raise ValueError(
+            f"input adapter detection for {source_uri!r} is ambiguous at confidence "
+            f"{best:g}: {winners}; pass adapter= explicitly"
+        )
+    return winners[0]