setspec 0.2.0__py3-none-any.whl

setspec/metrics.py

@@ -0,0 +1,184 @@
+"""Contract module — the measured value that every benchmark metric reduces to.
+
+Imports pydantic and :mod:`baseaicore`; performs no I/O and computes nothing. SetSpec carries
+statistics, it never produces them: the aggregation named here was performed by the producer, and
+this model's job is to make the producer state what it did rather than hand over a bare number
+(spec §3, non-goals).
+
+The invariants below are the schema-level half of
+ADR-0016 §6. That ADR requires unsupported
+samples to be *excluded* from a statistic and the surviving sample count to be reported next to it;
+a model that let a caller write ``value=0.0, sample_count=0`` would make the rule advisory. Here it
+is structural — the two fields cannot disagree, so a metric that was never measured cannot be
+serialized as one that measured zero.
+"""
+
+from __future__ import annotations
+
+from enum import StrEnum
+from typing import Self
+
+from baseaicore import UNSUPPORTED, is_supported
+from pydantic import Field, model_validator
+
+from setspec.base import PayloadDefinition, WireEnum, payload_models
+from setspec.serialization import MeasurementField
+
+__all__ = [
+    "Aggregation",
+    "MetricValueFields",
+    "MetricValueIn",
+    "MetricValueOut",
+]
+
+_MINIMUM_SAMPLES_FOR_DISPERSION = 2
+
+_METRIC_KEY_PATTERN = r"^[a-z][a-z0-9_]*(\.[a-z][a-z0-9_]*)*$"
+"""Lower snake case, starting with a letter, optionally dot-separated into segments.
+
+Enforced rather than documented because a metric key is an identifier a consumer *matches on*:
+``ttft_ms`` and ``TTFT_ms`` from two producers would be two metrics to every reader and one metric
+to every author. The suite's own rule — same concept, same name, everywhere — needs the wire format
+to be the place it cannot be broken.
+
+**Dots are for namespacing, and a real producer needs them**: FreeWeight's user-authored goal suites
+emit one metric per criterion as ``criterion.<key>``, where the segment after the dot is the
+author's own slug. A flat pattern would either reject those keys or push the producer to flatten
+them into ``criterion_<key>``, which collides with a criterion actually named ``key``. Each segment
+follows the same rule as a whole key, so ``criterion.house_voice`` is legal and
+``criterion.House Voice`` is not."""
+
+
+class Aggregation(StrEnum):
+    """How the samples behind a metric were reduced to one number.
+
+    A ``StrEnum`` so it serializes and logs as its own name rather than an opaque integer
+    (coding standards §2). The value is what appears on the wire.
+
+    Recording this is not bookkeeping: a mean and a p95 of the same samples answer different
+    questions, and a consumer that compares one against the other is comparing nothing. The
+    producer knows which it computed; the wire format makes it say so.
+    """
+
+    SINGLE = "single"
+    """One observation, not aggregated. ``sample_count`` is 1 and dispersion is unsupported."""
+
+    MEAN = "mean"
+    """Arithmetic mean of the supported samples."""
+
+    MEDIAN = "median"
+    """50th percentile — the value to prefer over ``MEAN`` where outliers are expected."""
+
+    MIN = "min"
+    """Smallest supported sample."""
+
+    MAX = "max"
+    """Largest supported sample."""
+
+    SUM = "sum"
+    """Total across the supported samples."""
+
+    COUNT = "count"
+    """How many events occurred — a tally, distinct from ``sample_count``, which says how many
+    observations produced this statistic."""
+
+    STDDEV = "stddev"
+    """Standard deviation of the supported samples, reported as the value in its own right."""
+
+    P50 = "p50"
+    """50th percentile — the same statistic as ``MEDIAN``, spelled the way a percentile family
+    (``p50``/``p95``/``p99``) is usually reported together; both members exist so a producer never
+    has to translate its own vocabulary to satisfy this one."""
+
+    P95 = "p95"
+    """95th percentile."""
+
+    P99 = "p99"
+    """99th percentile."""
+
+    RATIO = "ratio"
+    """A proportion in ``[0, 1]`` that is not a mean of pass/fail samples but a direct ratio —
+    e.g. a memory-overhead ratio computed from two other measurements."""
+
+    RAW = "raw"
+    """A single unaggregated reading, kept distinct from ``SINGLE``: ``SINGLE`` still promises a
+    real, comparable measurement of the metric's stated unit, while ``RAW`` marks a value passed
+    through without this build knowing whether it was ever meant to be aggregated at all."""
+
+
+class MetricValueFields(PayloadDefinition):
+    """Field definitions for ``metric.value``; use :data:`MetricValueOut` / :data:`MetricValueIn`.
+
+    A measured quantity, the statistic that produced it, and enough context for a consumer to know
+    whether comparing it to another one is meaningful.
+
+    Attributes:
+        metric_key: Which metric this is — ``"decode_tokens_per_second"``, ``"task_success"``.
+            Required, because both :class:`~setspec.benchmark.v1.BenchmarkResultFields` and
+            :class:`~setspec.benchmark.v1.BenchmarkRunSummaryFields` carry *sequences* of this
+            model: without a key, a consumer receives a list of numbers it cannot attribute,
+            chart, compare across runs, or check for the metric it was looking for. Lower snake
+            case, so one metric has one spelling everywhere in the suite.
+        value: The measurement, or ``UNSUPPORTED`` when this environment could not provide one.
+            Never ``null`` and never ``0`` as a stand-in for absence
+            (ADR-0016 §4).
+        unit: The unit the value is in — ``"ms"``, ``"tokens_per_second"``, ``"bytes"``,
+            ``"ratio"``. Required and non-empty: a number whose unit lives only in a field name
+            somewhere upstream is a number that will eventually be compared against a different
+            one (coding standards §3). Dimensionless quantities say so explicitly rather than
+            passing an empty string.
+        aggregation: Which statistic ``value`` is.
+        higher_is_better: Whether a larger value is a better result. Carried per metric because
+            the answer differs between metrics in the same payload — throughput and latency point
+            in opposite directions — and a consumer ranking results cannot infer it from the unit.
+        sample_count: How many **supported** samples produced ``value``. Unsupported samples are
+            excluded from the statistic and from this count (ADR-0016 §6), so it is the honest
+            denominator, not the number of attempts.
+        dispersion: Spread of those samples, as a standard deviation in the same unit as
+            ``value``. ``UNSUPPORTED`` when fewer than two supported samples exist, because the
+            spread of a single observation is undefined rather than zero.
+    """
+
+    metric_key: str = Field(min_length=1, pattern=_METRIC_KEY_PATTERN)
+    value: MeasurementField
+    unit: str = Field(min_length=1)
+    aggregation: WireEnum[Aggregation]
+    higher_is_better: bool
+    sample_count: int = Field(ge=0)
+    dispersion: MeasurementField
+
+    @model_validator(mode="after")
+    def _check_sample_coherence(self) -> Self:
+        """Enforce ADR-0016 §6: the value and its sample count must tell the same story.
+
+        Raises:
+            ValueError: If a real value claims no samples, if an unsupported value claims some, or
+                if a dispersion is reported for fewer than two samples.
+        """
+        if is_supported(self.value) and self.sample_count < 1:
+            raise ValueError(
+                "a metric with a real value must report at least one supported sample; "
+                "sample_count=0 with a number in `value` means the number came from nowhere "
+                "(ADR-0016 §6)"
+            )
+        if not is_supported(self.value) and self.sample_count != 0:
+            raise ValueError(
+                f"an unsupported metric has no supported samples, but sample_count is "
+                f"{self.sample_count}. A metric with no supported samples is itself unsupported — "
+                "report the attempts elsewhere, not as the denominator of a statistic that was "
+                "never computed (ADR-0016 §6)"
+            )
+        if (
+            self.dispersion is not UNSUPPORTED
+            and self.sample_count < _MINIMUM_SAMPLES_FOR_DISPERSION
+        ):
+            raise ValueError(
+                f"dispersion needs at least {_MINIMUM_SAMPLES_FOR_DISPERSION} supported samples; "
+                f"sample_count is {self.sample_count}. The spread of a single observation is "
+                "undefined, not zero — report it as 'unsupported'"
+            )
+        return self
+
+
+MetricValueOut, MetricValueIn = payload_models(MetricValueFields)
+"""The ``metric.value`` payload pair: ``Out`` for writers, ``In`` for readers."""

setspec/model/v1.py

@@ -0,0 +1,169 @@
+"""Contract module — ``model.identity`` v1: which weights, plus what a provider says about them.
+
+Imports pydantic and :mod:`baseaicore`; performs no I/O. Exchange form of
+:class:`baseaicore.ModelIdentity` (the identity triple) and :class:`baseaicore.ModelDescriptor`
+(the refreshable metadata a provider reports about those weights), combined into one payload
+because ADR-0022 §1 always
+carries them together as ``capability.evidence.model``.
+
+**Status: draft (`1.0`).** Registered in :data:`setspec.envelope.SUPPORTED_SCHEMAS` so FreeWeight
+has a concrete model to build against, but not yet frozen — see
+[development plan Phase 2](../../../docs/packages/setspec/development-plan.md) and
+[Phase 4](../../../docs/packages/setspec/development-plan.md), which promotes this to `1.0` only
+after FreeWeight has produced real results against it. A field may still be added, tightened or
+reshaped by that promotion without a major version bump signalling it in advance.
+
+Deliberately omitted: :attr:`baseaicore.ModelDescriptor.raw`, the untouched provider response.
+It carries no contract — its own docstring says nothing above the normalizer may read it for
+business logic — and freezing its presence on the wire would promise a shape for a value this
+package cannot describe.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Self
+
+from baseaicore import (
+    UNSUPPORTED,
+    IdentityConfidence,
+    ModelCapabilityFlag,
+    ModelIdentity,
+    ProviderKind,
+)
+from baseaicore import ValidationError as SuiteValidationError
+from pydantic import Field, model_validator
+
+from setspec.base import PayloadDefinition, WireEnum, WireSequence, payload_models
+from setspec.serialization import MeasurementField, TimestampField
+
+__all__ = [
+    "ModelIdentityFields",
+    "ModelIdentityIn",
+    "ModelIdentityOut",
+]
+
+
+class ModelIdentityFields(PayloadDefinition):
+    """Field definitions for ``model.identity``; use :data:`ModelIdentityOut` /
+    :data:`ModelIdentityIn`.
+
+    :attr:`canonical_id` and :attr:`identity_confidence` are materialized rather than left for a
+    reader to derive, so a consumer can display or index by them without reconstructing a
+    :class:`baseaicore.ModelIdentity`. Both are still validated against the identity triple on
+    every construction — see :meth:`_check_identity_coherence` — so "provenance completeness
+    enforced by the schema" (Phase 2 gold standard) applies to the derived fields too, not only
+    the fields nothing computes.
+
+    Attributes:
+        provider_kind: Which kind of provider serves these weights ([ADR-0008]
+            (0008 canonical model identity)). Reuses
+            :class:`baseaicore.ProviderKind` directly rather than a shadow enum, so a new provider
+            kind reaches this schema the moment BaseAiCore adds it.
+        provider_model_name: Exactly as the provider names it, case and punctuation preserved.
+        artifact_digest: ``"sha256:"`` + 64 lowercase hex characters, or ``None`` when the
+            provider exposes none.
+        identity_confidence: ``digest`` iff ``artifact_digest`` is present, ``name_only``
+            otherwise — checked, not merely documented.
+        canonical_id: ``{provider_kind}/{provider_model_name}@{digest_short}``
+            (ADR-0024). Lossy and
+            display-only; never parsed back into its parts.
+        observed_at: When this descriptor snapshot was read from the provider.
+        family: The model family name, e.g. ``"qwen3.5"``.
+        architecture: The architecture name, e.g. ``"transformer"``, ``"mamba"``.
+        parameter_count: Total parameter count.
+        active_parameter_count: MoE active parameters per token; equal to
+            ``parameter_count`` for a dense model.
+        expert_count: Number of experts, for a mixture-of-experts model.
+        quantization: Weight quantization, e.g. ``"Q8_0"``.
+        weight_format: File format, e.g. ``"gguf"``, ``"safetensors"``.
+        size_bytes: On-disk size of the weights.
+        max_context: The context length the model *advertises* — not the context a provider is
+            configured to serve, which is ``execution.served_context`` on a benchmark result
+            (ADR-0023 §4).
+        embedding_dim: Hidden/embedding dimension.
+        layers: Transformer layer count.
+        attention_heads: Attention head count.
+        kv_heads: Key/value head count.
+        head_dim: Dimension of each attention head.
+        vocab_size: Tokenizer vocabulary size.
+        rope_config: RoPE scaling configuration, in the provider's own shape.
+        sliding_window: Sliding-attention window size, if the architecture uses one.
+        declared_capabilities: What the provider *claims* this model can do — never conflated
+            with a measured capability from ``capability.evidence``.
+        license_text: The model's license, if the provider exposes one.
+    """
+
+    provider_kind: WireEnum[ProviderKind]
+    provider_model_name: str = Field(min_length=1)
+    artifact_digest: str | None = None
+    identity_confidence: WireEnum[IdentityConfidence]
+    canonical_id: str = Field(min_length=1)
+
+    observed_at: TimestampField
+    family: str | None = None
+    architecture: str | None = None
+    parameter_count: MeasurementField = UNSUPPORTED
+    active_parameter_count: MeasurementField = UNSUPPORTED
+    expert_count: MeasurementField = UNSUPPORTED
+    quantization: str | None = None
+    weight_format: str | None = None
+    size_bytes: MeasurementField = UNSUPPORTED
+    max_context: MeasurementField = UNSUPPORTED
+    embedding_dim: MeasurementField = UNSUPPORTED
+    layers: MeasurementField = UNSUPPORTED
+    attention_heads: MeasurementField = UNSUPPORTED
+    kv_heads: MeasurementField = UNSUPPORTED
+    head_dim: MeasurementField = UNSUPPORTED
+    vocab_size: MeasurementField = UNSUPPORTED
+    rope_config: dict[str, Any] | None = None
+    sliding_window: MeasurementField = UNSUPPORTED
+    declared_capabilities: WireSequence[WireEnum[ModelCapabilityFlag]] = ()
+    license_text: str | None = None
+
+    @model_validator(mode="after")
+    def _check_identity_coherence(self) -> Self:
+        """Recompute the identity triple's derived fields and require them to agree.
+
+        Unlike a machine fingerprint — whose inclusion policy is allowed to change under a
+        profile that must still reconstruct exactly as it was written — a canonical ID and an
+        identity confidence are pure functions of the triple with no such historical caveat
+        (:class:`baseaicore.ModelIdentity` never re-verifies a stored fingerprint for that reason,
+        but always recomputes ``canonical_id``). Recomputing here is therefore safe, not merely
+        convenient, and it catches a producer that materialized the derived fields inconsistently
+        with the triple that stands next to them.
+
+        Raises:
+            ValueError: If ``provider_model_name`` or ``artifact_digest`` fails
+                :class:`baseaicore.ModelIdentity`'s own validation, or if ``canonical_id`` or
+                ``identity_confidence`` disagrees with what the triple recomputes.
+        """
+        try:
+            identity = ModelIdentity(
+                provider_kind=self.provider_kind,
+                provider_model_name=self.provider_model_name,
+                artifact_digest=self.artifact_digest,
+            )
+        except SuiteValidationError as exc:
+            # baseaicore.ValidationError is a SuiteError, not a ValueError; pydantic only
+            # aggregates ValueError/AssertionError into its own report (setspec.serialization
+            # hits the same seam for timestamps).
+            raise ValueError(str(exc)) from exc
+        if identity.canonical_id != self.canonical_id:
+            raise ValueError(
+                f"canonical_id {self.canonical_id!r} does not match the identity triple, which "
+                f"recomputes to {identity.canonical_id!r}. canonical_id is a pure function of "
+                "provider_kind, provider_model_name and artifact_digest (ADR-0024) — it is "
+                "carried on the wire for convenience, not as an independent fact."
+            )
+        if identity.identity_confidence.value != self.identity_confidence:
+            raise ValueError(
+                f"identity_confidence {self.identity_confidence!r} does not match the identity "
+                f"triple: artifact_digest is "
+                f"{'present' if self.artifact_digest is not None else 'absent'}, which makes "
+                f"this identity {identity.identity_confidence.value!r}."
+            )
+        return self
+
+
+ModelIdentityOut, ModelIdentityIn = payload_models(ModelIdentityFields)
+"""The ``model.identity`` payload pair: ``Out`` for writers, ``In`` for readers."""

setspec/provenance.py

@@ -0,0 +1,59 @@
+"""Contract module — provenance building blocks shared by more than one versioned payload.
+
+Imports pydantic and :mod:`baseaicore`; performs no I/O.
+
+This module exists for the same reason :mod:`setspec.metrics` does. A sub-model used by two
+payload types belongs to neither of them: if ``EnvironmentFields`` lived in
+:mod:`setspec.benchmark.v1`, then the day ``benchmark.result`` needs a changed environment shape,
+whoever edits that class would silently change ``capability.evidence`` v1 too — a frozen contract
+mutating because someone edited a different payload's module, which is precisely the drift
+ADR-0009 exists to prevent. A shared block gets
+a neutral home so that changing it is an obviously cross-cutting act rather than an accident.
+
+Nothing here is a wire payload in its own right: none of these names appears in
+:data:`~setspec.envelope.SUPPORTED_SCHEMAS`, and none is generated into an ``Out``/``In`` pair.
+They are field groups that versioned payloads embed, which is why they carry no version of their
+own — their version is whichever payload's version they appear inside.
+"""
+
+from __future__ import annotations
+
+from baseaicore import ProviderKind
+from pydantic import Field
+
+from setspec.base import PayloadDefinition, WireEnum
+
+__all__ = ["EnvironmentFields"]
+
+
+class EnvironmentFields(PayloadDefinition):
+    """Provider and drift-sensitive environment facts at the moment of measurement.
+
+    Unifies two bullets that
+    Machine Identity §6 and
+    ADR-0022 §1 describe separately
+    but with identical content — "provider kind + provider version" and "GPU driver, CUDA, OS
+    version at measurement" — into the one nested object ADR-0022 already names ``environment`` on
+    ``capability.evidence``, so a benchmark result and the evidence aggregated from it carry
+    environment facts in the same shape rather than two shapes a consumer has to reconcile.
+
+    Every field but the provider's own identity is a **drift signal, not identity**: a driver
+    upgrade or an OS patch must never re-identify a machine (that is the machine fingerprint's
+    job, and it deliberately excludes all of these), but it does reduce confidence in performance
+    evidence measured before it
+    (ADR-0017's
+    ``environment_factor``). Recording them is what makes that reduction computable at all.
+
+    Attributes:
+        provider_kind: Which kind of provider served the model for this measurement.
+        provider_version: The provider's own version string, e.g. ``"0.32.13"``.
+        gpu_driver_version: A drift signal; ``None`` when no GPU was involved.
+        cuda_version: The CUDA/ROCm toolkit version. A drift signal.
+        os_version: A drift signal.
+    """
+
+    provider_kind: WireEnum[ProviderKind]
+    provider_version: str = Field(min_length=1)
+    gpu_driver_version: str | None = None
+    cuda_version: str | None = None
+    os_version: str | None = None