baseltest 0.7.0__py3-none-any.whl

baseltest/__init__.py

@@ -0,0 +1,35 @@
+"""baseltest: probabilistic testing for stochastic services.
+
+Python-native counterpart to punit (Java) and feotest (Rust) in the
+mavai framework family — statistical inference over repeated samples,
+not a single pass/fail assertion.
+
+The common entry points are re-exported here: construct a :class:`Bindings`,
+then call :func:`run`, :func:`explore`, :func:`optimize`, or
+:func:`check_contract` on a contract file. These are the declarative
+authoring surface (:mod:`baseltest.declarative`), promoted to the package
+root for convenience; that module also carries the narrower stepper/scorer
+context types, and :mod:`baseltest.contract` is the surface for authoring a
+service contract in Python directly.
+"""
+
+from baseltest._version import __version__
+from baseltest.contract import FileInput, MessageParts
+from baseltest.declarative import (
+    Bindings,
+    check_contract,
+    explore,
+    optimize,
+    run,
+)
+
+__all__ = [
+    "Bindings",
+    "FileInput",
+    "MessageParts",
+    "__version__",
+    "check_contract",
+    "explore",
+    "optimize",
+    "run",
+]

baseltest/_version.py

@@ -0,0 +1,11 @@
+"""Single source of the package version, importable without the API surface.
+
+Lives apart from the package ``__init__`` so a lower layer (the reporting
+renderers stamp the version into the verdict record) can read the version
+without importing the top-level authoring surface that ``__init__``
+re-exports — which would make a lower layer depend on a higher one.
+"""
+
+from importlib import metadata
+
+__version__ = metadata.version("baseltest")

baseltest/baseline/__init__.py

@@ -0,0 +1,46 @@
+"""The baseline artefact: the durable record of a measurement run.
+
+A measurement persists what was observed -- per-criterion counts and rates,
+identity, and provenance -- as a YAML artefact other tooling can later read.
+This package owns the artefact's schema and is its **single writer**: the
+serialisation lives here and nowhere else. The writer accepts
+characterisation data as input, so any caller that has measured something
+-- this framework's own measure runs, or an external characteriser -- hands
+its data over rather than emitting the format itself.
+
+Nothing in this package reads baselines back for threshold derivation; the
+artefact is a durable record whose consumption is a deliberately separate,
+later capability.
+"""
+
+from .reader import (
+    BaselineResolution,
+    StoredBaseline,
+    StoredCriterion,
+    StoredLatency,
+    read_baseline,
+    resolve_baseline,
+)
+from .record import (
+    BaselineRecord,
+    CriterionCharacterisation,
+    JudgementState,
+    NormativeJudgement,
+)
+from .writer import baseline_filename, render_baseline, write_baseline
+
+__all__ = [
+    "BaselineRecord",
+    "BaselineResolution",
+    "CriterionCharacterisation",
+    "JudgementState",
+    "NormativeJudgement",
+    "StoredBaseline",
+    "StoredCriterion",
+    "StoredLatency",
+    "baseline_filename",
+    "read_baseline",
+    "render_baseline",
+    "resolve_baseline",
+    "write_baseline",
+]

baseltest/baseline/reader.py

@@ -0,0 +1,247 @@
+"""Read-side of the baseline artefact: parsing and resolution.
+
+The single-writer rule is untouched — reading is not writing. Because the
+artefact is emitted by exactly one writer (:mod:`.writer`), the parser here
+accepts precisely that emission grammar: two-space indentation, one
+``key: value`` per line or one ``- item`` per line under a list key, every
+string JSON-quoted. No third-party dependency; the scalars are JSON, the
+structure is indentation.
+
+Both artefact generations read back: ``baseltest-baseline-2`` (current)
+and ``baseltest-baseline-1`` (no ``latency:`` block — a version-1 artefact
+simply characterises the functional dimension only).
+
+Resolution is strict identity of what was measured: same contract, same
+inputs fingerprint, same covariates (the recorded provenance, minus the
+volatile keys). A near-miss is reported with the reason it did not match —
+a config drift must never silently downgrade a judged criterion.
+"""
+
+import json
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from baseltest.engine import LatencyBasis
+
+from .writer import SCHEMA_VERSION
+
+_READABLE_SCHEMAS = frozenset({SCHEMA_VERSION, "baseltest-baseline-1"})
+
+# Provenance keys that legitimately differ between the measure run and a
+# later test run: they identify the run, not the thing measured.
+_VOLATILE_PROVENANCE = frozenset({"runMode", "taskFile"})
+
+
+@dataclass(frozen=True, slots=True)
+class StoredCriterion:
+    """One criterion's recorded evidence, as read back from the artefact."""
+
+    successes: int
+    trials: int
+
+
+@dataclass(frozen=True, slots=True)
+class StoredLatency:
+    """The artefact's latency block, read back for bound derivation.
+
+    The sorted vector is the payload a later test derives its bound from;
+    the percentiles are the measurement run's descriptive summary.
+    """
+
+    basis: LatencyBasis
+    contributing_samples: int
+    total_samples: int
+    percentiles: tuple[tuple[str, int], ...]
+    sorted_passing_latencies_ms: tuple[int, ...]
+
+
+@dataclass(frozen=True, slots=True)
+class StoredBaseline:
+    """The artefact's content, read back for resolution and judgement."""
+
+    path: Path
+    contract_id: str
+    sample_count: int
+    inputs_identity: str
+    generated_at: str
+    provenance: dict[str, str]
+    criteria: dict[str, StoredCriterion]
+    latency: StoredLatency | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class BaselineResolution:
+    """The outcome of looking for a matching baseline.
+
+    Exactly one of ``baseline`` / ``reason`` is meaningful: a match carries
+    the stored baseline; a non-match carries the honest reason.
+    """
+
+    baseline: StoredBaseline | None = None
+    reason: str | None = None
+    mismatched_keys: tuple[str, ...] = field(default=())
+
+    @property
+    def matched(self) -> bool:
+        return self.baseline is not None
+
+
+def _parse_lines(lines: list[str]) -> dict[str, Any]:
+    """Parse the writer's emission grammar into nested mappings and lists.
+
+    A ``key:`` line opens a nested container that starts as a mapping and
+    becomes a list on its first ``- item`` line — the writer only ever
+    emits homogeneous containers, so the switch is unambiguous.
+    """
+    root: dict[str, Any] = {}
+    # (indent, container, parent, key-in-parent); the root has no parent.
+    stack: list[tuple[int, Any, dict[str, Any] | None, str | None]] = [(0, root, None, None)]
+    for raw in lines:
+        if not raw.strip():
+            continue
+        indent = len(raw) - len(raw.lstrip(" "))
+        if indent % 2 != 0:
+            raise ValueError(f"malformed indentation: {raw!r}")
+        line = raw.strip()
+        while stack and stack[-1][0] > indent:
+            stack.pop()
+        top_indent, container, parent, parent_key = stack[-1]
+        if line.startswith("- "):
+            if isinstance(container, dict):
+                if container or parent is None or parent_key is None:
+                    raise ValueError(f"malformed list item: {raw!r}")
+                container = []
+                parent[parent_key] = container
+                stack[-1] = (top_indent, container, parent, parent_key)
+            container.append(json.loads(line[2:]))
+        elif isinstance(container, list):
+            raise ValueError(f"malformed line inside a list: {raw!r}")
+        else:
+            key, value_text = _split_entry(line)
+            if value_text is None:
+                child: dict[str, Any] = {}
+                container[key] = child
+                stack.append((indent + 2, child, container, key))
+            else:
+                container[key] = json.loads(value_text)
+    return root
+
+
+_DECODER = json.JSONDecoder()
+
+
+def _split_entry(line: str) -> tuple[str, str | None]:
+    """One emitted mapping line into its key and value text.
+
+    Returns ``(key, None)`` for a block-opening ``key:`` line, otherwise
+    ``(key, value_text)``. A JSON-quoted key is decoded, never searched
+    for a separator: the key itself may contain ``": "`` (a
+    failure-reason string quoting a regex, a covariate value).
+    """
+    if line.startswith('"'):
+        try:
+            key, end = _DECODER.raw_decode(line)
+        except json.JSONDecodeError as error:
+            raise ValueError(f"malformed line: {line!r}") from error
+        rest = line[end:]
+        if rest == ":":
+            return str(key), None
+        if rest.startswith(": ") and rest[2:]:
+            return str(key), rest[2:]
+        raise ValueError(f"malformed line: {line!r}")
+    if line.endswith(":"):
+        return line[:-1], None
+    key, _, value_text = line.partition(": ")
+    if not value_text:
+        raise ValueError(f"malformed line: {line!r}")
+    return key, value_text
+
+
+def _parse_latency(body: dict[str, Any] | None) -> StoredLatency | None:
+    if body is None:
+        return None
+    percentiles = tuple(
+        (key, int(value))
+        for key, value in body.items()
+        if key.startswith("p") and key.endswith("Ms")
+    )
+    vector = tuple(int(v) for v in body.get("sortedPassingLatenciesMs", []))
+    return StoredLatency(
+        basis=LatencyBasis(body["basis"]),
+        contributing_samples=int(body["contributingSamples"]),
+        total_samples=int(body["totalSamples"]),
+        percentiles=percentiles,
+        sorted_passing_latencies_ms=vector,
+    )
+
+
+def read_baseline(path: Path) -> StoredBaseline:
+    """Read one artefact back.
+
+    Raises:
+        ValueError: The file is not a readable baseline artefact (unknown
+            schema generation, malformed emission).
+        OSError: The file cannot be read.
+    """
+    data = _parse_lines(path.read_text(encoding="utf-8").splitlines())
+    schema = data.get("schemaVersion")
+    if schema not in _READABLE_SCHEMAS:
+        readable = ", ".join(sorted(_READABLE_SCHEMAS))
+        raise ValueError(f"{path.name}: schema {schema!r} is not one of: {readable}")
+    criteria: dict[str, StoredCriterion] = {}
+    for name, body in data.get("criteria", {}).items():
+        criteria[name] = StoredCriterion(
+            successes=int(body["successes"]), trials=int(body["trials"])
+        )
+    provenance = {str(k): str(v) for k, v in data.get("provenance", {}).items()}
+    return StoredBaseline(
+        path=path,
+        contract_id=str(data["contractId"]),
+        sample_count=int(data["sampleCount"]),
+        inputs_identity=str(data["inputsIdentity"]),
+        generated_at=str(data.get("generatedAt", "")),
+        provenance=provenance,
+        criteria=criteria,
+        latency=_parse_latency(data.get("latency")),
+    )
+
+
+def resolve_baseline(
+    baseline_dir: Path,
+    contract_id: str,
+    inputs_identity: str,
+    provenance: dict[str, str],
+) -> BaselineResolution:
+    """Find the baseline matching what this run would measure.
+
+    Matching is strict identity: the deterministic filename locates the
+    candidate (same contract, same inputs fingerprint), and the recorded
+    covariates — the provenance minus volatile run-identity keys — must be
+    equal. Any difference is a non-match, and the resolution says which
+    keys differed: a drifted configuration is surfaced, never silently
+    treated as "no baseline".
+    """
+    candidate = baseline_dir / f"{contract_id}-{inputs_identity[:12]}.yaml"
+    if not candidate.is_file():
+        return BaselineResolution(reason=f"no baseline found (expected {candidate.as_posix()})")
+    try:
+        stored = read_baseline(candidate)
+    except (ValueError, OSError, json.JSONDecodeError) as error:
+        return BaselineResolution(reason=f"baseline {candidate.name} is unreadable: {error}")
+    if stored.inputs_identity != inputs_identity:
+        return BaselineResolution(reason=f"baseline {candidate.name} records different inputs")
+    theirs = {k: v for k, v in stored.provenance.items() if k not in _VOLATILE_PROVENANCE}
+    ours = {k: v for k, v in provenance.items() if k not in _VOLATILE_PROVENANCE}
+    if theirs != ours:
+        differing = sorted(
+            key for key in set(theirs) | set(ours) if theirs.get(key) != ours.get(key)
+        )
+        return BaselineResolution(
+            reason=(
+                f"baseline {candidate.name} was measured under a different "
+                f"configuration (differing: {', '.join(differing)})"
+            ),
+            mismatched_keys=tuple(differing),
+        )
+    return BaselineResolution(baseline=stored)

baseltest/baseline/record.py

@@ -0,0 +1,152 @@
+"""The baseline record: what a measurement run durably states about a service."""
+
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+from datetime import datetime
+from enum import StrEnum
+from types import MappingProxyType
+
+from baseltest.engine import LatencyBlock, RunResult, latency_block
+
+
+class JudgementState(StrEnum):
+    """A measurement-time normative judgement's outcome.
+
+    The schema also reserves ``unsupportable`` for callers whose sample size
+    was not validated up front; baseltest validates every run's size before
+    sampling, so it emits only ``met`` or ``failed``.
+    """
+
+    MET = "met"
+    FAILED = "failed"
+
+
+@dataclass(frozen=True, slots=True)
+class NormativeJudgement:
+    """The measurement-time judgement of one criterion against its declared threshold.
+
+    Purely documentary: a later reader sees not only what was measured but
+    how the measurement stood relative to a bar in force at measurement
+    time. It never affects how the artefact is consumed.
+
+    Attributes:
+        state: The :class:`JudgementState` reached against the bar.
+        stipulated_threshold: The declared threshold judged against.
+        confidence: The confidence level of the judgement.
+    """
+
+    state: JudgementState
+    stipulated_threshold: float
+    confidence: float
+
+
+@dataclass(frozen=True, slots=True)
+class CriterionCharacterisation:
+    """One criterion's measured characterisation.
+
+    Attributes:
+        successes: Passing trials.
+        trials: Total trials.
+        failure_distribution: Failure reasons and their counts; empty when
+            every trial passed.
+        judgement: The measurement-time judgement, when the criterion
+            declared a threshold; ``None`` otherwise.
+    """
+
+    successes: int
+    trials: int
+    failure_distribution: Mapping[str, int] = field(default_factory=dict)
+    judgement: NormativeJudgement | None = None
+
+    def __post_init__(self) -> None:
+        object.__setattr__(
+            self, "failure_distribution", MappingProxyType(dict(self.failure_distribution))
+        )
+
+    @property
+    def observed_rate(self) -> float:
+        """The observed pass rate. A recorded characterisation has at least
+        one trial."""
+        return self.successes / self.trials
+
+
+@dataclass(frozen=True, slots=True)
+class BaselineRecord:
+    """Everything the baseline artefact states.
+
+    Attributes:
+        contract_id: The measured contract's identity.
+        generated_at: Measurement time, UTC.
+        sample_count: The run's total sample count.
+        inputs_identity: Order-insensitive fingerprint of the input list.
+        criteria: Per-criterion characterisations, keyed by criterion name,
+            in declaration order.
+        provenance: Additional provenance the caller supplies (e.g. the
+            contract-format identifier, the resolved binding's name). String
+            keys and values; recorded verbatim.
+        latency: The gated aggregate-latency summary, carrying the full
+            ascending vector of passing-sample durations — the raw material
+            a later test needs to derive its own bound at its own
+            confidence. ``None`` when no sample passed or no per-sample
+            observations were recorded.
+        views: Descriptive fingerprints of declared view output schemas
+            that are NOT covariates, keyed by view name — visible and
+            diffable in the artefact, never compared by baseline
+            resolution (covariate fingerprints travel in ``provenance``
+            instead). Additive, optional field of the artefact schema.
+    """
+
+    contract_id: str
+    generated_at: datetime
+    sample_count: int
+    inputs_identity: str
+    criteria: Mapping[str, CriterionCharacterisation]
+    provenance: Mapping[str, str] = field(default_factory=dict)
+    latency: LatencyBlock | None = None
+    views: Mapping[str, str] = field(default_factory=dict)
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "views", MappingProxyType(dict(self.views)))
+
+    @staticmethod
+    def from_run_result(
+        result: RunResult,
+        provenance: Mapping[str, str] | None = None,
+        views: Mapping[str, str] | None = None,
+    ) -> "BaselineRecord":
+        """Build a record from a completed run.
+
+        Thresholded criteria carry their measurement-time judgement
+        (met/failed from the run's verdict); unthresholded criteria are
+        characterised without one.
+        """
+        criteria: dict[str, CriterionCharacterisation] = {}
+        for criterion_result in result.criterion_results:
+            judgement = None
+            if criterion_result.verdict is not None:
+                criterion = criterion_result.criterion
+                assert criterion.threshold is not None
+                judgement = NormativeJudgement(
+                    state=JudgementState.MET
+                    if criterion_result.verdict.value == "pass"
+                    else JudgementState.FAILED,
+                    stipulated_threshold=criterion.threshold,
+                    confidence=criterion.confidence,
+                )
+            tally = criterion_result.tally
+            criteria[criterion_result.name] = CriterionCharacterisation(
+                successes=tally.successes,
+                trials=tally.trials,
+                failure_distribution=dict(tally.failure_reasons),
+                judgement=judgement,
+            )
+        return BaselineRecord(
+            contract_id=result.contract_id,
+            generated_at=result.finished_at,
+            sample_count=result.plan.samples,
+            inputs_identity=result.inputs_identity,
+            criteria=criteria,
+            provenance=dict(provenance or {}),
+            latency=latency_block(result.samples),
+            views=dict(views or {}),
+        )

baseltest/baseline/writer.py

@@ -0,0 +1,132 @@
+"""The single writer: serialising a baseline record to the artefact schema.
+
+Schema ``baseltest-baseline-2`` (draft), emitted deterministically with no
+third-party dependency: the schema is this package's own, every emitted
+string is JSON-quoted (a JSON string is a valid YAML flow scalar), and key
+order is fixed, so identical records produce identical bytes. Version 2
+adds the ``latency:`` block — field-compatible with punit's baseline
+latency block — to a version-1 body that is otherwise unchanged.
+
+Illustrative artefact:
+
+.. code-block:: yaml
+
+    schemaVersion: "baseltest-baseline-2"
+    contractId: "refund-confirmation"
+    generatedAt: "2026-07-06T12:00:00+00:00"
+    sampleCount: 300
+    inputsIdentity: "3fd0..."
+    provenance:
+      taskFormat: "mavai-contract/1"
+      binding: "refund-service"
+    criteria:
+      "relevant":
+        observedPassRate: 0.98
+        successes: 294
+        trials: 300
+        failureDistribution:
+          "response does not contain 'refund'": 6
+        normativeJudgement:
+          state: "met"
+          stipulatedThreshold: 0.95
+          confidence: 0.95
+    latency:
+      basis: "passing-samples"
+      contributingSamples: 294
+      totalSamples: 300
+      p50Ms: 240
+      p90Ms: 480
+      p95Ms: 760
+      p99Ms: 1180
+      sortedPassingLatenciesMs:
+        - 118
+        - 121
+        # ... every contributing duration, ascending
+
+The ``latency:`` block appears when at least one sample passed and carries
+only the percentiles its contributing-sample count can support (p50 needs
+1, p90 needs 10, p95 needs 20, p99 needs 100), followed by the full
+ascending vector of passing-sample durations. The vector, not the
+percentiles, is what a later test consumes to derive a latency bound at
+its own confidence — nothing derived is persisted here.
+"""
+
+from pathlib import Path
+
+from baseltest.engine.artefact import latency_lines, quote
+from baseltest.engine.naming import bounded_key
+
+from .record import BaselineRecord, CriterionCharacterisation
+
+SCHEMA_VERSION = "baseltest-baseline-2"
+
+
+def _criterion_lines(name: str, c: CriterionCharacterisation) -> list[str]:
+    lines = [
+        f"  {quote(bounded_key(name))}:",
+        f"    observedPassRate: {c.observed_rate:.6f}",
+        f"    successes: {c.successes}",
+        f"    trials: {c.trials}",
+    ]
+    if c.failure_distribution:
+        lines.append("    failureDistribution:")
+        for reason in sorted(c.failure_distribution):
+            lines.append(f"      {quote(bounded_key(reason))}: {c.failure_distribution[reason]}")
+    if c.judgement is not None:
+        lines.extend(
+            [
+                "    normativeJudgement:",
+                f"      state: {quote(c.judgement.state)}",
+                f"      stipulatedThreshold: {c.judgement.stipulated_threshold}",
+                f"      confidence: {c.judgement.confidence}",
+            ]
+        )
+    return lines
+
+
+def render_baseline(record: BaselineRecord) -> str:
+    """Serialise a record to the artefact schema, deterministically."""
+    lines = [
+        f"schemaVersion: {quote(SCHEMA_VERSION)}",
+        f"contractId: {quote(record.contract_id)}",
+        f"generatedAt: {quote(record.generated_at.isoformat())}",
+        f"sampleCount: {record.sample_count}",
+        f"inputsIdentity: {quote(record.inputs_identity)}",
+    ]
+    if record.provenance:
+        lines.append("provenance:")
+        for key in sorted(record.provenance):
+            lines.append(f"  {quote(key)}: {quote(record.provenance[key])}")
+    if record.views:
+        lines.append("views:")
+        for view in sorted(record.views):
+            lines.append(f"  {quote(view)}:")
+            lines.append(f"    outputSchemaFingerprint: {quote(record.views[view])}")
+    lines.append("criteria:")
+    for name, characterisation in record.criteria.items():
+        lines.extend(_criterion_lines(name, characterisation))
+    if record.latency is not None:
+        lines.extend(latency_lines(record.latency))
+    return "\n".join(lines) + "\n"
+
+
+def baseline_filename(record: BaselineRecord) -> str:
+    """The artefact's canonical filename: contract identity plus input-set tail.
+
+    Stable per (contract, input set): re-measuring the same pairing
+    refreshes the artefact rather than accumulating copies.
+    """
+    return f"{record.contract_id}-{record.inputs_identity[:12]}.yaml"
+
+
+def write_baseline(record: BaselineRecord, directory: Path) -> Path:
+    """Write the artefact under ``directory``, creating it if needed.
+
+    Returns the written path. The artefact is on disk when this returns --
+    callers that assert on results afterwards get persistence-before-
+    assertion by construction.
+    """
+    directory.mkdir(parents=True, exist_ok=True)
+    path = directory / baseline_filename(record)
+    path.write_text(render_baseline(record), encoding="utf-8")
+    return path