baseltest 0.22.0__py3-none-win_amd64.whl

baseltest/__init__.py

@@ -0,0 +1,36 @@
+"""baseltest: probabilistic testing for stochastic services.
+
+Python-native counterpart to punit (Java) and feotest (Rust) in the
+mavai framework family — statistical inference over repeated samples,
+not a single pass/fail assertion.
+
+The common entry points are re-exported here: construct a :class:`Bindings`,
+then call :func:`run`, :func:`explore`, :func:`optimize`, or
+:func:`check_contract` on a contract file. These are the declarative
+authoring surface (:mod:`baseltest.declarative`), promoted to the package
+root for convenience; that module also carries the narrower stepper/scorer
+context types, and :mod:`baseltest.contract` is the surface for authoring a
+service contract in Python directly.
+"""
+
+from baseltest._version import __version__
+from baseltest.contract import FileInput, MessageParts, Reply
+from baseltest.declarative import (
+    Bindings,
+    check_contract,
+    explore,
+    optimize,
+    run,
+)
+
+__all__ = [
+    "Bindings",
+    "FileInput",
+    "MessageParts",
+    "Reply",
+    "__version__",
+    "check_contract",
+    "explore",
+    "optimize",
+    "run",
+]

baseltest/_version.py

@@ -0,0 +1,11 @@
+"""Single source of the package version, importable without the API surface.
+
+Lives apart from the package ``__init__`` so a lower layer (the reporting
+renderers stamp the version into the verdict record) can read the version
+without importing the top-level authoring surface that ``__init__``
+re-exports — which would make a lower layer depend on a higher one.
+"""
+
+from importlib import metadata
+
+__version__ = metadata.version("baseltest")

baseltest/baseline/__init__.py

@@ -0,0 +1,46 @@
+"""The baseline artefact: the durable record of a measurement run.
+
+A measurement persists what was observed -- per-criterion counts and rates,
+identity, and provenance -- as a YAML artefact other tooling can later read.
+This package owns the artefact's schema and is its **single writer**: the
+serialisation lives here and nowhere else. The writer accepts
+characterisation data as input, so any caller that has measured something
+-- this framework's own measure runs, or an external characteriser -- hands
+its data over rather than emitting the format itself.
+
+Nothing in this package reads baselines back for threshold derivation; the
+artefact is a durable record whose consumption is a deliberately separate,
+later capability.
+"""
+
+from .reader import (
+    BaselineResolution,
+    StoredBaseline,
+    StoredCriterion,
+    StoredLatency,
+    read_baseline,
+    resolve_baseline,
+)
+from .record import (
+    BaselineRecord,
+    CriterionCharacterisation,
+    JudgementState,
+    NormativeJudgement,
+)
+from .writer import baseline_filename, render_baseline, write_baseline
+
+__all__ = [
+    "BaselineRecord",
+    "BaselineResolution",
+    "CriterionCharacterisation",
+    "JudgementState",
+    "NormativeJudgement",
+    "StoredBaseline",
+    "StoredCriterion",
+    "StoredLatency",
+    "baseline_filename",
+    "read_baseline",
+    "render_baseline",
+    "resolve_baseline",
+    "write_baseline",
+]

baseltest/baseline/reader.py

@@ -0,0 +1,290 @@
+"""Read-side of the baseline artefact: parsing and resolution.
+
+The single-writer rule is untouched — reading is not writing. Because the
+artefact is emitted by exactly one writer (:mod:`.writer`), the parser here
+accepts precisely that emission grammar: two-space indentation, one
+``key: value`` per line or one ``- item`` per line under a list key, every
+string JSON-quoted. No third-party dependency; the scalars are JSON, the
+structure is indentation.
+
+Both artefact generations read back: ``baseltest-baseline-2`` (current)
+and ``baseltest-baseline-1`` (no ``latency:`` block — a version-1 artefact
+simply characterises the functional dimension only).
+
+Resolution is strict identity of what was measured: same contract, same
+inputs fingerprint, same covariates (the recorded provenance, minus the
+volatile keys). A near-miss is reported with the reason it did not match —
+a config drift must never silently downgrade a judged criterion.
+"""
+
+import hashlib
+import json
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from baseltest.engine import LatencyBasis
+
+from .writer import FINGERPRINT_KEY, SCHEMA_VERSION, baseline_filename_for
+
+_READABLE_SCHEMAS = frozenset({SCHEMA_VERSION})
+
+# Provenance keys that legitimately differ between the measure run and a
+# later test run: they identify the run, not the thing measured.
+
+
+@dataclass(frozen=True, slots=True)
+class StoredCriterion:
+    """One criterion's recorded evidence, as read back from the artefact."""
+
+    successes: int
+    trials: int
+
+
+@dataclass(frozen=True, slots=True)
+class StoredLatency:
+    """The artefact's latency block, read back for bound derivation.
+
+    The sorted vector is the payload a later test derives its bound from;
+    the percentiles are the measurement run's descriptive summary.
+    """
+
+    basis: LatencyBasis
+    contributing_samples: int
+    total_samples: int
+    percentiles: tuple[tuple[str, int], ...]
+    sorted_passing_latencies_ms: tuple[int, ...]
+
+
+@dataclass(frozen=True, slots=True)
+class StoredBaseline:
+    """The artefact's content, read back for resolution and judgement."""
+
+    path: Path
+    contract_id: str
+    service_name: str
+    sample_count: int
+    inputs_identity: str
+    generated_at: str
+    covariate_profile: dict[str, str]
+    factor_record: dict[str, str]
+    criteria: dict[str, StoredCriterion]
+    latency: StoredLatency | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class BaselineResolution:
+    """The outcome of looking for a matching baseline.
+
+    Exactly one of ``baseline`` / ``reason`` is meaningful: a match carries
+    the stored baseline; a non-match carries the honest reason.
+    """
+
+    baseline: StoredBaseline | None = None
+    reason: str | None = None
+    mismatched_keys: tuple[str, ...] = field(default=())
+
+    @property
+    def matched(self) -> bool:
+        return self.baseline is not None
+
+
+def _parse_lines(lines: list[str]) -> dict[str, Any]:
+    """Parse the writer's emission grammar into nested mappings and lists.
+
+    A ``key:`` line opens a nested container that starts as a mapping and
+    becomes a list on its first ``- item`` line — the writer only ever
+    emits homogeneous containers, so the switch is unambiguous.
+    """
+    root: dict[str, Any] = {}
+    # (indent, container, parent, key-in-parent); the root has no parent.
+    stack: list[tuple[int, Any, dict[str, Any] | None, str | None]] = [(0, root, None, None)]
+    for raw in lines:
+        if not raw.strip():
+            continue
+        indent = len(raw) - len(raw.lstrip(" "))
+        if indent % 2 != 0:
+            raise ValueError(f"malformed indentation: {raw!r}")
+        line = raw.strip()
+        while stack and stack[-1][0] > indent:
+            stack.pop()
+        top_indent, container, parent, parent_key = stack[-1]
+        if line.startswith("- "):
+            if isinstance(container, dict):
+                if container or parent is None or parent_key is None:
+                    raise ValueError(f"malformed list item: {raw!r}")
+                container = []
+                parent[parent_key] = container
+                stack[-1] = (top_indent, container, parent, parent_key)
+            container.append(json.loads(line[2:]))
+        elif isinstance(container, list):
+            raise ValueError(f"malformed line inside a list: {raw!r}")
+        else:
+            key, value_text = _split_entry(line)
+            if value_text is None:
+                child: dict[str, Any] = {}
+                container[key] = child
+                stack.append((indent + 2, child, container, key))
+            else:
+                container[key] = json.loads(value_text)
+    return root
+
+
+_DECODER = json.JSONDecoder()
+
+
+def _split_entry(line: str) -> tuple[str, str | None]:
+    """One emitted mapping line into its key and value text.
+
+    Returns ``(key, None)`` for a block-opening ``key:`` line, otherwise
+    ``(key, value_text)``. A JSON-quoted key is decoded, never searched
+    for a separator: the key itself may contain ``": "`` (a
+    failure-reason string quoting a regex, a covariate value).
+    """
+    if line.startswith('"'):
+        try:
+            key, end = _DECODER.raw_decode(line)
+        except json.JSONDecodeError as error:
+            raise ValueError(f"malformed line: {line!r}") from error
+        rest = line[end:]
+        if rest == ":":
+            return str(key), None
+        if rest.startswith(": ") and rest[2:]:
+            return str(key), rest[2:]
+        raise ValueError(f"malformed line: {line!r}")
+    if line.endswith(":"):
+        return line[:-1], None
+    key, _, value_text = line.partition(": ")
+    if not value_text:
+        raise ValueError(f"malformed line: {line!r}")
+    return key, value_text
+
+
+def _parse_latency(body: dict[str, Any] | None) -> StoredLatency | None:
+    if body is None:
+        return None
+    percentiles = tuple(
+        (key, int(value))
+        for key, value in body.items()
+        if key.startswith("p") and key.endswith("Ms")
+    )
+    vector = tuple(int(v) for v in body.get("sortedPassingLatenciesMs", []))
+    return StoredLatency(
+        basis=LatencyBasis(body["basis"]),
+        contributing_samples=int(body["contributingSamples"]),
+        total_samples=int(body["totalSamples"]),
+        percentiles=percentiles,
+        sorted_passing_latencies_ms=vector,
+    )
+
+
+def read_baseline(path: Path) -> StoredBaseline:
+    """Read one artefact back.
+
+    Raises:
+        ValueError: The file is not a readable baseline artefact (unknown
+            schema generation, malformed emission).
+        OSError: The file cannot be read.
+    """
+    text = path.read_text(encoding="utf-8")
+    data = _parse_lines(text.splitlines())
+    schema = data.get("schemaVersion")
+    if schema != SCHEMA_VERSION:
+        # The whole migration experience is this sentence: what was found,
+        # what is expected, and the verb that regenerates the artefact.
+        raise ValueError(
+            f"{path.name}: schema {schema!r} is not {SCHEMA_VERSION!r}. "
+            "Baselines written by an earlier baseltest are not read: re-run "
+            "`basel measure` against this contract to regenerate it."
+        )
+    _verify_fingerprint(path, text, data)
+    criteria: dict[str, StoredCriterion] = {}
+    for name, body in data.get("criteria", {}).items():
+        criteria[name] = StoredCriterion(
+            successes=int(body["successes"]), trials=int(body["trials"])
+        )
+    execution = data.get("execution", {})
+    return StoredBaseline(
+        path=path,
+        contract_id=str(data["serviceContractId"]),
+        service_name=str(data["serviceName"]),
+        sample_count=int(execution["samplesExecuted"]),
+        inputs_identity=str(data["inputsIdentity"]),
+        generated_at=str(data.get("generatedAt", "")),
+        covariate_profile={str(k): str(v) for k, v in data.get("covariateProfile", {}).items()},
+        factor_record={str(k): str(v) for k, v in data.get("factorRecord", {}).items()},
+        criteria=criteria,
+        latency=_parse_latency(data.get("latency")),
+    )
+
+
+def _verify_fingerprint(path: Path, text: str, data: dict[str, object]) -> None:
+    """Refuse a record whose content does not match its stated fingerprint.
+
+    The fingerprint covers the document with its own line absent, which is
+    the emitted body up to that final line.
+    """
+    stated = data.get(FINGERPRINT_KEY)
+    if not stated:
+        raise ValueError(f"{path.name}: no {FINGERPRINT_KEY} — the record cannot be trusted")
+    marker = f"\n{FINGERPRINT_KEY}:"
+    body = text[: text.rindex(marker) + 1] if marker in text else text
+    actual = "sha256:" + hashlib.sha256(body.encode("utf-8")).hexdigest()
+    if actual != stated:
+        raise ValueError(
+            f"{path.name}: {FINGERPRINT_KEY} does not match the record's content "
+            "— the file has been edited or truncated since it was written"
+        )
+
+
+def resolve_baseline(
+    baseline_dir: Path,
+    contract_id: str,
+    service_name: str,
+    inputs_identity: str,
+    covariate_profile: dict[str, str],
+) -> BaselineResolution:
+    """Find the baseline matching what this run would measure.
+
+    Matching is strict identity over the tuple (contract, service, inputs,
+    covariate profile). The deterministic filename locates the candidate and
+    the loaded record's own statement decides — the body is authoritative,
+    never the path. Any difference is a non-match, and the resolution says
+    which keys differed: a drifted configuration is surfaced, never silently
+    treated as "no baseline".
+
+    The covariate profile is compared as stated, rather than reconstructed
+    by subtracting volatile keys from provenance: identity and provenance
+    are separate fields in the artefact, so they no longer have to be
+    separated again on the way back in.
+    """
+    candidate = baseline_dir / baseline_filename_for(contract_id, service_name, inputs_identity)
+    if not candidate.is_file():
+        return BaselineResolution(reason=f"no baseline found (expected {candidate.as_posix()})")
+    try:
+        stored = read_baseline(candidate)
+    except (ValueError, OSError, json.JSONDecodeError) as error:
+        return BaselineResolution(reason=f"baseline {candidate.name} is unreadable: {error}")
+    if stored.inputs_identity != inputs_identity:
+        return BaselineResolution(reason=f"baseline {candidate.name} records different inputs")
+    if stored.service_name != service_name:
+        return BaselineResolution(
+            reason=(
+                f"baseline {candidate.name} records service {stored.service_name!r}, "
+                f"not {service_name!r}"
+            )
+        )
+    theirs = dict(stored.covariate_profile)
+    ours = dict(covariate_profile)
+    if theirs != ours:
+        differing = sorted(
+            key for key in set(theirs) | set(ours) if theirs.get(key) != ours.get(key)
+        )
+        return BaselineResolution(
+            reason=(
+                f"baseline {candidate.name} was measured under a different "
+                f"configuration (differing: {', '.join(differing)})"
+            ),
+            mismatched_keys=tuple(differing),
+        )
+    return BaselineResolution(baseline=stored)

baseltest/baseline/record.py

@@ -0,0 +1,239 @@
+"""The baseline record: what a measurement run durably states about a service."""
+
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+from datetime import datetime
+from enum import StrEnum
+from types import MappingProxyType
+
+from baseltest.contract import DeliveryCause, FailureAxis, PostconditionStanding
+from baseltest.engine import LatencyBlock, RunResult, latency_block
+from baseltest.engine.naming import bounded_excerpt
+from baseltest.statistics import DEFAULT_CONFIDENCE_LEVEL, wilson_lower_bound
+
+
+class CriterionMode(StrEnum):
+    """Whether a criterion estimates a proportion (companion §1.5).
+
+    An observational criterion states no rate and no bound: it estimates no
+    proportion at all, passing iff no failure was observed.
+    """
+
+    INFERENTIAL = "inferential"
+    OBSERVATIONAL = "observational"
+
+
+class CriterionProcedure(StrEnum):
+    """The inferential procedure a criterion is judged under."""
+
+    REGRESSION = "REGRESSION"
+    COMPLIANCE = "COMPLIANCE"
+
+
+class JudgementState(StrEnum):
+    """A measurement-time normative judgement's outcome.
+
+    The schema also reserves ``unsupportable`` for callers whose sample size
+    was not validated up front; baseltest validates every run's size before
+    sampling, so it emits only ``met`` or ``failed``.
+    """
+
+    MET = "met"
+    FAILED = "failed"
+
+
+@dataclass(frozen=True, slots=True)
+class NormativeJudgement:
+    """The measurement-time judgement of one criterion against its declared threshold.
+
+    Purely documentary: a later reader sees not only what was measured but
+    how the measurement stood relative to a bar in force at measurement
+    time. It never affects how the artefact is consumed.
+
+    Attributes:
+        state: The :class:`JudgementState` reached against the bar.
+        stipulated_threshold: The declared threshold judged against.
+        confidence: The confidence level of the judgement.
+    """
+
+    state: JudgementState
+    stipulated_threshold: float
+    confidence: float
+
+
+@dataclass(frozen=True, slots=True)
+class CriterionCharacterisation:
+    """One criterion's measured characterisation.
+
+    Attributes:
+        successes: Passing trials.
+        trials: Total trials.
+        failure_distribution: Failure reasons and their counts; empty when
+            every trial passed.
+        failure_axes: The companion's diagnostic axis per observed reason.
+        delivery_causes: The delivery cause per observed reason that was a
+            failed delivery — what the artefact states as that entry's
+            identity, since the reason itself is an interpolated message.
+        judgement: The measurement-time judgement, when the criterion
+            declared a threshold; ``None`` otherwise.
+        standings: The criterion's descriptive per-postcondition tally —
+            per ``(input, check)``, passed/failed/skipped counts and the
+            observed fraction, each row carrying its check's optional
+            flag. Triage data, additive in the artefact schema; never an
+            interval or a per-check verdict.
+        optional_slack: The criterion's declared optional-check failure
+            budget, verbatim as authored (``None`` when undeclared —
+            never ``"0"``). Additive in the artefact schema.
+    """
+
+    successes: int
+    trials: int
+    failure_distribution: Mapping[str, int] = field(default_factory=dict)
+    failure_axes: Mapping[str, FailureAxis] = field(default_factory=dict)
+    delivery_causes: Mapping[str, DeliveryCause] = field(default_factory=dict)
+    judgement: NormativeJudgement | None = None
+    standings: tuple[PostconditionStanding, ...] = ()
+    optional_slack: str | None = None
+    mode: CriterionMode = CriterionMode.INFERENTIAL
+    procedure: CriterionProcedure | None = CriterionProcedure.REGRESSION
+    wilson_lower_bound: float | None = None
+
+    def __post_init__(self) -> None:
+        object.__setattr__(
+            self, "failure_distribution", MappingProxyType(dict(self.failure_distribution))
+        )
+        object.__setattr__(self, "failure_axes", MappingProxyType(dict(self.failure_axes)))
+        object.__setattr__(self, "delivery_causes", MappingProxyType(dict(self.delivery_causes)))
+
+    @property
+    def observed_rate(self) -> float:
+        """The observed pass rate. A recorded characterisation has at least
+        one trial."""
+        return self.successes / self.trials
+
+
+@dataclass(frozen=True, slots=True)
+class BaselineRecord:
+    """Everything the baseline artefact states.
+
+    Attributes:
+        service_contract_id: The measured service contract's identity.
+        service_name: The name of the service that was invoked. Identity,
+            not provenance: a contract may exercise several services and a
+            service may be the subject of several contracts, so neither
+            names a record alone.
+        generated_at: Measurement time, UTC.
+        confidence_level: The level every ``wilson_lower_bound`` in this
+            record was computed at. Distinct from a criterion's stipulated
+            ``NormativeJudgement.confidence``, which need not agree.
+        inputs_identity: Order-insensitive fingerprint of the input list.
+        samples_planned: Trials asked for.
+        samples_executed: Trials actually run. Equal to ``samples_planned``
+            until early termination exists; sourced from the run either way,
+            never assumed by the writer.
+        termination_reason: Why the run ended.
+        covariate_profile: The resolved covariate values — identity.
+        factor_record: Run provenance. Never compared by resolution.
+        criteria: Per-criterion characterisations, keyed by criterion name,
+            in declaration order.
+        latency: The gated aggregate-latency summary, carrying the full
+            ascending vector of passing-sample durations — the raw material
+            a later test needs to derive its own bound at its own
+            confidence. ``None`` when no sample passed or no per-sample
+            observations were recorded.
+        views: Descriptive fingerprints of declared view output schemas
+            that are NOT covariates, keyed by view name — visible and
+            diffable in the artefact, never compared by baseline
+            resolution (covariate fingerprints travel in ``provenance``
+            instead). Additive, optional field of the artefact schema.
+    """
+
+    service_contract_id: str
+    service_name: str
+    generated_at: datetime
+    confidence_level: float
+    inputs_identity: str
+    samples_planned: int
+    samples_executed: int
+    criteria: Mapping[str, CriterionCharacterisation]
+    termination_reason: str = "COMPLETED"
+    covariate_profile: Mapping[str, str] = field(default_factory=dict)
+    factor_record: Mapping[str, str] = field(default_factory=dict)
+    #: How each input the measurement drove presents itself, in input
+    #: order. Informational: identity stays ``inputs_identity``.
+    inputs: tuple[str, ...] = ()
+    latency: LatencyBlock | None = None
+    views: Mapping[str, str] = field(default_factory=dict)
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "views", MappingProxyType(dict(self.views)))
+
+    @staticmethod
+    def from_run_result(
+        result: RunResult,
+        service_name: str,
+        covariate_profile: Mapping[str, str] | None = None,
+        factor_record: Mapping[str, str] | None = None,
+        views: Mapping[str, str] | None = None,
+        confidence_level: float = DEFAULT_CONFIDENCE_LEVEL,
+    ) -> "BaselineRecord":
+        """Build a record from a completed run.
+
+        Thresholded criteria carry their measurement-time judgement
+        (met/failed from the run's verdict); unthresholded criteria are
+        characterised without one.
+
+        The Wilson lower bound is computed here, where the counts and the
+        confidence level are both in hand — never in the writer, which
+        states what the record holds and derives nothing.
+        """
+        criteria: dict[str, CriterionCharacterisation] = {}
+        for criterion_result in result.criterion_results:
+            judgement = None
+            if criterion_result.verdict is not None:
+                criterion = criterion_result.criterion
+                assert criterion.threshold is not None
+                judgement = NormativeJudgement(
+                    state=JudgementState.MET
+                    if criterion_result.verdict.value == "pass"
+                    else JudgementState.FAILED,
+                    stipulated_threshold=criterion.threshold,
+                    confidence=criterion.confidence,
+                )
+            tally = criterion_result.tally
+            slack = criterion_result.criterion.optional_slack
+            criteria[criterion_result.name] = CriterionCharacterisation(
+                successes=tally.successes,
+                trials=tally.trials,
+                failure_distribution=dict(tally.failure_reasons),
+                failure_axes=dict(tally.failure_axes),
+                delivery_causes=dict(tally.delivery_causes),
+                judgement=judgement,
+                standings=criterion_result.standings,
+                optional_slack=slack.declared if slack is not None else None,
+                wilson_lower_bound=(
+                    wilson_lower_bound(tally.successes, tally.trials, confidence_level)
+                    if tally.trials
+                    else None
+                ),
+            )
+        # Every criterion sees every sample, so any criterion's trial count
+        # is the run's executed count (companion §1.4.5a). Sourced rather
+        # than assumed equal to the plan: when early termination lands, this
+        # already states the truth.
+        executed = max((r.tally.trials for r in result.criterion_results), default=0)
+        return BaselineRecord(
+            service_contract_id=result.contract_id,
+            service_name=service_name,
+            generated_at=result.finished_at,
+            confidence_level=confidence_level,
+            inputs_identity=result.inputs_identity,
+            samples_planned=result.plan.samples,
+            samples_executed=executed,
+            criteria=criteria,
+            covariate_profile=dict(covariate_profile or {}),
+            factor_record=dict(factor_record or {}),
+            inputs=tuple(bounded_excerpt(str(v)) for v in result.plan.inputs),
+            latency=latency_block(result.samples),
+            views=dict(views or {}),
+        )