baseltest 0.7.0__tar.gz

baseltest-0.7.0/.github/workflows/ci.yml

@@ -0,0 +1,27 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+jobs:
+  conformance:
+    name: Oracle conformance
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Install package
+        run: pip install -e ".[dev]"
+      - name: Run conformance suite against the pinned mavai-R fixtures
+        run: pytest tests/statistics/test_conformance.py --no-cov
+      - name: Upload conformance report
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: conformance-report
+          path: build/conformance-report.json
+          if-no-files-found: ignore

baseltest-0.7.0/.github/workflows/release.yml

@@ -0,0 +1,81 @@
+name: Release
+
+# Publishes a tagged release to PyPI using Trusted Publishing (OIDC) — no
+# API token is stored in the repository. Fires on a version tag (`vX.Y.Z`),
+# runs the test suite, verifies the tag matches the packaged version, builds
+# the sdist and wheel, and publishes them.
+#
+# One-time PyPI setup required before this can succeed (see the repository's
+# release notes / RELEASING doc): register `mavai-org/baseltest` as a trusted
+# publisher for the `baseltest` project on PyPI, with workflow `release.yml`
+# and environment `pypi`.
+
+on:
+  push:
+    tags:
+      - "v*"
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    name: Test before release
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Install package
+        run: pip install -e ".[dev]"
+      - name: Run the test suite
+        run: pytest --no-cov
+
+  build:
+    name: Build distributions
+    needs: test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Verify the tag matches the packaged version
+        run: |
+          tag="${GITHUB_REF_NAME#v}"
+          pkg="$(python -c "import tomllib; print(tomllib.load(open('pyproject.toml','rb'))['project']['version'])")"
+          if [ "$tag" != "$pkg" ]; then
+            echo "::error::tag ${GITHUB_REF_NAME} does not match pyproject version ${pkg}" >&2
+            exit 1
+          fi
+          echo "tag ${GITHUB_REF_NAME} matches packaged version ${pkg}"
+      - name: Build sdist and wheel
+        run: pipx run build
+      - name: Check distribution metadata
+        run: pipx run twine check dist/*
+      - name: Upload distributions
+        uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  publish:
+    name: Publish to PyPI
+    needs: build
+    runs-on: ubuntu-latest
+    # Trusted Publishing authenticates via this OIDC token; the `pypi`
+    # environment name must match the trusted-publisher config on PyPI.
+    environment:
+      name: pypi
+      url: https://pypi.org/p/baseltest
+    permissions:
+      id-token: write
+    steps:
+      - name: Download the built distributions
+        uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

baseltest-0.7.0/.gitignore

@@ -0,0 +1,70 @@
+### Python ###
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+### Virtual environments ###
+venv/
+.venv/
+env/
+ENV/
+
+### Testing / coverage ###
+.pytest_cache/
+.coverage
+.coverage.*
+htmlcov/
+coverage.xml
+*.cover
+.hypothesis/
+
+### Type checking / linting ###
+.mypy_cache/
+.dmypy.json
+dmypy.json
+.ruff_cache/
+
+### Packaging / build backends ###
+.tox/
+.nox/
+
+### IntelliJ / PyCharm ###
+.idea/
+*.iml
+*.ipr
+*.iws
+
+### VS Code ###
+.vscode/
+
+### Claude Code ###
+.claude/settings.local.json
+CLAUDE.md
+MAXIMS.md
+
+### Mac OS ###
+.DS_Store
+
+### Orchestrator-managed ###
+# Requirements / design / planning material for baseltest lives in
+# mavai-orchestrator, not in this repository.
+/plan/
+_baseltest/

baseltest-0.7.0/.import_linter_cache/.gitignore

@@ -0,0 +1,2 @@
+# Automatically created by Grimp.
+*

baseltest-0.7.0/.import_linter_cache/300451ce89563663f2df89c392b36750da9c9358.data.json

@@ -0,0 +1 @@
+{"baseltest.declarative._sizing._criteria":[["baseltest.baseline",12,"from baseltest.baseline import BaselineResolution, StoredBaseline, resolve_baseline"],["baseltest.declarative._registry",22,"from .._registry import Registry"],["baseltest.engine",13,"from baseltest.engine import inputs_fingerprint"],["baseltest.statistics",14,"from baseltest.statistics import check_feasibility"],["baseltest.declarative._sizing._rates",19,"from ._rates import _effective_rate"],["baseltest.declarative._parser",16,"from .._parser import FORMAT_IDENTIFIER, ContractDeclaration, CriterionDeclaration"],["baseltest.declarative._sizing._model",18,"from ._model import _EmpiricalCriterion"],["baseltest.declarative._services",17,"from .._services import ServiceDefinition"]],"baseltest.statistics.proportion":[],"baseltest.declarative._sizing._modes":[["baseltest.declarative._sizing._render",25,"from ._render import _explanation, _json_payload, _sizing_table"],["baseltest.declarative._sizing._pricing",22,"from ._pricing import _governing_samples, _over_reach_message, _priced_claim"],["baseltest.declarative._sizing._model",14,"from ._model import ("],["baseltest.declarative._parser",12,"from .._parser import ContractDeclaration"],["baseltest.declarative._sizing._rates",24,"from ._rates import _percent"],["baseltest.statistics",10,"from baseltest.statistics import detectable_rate"],["baseltest.declarative._sizing._criteria",13,"from ._criteria import _normative_minimum"],["baseltest.declarative._sizing._prompts",23,"from ._prompts import _Interaction"]],"baseltest.reporting.verdict_reader":[["baseltest.reporting.verdict_xml",14,"from .verdict_xml import _NAMESPACE"],["baseltest.reporting.run_design",13,"from .run_design import BaselineDisclosure, ClaimDisclosure, RunDesign"]],"baseltest.observation.record":[["baseltest.engine.naming",17,"from baseltest.engine.naming import bounded_excerpt, bounded_key, per_input_index"],["baseltest.engine",16,"from baseltest.engine import LatencyBlock, RunResult, SampleRecord, latency_block"]],"baseltest.statistics":[["baseltest.statistics.proportion",31,"from .proportion import proportion_standard_error, proportion_variance"],["baseltest.statistics.power",30,"from .power import achieved_power, required_sample_size"],["baseltest.statistics.feasibility",21,"from .feasibility import FeasibilityCheck, check_feasibility"],["baseltest.statistics.verdict",43,"from .verdict import ("],["baseltest.statistics.threshold",34,"from .threshold import ("],["baseltest.statistics._constants",20,"from ._constants import DEFAULT_CONFIDENCE_LEVEL, DEFAULT_POWER"],["baseltest.statistics.wilson",50,"from .wilson import ("],["baseltest.statistics.sizing",32,"from .sizing import detectable_rate, power_at, required_samples_for_power"],["baseltest.statistics.summary",33,"from .summary import MultiRunSummary, RunOutcome, summarize_runs"],["baseltest.statistics.latency",22,"from .latency import ("]],"baseltest.declarative._runner._report":[["baseltest.declarative._disclosure",12,"from .._disclosure import sizing_disclosure"],["baseltest.declarative._errors",13,"from .._errors import ContractConfigurationError"],["baseltest.declarative._runner._shared",14,"from ._shared import DEFAULT_REPORTS_DIR, DEFAULT_VERDICT_DIR, MAVAI_EXPLORE_POINTER"],["baseltest.reporting",10,"from baseltest.reporting import read_verdict_directory, render_test_report"]],"baseltest.declarative._sizing._model":[["baseltest.contract",11,"from baseltest.contract import BaseltestError"]],"baseltest.declarative._cli":[["baseltest.declarative._providers",17,"from ._providers import ProviderResponseError"],["baseltest.engine",7,"from baseltest.engine import ("],["baseltest.declarative._errors",16,"from ._errors import ContractConfigurationError"],["baseltest.declarative._sizing",32,"from ._sizing import ResolvedSizing, SizingRefusalError, resolve_test_sizing"],["baseltest.reporting",14,"from baseltest.reporting import render_infeasible"],["baseltest.declarative._runner",18,"from ._runner import ("]],"baseltest.declarative._parser._latency":[["baseltest.contract",10,"from baseltest.contract import PERCENTILE_LEVELS"],["baseltest.declarative._parser._shape",13,"from ._shape import _fail, _require_mapping"],["baseltest.declarative._parser._model",12,"from ._model import LatencyDeclaration"]],"baseltest.declarative._instantiate._latency":[["baseltest.baseline",9,"from baseltest.baseline import BaselineResolution"],["baseltest.engine",16,"from baseltest.engine import minimum_contributing_samples"],["baseltest.statistics",17,"from baseltest.statistics import bound_existence_minimum, derive_latency_threshold"],["baseltest.contract",10,"from baseltest.contract import ("],["baseltest.declarative._errors",19,"from .._errors import ContractConfigurationError"],["baseltest.declarative._parser",20,"from .._parser import ContractDeclaration"]],"baseltest.declarative._runner._optimize":[["baseltest.declarative._errors",12,"from .._errors import ContractConfigurationError"],["baseltest.declarative._runner._shared",20,"from ._shared import DEFAULT_OPTIMIZATIONS_DIR"],["baseltest.declarative._instantiate",13,"from .._instantiate import optimize_definition"],["baseltest.declarative._runner._optimize_loop",19,"from ._optimize_loop import OptimizationOutcome, _drive_optimization"],["baseltest.declarative._services",18,"from .._services import ServiceDefinition, discover_services"],["baseltest.reporting",10,"from baseltest.reporting import render_run_plan"],["baseltest.declarative._parser",15,"from .._parser import load_contract"],["baseltest.declarative._optimize",14,"from .._optimize import OptimizationDeclaration"],["baseltest.declarative._registrations",16,"from .._registrations import discover_registrations"],["baseltest.declarative._registry",17,"from .._registry import Bindings"]],"baseltest.statistics.wilson":[["baseltest.statistics._constants",33,"from ._constants import DEFAULT_CONFIDENCE_LEVEL"],["baseltest.statistics._validation",34,"from ._validation import validate_confidence_level, validate_counts"]],"baseltest.declarative._sizing._prompts":[["baseltest.declarative._parser",11,"from .._parser import ContractDeclaration"],["baseltest.declarative._sizing._rates",13,"from ._rates import _parse_rate, _percent"],["baseltest.declarative._sizing._model",12,"from ._model import SizingRefusalError, _EmpiricalCriterion"]],"baseltest.declarative._steppers._contract":[["baseltest.declarative._signatures",17,"from .._signatures import ("],["baseltest.declarative._steppers._context",24,"from ._context import IterationSummary, OptimizeContext"],["baseltest.declarative._errors",16,"from .._errors import ContractConfigurationError"]],"baseltest.baseline":[["baseltest.baseline.reader",16,"from .reader import ("],["baseltest.baseline.record",24,"from .record import ("],["baseltest.baseline.writer",30,"from .writer import baseline_filename, render_baseline, write_baseline"]],"baseltest.declarative._steppers._prompt_engineer":[["baseltest.declarative._steppers._context",13,"from ._context import FailureDetail, IterationResult, OptimizeContext"],["baseltest.declarative._errors",12,"from .._errors import ContractConfigurationError"],["baseltest.declarative._steppers._contract",14,"from ._contract import StepFunction, StepProposal"],["baseltest.declarative._providers",56,"from .._providers import build_invoker, resolve_provider"],["baseltest.declarative._services",57,"from .._services import LanguageModelParameters"]],"baseltest.engine.artefact":[["baseltest.engine.latency",20,"from .latency import LatencyBlock"],["baseltest.engine.naming",21,"from .naming import bounded_key"]],"baseltest.declarative":[["baseltest.declarative._runner",44,"from ._runner import explore, optimize, run"],["baseltest.declarative._runner",43,"from ._runner import check as check_contract"],["baseltest.declarative._steppers",45,"from ._steppers import ("],["baseltest.declarative._registry",42,"from ._registry import Bindings"]],"baseltest.declarative._registry._service_types":[["baseltest.declarative._signatures",16,"from .._signatures import SCALAR_TYPES as _SCALAR_TYPES"],["baseltest.declarative._signatures",19,"from .._signatures import snake as _snake"],["baseltest.declarative._registry._guards",22,"from ._guards import RESERVED_COVARIATE_KEYS"],["baseltest.declarative._services",161,"from .._services import _language_model_type"],["baseltest.declarative._signatures",17,"from .._signatures import kebab as _kebab"],["baseltest.declarative._types",21,"from .._types import ServiceTypeContract"],["baseltest.declarative._errors",15,"from .._errors import ContractConfigurationError"],["baseltest.declarative._signatures",20,"from .._signatures import value_fits as _value_fits"],["baseltest.declarative._signatures",18,"from .._signatures import rendered_signature as _rendered_signature"]],"baseltest.declarative._runner._shared":[],"baseltest.declarative._signatures":[],"baseltest.declarative._disclosure":[["baseltest.statistics",13,"from baseltest.statistics import DEFAULT_POWER, detectable_rate"],["baseltest.reporting",12,"from baseltest.reporting import SizingDisclosure, VerdictRecord"]],"baseltest.reporting":[["baseltest.reporting.verdict_reader",25,"from .verdict_reader import ("],["baseltest.reporting.console",10,"from .console import ("],["baseltest.reporting.run_design",17,"from .run_design import ("],["baseltest.reporting.test_report",24,"from .test_report import render_test_report"],["baseltest.reporting.verdict_xml",31,"from .verdict_xml import render_verdict_record, write_verdict_record"]],"baseltest.declarative._providers":[["baseltest.declarative._providers._apertus",30,"from . import _anthropic, _apertus, _litellm, _mistral, _ollama, _openai"],["baseltest.declarative._providers._ollama",30,"from . import _anthropic, _apertus, _litellm, _mistral, _ollama, _openai"],["baseltest.declarative._services",44,"from .._services import LanguageModelParameters"],["baseltest.declarative._providers._protocol",32,"from ._protocol import ("],["baseltest.declarative._providers._openai",30,"from . import _anthropic, _apertus, _litellm, _mistral, _ollama, _openai"],["baseltest.declarative._providers._anthropic",30,"from . import _anthropic, _apertus, _litellm, _mistral, _ollama, _openai"],["baseltest.contract",27,"from baseltest.contract import BaseltestError, ServiceDeliveryError"],["baseltest.declarative._providers._mistral",30,"from . import _anthropic, _apertus, _litellm, _mistral, _ollama, _openai"],["baseltest.declarative._providers._media",31,"from ._media import CAPABILITY_FOR, media_kinds_present"],["baseltest.declarative._errors",29,"from .._errors import ContractConfigurationError"],["baseltest.declarative._providers._litellm",30,"from . import _anthropic, _apertus, _litellm, _mistral, _ollama, _openai"]],"baseltest.reporting.console":[["baseltest.engine",6,"from baseltest.engine import ("]],"baseltest.declarative._parser._contract":[["baseltest.declarative._parser._inputs",15,"from ._inputs import _parse_inputs"],["baseltest.declarative._parser._model",17,"from ._model import ContractDeclaration"],["baseltest.declarative._parser._latency",16,"from ._latency import _parse_latency"],["baseltest.engine",11,"from baseltest.engine import Intent"],["baseltest.statistics",12,"from baseltest.statistics import DEFAULT_CONFIDENCE_LEVEL"],["baseltest.declarative._parser._structure",19,"from ._structure import _check_top_level_keys, _parse_transforms"],["baseltest.declarative._parser._criteria",14,"from ._criteria import _parse_criterion"],["baseltest.declarative._parser._shape",18,"from ._shape import _fail, _load_yaml, _require_mapping, _require_string"]],"baseltest.contract.errors":[],"baseltest.statistics.latency":[],"baseltest.observation":[["baseltest.observation.record",12,"from .record import CriterionStatistics, FailureEntry, RunObservation"],["baseltest.observation.emit",11,"from .emit import observation_lines"]],"baseltest.engine.run.sample":[["baseltest.engine.defect",27,"from ..defect import DefectDiagnosisError"],["baseltest.engine.naming",28,"from ..naming import bounded_excerpt"],["baseltest.contract",15,"from baseltest.contract import ("],["baseltest.engine.run.model",29,"from .model import SampleRecord"]],"baseltest.declarative._steppers._numeric":[["baseltest.declarative._errors",9,"from .._errors import ContractConfigurationError"]],"baseltest.exploration":[["baseltest.exploration.writer",11,"from .writer import ("]],"baseltest.declarative._instantiate._compose":[["baseltest.declarative._instantiate._baseline",21,"from ._baseline import BaselineContext, _empirical_criteria, _resolve_matching_baseline"],["baseltest.declarative._instantiate._postconditions",23,"from ._postconditions import _build_criterion, _expected_postconditions"],["baseltest.declarative._instantiate._sizing_policy",25,"from ._sizing_policy import RunSizing, _resolve_run_size"],["baseltest.declarative._errors",16,"from .._errors import ContractConfigurationError"],["baseltest.declarative._services",20,"from .._services import ServiceDefinition"],["baseltest.declarative._registry",18,"from .._registry import Registry"],["baseltest.declarative._schema_walk",19,"from .._schema_walk import validate_declared_paths"],["baseltest.declarative._instantiate._views",26,"from ._views import _build_views"],["baseltest.declarative._instantiate._service",24,"from ._service import _resolve_service, _splat_tuple_invoke, _validate_inputs, _validate_media"],["baseltest.engine",14,"from baseltest.engine import RunKind, RunPlan"],["baseltest.contract",13,"from baseltest.contract import ServiceContract"],["baseltest.declarative._parser",17,"from .._parser import ContractDeclaration"],["baseltest.declarative._instantiate._latency",22,"from ._latency import _latency_bar"]],"baseltest.declarative._providers._openai":[["baseltest.declarative._services",15,"from .._services import LanguageModelParameters"],["baseltest.declarative._providers._protocol",5,"from ._protocol import ("]],"baseltest.declarative._instantiate._service":[["baseltest.contract",12,"from baseltest.contract import FileInput, MessageParts"],["baseltest.declarative._errors",14,"from .._errors import ContractConfigurationError"],["baseltest.declarative._signatures",18,"from .._signatures import value_fits as _value_fits"],["baseltest.declarative._providers",15,"from .._providers import require_media, resolve_provider"],["baseltest.declarative._services",17,"from .._services import LanguageModelParameters, ServiceDefinition"],["baseltest.declarative._registry",16,"from .._registry import Registry"]],"baseltest.declarative._instantiate._optimize_point":[["baseltest.declarative._instantiate._postconditions",23,"from ._postconditions import _build_criterion, _expected_postconditions"],["baseltest.declarative._registry",21,"from .._registry import Registry"],["baseltest.declarative._instantiate._service",24,"from ._service import _resolve_service, _splat_tuple_invoke, _validate_inputs, _validate_media"],["baseltest.declarative._instantiate._views",25,"from ._views import _build_views"],["baseltest.contract",16,"from baseltest.contract import ServiceContract"],["baseltest.declarative._services",22,"from .._services import ServiceDefinition, configuration_values"],["baseltest.declarative._errors",19,"from .._errors import ContractConfigurationError"],["baseltest.engine",17,"from baseltest.engine import Intent, RunKind, RunPlan"],["baseltest.declarative._parser",20,"from .._parser import ContractDeclaration"]],"baseltest.declarative._sizing._rates":[["baseltest.statistics",10,"from baseltest.statistics import wilson_lower_bound"],["baseltest.declarative._sizing._model",12,"from ._model import SizingRefusalError"]],"baseltest.engine.run.feasibility":[["baseltest.statistics",12,"from baseltest.statistics import check_feasibility"],["baseltest.contract",11,"from baseltest.contract import BaseltestError, ServiceContract"],["baseltest.engine.run.model",14,"from .model import InfeasibleCriterion, Intent, RunKind, RunPlan"]],"baseltest.declarative._runner._explore":[["baseltest.declarative._parser",20,"from .._parser import load_contract"],["baseltest.declarative._instantiate",19,"from .._instantiate import instantiate_explore"],["baseltest.declarative._registry",22,"from .._registry import Bindings"],["baseltest.exploration",15,"from baseltest.exploration import exploration_stem, write_exploration"],["baseltest.declarative._registrations",21,"from .._registrations import discover_registrations"],["baseltest.reporting",17,"from baseltest.reporting import render_explorations, render_run_plan"],["baseltest.declarative._runner._shared",24,"from ._shared import DEFAULT_EXPLORATIONS_DIR, _tty_progress"],["baseltest.declarative._services",23,"from .._services import discover_services"],["baseltest.engine",14,"from baseltest.engine import DefectDiagnosisError, RunResult, execute"],["baseltest.observation",16,"from baseltest.observation import RunObservation"]],"baseltest.declarative._registry._guards":[["baseltest.declarative._errors",10,"from .._errors import ContractConfigurationError"]],"baseltest.declarative._providers._ollama":[["baseltest.declarative._providers._protocol",12,"from ._protocol import Provider, no_constraint, plain_headers"],["baseltest.declarative._services",15,"from .._services import LanguageModelParameters"],["baseltest.declarative._providers._media",11,"from ._media import b64, message_parts"],["baseltest.contract",9,"from baseltest.contract import FileInput, MediaKind"]],"baseltest.declarative._sizing._flags":[["baseltest.declarative._sizing._model",8,"from ._model import SizingRefusalError"],["baseltest.declarative._sizing._rates",9,"from ._rates import _parse_rate"]],"baseltest.declarative._providers._anthropic":[["baseltest.declarative._services",35,"from .._services import LanguageModelParameters"],["baseltest.declarative._providers._protocol",32,"from ._protocol import Provider"],["baseltest.contract",29,"from baseltest.contract import FileInput, MediaKind, ServiceDeliveryError"],["baseltest.declarative._providers._media",31,"from ._media import b64, content_blocks, mime_type, unexpected_kind"]],"baseltest.statistics._constants":[],"baseltest.declarative._sizing._render":[["baseltest.statistics",9,"from baseltest.statistics import detectable_rate, power_at, wilson_lower_bound_from_rate"],["baseltest.declarative._sizing._model",11,"from ._model import SizingClaim"],["baseltest.declarative._sizing._rates",12,"from ._rates import _percent"]],"baseltest.reporting.test_report":[["baseltest.reporting.verdict_reader",14,"from .verdict_reader import LatencyRecord, VerdictRecord"],["baseltest.reporting.report_html",12,"from .report_html import document_head, escape, footer, timestamp, verdict_css_class"],["baseltest.reporting.run_design",13,"from .run_design import APPROACH_GLOSSES, SizingDisclosure"]],"baseltest.declarative._parser._shape":[["baseltest.declarative._errors",13,"from .._errors import ContractConfigurationError"]],"baseltest.declarative._parser._structure":[["baseltest.declarative._parser._shape",11,"from ._shape import _SEAM_POINTER, _fail, _require_mapping"],["baseltest.declarative._parser._model",10,"from ._model import FORMAT_IDENTIFIER, RAW_VIEW"]],"baseltest.statistics.feasibility":[["baseltest.statistics.wilson",26,"from .wilson import wilson_lower_bound_from_rate"],["baseltest.statistics._constants",25,"from ._constants import DEFAULT_CONFIDENCE_LEVEL, SOUNDNESS_FLOOR_CONFIDENCE"]],"baseltest._version":[],"baseltest.declarative._parser._forms":[["baseltest.declarative._parser._model",10,"from ._model import RAW_VIEW, Form, FormDeclaration"],["baseltest.declarative._parser._shape",11,"from ._shape import _fail"]],"baseltest.contract.postconditions":[],"baseltest.declarative._instantiate._explore":[["baseltest.declarative._errors",17,"from .._errors import ContractConfigurationError"],["baseltest.declarative._instantiate._views",24,"from ._views import _build_views"],["baseltest.declarative._parser",18,"from .._parser import ContractDeclaration"],["baseltest.declarative._instantiate._service",22,"from ._service import _resolve_service, _splat_tuple_invoke, _validate_inputs, _validate_media"],["baseltest.declarative._services",20,"from .._services import ServiceDefinition, configuration_values, factor_values"],["baseltest.declarative._instantiate._sizing_policy",23,"from ._sizing_policy import DEFAULT_SAMPLES, RunSizing"],["baseltest.declarative._registry",19,"from .._registry import Registry"],["baseltest.declarative._instantiate._postconditions",21,"from ._postconditions import _build_criterion, _expected_postconditions"],["baseltest.contract",14,"from baseltest.contract import ServiceContract"],["baseltest.engine",15,"from baseltest.engine import Intent, RunKind, RunPlan"]],"baseltest.declarative._materialise":[["baseltest.declarative._parser",9,"from ._parser import ContractDeclaration, CriterionDeclaration, Form, FormDeclaration"]],"baseltest.optimization.record":[["baseltest.engine",15,"from baseltest.engine import RunResult"],["baseltest.observation",16,"from baseltest.observation import RunObservation"]],"baseltest.declarative._instantiate._views":[["baseltest.declarative._structured",18,"from .._structured import STOCK_TRANSFORMS as STOCK_TRANSFORM_FNS"],["baseltest.declarative._parser",16,"from .._parser import ContractDeclaration"],["baseltest.contract",14,"from baseltest.contract import TransformError"],["baseltest.declarative._registry",17,"from .._registry import Registry"]],"baseltest.declarative._services._parse":[["baseltest.declarative._services._model",18,"from ._model import ("],["baseltest.declarative._registry",26,"from .._registry import Registry"],["baseltest.declarative._types",17,"from .._types import ServiceTypeContract"],["baseltest.declarative._optimize",16,"from .._optimize import OptimizationDeclaration, parse_optimizations"]],"baseltest.declarative._registrations":[["baseltest.declarative._registry",20,"from ._registry import Bindings, Registry"],["baseltest.declarative._errors",19,"from ._errors import ContractConfigurationError"]],"baseltest.declarative._instantiate._sizing_policy":[["baseltest.contract",10,"from baseltest.contract import Criterion"],["baseltest.statistics",12,"from baseltest.statistics import check_feasibility"],["baseltest.engine",11,"from baseltest.engine import Intent, RunKind"],["baseltest.declarative._errors",14,"from .._errors import ContractConfigurationError"]],"baseltest.engine.run.execute":[["baseltest.engine.latency",21,"from ..latency import evaluate_latency"],["baseltest.engine.run.sample",26,"from .sample import _run_one_sample, _SampleOutcome"],["baseltest.engine.run.model",25,"from .model import CriterionResult, RunPlan, RunResult, SampleRecord"],["baseltest.engine.run.identity",23,"from .identity import inputs_fingerprint"],["baseltest.statistics.verdict",19,"from baseltest.statistics.verdict import Verdict"],["baseltest.engine.run.judge",24,"from .judge import _judge"],["baseltest.engine.run.feasibility",22,"from .feasibility import _preflight"],["baseltest.contract",18,"from baseltest.contract import CriterionTally, ServiceContract"]],"baseltest.declarative._registry._bindings":[["baseltest.declarative._registry._core",18,"from ._core import Registry"],["baseltest.declarative._steppers",17,"from .._steppers import ScorerFunction, StepFunction"]],"baseltest.declarative._runner._load":[["baseltest.declarative._registry",15,"from .._registry import Bindings, Registry"],["baseltest.declarative._services",16,"from .._services import ServiceDefinition, discover_services"],["baseltest.declarative._parser",13,"from .._parser import ContractDeclaration, load_contract"],["baseltest.declarative._registrations",14,"from .._registrations import discover_registrations"]],"baseltest.declarative._services._language_model":[["baseltest.declarative._services._model",28,"from ._model import _fail"],["baseltest.declarative._providers",18,"from .._providers import ("],["baseltest.declarative._types",27,"from .._types import ServiceTypeContract"]],"baseltest.exploration.writer":[["baseltest.observation",82,"from baseltest.observation import RunObservation, observation_lines"],["baseltest.engine.artefact",81,"from baseltest.engine.artefact import factor_lines, quote"]],"baseltest.statistics.verdict":[["baseltest.statistics._validation",36,"from ._validation import validate_counts"],["baseltest.statistics._constants",35,"from ._constants import DEFAULT_CONFIDENCE_LEVEL"]],"baseltest.declarative._parser._criteria":[["baseltest.declarative._parser._forms",10,"from ._forms import _FORM_KEYS, _parse_form_entry"],["baseltest.declarative._parser._model",11,"from ._model import CriterionDeclaration, FormDeclaration"],["baseltest.declarative._parser._shape",12,"from ._shape import _SEAM_POINTER, _fail, _require_mapping"]],"baseltest.declarative._optimize":[["baseltest.declarative._steppers",23,"from ._steppers import ("],["baseltest.declarative._registry",32,"from ._registry import Registry"],["baseltest.declarative._errors",22,"from ._errors import ContractConfigurationError"],["baseltest.declarative._types",29,"from ._types import ServiceTypeContract"],["baseltest.optimization",20,"from baseltest.optimization import Objective"]],"baseltest.declarative._parser._inputs":[["baseltest.declarative._parser._shape",20,"from ._shape import _fail, _require_mapping"],["baseltest.declarative._parser._model",19,"from ._model import Form, FormDeclaration"],["baseltest.declarative._parser._forms",18,"from ._forms import _parse_form_entry"],["baseltest.contract",16,"from baseltest.contract import FileInput, MediaKind, MessageParts"]],"baseltest.baseline.record":[["baseltest.engine",9,"from baseltest.engine import LatencyBlock, RunResult, latency_block"]],"baseltest.observation.emit":[["baseltest.engine.naming",13,"from baseltest.engine.naming import bounded_key"],["baseltest.observation.record",15,"from .record import RunObservation"],["baseltest.engine.artefact",12,"from baseltest.engine.artefact import latency_lines, quote"]],"baseltest.engine.latency":[["baseltest.engine.run",46,"from .run import SampleRecord"],["baseltest.contract",18,"from baseltest.contract import PERCENTILE_LEVELS, LatencyBar, LatencyBound"],["baseltest.statistics",19,"from baseltest.statistics import latency_percentile"],["baseltest.statistics.verdict",20,"from baseltest.statistics.verdict import Verdict"]],"baseltest.declarative._runner._optimize_loop":[["baseltest.declarative._parser",29,"from .._parser import ContractDeclaration"],["baseltest.declarative._services",32,"from .._services import _resolved_point as _configuration_identity"],["baseltest.reporting",24,"from baseltest.reporting import render_optimization_run"],["baseltest.declarative._services",31,"from .._services import ServiceDefinition"],["baseltest.declarative._steppers",33,"from .._steppers import ("],["baseltest.declarative._instantiate",27,"from .._instantiate import instantiate_optimize_point"],["baseltest.optimization",16,"from baseltest.optimization import ("],["baseltest.declarative._errors",26,"from .._errors import ContractConfigurationError"],["baseltest.declarative._optimize",28,"from .._optimize import OptimizationDeclaration"],["baseltest.declarative._runner._shared",42,"from ._shared import _tty_progress"],["baseltest.engine",15,"from baseltest.engine import DefectDiagnosisError, RunResult, execute, latency_block"],["baseltest.declarative._registry",30,"from .._registry import Registry"]],"baseltest.declarative._instantiate._baseline":[["baseltest.declarative._parser",25,"from .._parser import FORMAT_IDENTIFIER, ContractDeclaration, CriterionDeclaration"],["baseltest.statistics",23,"from baseltest.statistics import derive_sample_size_first, wilson_lower_bound"],["baseltest.baseline",15,"from baseltest.baseline import ("],["baseltest.contract",21,"from baseltest.contract import Criterion, Postcondition, ThresholdProvenance"],["baseltest.declarative._registry",26,"from .._registry import Registry"],["baseltest.declarative._instantiate._postconditions",27,"from ._postconditions import _build_criterion"],["baseltest.engine",22,"from baseltest.engine import inputs_fingerprint"]],"baseltest.declarative._runner":[["baseltest.declarative._runner._shared",17,"from ._shared import ("],["baseltest.declarative._runner._run",16,"from ._run import run"],["baseltest.declarative._runner._optimize_loop",14,"from ._optimize_loop import OptimizationOutcome"],["baseltest.declarative._runner._report",15,"from ._report import report"],["baseltest.declarative._runner._check",10,"from ._check import check"],["baseltest.declarative._runner._shared",24,"from ._shared import _tty_progress as _tty_progress"],["baseltest.declarative._runner._explore",11,"from ._explore import AbortedConfiguration, ConfigurationExploration, ExplorationRun, explore"],["baseltest.declarative._runner._load",12,"from ._load import LoadedContract, load_for_run"],["baseltest.declarative._runner._optimize",13,"from ._optimize import optimize"]],"baseltest.declarative._providers._litellm":[["baseltest.declarative._providers._protocol",38,"from ._protocol import ("],["baseltest.declarative._services",49,"from .._services import LanguageModelParameters"]],"baseltest.declarative._registry._transform":[["baseltest.declarative._errors",21,"from .._errors import ContractConfigurationError"],["baseltest.declarative._structured",22,"from .._structured import STOCK_TRANSFORMS"]],"baseltest.declarative._steppers._builtins":[["baseltest.declarative._steppers._prompt_engineer",6,"from ._prompt_engineer import _prompt_engineer"],["baseltest.declarative._steppers._refining_grid",7,"from ._refining_grid import _refining_grid"],["baseltest.declarative._steppers._context",3,"from ._context import IterationSummary"],["baseltest.declarative._steppers._linear_sweep",5,"from ._linear_sweep import _linear_sweep"],["baseltest.declarative._steppers._contract",4,"from ._contract import ScorerFunction, StepperRegistration"]],"baseltest.declarative._errors":[["baseltest.contract",3,"from baseltest.contract import BaseltestError"]],"baseltest.declarative._types":[],"baseltest.declarative._registry":[["baseltest.declarative._registry._transform",23,"from ._transform import _STOCK_TRANSFORMS as _STOCK_TRANSFORMS"],["baseltest.declarative._registry._guards",22,"from ._guards import RESERVED_COVARIATE_KEYS"],["baseltest.declarative._registry._transform",24,"from ._transform import TransformRegistration"],["baseltest.declarative._registry._bindings",20,"from ._bindings import Bindings"],["baseltest.declarative._registry._core",21,"from ._core import Registry"]],"baseltest.reporting.report_html":[],"baseltest.contract":[["baseltest.contract.evaluation",23,"from .evaluation import ("],["baseltest.contract.errors",22,"from .errors import BaseltestError, ContractValidationError, PreconditionError"],["baseltest.contract.postconditions",45,"from .postconditions import ("],["baseltest.contract.model",32,"from .model import ("]],"baseltest.engine.run.judge":[["baseltest.contract",3,"from baseltest.contract import Criterion, CriterionTally"],["baseltest.statistics.wilson",5,"from baseltest.statistics.wilson import wilson_lower_bound"],["baseltest.statistics.verdict",4,"from baseltest.statistics.verdict import Verdict, evaluate_regression"]],"baseltest.engine.run.attainment":[["baseltest.statistics.verdict",14,"from baseltest.statistics.verdict import Verdict"],["baseltest.statistics.wilson",15,"from baseltest.statistics.wilson import wilson_lower_bound"],["baseltest.engine.run.model",17,"from .model import CriterionResult"]],"baseltest.reporting.run_design":[],"baseltest.reporting.verdict_xml":[["baseltest.engine",16,"from baseltest.engine import CriterionResult, RunResult"],["baseltest.reporting.run_design",18,"from .run_design import RunDesign"],["baseltest._version",15,"from baseltest._version import __version__"]],"baseltest.declarative._services._model":[["baseltest.declarative._errors",11,"from .._errors import ContractConfigurationError"],["baseltest.declarative._optimize",12,"from .._optimize import OptimizationDeclaration"],["baseltest.declarative._types",13,"from .._types import ServiceTypeContract"]],"baseltest.declarative._instantiate._postconditions":[["baseltest.declarative._structured",28,"from .._structured import compile_jsonpath, compile_xpath, path_qualified"],["baseltest.contract",13,"from baseltest.contract import ("],["baseltest.declarative._parser",26,"from .._parser import RAW_VIEW, CriterionDeclaration, Form, FormDeclaration"],["baseltest.declarative._registry",27,"from .._registry import Registry"],["baseltest.engine.naming",24,"from baseltest.engine.naming import bounded_excerpt, per_input_name"]],"baseltest.declarative._steppers._context":[],"baseltest.declarative._services":[["baseltest.declarative._services._language_model",25,"from ._language_model import ("],["baseltest.declarative._services._language_model",32,"from ._language_model import _validate_configuration as _validate_configuration"],["baseltest.declarative._services._parse",35,"from ._parse import discover_services, parse_services"],["baseltest.declarative._services._language_model",31,"from ._language_model import _language_model_type as _language_model_type"],["baseltest.declarative._services._model",33,"from ._model import ServiceDefinition, configuration_values, factor_values"],["baseltest.declarative._services._parse",34,"from ._parse import _resolved_point as _resolved_point"]],"baseltest.declarative._sizing._resolve":[["baseltest.declarative._sizing._flags",20,"from ._flags import _parse_tolerate_flags, _refuse_contradictory_sizing_flags"],["baseltest.declarative._sizing._modes",22,"from ._modes import _explicit_samples_mode, _over_reach_mode, _risk_driven_mode"],["baseltest.declarative._sizing._rates",29,"from ._rates import _parse_rate"],["baseltest.declarative._sizing._prompts",23,"from ._prompts import ("],["baseltest.declarative._services",18,"from .._services import ServiceDefinition"],["baseltest.declarative._parser",17,"from .._parser import ContractDeclaration"],["baseltest.declarative._registry",32,"from .._registry import Registry"],["baseltest.statistics",15,"from baseltest.statistics import DEFAULT_POWER"],["baseltest.declarative._sizing._model",21,"from ._model import ResolvedSizing, SizingRefusalError, _EmpiricalCriterion"],["baseltest.declarative._sizing._criteria",19,"from ._criteria import _sizeable_criteria, resolve_contract_baseline"]],"baseltest.baseline.reader":[["baseltest.baseline.writer",27,"from .writer import SCHEMA_VERSION"],["baseltest.engine",25,"from baseltest.engine import LatencyBasis"]],"baseltest.declarative._schema_walk":[["baseltest.declarative._errors",34,"from ._errors import ContractConfigurationError"],["baseltest.declarative._structured",283,"from ._structured import compile_jsonpath"],["baseltest.declarative._parser",281,"from ._parser import RAW_VIEW"],["baseltest.declarative._registry",37,"from ._registry import Registry"],["baseltest.declarative._registry",282,"from ._registry import _STOCK_TRANSFORMS"],["baseltest.engine.naming",32,"from baseltest.engine.naming import bounded_excerpt"]],"baseltest.contract.model":[["baseltest.statistics",10,"from baseltest.statistics import DEFAULT_CONFIDENCE_LEVEL"],["baseltest.contract.postconditions",13,"from .postconditions import Postcondition"],["baseltest.contract.errors",12,"from .errors import BaseltestError"]],"baseltest.declarative._parser":[["baseltest.declarative._parser._model",20,"from ._model import ("],["baseltest.declarative._parser._latency",19,"from ._latency import _PERCENTILE_KEYS as _PERCENTILE_KEYS"],["baseltest.declarative._parser._contract",18,"from ._contract import load_contract, parse_contract"]],"baseltest.engine.naming":[],"baseltest.declarative._steppers._linear_sweep":[["baseltest.declarative._errors",11,"from .._errors import ContractConfigurationError"],["baseltest.declarative._steppers._contract",13,"from ._contract import StepFunction"],["baseltest.declarative._steppers._numeric",14,"from ._numeric import _NUMERIC_TOLERANCE, _numeric"],["baseltest.declarative._steppers._context",12,"from ._context import OptimizeContext"]],"baseltest.statistics.summary":[["baseltest.statistics.verdict",20,"from .verdict import Verdict"]],"baseltest.baseline.writer":[["baseltest.engine.artefact",56,"from baseltest.engine.artefact import latency_lines, quote"],["baseltest.engine.naming",57,"from baseltest.engine.naming import bounded_key"],["baseltest.baseline.record",59,"from .record import BaselineRecord, CriterionCharacterisation"]],"baseltest.statistics.sizing":[["baseltest.statistics.wilson",36,"from .wilson import wilson_lower_bound_from_rate"],["baseltest.statistics._constants",34,"from ._constants import DEFAULT_CONFIDENCE_LEVEL, DEFAULT_POWER"],["baseltest.statistics._validation",35,"from ._validation import validate_unit_interval"]],"baseltest.declarative._providers._protocol":[["baseltest.declarative._providers._media",21,"from ._media import b64 as _b64"],["baseltest.contract",12,"from baseltest.contract import FileInput, MediaKind"],["baseltest.declarative._services",24,"from .._services import LanguageModelParameters"],["baseltest.declarative._providers._media",14,"from ._media import ("]],"baseltest.engine.defect":[],"baseltest.declarative._providers._mistral":[["baseltest.contract",3,"from baseltest.contract import MediaKind"],["baseltest.declarative._providers._protocol",5,"from ._protocol import ("]],"baseltest.optimization.writer":[["baseltest.observation",16,"from baseltest.observation import observation_lines"],["baseltest.optimization.record",18,"from .record import OptimizationRecord"],["baseltest.engine.artefact",15,"from baseltest.engine.artefact import factor_lines, quote, scalar"]],"baseltest.declarative._providers._media":[["baseltest.contract",16,"from baseltest.contract import FileInput, MediaKind, MessageParts"]],"baseltest.declarative._providers._apertus":[["baseltest.declarative._providers._protocol",14,"from ._protocol import ("]],"baseltest.declarative._sizing._pricing":[["baseltest.declarative._sizing._model",11,"from ._model import SizingClaim, _EmpiricalCriterion"],["baseltest.statistics",9,"from baseltest.statistics import required_samples_for_power"],["baseltest.declarative._sizing._rates",12,"from ._rates import _percent"]],"baseltest.optimization":[["baseltest.optimization.writer",19,"from .writer import render_optimization, write_optimization"],["baseltest.optimization.record",12,"from .record import ("]],"baseltest.contract.evaluation":[["baseltest.contract.model",21,"from .model import Criterion, TransformError"],["baseltest.statistics",19,"from baseltest.statistics import proportion_standard_error, proportion_variance"]],"baseltest":[["baseltest.contract",17,"from baseltest.contract import FileInput, MessageParts"],["baseltest._version",16,"from baseltest._version import __version__"],["baseltest.declarative",18,"from baseltest.declarative import ("]],"baseltest.declarative._registry._core":[["baseltest.declarative._registry._transform",33,"from ._transform import _STOCK_TRANSFORMS, TransformRegistration, _loaded_schema"],["baseltest.declarative._registry._guards",26,"from ._guards import _register, _validated_covariates"],["baseltest.declarative._registry._service_types",27,"from ._service_types import ("],["baseltest.declarative._types",25,"from .._types import ServiceTypeContract"],["baseltest.declarative._steppers",17,"from .._steppers import ("],["baseltest.declarative._errors",16,"from .._errors import ContractConfigurationError"]],"baseltest.declarative._sizing":[["baseltest.declarative._sizing._model",26,"from ._model import ResolvedSizing, SizingClaim, SizingRefusalError"],["baseltest.declarative._sizing._resolve",27,"from ._resolve import resolve_test_sizing"]],"baseltest.engine":[["baseltest.engine.latency",14,"from .latency import ("],["baseltest.statistics.verdict",11,"from baseltest.statistics.verdict import Verdict"],["baseltest.engine.run",24,"from .run import ("],["baseltest.engine.defect",13,"from .defect import TRANSFORM_CONTRACT_NOTE, DefectDiagnosisError"],["baseltest.engine.run.execute",24,"from .run import ("]],"baseltest.statistics._validation":[],"baseltest.declarative._structured":[["baseltest.declarative._errors",34,"from ._errors import ContractConfigurationError"],["baseltest.contract",32,"from baseltest.contract import Postcondition, PostconditionResult, TransformError"]],"baseltest.declarative._instantiate":[["baseltest.declarative._instantiate._optimize_point",19,"from ._optimize_point import OptimizePoint, instantiate_optimize_point, optimize_definition"],["baseltest.declarative._instantiate._baseline",16,"from ._baseline import BaselineContext"],["baseltest.declarative._instantiate._views",21,"from ._views import descriptive_view_fingerprints"],["baseltest.declarative._instantiate._explore",18,"from ._explore import ExploreConfiguration, instantiate_explore"],["baseltest.declarative._instantiate._compose",17,"from ._compose import Instantiation, instantiate"],["baseltest.declarative._instantiate._sizing_policy",20,"from ._sizing_policy import RunSizing"]],"baseltest.declarative._steppers":[["baseltest.declarative._steppers._contract",48,"from ._contract import ("],["baseltest.declarative._steppers._context",40,"from ._context import ("],["baseltest.declarative._steppers._builtins",39,"from ._builtins import builtin_scorers, builtin_stepper_registrations"]],"baseltest.declarative._runner._check":[["baseltest.declarative._instantiate._service",13,"from .._instantiate._service import _validate_inputs"],["baseltest.declarative._services",18,"from .._services import discover_services"],["baseltest.declarative._schema_walk",17,"from .._schema_walk import validate_declared_paths"],["baseltest.declarative._registry",16,"from .._registry import Bindings"],["baseltest.declarative._registrations",15,"from .._registrations import discover_registrations"],["baseltest.declarative._parser",14,"from .._parser import load_contract"],["baseltest.declarative._instantiate",12,"from .._instantiate import instantiate"],["baseltest.engine",10,"from baseltest.engine import RunKind"]],"baseltest.engine.run.identity":[["baseltest.contract",8,"from baseltest.contract import FileInput, MessageParts"]],"baseltest.declarative._parser._model":[["baseltest.engine",13,"from baseltest.engine import Intent"]],"baseltest.statistics.power":[["baseltest.statistics._validation",19,"from ._validation import validate_unit_interval"]],"baseltest.engine.run.model":[["baseltest.engine.latency",17,"from ..latency import LatencyEvaluation"],["baseltest.contract",13,"from baseltest.contract import Criterion, CriterionTally, Outcome"],["baseltest.statistics.verdict",14,"from baseltest.statistics.verdict import Verdict"]],"baseltest.engine.run":[["baseltest.engine.run.feasibility",10,"from .feasibility import InfeasibleRunError, derive_minimum_samples"],["baseltest.engine.run.identity",11,"from .identity import inputs_fingerprint"],["baseltest.engine.run.attainment",8,"from .attainment import BarAttainment, bar_attainment"],["baseltest.engine.run.model",12,"from .model import ("],["baseltest.engine.run.execute",9,"from .execute import execute"]],"baseltest.statistics.threshold":[["baseltest.statistics._validation",35,"from ._validation import validate_confidence_level"],["baseltest.statistics._constants",34,"from ._constants import DEFAULT_CONFIDENCE_LEVEL, DEFAULT_POWER, SOUNDNESS_FLOOR_CONFIDENCE"],["baseltest.statistics.wilson",37,"from .wilson import wilson_lower_bound, wilson_lower_bound_from_rate"],["baseltest.statistics.power",36,"from .power import required_sample_size"]],"baseltest.declarative._steppers._refining_grid":[["baseltest.declarative._steppers._numeric",19,"from ._numeric import _NUMERIC_TOLERANCE, _grid_values, _numeric"],["baseltest.declarative._steppers._contract",18,"from ._contract import Phase, StepFunction, StepProposal"],["baseltest.declarative._errors",16,"from .._errors import ContractConfigurationError"],["baseltest.declarative._steppers._context",17,"from ._context import OptimizeContext"],["baseltest.statistics",14,"from baseltest.statistics import DEFAULT_CONFIDENCE_LEVEL, wilson_interval"]],"baseltest.declarative._runner._run":[["baseltest.declarative._errors",25,"from .._errors import ContractConfigurationError"],["baseltest.reporting",11,"from baseltest.reporting import ("],["baseltest.declarative._runner._shared",31,"from ._shared import DEFAULT_BASELINE_DIR, _tty_progress"],["baseltest.declarative._registry",28,"from .._registry import Bindings"],["baseltest.declarative._instantiate",26,"from .._instantiate import BaselineContext, descriptive_view_fingerprints, instantiate"],["baseltest.declarative._runner._load",30,"from ._load import LoadedContract, load_for_run"],["baseltest.declarative._disclosure",24,"from .._disclosure import sizing_disclosure"],["baseltest.baseline",9,"from baseltest.baseline import BaselineRecord, write_baseline"],["baseltest.declarative._parser",27,"from .._parser import FORMAT_IDENTIFIER"],["baseltest.engine",10,"from baseltest.engine import RunKind, RunResult, execute"],["baseltest.declarative._sizing",29,"from .._sizing import ResolvedSizing"]]}

baseltest-0.7.0/.import_linter_cache/CACHEDIR.TAG

@@ -0,0 +1,3 @@
+Signature: 8a477f597d28d172789f06886806bc55
+# This file is a cache directory tag automatically created by Grimp.
+# For information about cache directory tags see https://bford.info/cachedir/

baseltest-0.7.0/.import_linter_cache/baseltest.meta.json

@@ -0,0 +1 @@
+{"baseltest.declarative._providers._litellm": 1784829529.8269892, "baseltest.declarative._providers._ollama": 1784829044.6770706, "baseltest.observation.emit": 1784730023.5673316, "baseltest.baseline.reader": 1784624486.3295696, "baseltest.declarative._sizing._resolve": 1784700933.2291906, "baseltest.declarative._instantiate._latency": 1784697825.7258992, "baseltest.reporting.console": 1784724278.0934372, "baseltest.declarative._structured": 1784660834.7805767, "baseltest.baseline.writer": 1784721732.8947453, "baseltest.declarative._steppers._context": 1784711183.7829256, "baseltest.declarative._services._model": 1784704043.7597773, "baseltest.declarative._cli": 1784717739.0876503, "baseltest.declarative._instantiate._optimize_point": 1784830564.037318, "baseltest.declarative._providers._openai": 1784829529.8268123, "baseltest": 1784828119.1467574, "baseltest.baseline.record": 1784642532.6878533, "baseltest.observation": 1784730023.5668638, "baseltest.engine.artefact": 1784721732.8950317, "baseltest.exploration": 1784731446.1992078, "baseltest.declarative._registry._bindings": 1784828295.6439784, "baseltest.optimization": 1784624486.3313675, "baseltest.statistics.latency": 1783627706.2529247, "baseltest.declarative._sizing._rates": 1784700933.2275133, "baseltest.statistics.summary": 1783342255.9972599, "baseltest.declarative": 1784711183.777789, "baseltest.declarative._parser._structure": 1784701927.6573, "baseltest.declarative._sizing._criteria": 1784700933.2240186, "baseltest.statistics.proportion": 1784724278.093899, "baseltest.optimization.writer": 1784730023.5684454, "baseltest.declarative._parser._latency": 1784701927.656845, "baseltest.contract.postconditions": 1784796507.7300982, "baseltest.declarative._instantiate._views": 1784697825.7271893, "baseltest.declarative._instantiate._postconditions": 1784697825.7264152, "baseltest.declarative._signatures": 1784230065.6745944, "baseltest.declarative._services": 1784704043.7589154, "baseltest.statistics.sizing": 1784647895.008006, "baseltest.statistics.verdict": 1784647895.008318, "baseltest.statistics.power": 1784642532.6949065, "baseltest.declarative._sizing._prompts": 1784700933.2273402, "baseltest.declarative._providers._media": 1784828823.8133955, "baseltest.engine.run.execute": 1784662118.0305192, "baseltest.engine.run.identity": 1784828154.437675, "baseltest.declarative._registrations": 1784711183.778993, "baseltest.engine.defect": 1784576130.4987924, "baseltest.reporting.report_html": 1784179100.0100958, "baseltest.declarative._steppers._linear_sweep": 1784711183.7834938, "baseltest.engine.run.judge": 1784652381.840744, "baseltest.statistics._constants": 1784647895.0077252, "baseltest.declarative._instantiate._baseline": 1784697825.7248998, "baseltest.engine.run.sample": 1784662118.0307624, "baseltest.declarative._providers._anthropic": 1784829008.5363247, "baseltest.reporting.run_design": 1783873845.9452946, "baseltest.declarative._optimize": 1784711183.7788277, "baseltest.contract.model": 1784828110.9005754, "baseltest.declarative._runner._shared": 1784705746.816538, "baseltest.declarative._registry._transform": 1784701927.6581495, "baseltest.engine.run.feasibility": 1784652381.8404539, "baseltest.declarative._providers._apertus": 1784296847.7761314, "baseltest.declarative._sizing._modes": 1784700933.2269435, "baseltest.declarative._types": 1784828295.6442204, "baseltest.statistics": 1784724278.0937877, "baseltest.declarative._parser._forms": 1784701927.6564765, "baseltest.declarative._steppers._numeric": 1784711183.7837136, "baseltest.declarative._runner._check": 1784711183.7801807, "baseltest.declarative._runner._explore": 1784730023.5657048, "baseltest.optimization.record": 1784730023.568026, "baseltest.statistics._validation": 1784642532.6940644, "baseltest.baseline": 1784731446.198929, "baseltest.declarative._runner._load": 1784717739.0880518, "baseltest.engine.run": 1784656883.865689, "baseltest.declarative._runner._optimize_loop": 1784711183.7817612, "baseltest.contract.evaluation": 1784796507.7297335, "baseltest.declarative._parser._contract": 1784827427.3974364, "baseltest.declarative._sizing._pricing": 1784700933.2271376, "baseltest.declarative._registry": 1784711183.7792487, "baseltest.declarative._services._parse": 1784828295.644096, "baseltest.declarative._parser._criteria": 1784701927.6561775, "baseltest.declarative._instantiate._compose": 1784830564.0370176, "baseltest.declarative._instantiate": 1784697825.724575, "baseltest.declarative._parser._inputs": 1784828145.4987104, "baseltest.declarative._disclosure": 1784647895.0063276, "baseltest.declarative._runner": 1784717739.0879042, "baseltest.declarative._sizing._render": 1784700933.2277975, "baseltest.reporting.verdict_reader": 1784647895.0073047, "baseltest.statistics.threshold": 1784647895.0081513, "baseltest.declarative._steppers._prompt_engineer": 1784711183.7843645, "baseltest.engine.naming": 1784311537.804023, "baseltest.declarative._steppers._builtins": 1784711183.7827446, "baseltest.declarative._steppers": 1784711183.7825606, "baseltest.declarative._instantiate._explore": 1784830564.03718, "baseltest.declarative._runner._report": 1784705746.816261, "baseltest.declarative._providers._mistral": 1784830564.037437, "baseltest.declarative._schema_walk": 1784694483.4969451, "baseltest.reporting.test_report": 1784610771.8111923, "baseltest.declarative._sizing._model": 1784700933.2267623, "baseltest.declarative._providers": 1784829589.424779, "baseltest.observation.record": 1784730023.5676389, "baseltest._version": 1784731446.1986468, "baseltest.engine.latency": 1784828295.6438906, "baseltest.contract.errors": 1784636632.259277, "baseltest.declarative._runner._run": 1784717739.088279, "baseltest.reporting": 1784656883.865972, "baseltest.declarative._instantiate._service": 1784830564.036805, "baseltest.declarative._services._language_model": 1784829589.4253302, "baseltest.declarative._runner._optimize": 1784711183.7811997, "baseltest.declarative._sizing": 1784700933.2238023, "baseltest.statistics.wilson": 1784647895.0084746, "baseltest.declarative._registry._guards": 1784701927.6579168, "baseltest.declarative._registry._core": 1784711183.7799122, "baseltest.contract": 1784828119.1466117, "baseltest.declarative._instantiate._sizing_policy": 1784697825.726926, "baseltest.declarative._parser": 1784701927.6558995, "baseltest.engine.run.attainment": 1784656883.865816, "baseltest.declarative._materialise": 1784624486.3301847, "baseltest.declarative._providers._protocol": 1784828898.9633727, "baseltest.declarative._registry._service_types": 1784701927.6580393, "baseltest.declarative._steppers._refining_grid": 1784711183.7845929, "baseltest.declarative._errors": 1784636632.2596185, "baseltest.declarative._steppers._contract": 1784828295.643966, "baseltest.declarative._parser._shape": 1784701927.6571376, "baseltest.reporting.verdict_xml": 1784731446.1995153, "baseltest.statistics.feasibility": 1784647895.0078654, "baseltest.engine.run.model": 1784735603.3854725, "baseltest.declarative._sizing._flags": 1784700933.224295, "baseltest.engine": 1784656883.865531, "baseltest.exploration.writer": 1784730023.5665224, "baseltest.declarative._parser._model": 1784701927.6569657}

baseltest-0.7.0/AGENTS.md

@@ -0,0 +1,141 @@
+# AGENTS.md
+
+Guidance for coding agents (and human contributors) working in this repository.
+
+## Project overview
+
+`baseltest` is a Python-native framework for probabilistic testing of
+stochastic services (LLMs, ML models, randomized algorithms, network-dependent
+services). It is the Python member of the mavai framework family, alongside
+`punit` (Java) and `feotest` (Rust): same statistical methodology, idiomatic
+per-language implementation, not a port.
+
+The project is early-stage: only packaging and tooling scaffolding exists so
+far. Internal architecture (module layout beyond the package root, statistical
+approach, API shape) is not yet decided and should not be assumed from
+`punit` or `feotest` — it will be worked out via orchestrator directives as
+implementation begins.
+
+## Environment setup
+
+```bash
+python -m venv venv
+source venv/bin/activate
+pip install -e ".[dev]"
+```
+
+Python 3.11+ is required (see `pyproject.toml`).
+
+## Build, test, and quality commands
+
+```bash
+# Run the test suite
+pytest
+
+# Run a single test file
+pytest tests/test_something.py
+
+# Run a single test by name (substring match)
+pytest -k test_name
+
+# Run with coverage report (also the default via pyproject addopts)
+pytest --cov=baseltest --cov-report=term-missing
+
+# Lint
+ruff check .
+
+# Format
+ruff format .
+
+# Format check (CI-friendly, no rewrite)
+ruff format --check .
+
+# Static type check
+mypy src
+
+# Build distributable artifacts
+python -m build
+```
+
+Run `ruff check .`, `ruff format --check .`, `mypy src`, and `pytest` before
+opening a pull request — the same set CI enforces.
+
+## Current layout
+
+```
+src/baseltest/   # package root — __init__.py + py.typed only, so far
+tests/           # conftest.py only, so far
+```
+
+Do not pre-create subpackages or modules ahead of an actual directive to
+implement something — the internal architecture (module layout, statistical
+approach, API shape, exception vs. result-object conventions for expected
+failures) is an open design question, not settled by this scaffold.
+
+## Conventions
+
+- **Idiomatic Python, not a Java (or Rust) lookalike.** This carries the same
+  weight it did for `feotest`'s Rust idiom rule — punit's implementation is a
+  specification of *what* the statistics/behaviour must do, never a template
+  for *how* to structure it. Concretely:
+  - Prefer plain functions and `@dataclass` (or `NamedTuple`) over classes
+    that exist only to hold immutable state and one computation method —
+    that shape is a Java-ism, not a Python one.
+  - Do not port a one-class-per-concern Java package layout 1:1. Group by
+    cohesive Python module instead (see punit's `statistics` package vs. any
+    `baseltest` equivalent — different file counts and boundaries are
+    expected and correct).
+  - No getter/setter methods, no builder-pattern classes for simple value
+    construction, no `Optional`-wrapper idioms — use plain attributes,
+    keyword arguments with defaults, and `X | None`.
+  - Prefer composition via plain functions and modules over inheritance
+    hierarchies or interface/impl pairs manufactured for a single
+    implementation.
+  - Errors: see punit's `Outcome<T>` convention in the orchestrator
+    `CLAUDE.md` — that convention is Java-specific and does not transfer.
+    Python idiom uses exceptions for defects and, where an anticipated
+    negative outcome must not abort a longer-running process (e.g. one
+    sample among many), a plain data object — not a ported `Outcome` sealed
+    type or `Result`-style wrapper.
+  - Documentation and naming should read as Python: `snake_case` functions
+    and variables, `PascalCase` only for actual classes, no `I`-prefixed
+    interfaces, no Hungarian-notation leftovers. Do not reference Java,
+    JUnit, or punit's class names in docstrings or comments — write for a
+    Python audience that doesn't need to know the framework's origins.
+  - When in doubt, ask: "would a Python developer who has never seen punit
+    recognise this as normal, well-written Python?" If the answer is no,
+    the shape is wrong even if the math is right.
+- Target Python 3.11+; use modern typing (`X | Y` unions, `list[str]`, no
+  `from __future__ import annotations` needed).
+- Every public function and class carries type hints; `mypy --strict` must
+  pass.
+- Every public module, class, and function carries a docstring
+  (Google or NumPy style — pick one and stay consistent within a module).
+- Formatting and linting are enforced by `ruff` (see `[tool.ruff]` in
+  `pyproject.toml`), not by hand; do not hand-format around `ruff format`'s
+  output.
+- Observe a consistent level of abstraction in any given unit of
+  functionality: a public method expressing high-level business logic should
+  not contain low-level detail (string parsing, file I/O) inline — delegate
+  to a helper that is tested in its own right. Small, trivial exceptions are
+  fine where the extra abstraction would not pay for itself.
+- All non-trivial functionality is covered by unit tests; functionality
+  dependent on resources outside the test's control gets an integration test
+  instead, kept separately identifiable (e.g. `@pytest.mark.integration`).
+- Construct test assertions as plain `assert` statements (pytest's assertion
+  rewriting gives readable failure output without a separate assertion
+  library).
+
+## Requirement-code isolation
+
+Internal feature-tracking codes used in orchestrator planning documents
+(short letter-prefix codes such as `PT13`, `EX06`, `RP01`) must never appear
+in this repository's source — production or test. A reader of this
+open-source code has no context for them. Refer to features by their domain
+name ("baseline integrity check", "passing-only latency block") instead.
+
+## License and contributions
+
+Apache License 2.0 (see `LICENSE`). Contributions require a `Signed-off-by`
+line per the Developer Certificate of Origin — see `CONTRIBUTING.md` and
+`dco.txt`.

baseltest-0.7.0/CONTRIBUTING.md

@@ -0,0 +1,60 @@
+# Contributing to baseltest
+
+Thank you for your interest in contributing to baseltest. This document
+describes how to submit contributions and the legal terms that apply.
+
+## License
+
+baseltest is licensed under the [Apache License, Version 2.0](LICENSE).
+All contributions are accepted under the same license.
+
+## Developer Certificate of Origin
+
+All contributions are subject to the
+[Developer Certificate of Origin (DCO)](https://developercertificate.org/).
+The DCO text is available verbatim in the [dco.txt](dco.txt) file in the
+root of this repository.
+
+By signing off your commits, you certify that you wrote the contribution
+or otherwise have the right to submit it under the project's license. No
+separate contributor agreement is required.
+
+### Signing your commits
+
+Add a `Signed-off-by` line to every commit message:
+
+```
+Signed-off-by: Your Name <your.email@example.com>
+```
+
+The easiest way is to use the `-s` (or `--signoff`) flag with `git commit`:
+
+```
+git commit -s -m "Your commit message"
+```
+
+The name and email must match the values configured in your git
+identity (`git config user.name` and `git config user.email`).
+
+To sign off a series of existing commits before pushing, use:
+
+```
+git rebase --signoff <base-branch>
+```
+
+Unsigned commits will be blocked by automated checks on pull requests.
+
+## Reporting issues
+
+Please use [GitHub Issues](https://github.com/mavai-org/baseltest/issues) for
+bug reports and feature requests. Include a minimal reproducer where
+possible.
+
+## Pull requests
+
+- Fork the repository and create a topic branch from `main`.
+- Keep changes focused; one logical change per pull request.
+- Ensure all commits are signed off (see above).
+- Run the test suite locally (`pytest`) and the linters (`ruff check`,
+  `ruff format --check`, `mypy src`) before opening the pull request.
+- Reference any related issue in the pull request description.