proofstep-core 0.1.0__tar.gz

proofstep_core-0.1.0/.gitignore

@@ -0,0 +1,57 @@
+
+__pycache__/
+.coverage
+.coverage.*
+.docker-data/
+.DS_Store
+.e2e-api.log
+.env
+.env.*
+.env.prod
+# Exceptions, and they must come *after* the patterns above: git takes the last matching rule, so a
+# negation written earlier in the file is silently overridden. That is not a hypothetical — the `!`
+# line used to sit at the top, `.env.*` re-ignored it, and `.env.prod.example` was never committed.
+# `scripts/init_secrets.sh` reads that file, so the first documented step of self-hosting failed for
+# anyone who cloned the repository. It was caught by CI running the same step.
+!.env.example
+!.env.prod.example
+.hypothesis/
+.idea/
+.mypy_cache/
+.next/
+.proofstep/
+.pytest_cache/
+.ruff_cache/
+.turbo/
+.venv/
+.vscode/
+*.egg-info/
+*.key
+*.pem
+*.proofstep.local.yaml
+*.py[cod]
+*.swp
+*.tsbuildinfo
+# Database dumps. Never committed: they contain every tenant's data, and a backup in a git history
+# Docker volumes
+# Editors / OS
+# is a backup with no access control and no retention.
+# Node
+# Proofstep local state
+# Python
+# Reports a local run drops in the working tree. The name comes from the suite, so the pattern has
+# Secrets and local config — never commit these
+# The e2e stack's server log, written next to the repo so a CI failure can print it.
+# to cover all of them rather than the default filename only.
+~/.proofstep/
+backups/
+build/
+coverage.xml
+credentials.json
+dist/
+htmlcov/
+node_modules/
+out/
+proofstep-*.json
+secrets/
+venv/

proofstep_core-0.1.0/PKG-INFO

@@ -0,0 +1,39 @@
+Metadata-Version: 2.5
+Name: proofstep-core
+Version: 0.1.0
+Summary: Proofstep evaluation engine — datasets, evaluators, runner, aggregation, gates
+Project-URL: Homepage, https://github.com/IlaKhan17/proofstep
+Project-URL: Documentation, https://github.com/IlaKhan17/proofstep/tree/main/docs
+Project-URL: Repository, https://github.com/IlaKhan17/proofstep
+Project-URL: Issues, https://github.com/IlaKhan17/proofstep/issues
+License-Expression: Apache-2.0
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Quality Assurance
+Classifier: Topic :: Software Development :: Testing
+Classifier: Typing :: Typed
+Requires-Python: >=3.11
+Requires-Dist: proofstep-types
+Requires-Dist: pydantic>=2.9
+Description-Content-Type: text/markdown
+
+# proofstep-core
+
+**Evaluators, aggregation, gates, statistics** — part of [Proofstep](https://github.com/IlaKhan17/proofstep), the CI gate for AI
+agents that knows the difference between a regression and a bad day.
+
+The pure evaluation library: deterministic and statistical evaluators, LLM-judge harnesses with
+calibration, score aggregation, quality gates, and paired significance testing.
+
+No HTTP, no database, no provider SDKs — enforced in CI by an import-linter contract. That boundary
+is what makes local mode, CI mode, and server mode the same code path, and it is why the CLI's exit
+code and a dashboard's verdict cannot drift apart.
+
+## Documentation
+
+Full documentation lives in the [repository](https://github.com/IlaKhan17/proofstep/tree/main/docs).
+
+Apache-2.0.

proofstep_core-0.1.0/README.md

@@ -0,0 +1,17 @@
+# proofstep-core
+
+**Evaluators, aggregation, gates, statistics** — part of [Proofstep](https://github.com/IlaKhan17/proofstep), the CI gate for AI
+agents that knows the difference between a regression and a bad day.
+
+The pure evaluation library: deterministic and statistical evaluators, LLM-judge harnesses with
+calibration, score aggregation, quality gates, and paired significance testing.
+
+No HTTP, no database, no provider SDKs — enforced in CI by an import-linter contract. That boundary
+is what makes local mode, CI mode, and server mode the same code path, and it is why the CLI's exit
+code and a dashboard's verdict cannot drift apart.
+
+## Documentation
+
+Full documentation lives in the [repository](https://github.com/IlaKhan17/proofstep/tree/main/docs).
+
+Apache-2.0.

proofstep_core-0.1.0/pyproject.toml

@@ -0,0 +1,38 @@
+[project]
+name = "proofstep-core"
+version = "0.1.0"
+description = "Proofstep evaluation engine — datasets, evaluators, runner, aggregation, gates"
+readme = "README.md"
+requires-python = ">=3.11"
+license = "Apache-2.0"
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: Apache Software License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Software Development :: Testing",
+    "Topic :: Software Development :: Quality Assurance",
+    "Typing :: Typed",
+]
+
+dependencies = [
+    "proofstep-types",
+    "pydantic>=2.9",
+]
+
+[project.urls]
+Homepage = "https://github.com/IlaKhan17/proofstep"
+Documentation = "https://github.com/IlaKhan17/proofstep/tree/main/docs"
+Repository = "https://github.com/IlaKhan17/proofstep"
+Issues = "https://github.com/IlaKhan17/proofstep/issues"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/proofstep_core"]
+
+[tool.uv.sources]
+proofstep-types = { workspace = true }

proofstep_core-0.1.0/src/proofstep_core/__init__.py

@@ -0,0 +1,59 @@
+"""Proofstep evaluation engine (pure library — no I/O).
+
+No HTTP, no database, no provider SDKs. Model access arrives through an injected
+`ModelClient` protocol. That boundary is what makes local mode, CI mode, and server
+mode the same code path, and it is enforced in CI by `.importlinter`.
+"""
+
+from importlib import metadata as _metadata
+
+from proofstep_core.aggregate import aggregate_scores, scores_for
+from proofstep_core.compare import Comparison, ExampleRegression, compare_metrics
+from proofstep_core.dataset import Dataset
+from proofstep_core.gates import GateReport, evaluate_gates
+from proofstep_core.paths import PathError, resolve, resolve_in_context
+from proofstep_core.runner import EvalResult, RunConfig, run_suite
+from proofstep_core.suite import EvalSuite, FunctionEvaluator, evaluate
+from proofstep_core.types import (
+    CorpusEvaluator,
+    EvalContext,
+    Evaluator,
+    EvaluatorBase,
+    Message,
+    ModelClient,
+    ModelResponse,
+    Task,
+)
+
+# Read from the installed distribution rather than written here twice. A hand-maintained
+# copy drifts the first time a release bumps one and not the other — which it already did,
+# reporting 0.1.0.dev0 from a 0.1.0 wheel.
+__version__ = _metadata.version("proofstep-core")
+
+__all__ = [
+    "Comparison",
+    "CorpusEvaluator",
+    "Dataset",
+    "EvalContext",
+    "EvalResult",
+    "EvalSuite",
+    "Evaluator",
+    "EvaluatorBase",
+    "ExampleRegression",
+    "FunctionEvaluator",
+    "GateReport",
+    "Message",
+    "ModelClient",
+    "ModelResponse",
+    "PathError",
+    "RunConfig",
+    "Task",
+    "aggregate_scores",
+    "compare_metrics",
+    "evaluate",
+    "evaluate_gates",
+    "resolve",
+    "resolve_in_context",
+    "run_suite",
+    "scores_for",
+]

proofstep_core-0.1.0/src/proofstep_core/aggregate.py

@@ -0,0 +1,140 @@
+"""Roll per-example scores up into metrics.
+
+The load-bearing rule here: **errored evaluations are excluded from the mean and
+counted separately.** A judge that timed out is not a failing example, and averaging
+infrastructure failures in as zeros is the fastest way to make a metric untrustworthy
+(docs/EVALUATION_ENGINE.md §1).
+"""
+
+from __future__ import annotations
+
+from collections import defaultdict
+from collections.abc import Iterable, Sequence
+
+from proofstep_core.stats import bootstrap_ci, mean, stddev
+from proofstep_types import ExampleResult, Metric, Score
+
+# A slice is identified by its sorted key/value pairs so it can be a dict key.
+SliceKey = tuple[tuple[str, str], ...]
+
+
+def _slice_key(slice_: dict[str, str] | None) -> SliceKey:
+    return tuple(sorted(slice_.items())) if slice_ else ()
+
+
+def _unkey(key: SliceKey) -> dict[str, str] | None:
+    return dict(key) if key else None
+
+
+def aggregate_scores(
+    results: Iterable[ExampleResult],
+    *,
+    slice_by: Sequence[str] = (),
+    confidence_intervals: bool = True,
+    seed: int = 42,
+) -> list[Metric]:
+    """Aggregate every score across every result into metrics.
+
+    ``slice_by`` names metadata keys to additionally break each metric down by. This
+    is what surfaces a rare-class collapse that the aggregate hides: slicing
+    ``per_class_recall`` by ``class`` makes the unsubscribe number *visible* even
+    when nobody thought to gate on it.
+    """
+    buckets: dict[tuple[str, SliceKey], list[float]] = defaultdict(list)
+    errors: dict[tuple[str, SliceKey], int] = defaultdict(int)
+
+    for result in results:
+        extra = _extra_slices(result, slice_by)
+        for score in result.scores:
+            own = _slice_key(score.slice)
+            for slice_key in {own, *[_merge(own, s) for s in extra]}:
+                bucket = (score.metric, slice_key)
+                if score.errored:
+                    errors[bucket] += 1
+                elif score.value is not None:
+                    buckets[bucket].append(score.value)
+
+    # A metric that produced nothing but errors must still appear, with count 0 —
+    # otherwise a gate on it sees "metric missing" and cannot distinguish a typo
+    # from a wholly broken evaluator.
+    metrics: list[Metric] = []
+    for bucket in sorted(set(buckets) | set(errors)):
+        metric_key, slice_key = bucket
+        values = buckets.get(bucket, [])
+        error_count = errors.get(bucket, 0)
+        metrics.append(
+            _build_metric(
+                metric_key,
+                values,
+                error_count,
+                _unkey(slice_key),
+                confidence_intervals=confidence_intervals,
+                seed=seed,
+            )
+        )
+    return metrics
+
+
+def _build_metric(
+    key: str,
+    values: list[float],
+    error_count: int,
+    slice_: dict[str, str] | None,
+    *,
+    confidence_intervals: bool,
+    seed: int,
+) -> Metric:
+    if not values:
+        return Metric(key=key, value=0.0, count=0, error_count=error_count, slice=slice_)
+
+    ci_low = ci_high = None
+    # Bootstrapping a handful of points produces an interval that says nothing;
+    # reporting one anyway would imply precision that isn't there.
+    if confidence_intervals and len(values) >= 5:
+        ci_low, ci_high = bootstrap_ci(values, seed=seed)
+
+    return Metric(
+        key=key,
+        value=mean(values),
+        count=len(values),
+        error_count=error_count,
+        stddev=stddev(values),
+        ci_low=ci_low,
+        ci_high=ci_high,
+        slice=slice_,
+    )
+
+
+def _extra_slices(result: ExampleResult, slice_by: Sequence[str]) -> list[SliceKey]:
+    keys: list[SliceKey] = []
+    for dimension in slice_by:
+        value = result.metadata.get(dimension)
+        if value is None and result.expected is not None:
+            value = result.expected.get(dimension)
+        if value is not None:
+            keys.append(((dimension, str(value)),))
+    return keys
+
+
+def _merge(a: SliceKey, b: SliceKey) -> SliceKey:
+    return tuple(sorted({**dict(a), **dict(b)}.items()))
+
+
+def scores_for(results: Iterable[ExampleResult], metric_key: str) -> list[float]:
+    """Every non-errored value for one metric, in result order.
+
+    Used by comparison to bootstrap a CI on the delta between two runs.
+    """
+    return [
+        score.value
+        for result in results
+        for score in result.scores
+        if score.metric == metric_key and score.counts_toward_mean and score.value is not None
+    ]
+
+
+def group_by_metric(scores: Iterable[Score]) -> dict[str, list[Score]]:
+    grouped: dict[str, list[Score]] = defaultdict(list)
+    for score in scores:
+        grouped[score.metric].append(score)
+    return dict(grouped)