proofstep-cli 0.1.0__py3-none-any.whl

proofstep_cli/suite/schema.py

@@ -0,0 +1,343 @@
+"""The suite YAML schema.
+
+Versioned by `apiVersion` from the first release. Suite files live in users' repos
+and must not break on upgrade, which costs nothing to promise now and is expensive
+to retrofit later (docs/OPEN_QUESTIONS.md Q11).
+
+`extra="forbid"` throughout: a typo'd key that is silently ignored produces a suite
+which looks configured and is not. `capture_args: true` misspelled as `capture_arg`
+should be an error, not a shrug.
+"""
+
+from __future__ import annotations
+
+import difflib
+from typing import Any, Literal
+
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+API_VERSION = "proofstep.dev/v1"
+
+ReportFormat = Literal["terminal", "json"]
+
+EVALUATOR_TYPES = (
+    "exact_match",
+    "json_schema",
+    "regex",
+    "contains",
+    "length",
+    "numeric_range",
+    "set_comparison",
+    "llm_judge",
+    "trajectory",
+    "operational",
+    "classification",
+    "ranking",
+    "calibration",
+    "discrimination",
+)
+
+
+class DatasetRef(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    name: str | None = None
+    version: str = "latest-locked"
+    path: str | None = Field(
+        default=None, description="Local JSONL/CSV file; mutually exclusive with `name`"
+    )
+    split: str | None = None
+    limit: int | None = Field(default=None, ge=1)
+
+    @model_validator(mode="after")
+    def _one_source(self) -> DatasetRef:
+        if bool(self.name) == bool(self.path):
+            msg = "dataset needs exactly one of `name` (server) or `path` (local file)"
+            raise ValueError(msg)
+        return self
+
+    @property
+    def is_local(self) -> bool:
+        return self.path is not None
+
+
+class TaskRef(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    entrypoint: str = Field(pattern=r"^[\w.]+:[\w.]+$", description="module:function")
+    timeout_s: float = Field(default=120.0, gt=0)
+    retries: int = Field(default=2, ge=0)
+
+
+class Execution(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    concurrency: int = Field(default=8, ge=1, le=256)
+    judge_concurrency: int = Field(default=4, ge=1, le=64)
+    max_error_rate: float = Field(default=0.10, ge=0, le=1)
+    seed: int = 42
+    slice_by: list[str] = Field(default_factory=list)
+    max_cost: float | None = Field(default=None, gt=0)
+
+
+class Scale(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    min: int = 1
+    max: int = 5
+    normalize: bool = True
+
+
+class Calibration(BaseModel):
+    """One judge's calibration: which labelled set, and what it has to achieve.
+
+    Thresholds live with the evaluator rather than with the gate because they are a
+    property of the measurement — an unsubscribe judge needs a tighter false-pass rate
+    than a tone judge, whatever gate set happens to reference it. Whether falling short
+    *blocks* is the gate set's decision (`calibration.require`).
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    #: Path to the labelled JSONL, relative to the suite file.
+    dataset: str
+    version: str = "latest-locked"
+    #: Labels that count as "passing", which is what makes the false-pass and false-fail
+    #: rates computable. Without it those two numbers are unmeasured, and they are the
+    #: ones that matter most.
+    passing_labels: list[str] = Field(default_factory=list)
+    min_agreement: float | None = Field(default=None, ge=0, le=1)
+    min_kappa: float | None = Field(default=None, ge=-1, le=1)
+    max_false_pass_rate: float | None = Field(default=None, ge=0, le=1)
+    max_false_fail_rate: float | None = Field(default=None, ge=0, le=1)
+    min_examples: int | None = Field(default=None, ge=1)
+    min_per_class: int | None = Field(default=None, ge=1)
+    allow_position_bias: bool = False
+    required: bool = False
+
+
+class CalibrationPolicy(BaseModel):
+    """Suite-level calibration settings.
+
+    `require` is the enforcement switch: `false` means an uncalibrated gated judge only
+    warns, `true` means it fails the run, and a mapping overrides the thresholds that
+    apply when an evaluator does not state its own.
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    #: Where calibration records live, relative to the suite file. They are committed to
+    #: git so `require` works in CI with no server — see calibration_store.py.
+    directory: str = "calibrations"
+    require: bool | dict[str, Any] = False
+
+
+class EvaluatorSpec(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    name: str = Field(pattern=r"^[a-z][a-z0-9_]*$")
+    type: str
+
+    # Deterministic
+    field: str | None = None
+    expected_field: str | None = None
+    normalize: Literal["none", "case", "whitespace", "punctuation", "all"] = "none"
+    schema_: dict[str, Any] | None = Field(default=None, alias="schema")
+    schema_path: str | None = None
+    allow: list[str] = Field(default_factory=list)
+    deny: list[str] = Field(default_factory=list)
+    substrings: list[str] = Field(default_factory=list)
+    mode: str | None = None
+    case_sensitive: bool = True
+    unit: Literal["chars", "words"] = "chars"
+    minimum: float | None = None
+    maximum: float | None = None
+    inclusive: bool = True
+
+    # Judge
+    rubric: str | None = None
+    rubric_path: str | None = None
+    model: str | None = None
+    temperature: float = 0.0
+    seed: int | None = 42
+    inputs: list[str] = Field(default_factory=list)
+    labels: list[str] = Field(default_factory=list)
+    #: Which of `labels` count as a pass, turning a classify judge's label into a gateable rate.
+    #: Declared on the evaluator rather than only under `calibration`, because it changes what the
+    #: judge scores and not merely how it is calibrated.
+    passing_labels: list[str] = Field(default_factory=list)
+    scale: Scale = Field(default_factory=Scale)
+    votes: int = 1
+    timeout_s: float = 60.0
+    max_retries: int = 2
+    calibration: Calibration | None = None
+
+    # Trajectory
+    policy: str | None = None
+
+    # Statistical / operational
+    prediction_field: str | None = None
+    label_field: str | None = None
+    averaging: Literal["macro", "micro", "weighted"] = "macro"
+    k: int = 10
+    ranking_field: str | None = None
+    relevant_field: str | None = None
+    confidence_field: str | None = None
+    #: A boolean output field saying whether the prediction was right, for cases where
+    #: correctness is not a prediction/label comparison.
+    correct_field: str | None = None
+    percentiles: list[int] = Field(default_factory=lambda: [50, 95, 99])
+
+    @model_validator(mode="after")
+    def _known_type(self) -> EvaluatorSpec:
+        if self.type not in EVALUATOR_TYPES:
+            closest = _closest(self.type, EVALUATOR_TYPES)
+            hint = f" Did you mean {closest!r}?" if closest else ""
+            msg = (
+                f"evaluator {self.name!r} has unknown type {self.type!r}.{hint} "
+                f"Valid types: {', '.join(sorted(EVALUATOR_TYPES))}"
+            )
+            raise ValueError(msg)
+        return self
+
+    @model_validator(mode="after")
+    def _judges_declare_inputs(self) -> EvaluatorSpec:
+        if self.type != "llm_judge":
+            return self
+        if not self.inputs:
+            # A judge handed the whole example can read `expected` and grade against
+            # the answer key. Enumerating inputs is the only thing that prevents it.
+            msg = (
+                f"judge {self.name!r} must declare `inputs`: without it the judge can see "
+                "`expected` and grade against the answer key"
+            )
+            raise ValueError(msg)
+        if not self.rubric and not self.rubric_path:
+            msg = f"judge {self.name!r} needs `rubric` or `rubric_path`"
+            raise ValueError(msg)
+        if not self.model:
+            # Unpinned means the provider can change the model underneath you and
+            # invalidate every historical number without any signal.
+            msg = (
+                f"judge {self.name!r} must pin a `model`: an unpinned judge silently "
+                "invalidates every historical comparison when the provider updates it"
+            )
+            raise ValueError(msg)
+        return self
+
+    @model_validator(mode="after")
+    def _trajectory_needs_a_policy(self) -> EvaluatorSpec:
+        if self.type == "trajectory" and not self.policy:
+            msg = f"trajectory evaluator {self.name!r} needs a `policy` path"
+            raise ValueError(msg)
+        return self
+
+
+class GateSpec(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    minimum: float | None = None
+    maximum: float | None = None
+    max_regression: float | None = None
+    max_relative_regression: float | None = None
+    blocking: bool = True
+    slice: dict[str, str] | None = None
+    require_baseline: bool = False
+    #: Alpha for a paired significance test. With it, a regression must be both bigger than the
+    #: threshold *and* distinguishable from noise before it fails the build — because at the sample
+    #: sizes eval suites actually run at, most measured regressions are neither.
+    significance: float | None = Field(default=None, ge=0.0, le=1.0)
+    #: Report ERROR when the run was too small to have detected the regression this gate guards.
+    #: A green check from a test that could never have failed is worse than no check.
+    require_power: bool = False
+
+    @model_validator(mode="after")
+    def _has_a_condition(self) -> GateSpec:
+        if all(
+            v is None
+            for v in (
+                self.minimum,
+                self.maximum,
+                self.max_regression,
+                self.max_relative_regression,
+            )
+        ):
+            msg = "declares no condition; a gate that cannot fail gives false assurance"
+            raise ValueError(msg)
+        return self
+
+
+class BaselineSpec(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    strategy: Literal["latest_on_branch", "run_id", "none"] = "latest_on_branch"
+    branch: str = "main"
+    run_id: str | None = None
+    require_dataset_match: bool = True
+
+    @model_validator(mode="after")
+    def _run_id_present(self) -> BaselineSpec:
+        if self.strategy == "run_id" and not self.run_id:
+            msg = "baseline strategy 'run_id' needs a `run_id`"
+            raise ValueError(msg)
+        return self
+
+
+class ReportSpec(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    formats: list[ReportFormat] = Field(
+        default_factory=lambda: ["terminal", "json"]  # type: ignore[arg-type]
+    )
+    output: str = "proofstep-report.json"
+
+
+class Suite(BaseModel):
+    model_config = ConfigDict(extra="forbid", populate_by_name=True)
+
+    apiVersion: str = API_VERSION  # noqa: N815 — YAML field name
+    kind: Literal["EvalSuite"] = "EvalSuite"
+    name: str = Field(pattern=r"^[a-z][a-z0-9-]*$")
+    description: str | None = None
+    extends: str | None = None
+
+    dataset: DatasetRef
+    task: TaskRef | None = None
+    configuration: dict[str, Any] = Field(default_factory=dict)
+    execution: Execution = Field(default_factory=Execution)
+    evaluators: list[EvaluatorSpec] = Field(min_length=1)
+    gates: dict[str, GateSpec] = Field(default_factory=dict)
+    calibration: CalibrationPolicy = Field(default_factory=CalibrationPolicy)
+    baseline: BaselineSpec = Field(default_factory=BaselineSpec)
+    report: ReportSpec = Field(default_factory=ReportSpec)
+
+    @model_validator(mode="after")
+    def _supported_api_version(self) -> Suite:
+        if self.apiVersion != API_VERSION:
+            msg = (
+                f"unsupported apiVersion {self.apiVersion!r}; this CLI understands {API_VERSION!r}"
+            )
+            raise ValueError(msg)
+        return self
+
+    @model_validator(mode="after")
+    def _unique_evaluator_names(self) -> Suite:
+        seen: set[str] = set()
+        for evaluator in self.evaluators:
+            if evaluator.name in seen:
+                msg = f"duplicate evaluator name {evaluator.name!r}"
+                raise ValueError(msg)
+            seen.add(evaluator.name)
+        return self
+
+    @property
+    def judge_ratio(self) -> float:
+        if not self.evaluators:
+            return 0.0
+        judges = sum(1 for e in self.evaluators if e.type == "llm_judge")
+        return judges / len(self.evaluators)
+
+
+def _closest(value: str, candidates: tuple[str, ...]) -> str | None:
+    matches = difflib.get_close_matches(value, candidates, n=1, cutoff=0.6)
+    return matches[0] if matches else None

proofstep_cli-0.1.0.dist-info/METADATA

@@ -0,0 +1,58 @@
+Metadata-Version: 2.5
+Name: proofstep-cli
+Version: 0.1.0
+Summary: Proofstep command-line interface — run suites, gate CI, compare experiments
+Project-URL: Homepage, https://github.com/IlaKhan17/proofstep
+Project-URL: Documentation, https://github.com/IlaKhan17/proofstep/tree/main/docs
+Project-URL: Repository, https://github.com/IlaKhan17/proofstep
+Project-URL: Issues, https://github.com/IlaKhan17/proofstep/issues
+License-Expression: Apache-2.0
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Quality Assurance
+Classifier: Topic :: Software Development :: Testing
+Classifier: Typing :: Typed
+Requires-Python: >=3.11
+Requires-Dist: httpx>=0.27
+Requires-Dist: proofstep
+Requires-Dist: proofstep-core
+Requires-Dist: proofstep-trajectory
+Requires-Dist: proofstep-types
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: rich>=13.9
+Requires-Dist: typer>=0.15
+Description-Content-Type: text/markdown
+
+# proofstep-cli
+
+**The `proofstep` command** — part of [Proofstep](https://github.com/IlaKhan17/proofstep), the CI gate for AI
+agents that knows the difference between a regression and a bad day.
+
+Run an evaluation suite, apply its gates, and exit non-zero when a protected metric regresses.
+
+```bash
+proofstep eval evals/suites/reply-intent.yaml
+echo $?   # 0 merge · 1 a blocking gate failed · 2 execution error · 3 the suite is wrong
+```
+
+The exit code is the contract. Everything else — the terminal table, the JSON report, the
+pull-request comment — exists to explain it.
+
+Gates can be statistically honest rather than just thresholded:
+
+```yaml
+gates:
+  intent_accuracy:
+    max_regression: 0.02
+    significance: 0.05      # only fail if the drop is distinguishable from noise
+    require_power: true     # and ERROR if this run could never have detected it
+```
+
+## Documentation
+
+Full documentation lives in the [repository](https://github.com/IlaKhan17/proofstep/tree/main/docs).
+
+Apache-2.0.

proofstep_cli-0.1.0.dist-info/RECORD

@@ -0,0 +1,21 @@
+proofstep_cli/__init__.py,sha256=kZDCFB-6Wudptp_a3AK2xCFx-s2muRDAL_oxo_J6BsI,341
+proofstep_cli/calibration.py,sha256=3_iQ7aNBY3o0yYOe2ejS8xfvel_tfFgovoOm6AEoiMc,11572
+proofstep_cli/calibration_store.py,sha256=wjxopf2nAl1wKOXAfkivjD6u2E7PhIoTmLloFfZoDHw,7946
+proofstep_cli/main.py,sha256=4wtRYvoWWpO8TWOC9E4pvtiDX5JtzoQzymHuW5I5rjQ,21821
+proofstep_cli/publish.py,sha256=5XZ-eg40uoUXYXOx5qCzT0nfIi2KofT7f2XmJTwb-Qk,22955
+proofstep_cli/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+proofstep_cli/registry.py,sha256=8X8IL8U_8tTCthI5pycy_IUxgGpddTK649S9XerLcyk,7951
+proofstep_cli/runner.py,sha256=8MYXZTXCxFyXObE2mWZEBLzgD_IBvyB4qVrkpj92UgY,12651
+proofstep_cli/commands/__init__.py,sha256=QN-uD3qaiBO7K7E4QEfRPnoh3rvXcIe89veVtfE8c38,23
+proofstep_cli/render/__init__.py,sha256=bmQ3dGBiHqSPwumamEebbTyZSdWgUDms_jPjo1MBB0Y,24
+proofstep_cli/render/calibration.py,sha256=r9Zidu3Dw4AMe7aOj24DayDGJmiQJnQIKvZhudPXa8o,5108
+proofstep_cli/render/markdown.py,sha256=Y58yri1ynyD47vE0VTAgMBVqdIYyzVMjrZ1rSnRBYlk,10651
+proofstep_cli/render/report.py,sha256=44BKaBO5Li42t1XzEGYqnXPoKCIgThGipyUyPBlw5Jc,6708
+proofstep_cli/render/terminal.py,sha256=Ax2PBj1gtWExN6yS5IQpo8BFUy_-uQSYmZDrv65BNGI,10343
+proofstep_cli/suite/__init__.py,sha256=CEqbhLYlg7SJoIKNYdV4W5m7MrwQa21mG7OhK5TGNAM,27
+proofstep_cli/suite/loader.py,sha256=llGUMwSt5LEB__hz3Pv7SpmHt1d0Qqj1cXGSrRi3VN4,14050
+proofstep_cli/suite/schema.py,sha256=p6iL4Mgz9baqqjVz9TtpHGgLgDtb3RrmbOBlw1Ji0Ow,12459
+proofstep_cli-0.1.0.dist-info/METADATA,sha256=3ueFoVrFRD0nygOEwLm5EcbgvjoJEy71zcDrHOe1nmQ,2148
+proofstep_cli-0.1.0.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
+proofstep_cli-0.1.0.dist-info/entry_points.txt,sha256=kvjBHqD1IOyphRKMklvkz_Yrhe4VxUY5bZkuK35yrNY,53
+proofstep_cli-0.1.0.dist-info/RECORD,,

proofstep_cli-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.0
+Root-Is-Purelib: true
+Tag: py3-none-any

proofstep_cli-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+proofstep = proofstep_cli.main:app