oathgate 0.0.2__py3-none-any.whl

oathgate/__init__.py

@@ -0,0 +1,3 @@
+"""oathgate — freeze the measurement ruler before you run the eval."""
+
+__version__ = "0.0.1"

oathgate/cli.py

@@ -0,0 +1,107 @@
+"""oathgate — freeze the measurement ruler before you run the eval."""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+
+# Only these keys describe the *ruler*: how a result is measured.
+# The system under test (prompt, model, agent, temperature) is deliberately
+# excluded — you are allowed to change it, that is the whole point.
+RULER_KEYS = ("metrics", "thresholds", "dataset", "references")
+
+LOCK_NAME = "oath.lock.json"
+
+
+def _canonical(spec: dict) -> str:
+    ruler = {k: spec[k] for k in RULER_KEYS if k in spec}
+    return json.dumps(ruler, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
+
+
+def _digest(spec: dict) -> str:
+    return hashlib.sha256(_canonical(spec).encode("utf-8")).hexdigest()
+
+
+def _load(path: Path) -> dict:
+    if not path.exists():
+        sys.exit(f"oathgate: spec not found: {path}")
+    try:
+        return json.loads(path.read_text(encoding="utf-8"))
+    except json.JSONDecodeError as exc:
+        sys.exit(f"oathgate: cannot parse {path}: {exc}")
+
+
+def cmd_freeze(args: argparse.Namespace) -> int:
+    spec_path = Path(args.spec)
+    spec = _load(spec_path)
+
+    missing = [k for k in RULER_KEYS if k not in spec]
+    if missing:
+        sys.exit(f"oathgate: spec is missing required keys: {', '.join(missing)}")
+
+    if not args.predict:
+        sys.exit("oathgate: refusing to freeze without a prediction (--predict)")
+
+    lock = {
+        "version": 1,
+        "spec": str(spec_path),
+        "ruler_sha256": _digest(spec),
+        "prediction": args.predict,
+        "frozen_at": datetime.now(timezone.utc).isoformat(timespec="seconds"),
+    }
+    lock_path = Path(args.lock)
+    lock_path.write_text(json.dumps(lock, indent=2) + "\n", encoding="utf-8")
+    print(f"frozen  {lock['ruler_sha256'][:12]}  -> {lock_path}")
+    print(f"oath    {args.predict}")
+    return 0
+
+
+def cmd_check(args: argparse.Namespace) -> int:
+    lock_path = Path(args.lock)
+    if not lock_path.exists():
+        sys.exit(f"oathgate: no lock file ({lock_path}). Run `oathgate freeze` first.")
+
+    lock = json.loads(lock_path.read_text(encoding="utf-8"))
+    spec = _load(Path(args.spec or lock["spec"]))
+    current = _digest(spec)
+
+    if current != lock["ruler_sha256"]:
+        print("BLOCKED: the measurement ruler changed after the oath was taken.")
+        print(f"  frozen:  {lock['ruler_sha256'][:12]}  ({lock['frozen_at']})")
+        print(f"  current: {current[:12]}")
+        print("  Re-freeze deliberately, or restore the spec. Do not do it silently.")
+        return 1
+
+    print(f"ok      {current[:12]}  ruler unchanged")
+    print(f"oath    {lock['prediction']}")
+    return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="oathgate",
+        description="Freeze the eval scoring spec and your prediction before the run.",
+    )
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    f = sub.add_parser("freeze", help="freeze the ruler and record a prediction")
+    f.add_argument("spec", help="path to the eval spec (JSON)")
+    f.add_argument("--predict", required=True, help="what you expect to happen")
+    f.add_argument("--lock", default=LOCK_NAME, help=f"lock file (default: {LOCK_NAME})")
+    f.set_defaults(func=cmd_freeze)
+
+    c = sub.add_parser("check", help="verify the ruler is unchanged before a run")
+    c.add_argument("spec", nargs="?", help="path to the eval spec (default: from lock)")
+    c.add_argument("--lock", default=LOCK_NAME, help=f"lock file (default: {LOCK_NAME})")
+    c.set_defaults(func=cmd_check)
+
+    args = parser.parse_args(argv)
+    return args.func(args)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

oathgate-0.0.2.dist-info/METADATA

@@ -0,0 +1,97 @@
+Metadata-Version: 2.5
+Name: oathgate
+Version: 0.0.2
+Summary: Freeze the eval scoring spec and your prediction before the run.
+Project-URL: Homepage, https://github.com/UladzKha/oathgate
+Project-URL: Issues, https://github.com/UladzKha/oathgate/issues
+Author: Uladzimir Khadakouski
+License: MIT
+License-File: LICENSE
+Keywords: benchmark,evals,evaluation,llm,reproducibility
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+
+# oathgate
+
+**Take the oath before you run the eval.**
+
+`oathgate` blocks an evaluation run until two things are on the record:
+
+1. **The measurement ruler is frozen** — metrics, thresholds, dataset composition and references are hashed.
+2. **A prediction is written down** — what you expect to happen, before you know what happened.
+
+Nothing here measures your model. It measures whether you were honest about how you measured.
+
+## The problem
+
+Evals drift. Not because anyone lies, but because the ruler is soft: a threshold moves from 0.85 to 0.80, three hard cases quietly leave the set, a metric is swapped for a friendlier one — and the number goes up. Every step is defensible on its own. The result is a benchmark that only ever improves.
+
+The fix is not more rigour in the moment. It is making the ruler expensive to change _after_ you have seen the outcome.
+
+## What gets hashed
+
+Only the ruler:
+
+| Hashed                      | Not hashed                |
+| --------------------------- | ------------------------- |
+| `metrics`                   | prompt                    |
+| `thresholds`                | model / checkpoint        |
+| `dataset` (composition)     | agent scaffold            |
+| `references` (gold answers) | temperature, seeds, infra |
+
+This split is the whole design. You are _supposed_ to change the system under test — that is the experiment. The ruler is what has to hold still for the comparison to mean anything.
+
+## Usage
+
+Freeze the ruler and commit to a prediction:
+
+```console
+$ oathgate freeze examples/spec.json --predict "f1 lands between 0.88 and 0.92; exact_match misses the 0.80 bar"
+frozen  4f2a91c0d3e8  -> oath.lock.json
+oath    f1 lands between 0.88 and 0.92; exact_match misses the 0.80 bar
+```
+
+Then, as the first step of your run:
+
+```console
+$ oathgate check && python run_eval.py
+ok      4f2a91c0d3e8  ruler unchanged
+oath    f1 lands between 0.88 and 0.92; exact_match misses the 0.80 bar
+```
+
+If someone nudged a threshold in between, the gate refuses and exits non-zero:
+
+```console
+$ oathgate check
+BLOCKED: the measurement ruler changed after the oath was taken.
+  frozen:  4f2a91c0d3e8  (2026-08-31T09:14:00+00:00)
+  current: b71e05ad9c2f
+  Re-freeze deliberately, or restore the spec. Do not do it silently.
+```
+
+Re-freezing is allowed. It just cannot happen by accident, and it leaves a timestamp.
+
+## Install
+
+```console
+pip install oathgate
+```
+
+## Spec format
+
+A JSON file with the four ruler keys. Anything else — including a `system` block describing what you are testing — is ignored by the hash and free to change.
+
+See [`examples/spec.json`](https://github.com/UladzKha/oathgate/blob/main/examples/spec.json).
+
+## Status
+
+Early. The CLI is the whole surface right now; an MCP wrapper is planned so agents can be held to the same gate.
+
+## Licence
+
+MIT

oathgate-0.0.2.dist-info/RECORD

@@ -0,0 +1,7 @@
+oathgate/__init__.py,sha256=zrLovLYUKPjjJamxwtNiO5MfPzvNTPKtJ0b-t1S5sfY,95
+oathgate/cli.py,sha256=Ce8SxpO_CLSGTRTMZNEE0B2cVoyyGxk8l1ZS6Xh2Q8s,3779
+oathgate-0.0.2.dist-info/METADATA,sha256=3e9d-OeFXdh3NlmAF2SPUfc7K8-LyivLFCABzdjLQ1Y,3514
+oathgate-0.0.2.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
+oathgate-0.0.2.dist-info/entry_points.txt,sha256=380Z8qNh4DjznwbDwJ2TgDtXh7ndp_JQVwdGP1QLUMQ,47
+oathgate-0.0.2.dist-info/licenses/LICENSE,sha256=hL3Xx-WseS6-fKaqSCbFcvdKOt2cZ4N1bnfs1HrtCHY,1093
+oathgate-0.0.2.dist-info/RECORD,,

oathgate-0.0.2.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.0
+Root-Is-Purelib: true
+Tag: py3-none-any

oathgate-0.0.2.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+oathgate = oathgate.cli:main

oathgate-0.0.2.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Uladzimir Khadakouski
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT Uladzimir KhadakouskiS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.