opendxp 0.1.0__py3-none-any.whl

opendxp/__init__.py

@@ -0,0 +1,26 @@
+"""OpenDXP: a portable runtime standard for calibrated single-pass decision models.
+
+This package is the reference implementation: the runtimes that run any
+conforming package without model-specific code, the converters that make
+packages from the models' own checkpoints, and the conformance tools.
+
+    import opendxp
+    model = opendxp.load("path/to/package")          # device chosen from what is available
+    model.predict(state, questions)                 # {"answers": {...}, "usage": {...}}
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+__version__ = "0.1.0"
+
+
+def load(package_dir: Any, **options: Any) -> Any:
+    """Open a package and start the reference runtime for its profile."""
+    from opendxp.runtime import load as _load
+
+    return _load(package_dir, **options)
+
+
+__all__ = ["__version__", "load"]

opendxp/__main__.py

@@ -0,0 +1,3 @@
+from opendxp.cli import main
+
+raise SystemExit(main())

opendxp/answers.py

@@ -0,0 +1,96 @@
+"""From scores to System One answers: the calibrated softmax, confidence, answer objects."""
+
+from __future__ import annotations
+
+import math
+from collections.abc import Sequence
+from typing import Any
+
+import numpy as np
+
+from opendxp.errors import OpenDXPError
+from opendxp.request import Question
+from opendxp.spec import DECIMALS
+
+
+def softmax(logits: Sequence[float], temperature: float = 1.0) -> list[float]:
+    """softmax(z / T) in float64, over the logits given (the valid options only)."""
+    z = np.asarray(logits, dtype=np.float64) / float(temperature)
+    if not np.all(np.isfinite(z)):
+        raise OpenDXPError("the model returned non-finite scores")
+    e = np.exp(z - z.max())
+    return (e / e.sum()).tolist()
+
+
+def normalise(values: Sequence[float]) -> list[float]:
+    total = float(sum(values))
+    if total <= 0:
+        return [1.0 / len(values)] * len(values)
+    return [float(v) / total for v in values]
+
+
+def confidence(rule: str, p: Sequence[float]) -> float | None:
+    """The confidence rules a package can declare (SPEC.md, "Answers")."""
+    n = len(p)
+    if rule == "none":
+        return None
+    if rule == "max-probability":
+        return float(max(p))
+    if rule == "entropy":
+        if n < 2:
+            return 1.0
+        h = -sum(x * math.log(min(max(x, 1e-12), 1.0)) for x in p)
+        return min(1.0, max(0.0, 1.0 - h / math.log(n)))
+    if rule == "typesafe":
+        if n < 2:
+            return 1.0
+        return min(1.0, max(0.0, (n * max(p) - 1) / (n - 1)))
+    if rule == "typesafe-ordinal":
+        if n < 2:
+            return 1.0
+        k = max(range(n), key=p.__getitem__)
+        spread = sum(x * abs(i - k) for i, x in enumerate(p))
+        uniform = sum(abs(i - (n - 1) / 2) for i in range(n)) / n
+        return min(1.0, max(0.0, 1.0 - spread / uniform))
+    raise OpenDXPError(f"unknown confidence rule {rule!r}")
+
+
+def argmax(p: Sequence[float]) -> int:
+    """The first index of the largest value."""
+    return max(range(len(p)), key=p.__getitem__)
+
+
+def answer(
+    question: Question,
+    p: Sequence[float],
+    rules: dict[str, str] | None = None,
+    decimals: int | None = DECIMALS,
+) -> dict[str, Any]:
+    """One answer in the System One format; probabilities rounded to `decimals` if set."""
+
+    def r(x: float) -> float:
+        return round(float(x), decimals) if decimals is not None else float(x)
+
+    rule = (rules or {}).get(question.type, "none")
+    conf = confidence(rule, p)
+    keys = question.keys
+    best = argmax(p)
+    out: dict[str, Any]
+    if question.type == "choice":
+        out = {
+            "type": "choice",
+            "choice": keys[best],
+            "probabilities": {k: r(x) for k, x in zip(keys, p, strict=True)},
+        }
+    elif question.type == "score":
+        out = {
+            "type": "score",
+            "score": r(sum(i * x for i, x in enumerate(p))),
+            "legend": question.legend(),
+            "probabilities": {k: r(x) for k, x in zip(keys, p, strict=True)},
+        }
+    else:
+        out = {"type": "noul", "noul": r(p[1])}
+    if conf is not None:
+        out["confidence"] = r(conf)
+    return out

opendxp/calibration.py

@@ -0,0 +1,71 @@
+"""calibration.json: the temperature each answer is divided by before the softmax.
+
+A temperature is looked up by question type and option count: the first entry
+of `by_option_count` whose type matches and whose [min, max] range holds the
+count wins; otherwise the per-type temperature; otherwise 1.
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from opendxp.errors import PackageError
+from opendxp.spec import QUESTION_TYPES
+
+
+@dataclass(frozen=True)
+class Bucket:
+    type: str
+    min: int
+    max: int | None
+    temperature: float
+
+    def holds(self, qtype: str, count: int) -> bool:
+        return qtype == self.type and count >= self.min and (self.max is None or count <= self.max)
+
+
+class Calibration:
+    def __init__(self, data: dict[str, Any] | None = None) -> None:
+        data = data or {}
+        self.data = data
+        self.by_type: dict[str, float] = {}
+        for qtype, value in (data.get("temperature") or {}).items():
+            if qtype not in QUESTION_TYPES:
+                raise PackageError(f"calibration: unknown question type {qtype!r}")
+            self.by_type[qtype] = _positive(value, f"temperature.{qtype}")
+        self.buckets: list[Bucket] = []
+        for i, entry in enumerate(data.get("by_option_count") or []):
+            qtype = entry.get("type")
+            if qtype not in QUESTION_TYPES:
+                raise PackageError(f"calibration: by_option_count[{i}] has no valid type")
+            low, high = entry.get("min", 2), entry.get("max")
+            if not isinstance(low, int) or (high is not None and not isinstance(high, int)):
+                raise PackageError(f"calibration: by_option_count[{i}] min/max must be integers")
+            if high is not None and high < low:
+                raise PackageError(f"calibration: by_option_count[{i}] has max < min")
+            self.buckets.append(
+                Bucket(qtype, low, high, _positive(entry.get("temperature"), f"bucket {i}"))
+            )
+
+    @classmethod
+    def from_file(cls, path: Path) -> Calibration:
+        return cls(json.loads(Path(path).read_text()))
+
+    def temperature(self, qtype: str, count: int) -> float:
+        for bucket in self.buckets:
+            if bucket.holds(qtype, count):
+                return bucket.temperature
+        return self.by_type.get(qtype, 1.0)
+
+
+def _positive(value: Any, where: str) -> float:
+    try:
+        number = float(value)
+    except (TypeError, ValueError):
+        raise PackageError(f"calibration: {where} is not a number") from None
+    if not number > 0 or number == float("inf"):
+        raise PackageError(f"calibration: {where} must be a positive finite number")
+    return number

opendxp/cli.py

@@ -0,0 +1,203 @@
+"""The `opendxp` command.
+
+opendxp export laya|julia|decider CHECKPOINT OUT_DIR
+opendxp conformance generate PACKAGE --native CHECKPOINT --runtime laya|julia|decider
+opendxp check PACKAGE [--device cpu|auto|coreml|cuda|openvino|qnn|directml|gpu]
+opendxp validate PACKAGE
+opendxp run PACKAGE --request request.json
+opendxp info
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+from typing import Any
+
+from opendxp import __version__
+
+
+def _print_json(data: Any) -> None:
+    print(json.dumps(data, ensure_ascii=False, indent=2))
+
+
+def cmd_export(args: argparse.Namespace) -> int:
+    mode = "copy" if args.copy else "link"
+    options: dict[str, Any] = {"weights_mode": mode}
+    if args.name:
+        options["name"] = args.name
+    if args.family == "laya":
+        from opendxp.export import laya as exporter
+    elif args.family == "julia":
+        from opendxp.export import julia as exporter
+
+        if args.max_tokens:
+            options["max_tokens"] = args.max_tokens
+        if args.official_onnx:
+            options["official_onnx"] = Path(args.official_onnx)
+    else:
+        from opendxp.export import decider as exporter
+    manifest = exporter.export(Path(args.checkpoint), Path(args.out_dir), **options)
+    print(f"wrote {Path(args.out_dir) / 'odxp.json'} ({manifest['profile']})")
+    print(
+        "next: opendxp conformance generate",
+        args.out_dir,
+        "--native",
+        args.checkpoint,
+        "--runtime",
+        args.family,
+    )
+    return 0
+
+
+def cmd_generate(args: argparse.Namespace) -> int:
+    from opendxp.conformance import DEFAULT_REQUESTS, generate, read_jsonl
+    from opendxp.native import load_native
+
+    requests = read_jsonl(Path(args.requests) if args.requests else DEFAULT_REQUESTS)
+    if args.limit:
+        requests = requests[: args.limit]
+    print(f"loading the model's own runtime ({args.runtime}) from {args.native}")
+    native = load_native(args.runtime, Path(args.native), threads=args.threads)
+    try:
+        summary = generate(Path(args.package), native, requests)
+    finally:
+        native.close()
+    _print_json(summary)
+    return 0
+
+
+def cmd_check(args: argparse.Namespace) -> int:
+    from opendxp.conformance import check
+
+    options: dict[str, Any] = {}
+    if args.optimization:
+        options["optimization"] = args.optimization
+    report = check(
+        Path(args.package),
+        device=args.device,
+        threads=args.threads,
+        hashes=not args.no_hashes,
+        **options,
+    )
+    path = Path(args.report) if args.report else Path(args.package) / f"check-{args.device}.json"
+    path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+    lat = report["latency_ms"]
+    agree = report["argmax_agreement"]
+    print(f"{report['package']['name']} [{report['package']['profile']}] on {args.device}")
+    print(
+        f"  cases {report['cases_passed']}/{report['cases']} passed, "
+        f"questions {report['questions']}, max |dp| {report['max_abs_dp']:.2e}, "
+        f"argmax {agree['agree']}/{agree['total']}, "
+        f"errors matched {report['errors']['matched']}/{report['errors']['expected']}"
+    )
+    print(
+        f"  latency: load {lat['load']} ms, request median {lat['request_median']} ms, "
+        f"p90 {lat['request_p90']} ms, per question {lat['question_median']} ms"
+    )
+    for problem in report["package_problems"]:
+        print(f"  package: {problem}")
+    for failure in report["failures"][:10]:
+        print(f"  FAIL {failure['id']}: {failure.get('reason') or failure.get('questions')}")
+    verdict = "PASS" if report["passed"] else "FAIL"
+    if report["passed"] and not report["compatible"]:
+        verdict += " (coverage below the minimum: not badge-eligible)"
+    print(f"  {verdict}; report: {path}")
+    return 0 if report["passed"] else 1
+
+
+def cmd_validate(args: argparse.Namespace) -> int:
+    from opendxp.validate import validate_package
+
+    problems = validate_package(Path(args.package), hashes=not args.no_hashes)
+    for problem in problems:
+        print(problem)
+    print("valid" if not problems else f"{len(problems)} problem(s)")
+    return 0 if not problems else 1
+
+
+def cmd_run(args: argparse.Namespace) -> int:
+    from opendxp.runtime import load
+
+    text = sys.stdin.read() if args.request == "-" else Path(args.request).read_text()
+    request = json.loads(text)
+    model = load(Path(args.package), device=args.device, threads=args.threads)
+    try:
+        _print_json(model.predict(request.get("state", ""), request["questions"]))
+    finally:
+        model.close()
+    return 0
+
+
+def cmd_info(args: argparse.Namespace) -> int:
+    from opendxp.providers import describe_machine
+
+    _print_json({"opendxp": __version__, **describe_machine()})
+    return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="opendxp", description="OpenDXP reference tools")
+    parser.add_argument("--version", action="version", version=f"opendxp {__version__}")
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    p = sub.add_parser("export", help="convert a model's checkpoint into an OpenDXP package")
+    p.add_argument("family", choices=["laya", "julia", "decider"])
+    p.add_argument("checkpoint")
+    p.add_argument("out_dir")
+    p.add_argument("--name", help="the package name (default: the model's registry name)")
+    p.add_argument("--copy", action="store_true", help="copy weights instead of hard-linking")
+    p.add_argument("--max-tokens", type=int, help="julia: total token budget (default 8192)")
+    p.add_argument(
+        "--official-onnx",
+        help="julia: map SupersonicLabs/Julia-1-ONNX's model.onnx instead of exporting a graph",
+    )
+    p.set_defaults(func=cmd_export)
+
+    p = sub.add_parser("conformance", help="conformance files")
+    csub = p.add_subparsers(dest="action", required=True)
+    g = csub.add_parser("generate", help="record the model's own answers into conformance.jsonl")
+    g.add_argument("package")
+    g.add_argument(
+        "--native", required=True, help="the checkpoint directory the model's code reads"
+    )
+    g.add_argument("--runtime", required=True, choices=["laya", "julia", "decider"])
+    g.add_argument("--requests", help="a requests JSONL (default: the OpenDXP 0.1 set)")
+    g.add_argument("--threads", type=int, default=4)
+    g.add_argument("--limit", type=int, help="only the first N requests (for a quick try)")
+    g.set_defaults(func=cmd_generate)
+
+    p = sub.add_parser("check", help="run the conformance file through the reference runtime")
+    p.add_argument("package")
+    p.add_argument(
+        "--device", default="cpu", help="cpu (the reference), auto, coreml, cuda, gpu, ..."
+    )
+    p.add_argument("--threads", type=int)
+    p.add_argument("--report", help="where to write the JSON report")
+    p.add_argument("--no-hashes", action="store_true", help="skip the sha256 checks")
+    p.add_argument("--optimization", choices=["disable", "basic", "extended", "all"])
+    p.set_defaults(func=cmd_check)
+
+    p = sub.add_parser("validate", help="check a package's files against the schemas and hashes")
+    p.add_argument("package")
+    p.add_argument("--no-hashes", action="store_true")
+    p.set_defaults(func=cmd_validate)
+
+    p = sub.add_parser("run", help="answer one request with a package")
+    p.add_argument("package")
+    p.add_argument("--request", required=True, help="a request JSON file, or - for stdin")
+    p.add_argument("--device", default="auto")
+    p.add_argument("--threads", type=int)
+    p.set_defaults(func=cmd_run)
+
+    p = sub.add_parser("info", help="this machine's backends and devices")
+    p.set_defaults(func=cmd_info)
+
+    args = parser.parse_args(argv)
+    return int(args.func(args))
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())