jevals 0.1.0__tar.gz

jevals-0.1.0/.gitignore

@@ -0,0 +1,26 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+*.egg-info/
+.eggs/
+dist/
+build/
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+.coverage
+htmlcov/
+.venv/
+venv/
+
+# Tools
+*.log
+
+# Editors
+.idea/
+.vscode/
+*.swp
+.DS_Store

jevals-0.1.0/PKG-INFO

@@ -0,0 +1,61 @@
+Metadata-Version: 2.5
+Name: jevals
+Version: 0.1.0
+Summary: Run suites of JSON evaluations and report a score.
+Project-URL: Homepage, https://pypi.org/project/jevals/
+Project-URL: Issues, https://pypi.org/project/jevals/
+Author-email: Gabriel Tinoco <gabriel@openlayer.com>
+License-Expression: MIT
+Keywords: eval,evaluation,jeval,json,suite,testing
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.10
+Requires-Dist: jeval>=0.1.0
+Description-Content-Type: text/markdown
+
+# jevals
+
+Run a suite of JSON evaluation cases and report a score.
+
+```bash
+pip install jevals
+```
+
+`jevals` depends on [`jeval`](https://pypi.org/project/jeval/) for the per-case checks.
+
+```bash
+jevals run examples/suite.json
+jevals run examples/cases
+```
+
+A suite is a JSON object with `cases`, a JSON list of cases, or a directory of `.json` files:
+
+```json
+{
+  "name": "checkout",
+  "cases": [
+    {
+      "id": "order-total",
+      "actual": {"total": 21, "currency": "USD"},
+      "checks": [
+        {"path": "total", "op": "eq", "value": 21},
+        {"path": "currency", "op": "eq", "value": "USD"}
+      ]
+    }
+  ]
+}
+```
+
+Exit codes: `0` all passed, `1` some failed, `2` usage or file error.
+
+```bash
+jevals run examples/suite.json --json
+jevals run examples/suite.json --fail-under 1.0
+```

jevals-0.1.0/README.md

@@ -0,0 +1,39 @@
+# jevals
+
+Run a suite of JSON evaluation cases and report a score.
+
+```bash
+pip install jevals
+```
+
+`jevals` depends on [`jeval`](https://pypi.org/project/jeval/) for the per-case checks.
+
+```bash
+jevals run examples/suite.json
+jevals run examples/cases
+```
+
+A suite is a JSON object with `cases`, a JSON list of cases, or a directory of `.json` files:
+
+```json
+{
+  "name": "checkout",
+  "cases": [
+    {
+      "id": "order-total",
+      "actual": {"total": 21, "currency": "USD"},
+      "checks": [
+        {"path": "total", "op": "eq", "value": 21},
+        {"path": "currency", "op": "eq", "value": "USD"}
+      ]
+    }
+  ]
+}
+```
+
+Exit codes: `0` all passed, `1` some failed, `2` usage or file error.
+
+```bash
+jevals run examples/suite.json --json
+jevals run examples/suite.json --fail-under 1.0
+```

jevals-0.1.0/pyproject.toml

@@ -0,0 +1,36 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "jevals"
+version = "0.1.0"
+description = "Run suites of JSON evaluations and report a score."
+readme = "README.md"
+license = "MIT"
+requires-python = ">=3.10"
+authors = [
+  { name = "Gabriel Tinoco", email = "gabriel@openlayer.com" },
+]
+keywords = ["eval", "evaluation", "json", "testing", "suite", "jeval"]
+classifiers = [
+  "Development Status :: 3 - Alpha",
+  "Intended Audience :: Developers",
+  "License :: OSI Approved :: MIT License",
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Programming Language :: Python :: 3.13",
+  "Topic :: Software Development :: Testing",
+]
+dependencies = [
+  "jeval>=0.1.0",
+]
+
+[project.urls]
+Homepage = "https://pypi.org/project/jevals/"
+Issues = "https://pypi.org/project/jevals/"
+
+[project.scripts]
+jevals = "jevals.cli:main"

jevals-0.1.0/src/jevals/__init__.py

@@ -0,0 +1,6 @@
+"""Run suites of JSON evaluations and report a score."""
+
+from jevals.suite import Suite, SuiteResult, load_suite
+
+__version__ = "0.1.0"
+__all__ = ["Suite", "SuiteResult", "load_suite", "__version__"]

jevals-0.1.0/src/jevals/__main__.py

@@ -0,0 +1,4 @@
+from jevals.cli import main
+
+if __name__ == "__main__":
+    raise SystemExit(main())

jevals-0.1.0/src/jevals/cli.py

@@ -0,0 +1,64 @@
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+from typing import TextIO
+
+from jeval import CaseError
+
+from jevals import __version__
+from jevals.suite import load_actuals, load_suite
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="jevals",
+        description="Run a suite of JSON evaluation cases and report a score.",
+    )
+    sub = parser.add_subparsers(dest="command")
+
+    run = sub.add_parser("run", help="Run a suite file or directory of cases")
+    run.add_argument("path", help="Suite JSON file or directory of .json cases")
+    run.add_argument(
+        "--actuals",
+        help="JSON file mapping case id to actual value (overrides case actuals).",
+    )
+    run.add_argument("--json", action="store_true", dest="as_json", help="Print JSON.")
+    run.add_argument(
+        "--fail-under",
+        type=float,
+        default=1.0,
+        help="Fail unless score (passed/total) is at least this value. Default: 1.0",
+    )
+
+    parser.add_argument("--version", action="version", version=f"jevals {__version__}")
+    args = parser.parse_args(argv)
+
+    if args.command != "run":
+        parser.print_help()
+        return 2
+
+    try:
+        suite = load_suite(Path(args.path))
+        actuals = load_actuals(Path(args.actuals)) if args.actuals else None
+        result = suite.run(actuals)
+    except (OSError, json.JSONDecodeError, CaseError, ValueError) as exc:
+        print(f"jevals: {exc}", file=sys.stderr)
+        return 2
+
+    _print(result, as_json=args.as_json, stream=sys.stdout)
+    if result.total == 0:
+        return 2
+    if result.score < args.fail_under:
+        return 1
+    return 0
+
+
+def _print(result, as_json: bool, stream: TextIO) -> None:
+    if as_json:
+        json.dump(result.to_dict(), stream, indent=2)
+        stream.write("\n")
+        return
+    stream.write(result.summary() + "\n")

jevals-0.1.0/src/jevals/suite.py

@@ -0,0 +1,160 @@
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from jeval import Case, CaseError, Result, evaluate
+
+
+@dataclass
+class Suite:
+    name: str
+    cases: list[Case]
+
+    def run(self, actuals: dict[str, Any] | None = None) -> SuiteResult:
+        results: list[Result] = []
+        for case in self.cases:
+            if actuals is None or case.id not in actuals:
+                results.append(evaluate(case))
+            else:
+                results.append(evaluate(case, actuals[case.id]))
+        return SuiteResult(name=self.name, results=results)
+
+
+@dataclass
+class SuiteResult:
+    name: str
+    results: list[Result] = field(default_factory=list)
+
+    @property
+    def passed(self) -> int:
+        return sum(1 for item in self.results if item.ok)
+
+    @property
+    def failed(self) -> int:
+        return sum(1 for item in self.results if not item.ok)
+
+    @property
+    def total(self) -> int:
+        return len(self.results)
+
+    @property
+    def ok(self) -> bool:
+        return self.total > 0 and self.failed == 0
+
+    @property
+    def score(self) -> float:
+        if self.total == 0:
+            return 0.0
+        return self.passed / self.total
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "name": self.name,
+            "ok": self.ok,
+            "passed": self.passed,
+            "failed": self.failed,
+            "total": self.total,
+            "score": self.score,
+            "results": [item.to_dict() for item in self.results],
+        }
+
+    def summary(self) -> str:
+        if self.total == 0:
+            return f"{self.name}: no cases"
+        status = "PASS" if self.ok else "FAIL"
+        lines = [
+            f"{status}  {self.name}  {self.passed} passed, {self.failed} failed  ({self.score:.0%})"
+        ]
+        for item in self.results:
+            if item.ok:
+                lines.append(f"  PASS  {item.case_id}")
+            else:
+                lines.append(f"  FAIL  {item.case_id}")
+                for check in item.failed:
+                    lines.append(f"        {check.message}")
+        return "\n".join(lines)
+
+
+def load_suite(path: Path) -> Suite:
+    path = path.resolve()
+    if path.is_dir():
+        cases = _load_dir(path)
+        return Suite(name=path.name, cases=cases)
+    data = json.loads(path.read_text(encoding="utf-8"))
+    return _suite_from_data(data, default_name=path.stem, source=path)
+
+
+def _load_dir(path: Path) -> list[Case]:
+    files = sorted(item for item in path.glob("*.json") if item.is_file())
+    if not files:
+        raise CaseError(f"no .json cases in {path}")
+    cases: list[Case] = []
+    for file in files:
+        data = json.loads(file.read_text(encoding="utf-8"))
+        cases.extend(_cases_from_data(data, default_id=file.stem, source=file))
+    return cases
+
+
+def _suite_from_data(data: Any, default_name: str, source: Path) -> Suite:
+    if isinstance(data, list):
+        return Suite(
+            name=default_name,
+            cases=_cases_from_data(data, default_id=default_name, source=source),
+        )
+    if not isinstance(data, dict):
+        raise CaseError(f"{source} must be a suite object or a list of cases")
+    if "cases" in data:
+        name = str(data.get("name") or data.get("id") or default_name)
+        raw_cases = data["cases"]
+        if not isinstance(raw_cases, list):
+            raise CaseError("cases must be a list")
+        cases = [
+            Case.from_dict(item, default_id=f"{name}-{index}")
+            if isinstance(item, dict)
+            else _reject_case(item)
+            for index, item in enumerate(raw_cases, start=1)
+        ]
+        return Suite(name=name, cases=cases)
+    return Suite(name=default_name, cases=[Case.from_dict(data, default_id=default_name)])
+
+
+def _cases_from_data(data: Any, default_id: str, source: Path) -> list[Case]:
+    if isinstance(data, list):
+        return [
+            Case.from_dict(item, default_id=f"{default_id}-{index}")
+            if isinstance(item, dict)
+            else _reject_case(item)
+            for index, item in enumerate(data, start=1)
+        ]
+    if isinstance(data, dict) and "cases" in data:
+        return _suite_from_data(data, default_name=default_id, source=source).cases
+    if isinstance(data, dict):
+        return [Case.from_dict(data, default_id=default_id)]
+    raise CaseError(f"{source} is not a valid case or suite")
+
+
+def _reject_case(item: Any) -> Case:
+    raise CaseError(f"case must be an object, got {type(item).__name__}")
+
+
+def load_actuals(path: Path) -> dict[str, Any]:
+    data = json.loads(path.read_text(encoding="utf-8"))
+    if isinstance(data, dict):
+        return data
+    if isinstance(data, list):
+        actuals: dict[str, Any] = {}
+        for item in data:
+            if not isinstance(item, dict) or "id" not in item:
+                raise CaseError("actuals list items must be objects with an 'id'")
+            case_id = str(item["id"])
+            if "actual" in item:
+                actuals[case_id] = item["actual"]
+            elif "value" in item:
+                actuals[case_id] = item["value"]
+            else:
+                raise CaseError(f"actuals item {case_id!r} is missing 'actual'")
+        return actuals
+    raise CaseError("actuals must be an object mapping case id to value, or a list")

jevals-0.1.0/tests/test_jevals_cli.py

@@ -0,0 +1,44 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from jevals.cli import main
+
+
+def test_cli_run(tmp_path: Path) -> None:
+    suite = tmp_path / "suite.json"
+    suite.write_text(
+        json.dumps(
+            {
+                "name": "demo",
+                "cases": [
+                    {
+                        "id": "ok",
+                        "actual": {"n": 1},
+                        "checks": [{"path": "n", "op": "eq", "value": 1}],
+                    }
+                ],
+            }
+        ),
+        encoding="utf-8",
+    )
+    assert main(["run", str(suite)]) == 0
+    assert main(["run", str(suite), "--json"]) == 0
+
+
+def test_cli_fail_under(tmp_path: Path) -> None:
+    suite = tmp_path / "suite.json"
+    suite.write_text(
+        json.dumps(
+            {
+                "cases": [
+                    {"id": "a", "actual": 1, "expected": 1},
+                    {"id": "b", "actual": 1, "expected": 2},
+                ]
+            }
+        ),
+        encoding="utf-8",
+    )
+    assert main(["run", str(suite)]) == 1
+    assert main(["run", str(suite), "--fail-under", "0.4"]) == 0

jevals-0.1.0/tests/test_suite.py

@@ -0,0 +1,63 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from jevals.suite import Suite, load_actuals, load_suite
+from jeval import Case, Check
+
+
+def test_suite_score(tmp_path: Path) -> None:
+    suite_path = tmp_path / "suite.json"
+    suite_path.write_text(
+        json.dumps(
+            {
+                "name": "demo",
+                "cases": [
+                    {
+                        "id": "ok",
+                        "actual": {"n": 1},
+                        "checks": [{"path": "n", "op": "eq", "value": 1}],
+                    },
+                    {
+                        "id": "bad",
+                        "actual": {"n": 2},
+                        "checks": [{"path": "n", "op": "eq", "value": 1}],
+                    },
+                ],
+            }
+        ),
+        encoding="utf-8",
+    )
+    result = load_suite(suite_path).run()
+    assert result.passed == 1
+    assert result.failed == 1
+    assert result.score == 0.5
+    assert not result.ok
+
+
+def test_directory_suite(tmp_path: Path) -> None:
+    cases = tmp_path / "cases"
+    cases.mkdir()
+    (cases / "one.json").write_text(
+        json.dumps({"id": "one", "actual": {"ok": True}, "expected": {"ok": True}}),
+        encoding="utf-8",
+    )
+    result = load_suite(cases).run()
+    assert result.ok
+    assert result.total == 1
+
+
+def test_actuals_override() -> None:
+    suite = Suite(
+        name="override",
+        cases=[Case(id="n", checks=[Check(path="$", op="eq", value=3)], actual=1)],
+    )
+    assert not suite.run().ok
+    assert suite.run({"n": 3}).ok
+
+
+def test_load_actuals_list(tmp_path: Path) -> None:
+    path = tmp_path / "actuals.json"
+    path.write_text(json.dumps([{"id": "n", "actual": 3}]), encoding="utf-8")
+    assert load_actuals(path) == {"n": 3}