harbor-rewardkit 0.1.0__tar.gz

harbor_rewardkit-0.1.0/PKG-INFO

@@ -0,0 +1,73 @@
+Metadata-Version: 2.4
+Name: harbor-rewardkit
+Version: 0.1.0
+Summary: Lightweight grading toolkit for environment-based tasks.
+Keywords: grading,evaluation,rewards,llm,agents,benchmarks
+Author: benediktstroebl
+License-Expression: Apache-2.0
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Testing
+Requires-Dist: litellm>=1.83
+Requires-Dist: openpyxl>=3.1 ; extra == 'all'
+Requires-Dist: pillow>=10.0 ; extra == 'all'
+Requires-Dist: pillow>=10.0 ; extra == 'image'
+Requires-Dist: openpyxl>=3.1 ; extra == 'office'
+Requires-Python: >=3.12
+Project-URL: Repository, https://github.com/harbor-framework/harbor
+Project-URL: Issues, https://github.com/harbor-framework/harbor/issues
+Provides-Extra: all
+Provides-Extra: image
+Provides-Extra: office
+Description-Content-Type: text/markdown
+
+# rewardkit
+
+[![](https://dcbadge.limes.pink/api/server/https://discord.gg/6xWPKhGDbA)](https://discord.gg/6xWPKhGDbA)
+[![Docs](https://img.shields.io/badge/Docs-000000?style=for-the-badge&logo=mdbook&color=105864)](https://harborframework.com/docs/rewardkit)
+
+A lightweight grading toolkit for defining and running verifiers that output structured reward scores.
+
+## Installation
+
+```bash
+uv tool install rewardkit
+```
+
+## Example: Programmatic criteria
+
+```python
+# tests/check.py
+from rewardkit import criteria
+
+criteria.file_exists("output.txt")
+criteria.file_contains("output.txt", "hello")
+```
+
+## Example: LLM judge
+
+```toml
+# tests/quality.toml
+[judge]
+judge = "anthropic/claude-sonnet-4-6"
+files = ["/app/main.py"]
+
+[[criterion]]
+description = "Is the code correct?"
+type = "binary"
+```
+
+## Usage
+
+Add rewardkit to your `test.sh` file:
+
+```bash
+# tests/test.sh
+uvx rewardkit /tests
+```
+
+See the [documentation](https://harborframework.com/docs/rewardkit) and a full [working example](https://github.com/harbor-framework/harbor/tree/main/examples/tasks/reward-kit-example).

harbor_rewardkit-0.1.0/README.md

@@ -0,0 +1,46 @@
+# rewardkit
+
+[![](https://dcbadge.limes.pink/api/server/https://discord.gg/6xWPKhGDbA)](https://discord.gg/6xWPKhGDbA)
+[![Docs](https://img.shields.io/badge/Docs-000000?style=for-the-badge&logo=mdbook&color=105864)](https://harborframework.com/docs/rewardkit)
+
+A lightweight grading toolkit for defining and running verifiers that output structured reward scores.
+
+## Installation
+
+```bash
+uv tool install rewardkit
+```
+
+## Example: Programmatic criteria
+
+```python
+# tests/check.py
+from rewardkit import criteria
+
+criteria.file_exists("output.txt")
+criteria.file_contains("output.txt", "hello")
+```
+
+## Example: LLM judge
+
+```toml
+# tests/quality.toml
+[judge]
+judge = "anthropic/claude-sonnet-4-6"
+files = ["/app/main.py"]
+
+[[criterion]]
+description = "Is the code correct?"
+type = "binary"
+```
+
+## Usage
+
+Add rewardkit to your `test.sh` file:
+
+```bash
+# tests/test.sh
+uvx rewardkit /tests
+```
+
+See the [documentation](https://harborframework.com/docs/rewardkit) and a full [working example](https://github.com/harbor-framework/harbor/tree/main/examples/tasks/reward-kit-example).

harbor_rewardkit-0.1.0/pyproject.toml

@@ -0,0 +1,42 @@
+[project]
+name = "harbor-rewardkit"
+version = "0.1.0"
+description = "Lightweight grading toolkit for environment-based tasks."
+readme = "README.md"
+license = "Apache-2.0"
+authors = [
+    { name = "benediktstroebl" },
+]
+requires-python = ">=3.12"
+keywords = ["grading", "evaluation", "rewards", "llm", "agents", "benchmarks"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: Apache Software License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Software Development :: Testing",
+]
+dependencies = [
+    "litellm>=1.83",
+]
+
+[project.optional-dependencies]
+office = ["openpyxl>=3.1"]
+image = ["Pillow>=10.0"]
+all = ["openpyxl>=3.1", "Pillow>=10.0"]
+
+[project.urls]
+Repository = "https://github.com/harbor-framework/harbor"
+Issues = "https://github.com/harbor-framework/harbor/issues"
+
+[project.scripts]
+rewardkit = "rewardkit.__main__:main"
+
+[build-system]
+requires = ["uv_build>=0.10.8,<0.11.0"]
+build-backend = "uv_build"
+
+[tool.uv.build-backend]
+module-name = "rewardkit"

harbor_rewardkit-0.1.0/src/rewardkit/__init__.py

@@ -0,0 +1,37 @@
+from rewardkit.compare import ComparisonResult, compare, format_comparison
+from rewardkit.models import (
+    Aggregation,
+    AgentJudge,
+    Binary,
+    Criterion,
+    LLMJudge,
+    Likert,
+    Numeric,
+    OutputFormat,
+    Score,
+)
+from rewardkit.reward import Reward
+from rewardkit.runner import discover, run, run_multi
+from rewardkit.session import criterion
+from rewardkit.trajectory import format_trajectory
+
+__all__ = [
+    "Aggregation",
+    "AgentJudge",
+    "Binary",
+    "ComparisonResult",
+    "Criterion",
+    "LLMJudge",
+    "Likert",
+    "Numeric",
+    "OutputFormat",
+    "Reward",
+    "Score",
+    "compare",
+    "criterion",
+    "discover",
+    "format_comparison",
+    "format_trajectory",
+    "run",
+    "run_multi",
+]

harbor_rewardkit-0.1.0/src/rewardkit/__main__.py

@@ -0,0 +1,90 @@
+"""CLI entry point: ``python -m rewardkit``."""
+
+from __future__ import annotations
+
+import argparse
+
+from rewardkit.runner import run, run_multi
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        prog="rewardkit",
+        description="Discover and run folder-based rewards.",
+    )
+    parser.add_argument(
+        "tests_dirs",
+        nargs="+",
+        help="Path(s) to tests directory. Multiple dirs run independently and are compared.",
+    )
+    parser.add_argument(
+        "--workspace", default="/app", help="Workspace path (default: /app)"
+    )
+    parser.add_argument(
+        "--output",
+        default="/logs/verifier/reward.json",
+        help="Output JSON path (default: /logs/verifier/reward.json)",
+    )
+    parser.add_argument(
+        "--max-concurrent-programmatic",
+        "--mcprog",
+        "--mcp",
+        type=int,
+        default=8,
+        help="Max programmatic rewards to run in parallel (0 = unlimited, default: 8)",
+    )
+    parser.add_argument(
+        "--max-concurrent-llm",
+        "--mcllm",
+        "--mcl",
+        type=int,
+        default=8,
+        help="Max LLM judge calls to run in parallel (0 = unlimited, default: 8)",
+    )
+    parser.add_argument(
+        "--max-concurrent-agent",
+        "--mcagent",
+        "--mca",
+        type=int,
+        default=2,
+        help="Max agent judge calls to run in parallel (0 = unlimited, default: 2)",
+    )
+
+    args = parser.parse_args()
+    concurrency_kwargs = dict(
+        max_concurrent_programmatic=args.max_concurrent_programmatic,
+        max_concurrent_llm=args.max_concurrent_llm,
+        max_concurrent_agent=args.max_concurrent_agent,
+    )
+
+    if len(args.tests_dirs) == 1:
+        result = run(
+            args.tests_dirs[0],
+            workspace=args.workspace,
+            output=args.output,
+            **concurrency_kwargs,
+        )
+        for name, score in result.items():
+            print(f"{name}: {score}")
+    else:
+        # Multiple test suites: run independently, compare
+        per_dir = run_multi(
+            args.tests_dirs,
+            workspace=args.workspace,
+            output=args.output,
+            **concurrency_kwargs,
+        )
+        for label, scores in per_dir.items():
+            for name, score in scores.items():
+                print(f"{label}/{name}: {score}")
+
+        from rewardkit.compare import format_comparison
+
+        table = format_comparison(per_dir)
+        if table:
+            print()
+            print(table)
+
+
+if __name__ == "__main__":
+    main()

harbor_rewardkit-0.1.0/src/rewardkit/compare.py

@@ -0,0 +1,91 @@
+"""Comparison utilities for multi-dir reward results."""
+
+from __future__ import annotations
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class ComparisonResult(BaseModel):
+    """Comparison of reward scores across multiple test directories."""
+
+    model_config = ConfigDict(frozen=True)
+
+    labels: list[str]
+    per_reward: dict[str, dict[str, float]] = Field(default_factory=dict)
+
+
+def compare(
+    results: dict[str, dict[str, float]],
+) -> ComparisonResult:
+    """Compare results from multiple test directories.
+
+    Args:
+        results: Mapping of ``dir_label -> {reward_name: score}``.
+
+    Returns:
+        A :class:`ComparisonResult` with overlapping reward names mapped
+        to per-label scores.
+    """
+    labels = list(results.keys())
+    if len(labels) < 2:
+        return ComparisonResult(labels=labels)
+
+    all_names: set[str] = set()
+    for scores in results.values():
+        all_names.update(scores.keys())
+
+    per_reward: dict[str, dict[str, float]] = {}
+    for name in sorted(all_names):
+        entry: dict[str, float] = {}
+        for label in labels:
+            score = results[label].get(name)
+            if score is not None:
+                entry[label] = score
+        if len(entry) >= 2:
+            per_reward[name] = entry
+
+    return ComparisonResult(labels=labels, per_reward=per_reward)
+
+
+def format_comparison(results: dict[str, dict[str, float]]) -> str:
+    """Format a comparison table for printing to stdout.
+
+    Returns an empty string if there are fewer than 2 dirs or no overlapping
+    reward names.
+    """
+    cr = compare(results)
+    if not cr.per_reward:
+        return ""
+
+    labels = cr.labels
+    name_width = max(len("reward"), max(len(n) for n in cr.per_reward))
+    col_widths = {label: max(len(label), 6) for label in labels}
+
+    header = "reward".ljust(name_width)
+    for label in labels:
+        header += "  " + label.rjust(col_widths[label])
+    header += "  " + "diff".rjust(6)
+
+    sep = "-" * len(header)
+    lines = ["Comparison:", sep, header, sep]
+
+    for name, scores in cr.per_reward.items():
+        row = name.ljust(name_width)
+        values = []
+        for label in labels:
+            val = scores.get(label)
+            if val is not None:
+                row += "  " + f"{val:.4f}".rjust(col_widths[label])
+                values.append(val)
+            else:
+                row += "  " + "-".rjust(col_widths[label])
+        if len(values) >= 2:
+            diff = values[0] - values[-1]
+            sign = "+" if diff > 0 else ""
+            row += "  " + f"{sign}{diff:.4f}".rjust(6)
+        else:
+            row += "  " + "-".rjust(6)
+        lines.append(row)
+
+    lines.append(sep)
+    return "\n".join(lines)

harbor_rewardkit-0.1.0/src/rewardkit/criteria/__init__.py

@@ -0,0 +1,63 @@
+"""Criterion functions for rewardkit.
+
+All criteria — built-in and user-defined — are resolved via the global
+``_factory_registry`` in :mod:`rewardkit.session`.  User-defined criteria
+registered with ``@criterion`` override built-ins of the same name.
+"""
+
+import importlib as _importlib
+import sys as _sys
+
+from rewardkit.session import _builtin_names, _factory_registry
+
+# Import built-in criterion modules so their @criterion decorators execute
+# and populate _factory_registry.
+_BUILTIN_MODULES = [
+    "command_output_contains",
+    "command_output_matches",
+    "command_output_matches_regex",
+    "command_succeeds",
+    "csv_cell_equals",
+    "diff_ratio",
+    "file_contains",
+    "file_contains_regex",
+    "file_exists",
+    "file_matches",
+    "file_not_exists",
+    "files_equal",
+    "http_response_contains",
+    "http_status_equals",
+    "image_similarity",
+    "image_size_equals",
+    "json_key_equals",
+    "json_path_equals",
+    "sqlite_query_equals",
+    "trajectory_tool_not_used",
+    "trajectory_tool_used",
+    "trajectory_turn_count",
+    "xlsx_cell_equals",
+]
+
+for _name in _BUILTIN_MODULES:
+    _importlib.import_module(f"rewardkit.criteria.{_name}")
+
+# Mark currently registered names as built-in so user overrides produce a warning.
+_builtin_names.update(_factory_registry)
+
+# Python sets submodule attributes on the parent package.
+# Remove them so all lookups go through __getattr__, which checks
+# _factory_registry — this lets user-defined criteria override built-ins.
+_this = _sys.modules[__name__]
+for _name in _BUILTIN_MODULES:
+    delattr(_this, _name)
+
+del _name, _this, _importlib, _sys
+
+__all__ = list(_BUILTIN_MODULES)
+
+
+def __getattr__(name: str):  # noqa: ANN204
+    """Resolve criteria from the global factory registry."""
+    if name in _factory_registry:
+        return _factory_registry[name]
+    raise AttributeError(f"module 'rewardkit.criteria' has no attribute {name!r}")

harbor_rewardkit-0.1.0/src/rewardkit/criteria/_command.py

@@ -0,0 +1,30 @@
+"""Shared subprocess helper for command-based criteria."""
+
+from __future__ import annotations
+
+import subprocess
+from pathlib import Path
+
+
+def run_command(
+    workspace: Path,
+    cmd: str,
+    cwd: str | None = None,
+    timeout: int = 30,
+) -> subprocess.CompletedProcess[str] | None:
+    """Run a shell command in the workspace directory.
+
+    Returns the CompletedProcess on success, or None on timeout.
+    """
+    run_cwd = str(workspace / cwd) if cwd else str(workspace)
+    try:
+        return subprocess.run(
+            cmd,
+            shell=True,
+            cwd=run_cwd,
+            capture_output=True,
+            text=True,
+            timeout=timeout,
+        )
+    except subprocess.TimeoutExpired:
+        return None

harbor_rewardkit-0.1.0/src/rewardkit/criteria/_trajectory.py

@@ -0,0 +1,31 @@
+"""Shared helpers for trajectory-based criteria."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+
+def load_trajectory(path: str | Path) -> dict | None:
+    """Load an ATIF trajectory JSON file. Returns None on error."""
+    p = Path(path)
+    if not p.exists():
+        return None
+    try:
+        return json.loads(p.read_text())
+    except (json.JSONDecodeError, OSError):
+        return None
+
+
+def count_agent_turns(data: dict) -> int:
+    """Count the number of steps with source == 'agent'."""
+    return sum(1 for s in data.get("steps", []) if s.get("source") == "agent")
+
+
+def collect_tool_calls(data: dict) -> list[dict]:
+    """Collect all tool calls across all steps."""
+    calls: list[dict] = []
+    for step in data.get("steps", []):
+        for tc in step.get("tool_calls") or []:
+            calls.append(tc)
+    return calls

harbor_rewardkit-0.1.0/src/rewardkit/criteria/command_output_contains.py

@@ -0,0 +1,18 @@
+"""Criterion: check that a command's stdout contains a given string."""
+
+from pathlib import Path
+
+from rewardkit.criteria._command import run_command
+from rewardkit.session import criterion
+
+
+@criterion(description="Check that the stdout of `{cmd}` contains '{text}'")
+def command_output_contains(
+    workspace: Path,
+    cmd: str,
+    text: str,
+    cwd: str | None = None,
+    timeout: int = 30,
+) -> bool:
+    result = run_command(workspace, cmd, cwd=cwd, timeout=timeout)
+    return result is not None and text in result.stdout

harbor_rewardkit-0.1.0/src/rewardkit/criteria/command_output_matches.py

@@ -0,0 +1,18 @@
+"""Criterion: check that a command's stdout exactly matches a string."""
+
+from pathlib import Path
+
+from rewardkit.criteria._command import run_command
+from rewardkit.session import criterion
+
+
+@criterion(description="Check that output of `{cmd}` matches {expected!r}")
+def command_output_matches(
+    workspace: Path,
+    cmd: str,
+    expected: str,
+    cwd: str | None = None,
+    timeout: int = 30,
+) -> bool:
+    result = run_command(workspace, cmd, cwd=cwd, timeout=timeout)
+    return result is not None and result.stdout.strip() == expected.strip()

harbor_rewardkit-0.1.0/src/rewardkit/criteria/command_output_matches_regex.py

@@ -0,0 +1,19 @@
+"""Criterion: check that a command's stdout matches a regex pattern."""
+
+import re
+from pathlib import Path
+
+from rewardkit.criteria._command import run_command
+from rewardkit.session import criterion
+
+
+@criterion(description="Check that output of `{cmd}` matches regex '{pattern}'")
+def command_output_matches_regex(
+    workspace: Path,
+    cmd: str,
+    pattern: str,
+    cwd: str | None = None,
+    timeout: int = 30,
+) -> bool:
+    result = run_command(workspace, cmd, cwd=cwd, timeout=timeout)
+    return result is not None and re.search(pattern, result.stdout) is not None

harbor_rewardkit-0.1.0/src/rewardkit/criteria/command_succeeds.py

@@ -0,0 +1,17 @@
+"""Criterion: check that a shell command exits with code 0."""
+
+from pathlib import Path
+
+from rewardkit.criteria._command import run_command
+from rewardkit.session import criterion
+
+
+@criterion(description="Check that the command `{cmd}` exits with code 0")
+def command_succeeds(
+    workspace: Path,
+    cmd: str,
+    cwd: str | None = None,
+    timeout: int = 30,
+) -> bool:
+    result = run_command(workspace, cmd, cwd=cwd, timeout=timeout)
+    return result is not None and result.returncode == 0

harbor_rewardkit-0.1.0/src/rewardkit/criteria/csv_cell_equals.py

@@ -0,0 +1,33 @@
+"""Criterion: check that a specific CSV cell has the expected value."""
+
+import csv
+from pathlib import Path
+
+from rewardkit.session import criterion
+
+
+@criterion(description="Check that {path}[{row},{col}] == {expected!r}")
+def csv_cell_equals(
+    workspace: Path,
+    path: str,
+    row: int,
+    col: int | str,
+    expected: str,
+) -> bool:
+    try:
+        with (workspace / path).open(newline="") as f:
+            if isinstance(col, str):
+                reader = csv.DictReader(f)
+                for i, r in enumerate(reader):
+                    if i == row:
+                        return str(r.get(col, "")) == expected
+                return False
+            else:
+                for i, r in enumerate(csv.reader(f)):
+                    if i == row:
+                        if col < len(r):
+                            return r[col] == expected
+                        return False
+                return False
+    except (FileNotFoundError, OSError, csv.Error, KeyError, IndexError):
+        return False

harbor_rewardkit-0.1.0/src/rewardkit/criteria/diff_ratio.py

@@ -0,0 +1,15 @@
+"""Criterion: fuzzy text comparison returning a similarity ratio [0, 1]."""
+
+from difflib import SequenceMatcher
+from pathlib import Path
+
+from rewardkit.session import criterion
+
+
+@criterion(description="Similarity ratio for {path}")
+def diff_ratio(workspace: Path, path: str, expected: str) -> float:
+    try:
+        content = (workspace / path).read_text()
+    except (FileNotFoundError, OSError):
+        return 0.0
+    return SequenceMatcher(None, content.strip(), expected.strip()).ratio()

harbor_rewardkit-0.1.0/src/rewardkit/criteria/file_contains.py

@@ -0,0 +1,13 @@
+"""Criterion: check that a file contains a given string."""
+
+from pathlib import Path
+
+from rewardkit.session import criterion
+
+
+@criterion(description="Check that {path} contains the text '{text}'")
+def file_contains(workspace: Path, path: str, text: str) -> bool:
+    try:
+        return text in (workspace / path).read_text()
+    except (FileNotFoundError, OSError):
+        return False

harbor_rewardkit-0.1.0/src/rewardkit/criteria/file_contains_regex.py

@@ -0,0 +1,14 @@
+"""Criterion: check that a file contains text matching a regex pattern."""
+
+import re
+from pathlib import Path
+
+from rewardkit.session import criterion
+
+
+@criterion(description="Check that {path} contains text matching regex '{pattern}'")
+def file_contains_regex(workspace: Path, path: str, pattern: str) -> bool:
+    try:
+        return re.search(pattern, (workspace / path).read_text()) is not None
+    except (FileNotFoundError, OSError):
+        return False

harbor_rewardkit-0.1.0/src/rewardkit/criteria/file_exists.py

@@ -0,0 +1,10 @@
+"""Criterion: check that a file exists."""
+
+from pathlib import Path
+
+from rewardkit.session import criterion
+
+
+@criterion(description="Check that {path} exists in the workspace")
+def file_exists(workspace: Path, path: str) -> bool:
+    return (workspace / path).exists()