tf-rewardkit 1.6.3__py3-none-any.whl

rewardkit/__init__.py

@@ -0,0 +1,66 @@
+from rewardkit.agents import AgentAttempt, AgentBackend, register_agent
+from rewardkit.compare import ComparisonResult, compare, format_comparison
+from rewardkit.models import (
+    Aggregation,
+    AgentJudge,
+    Binary,
+    Criterion,
+    JevJudge,
+    LLMJudge,
+    Likert,
+    MCPServerConfig,
+    Numeric,
+    OutputFormat,
+    Rubric,
+    Score,
+)
+from rewardkit.reward import Reward
+from rewardkit.runner import discover, run, run_multi
+from rewardkit.session import criterion
+from rewardkit.trajectory import format_trajectory
+
+__all__ = [
+    "AgentAttempt",
+    "AgentBackend",
+    "Aggregation",
+    "AgentJudge",
+    "Binary",
+    "ComparisonResult",
+    "Criterion",
+    "JevJudge",
+    "LLMJudge",
+    "Likert",
+    "MCPServerConfig",
+    "Numeric",
+    "OutputFormat",
+    "Reward",
+    "Rubric",
+    "Score",
+    "compare",
+    "criterion",
+    "discover",
+    "format_comparison",
+    "format_trajectory",
+    "register_agent",
+    "run",
+    "run_multi",
+]
+
+
+def __getattr__(name: str):  # noqa: ANN204
+    """Allow criteria to be accessed directly, e.g. ``rk.file_exists(...)``."""
+    import rewardkit.criteria  # noqa: F401 — ensures built-in criteria are registered
+
+    from rewardkit.session import _factory_registry
+
+    if name in _factory_registry:
+        return _factory_registry[name]
+    raise AttributeError(f"module 'rewardkit' has no attribute {name!r}")
+
+
+def __dir__() -> list[str]:
+    import rewardkit.criteria  # noqa: F401
+
+    from rewardkit.session import _factory_registry
+
+    return list(set(__all__) | set(_factory_registry))

rewardkit/__main__.py

@@ -0,0 +1,154 @@
+"""CLI entry point: ``python -m rewardkit``."""
+
+from __future__ import annotations
+
+import argparse
+import logging
+import os
+import sys
+
+from rewardkit.runner import run, run_multi
+
+
+def _enable_cli_logging() -> None:
+    """让 rewardkit 自己的 INFO 日志落到 stderr。
+
+    包内一律用 ``logging.getLogger(__name__)`` 且从不挂 handler，缺省只有 WARNING
+    及以上经 lastResort 输出，于是 agent CLI 的复用/安装来源这类 INFO 记录在判分
+    日志里完全看不到，无法区分"没装"与"装了但没记录"。只挂 rewardkit 这一个命名
+    空间而不动 root：root 上加 handler 会把 litellm、prefect 的 INFO 一起放出来。
+    """
+    logger = logging.getLogger("rewardkit")
+    if logger.handlers:
+        return
+    handler = logging.StreamHandler(sys.stderr)
+    handler.setFormatter(logging.Formatter("%(levelname)s %(name)s: %(message)s"))
+    logger.addHandler(handler)
+    logger.setLevel(logging.INFO)
+
+
+def main() -> None:
+    _enable_cli_logging()
+    parser = argparse.ArgumentParser(
+        prog="rewardkit",
+        description="Discover and run folder-based rewards.",
+    )
+    parser.add_argument(
+        "tests_dirs",
+        nargs="+",
+        help="Path(s) to tests directory. Multiple dirs run independently and are compared.",
+    )
+    parser.add_argument(
+        "--workspace", default="/app", help="Workspace path (default: /app)"
+    )
+    parser.add_argument(
+        "--output",
+        default="/logs/verifier/reward.json",
+        help="Output JSON path (default: /logs/verifier/reward.json)",
+    )
+    parser.add_argument(
+        "--max-concurrent-programmatic",
+        "--mcprog",
+        "--mcp",
+        type=int,
+        default=8,
+        help="Max programmatic rewards to run in parallel (0 = unlimited, default: 8)",
+    )
+    parser.add_argument(
+        "--max-concurrent-llm",
+        "--mcllm",
+        "--mcl",
+        type=int,
+        default=8,
+        help="Max LLM judge calls to run in parallel (0 = unlimited, default: 8)",
+    )
+    parser.add_argument(
+        "--max-concurrent-agent",
+        "--mcagent",
+        "--mca",
+        type=int,
+        default=2,
+        help="Max agent judge calls to run in parallel (0 = unlimited, default: 2)",
+    )
+    parser.add_argument(
+        "--max-concurrent-workflow-judge",
+        "--mcwf",
+        type=int,
+        default=1,
+        help="Max workflow-judge graphs to run in parallel "
+        "(0 = unlimited, default: 1)",
+    )
+    parser.add_argument(
+        "--judge-env",
+        "--je",
+        action="append",
+        default=[],
+        metavar="KEY=VALUE",
+        help="Set an env var for the judge run (repeatable). Overrides parent env.",
+    )
+    parser.add_argument(
+        "--judge",
+        "-j",
+        default=None,
+        metavar="MODEL_OR_AGENT",
+        help="Override the rubric's [judge].judge field. "
+        "Equivalent to setting REWARDKIT_JUDGE.",
+    )
+    parser.add_argument(
+        "--model",
+        "-m",
+        default=None,
+        metavar="MODEL",
+        help="Override the rubric's [judge].model field (used when the judge is an agent). "
+        "Equivalent to setting REWARDKIT_MODEL.",
+    )
+
+    args = parser.parse_args()
+
+    for entry in args.judge_env:
+        key, sep, value = entry.partition("=")
+        if not sep or not key:
+            parser.error(f"--judge-env expects KEY=VALUE, got: {entry!r}")
+        os.environ[key] = value
+    if args.judge:
+        os.environ["REWARDKIT_JUDGE"] = args.judge
+    if args.model:
+        os.environ["REWARDKIT_MODEL"] = args.model
+    concurrency_kwargs = dict(
+        max_concurrent_programmatic=args.max_concurrent_programmatic,
+        max_concurrent_llm=args.max_concurrent_llm,
+        max_concurrent_agent=args.max_concurrent_agent,
+        max_concurrent_workflow_judge=args.max_concurrent_workflow_judge,
+    )
+
+    if len(args.tests_dirs) == 1:
+        result = run(
+            args.tests_dirs[0],
+            workspace=args.workspace,
+            output=args.output,
+            **concurrency_kwargs,
+        )
+        for name, score in result.items():
+            print(f"{name}: {score}")
+    else:
+        # Multiple test suites: run independently, compare
+        per_dir = run_multi(
+            args.tests_dirs,
+            workspace=args.workspace,
+            output=args.output,
+            **concurrency_kwargs,
+        )
+        for label, scores in per_dir.items():
+            for name, score in scores.items():
+                print(f"{label}/{name}: {score}")
+
+        from rewardkit.compare import format_comparison
+
+        table = format_comparison(per_dir)
+        if table:
+            print()
+            print(table)
+
+
+if __name__ == "__main__":
+    main()