tf-rewardkit 1.6.3__tar.gz → 1.6.4.dev0__tar.gz

{tf_rewardkit-1.6.3 → tf_rewardkit-1.6.4.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tf-rewardkit
-Version: 1.6.3
+Version: 1.6.4.dev0
 Summary: Lightweight grading toolkit for environment-based tasks.
 Keywords: grading,evaluation,rewards,llm,agents,benchmarks
 Author: benediktstroebl

{tf_rewardkit-1.6.3 → tf_rewardkit-1.6.4.dev0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "tf-rewardkit"
-version = "1.6.3"
+version = "1.6.4.dev0"
 description = "Lightweight grading toolkit for environment-based tasks."
 readme = "README.md"
 license = "Apache-2.0"

{tf_rewardkit-1.6.3 → tf_rewardkit-1.6.4.dev0}/src/rewardkit/workflow/__init__.py

@@ -11,7 +11,6 @@ Prefect 3 调度执行。
 - ``errors``    框架级 Failure 类型：错误形状的载体
 - ``context``   共享运行时服务
 - ``component`` 组件注册表与随框架组件
-- ``converter`` 转换工具：rubric.json → workflow.toml
 
 完整契约见 docs/judging-graph-protocol.md。
 """

tf_rewardkit-1.6.4.dev0/src/rewardkit/workflow/component/_judge_args.py

@@ -0,0 +1,198 @@
+"""args 传参判分组件（``llm_judge@1`` / ``agentic@1``）的共享私有helper。
+
+两个组件不经任何清单文件：评分项与调用配置全部由图 ``args`` 传入，一个
+node 对应一次 ``arun_llm`` / ``arun_agent`` 调用。本模块承载三块共享逻辑：
+
+- ``criteria`` 槽解析：TOML inline table 的数组，元素字段对齐存量 judge
+  TOML 的 ``[[criterion]]`` 表（``description`` 必填，其余选填、缺省值同
+  存量配置）；元素形状校验失败返回原因串由调用方按 unjudged 处置——评分
+  项配置问题是清单问题，不是节点运行失败；
+- 覆盖链解析：model 链 ``context.model_name``（CLI）> ``REWARDKIT_MODEL``
+  > 额外 env（如 ``REWARDKIT_JUDGE``）> args 槽 > 组件缺省；agent 链
+  ``context.judge``（CLI）> agent 专用 env > ``REWARDKIT_JUDGE``（仅当值
+  在 known_agents 注册表内才采用——部署里该变量常被设成 LLM 模型名）>
+  args 槽 > 组件缺省；
+- ``files`` 槽锚定：相对路径一律按 workspace 根解析（判分调用直读路径
+  相对进程 CWD，而 workspace 是独立注入的）。
+"""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from typing import Any
+
+from rewardkit.models import Binary, Criterion, Likert, Numeric, Rubric
+
+JUDGE_NAME_ENV = "REWARDKIT_JUDGE"
+"""通用 judge 名覆盖变量（LLM 侧作模型名通道，agent 侧须过注册表门槛）。"""
+
+MODEL_NAME_ENV = "REWARDKIT_MODEL"
+"""模型名覆盖变量（两个组件的 model 链共用）。"""
+
+_CRITERION_ENTRY_KEYS = frozenset(
+    {
+        "description",
+        "type",
+        "name",
+        "id",
+        "points",
+        "levels",
+        "min",
+        "max",
+        "weight",
+        "files",
+        "negate",
+        "optional",
+    }
+)
+
+
+def _output_format_for_entry(entry: dict[str, Any]) -> Any:
+    """元素 type/points/levels/min/max → OutputFormat（映射对齐存量
+    ``runner._build_criteria_from_toml``：likert → ``Likert(points)``、
+    numeric → ``Numeric(min, max)``、rubric → ``Rubric(levels)``、其余 →
+    ``Binary()``）。"""
+    ctype = entry.get("type", "binary")
+    if ctype == "likert":
+        return Likert(points=entry.get("points", 5))
+    if ctype == "numeric":
+        return Numeric(min=entry.get("min", 0.0), max=entry.get("max", 1.0))
+    if ctype == "rubric":
+        return Rubric(levels=tuple(entry.get("levels", ())))
+    return Binary()
+
+
+def _validate_entry(entry: dict[str, Any], index: int) -> str | None:
+    """单元素字段级校验；返回原因串或 None（合法）。"""
+    unknown = set(entry) - _CRITERION_ENTRY_KEYS
+    if unknown:
+        return f"criteria 元素 {index} 含未知字段：{sorted(unknown)}"
+    description = entry.get("description")
+    if not isinstance(description, str) or not description.strip():
+        return f"criteria 元素 {index} 缺少非空 description 字段"
+    ctype = entry.get("type", "binary")
+    if ctype not in ("binary", "likert", "numeric", "rubric"):
+        return f"criteria 元素 {index} 的 type 非法：{ctype!r}"
+    if "points" in entry and not isinstance(entry["points"], int):
+        return f"criteria 元素 {index} 的 points 必须是整数"
+    if "levels" in entry and not (
+        isinstance(entry["levels"], (list, tuple))
+        and all(isinstance(level, str) for level in entry["levels"])
+    ):
+        return f"criteria 元素 {index} 的 levels 必须是字符串数组"
+    for field in ("min", "max", "weight"):
+        if field in entry and not isinstance(entry[field], (int, float)):
+            return f"criteria 元素 {index} 的 {field} 必须是数值"
+    for field in ("negate", "optional"):
+        if field in entry and not isinstance(entry[field], bool):
+            return f"criteria 元素 {index} 的 {field} 必须是布尔"
+    if "files" in entry and not (
+        isinstance(entry["files"], (list, tuple))
+        and all(isinstance(item, str) for item in entry["files"])
+    ):
+        return f"criteria 元素 {index} 的 files 必须是字符串数组"
+    for field in ("name", "id"):
+        if field in entry and entry[field] is not None and not isinstance(entry[field], str):
+            return f"criteria 元素 {index} 的 {field} 必须是字符串"
+    return None
+
+
+def parse_criteria_entries(
+    entries: Any, workspace_root: Any
+) -> tuple[list[Criterion], list[float], str | None]:
+    """解析 ``criteria`` 槽，返回 ``(criteria, weights, 原因串)``。
+
+    原因串非 None 表示槽非法（调用方按 unjudged 处置）；合法时原因串为
+    None，weights 与 criteria 按下标一一对应（元素 ``weight`` 缺省 1.0）。
+    """
+    if isinstance(entries, (str, dict)) or not isinstance(entries, (list, tuple)):
+        return [], [], "criteria 必须是评分项表的数组（args 的 array 槽）"
+    if not entries:
+        return [], [], "criteria 不能为空数组"
+
+    criteria: list[Criterion] = []
+    weights: list[float] = []
+    for index, entry in enumerate(entries):
+        if not isinstance(entry, dict):
+            return [], [], f"criteria 元素 {index} 必须是表，收到 {type(entry).__name__}"
+        reason = _validate_entry(entry, index)
+        if reason:
+            return [], [], reason
+        try:
+            output_format = _output_format_for_entry(entry)
+            criterion = Criterion(
+                description=entry["description"],
+                output_format=output_format,
+                name=entry.get("name") or None,
+                id=entry.get("id"),
+                files=abs_files(entry.get("files", ()), workspace_root),
+                negate=entry.get("negate", False),
+                optional=entry.get("optional", False),
+            )
+        except Exception as exc:  # noqa: BLE001 - name/output_format 非法属清单问题
+            return [], [], f"criteria 元素 {index} 构造失败（name 或 output_format 非法）：{exc}"
+        criteria.append(criterion)
+        weights.append(float(entry.get("weight", 1.0)))
+    return criteria, weights, None
+
+
+def abs_files(entries: Any, workspace_root: Any) -> tuple[str, ...]:
+    """路径列表按 workspace 根锚定；绝对路径原样。"""
+    resolved = []
+    for entry in entries:
+        path = Path(str(entry))
+        resolved.append(str(path if path.is_absolute() else Path(workspace_root) / path))
+    return tuple(resolved)
+
+
+def resolve_arg_files(files: Any, workspace_root: Any) -> tuple[str, ...]:
+    """``files`` 槽：单路径字符串或路径列表 → 按 workspace 根锚定；
+    空值与不认识的形状落空元组。"""
+    if isinstance(files, str) and files:
+        return abs_files([files], workspace_root)
+    if isinstance(files, (list, tuple)) and files:
+        return abs_files(files, workspace_root)
+    return ()
+
+
+def resolve_model(
+    context: Any,
+    slot: str | None,
+    *,
+    extra_envs: tuple[str, ...] = (),
+    default: str | None = None,
+) -> str | None:
+    """model 覆盖链：``context.model_name``（CLI）> ``REWARDKIT_MODEL`` >
+    ``extra_envs`` 逐个 > args 槽 > ``default``。"""
+    candidates = [getattr(context, "model_name", None), os.environ.get(MODEL_NAME_ENV)]
+    candidates += [os.environ.get(name) for name in extra_envs]
+    candidates += [slot, default]
+    for value in candidates:
+        if value:
+            return str(value)
+    return default
+
+
+def resolve_agent(
+    context: Any,
+    slot: str | None,
+    *,
+    agent_env: str,
+    default: str,
+) -> str:
+    """agent 覆盖链：``context.judge``（CLI）> ``agent_env`` >
+    ``REWARDKIT_JUDGE``（仅当值在 known_agents 内才采用）> args 槽 >
+    ``default``。"""
+    from rewardkit.agents import known_agents
+
+    cli_judge = getattr(context, "judge", None)
+    if cli_judge:
+        return str(cli_judge)
+    env_agent = os.environ.get(agent_env)
+    if env_agent:
+        return env_agent
+    env_judge = os.environ.get(JUDGE_NAME_ENV)
+    if env_judge in known_agents():
+        return env_judge
+    return slot or default

tf_rewardkit-1.6.4.dev0/src/rewardkit/workflow/component/agentic.py

@@ -0,0 +1,136 @@
+"""agentic 判分组件 ``agentic@1``：编码智能体 judge 的图内入口（args 传参薄封装）。
+
+不读任何清单文件：评分项与调用配置全部由图 ``args`` 传入，一个 node
+一次 ``arun_agent`` 调用。
+
+- ``criteria`` 必填槽为评分项表的数组（元素字段与权重语义同
+  ``llm_judge@1``）；
+- 其余选填槽与 ``AgentJudge`` 字段逐字同名（``agent`` / ``model`` /
+  ``version`` / ``timeout`` / ``cwd`` / ``isolated`` / ``atif_trajectory``
+  / ``mode`` / ``mcp_servers``），另加 ``system_prompt``；槽不写即不传
+  该字段，落 ``AgentJudge`` 自身缺省（``isolated`` 缺省 False 与宿主
+  对齐，组件层不自造缺省）；槽值非法（如 agent 不在注册表内）按
+  unjudged 处置；
+- agent 覆盖链：``context.judge``（CLI --judge）> ``REWARDKIT_AGENT``
+  环境变量 > ``REWARDKIT_JUDGE``（**仅当值在 known_agents 注册表内才
+  采用**——该变量常被设成 LLM 模型名）> args ``agent`` > 组件缺省
+  ``codex``；agent 名的注册表校验由 AgentJudge 的 validator 兜底（非法
+  名显式报错，错误消息含已知 agent 列表）；
+- model 覆盖链：``context.model_name`` > ``REWARDKIT_MODEL`` 环境变量 >
+  args ``model`` > None（AgentJudge 自身缺省）；
+- ``cwd`` 槽缺省时由 workspace 根兜底（``arun_agent`` 自身语义：
+  ``judge.cwd or workspace``）；
+- 产出 ``{"scores": [{"id", "score", "reasoning", "warnings", "weight",
+  "optional"}, ...], "output", "warnings", "judge_logs"}``（信封语义同
+  ``llm_judge@1``）。agentic 没有门控职责，判 0 分也是有效判定，**调用
+  异常不由组件转 Failure**（冒泡为任务失败，由归因层收集）；
+  ``criteria`` 槽非法 → unjudged。
+
+真实 Codex 执行的验证在 AP 题包与原生格式端到端验证中走过；仓库内
+测试用假 backend 打桩。
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from rewardkit.judges import arun_agent
+from rewardkit.models import AgentJudge
+from rewardkit.workflow.component._judge_args import (
+    parse_criteria_entries,
+    resolve_agent,
+    resolve_model,
+)
+from rewardkit.workflow.component.registry import InputSlot, component
+
+AGENT_NAME_ENV = "REWARDKIT_AGENT"
+DEFAULT_AGENT = "codex"
+
+
+@component(
+    name="agentic",
+    version=1,
+    outputs=["verdict"],
+    inputs=[
+        InputSlot(
+            name="criteria",
+            accepted_types=("array",),
+            description="评分项表的数组，元素字段对齐存量 [[criterion]] 表",
+        ),
+        InputSlot(name="agent", required=False, accepted_types=("string",)),
+        InputSlot(name="model", required=False, accepted_types=("string",)),
+        InputSlot(name="version", required=False, accepted_types=("string",)),
+        InputSlot(name="timeout", required=False, accepted_types=("int",)),
+        InputSlot(name="cwd", required=False, accepted_types=("string",)),
+        InputSlot(name="isolated", required=False, accepted_types=("bool",)),
+        InputSlot(name="atif_trajectory", required=False, accepted_types=("string",)),
+        InputSlot(name="mode", required=False, accepted_types=("string",)),
+        InputSlot(name="mcp_servers", required=False, accepted_types=("array",)),
+        InputSlot(name="system_prompt", required=False, accepted_types=("string",)),
+    ],
+)
+async def agentic(
+    context,
+    criteria,
+    agent=None,
+    model=None,
+    version=None,
+    timeout=None,
+    cwd=None,
+    isolated=None,
+    atif_trajectory=None,
+    mode=None,
+    mcp_servers=None,
+    system_prompt=None,
+) -> dict[str, Any]:
+    """``arun_agent`` 的薄封装：评分项与调用配置全来自 args（见模块 docstring）。"""
+    root = context.workspace.root
+    criteria_objs, weights, reason = parse_criteria_entries(criteria, root)
+    if reason:
+        return {"unjudged": True, "reason": reason}
+
+    judge_kwargs: dict[str, Any] = {}
+    for key, value in (
+        ("version", version),
+        ("timeout", timeout),
+        ("cwd", cwd),
+        ("isolated", None if isolated is None else bool(isolated)),
+        ("atif_trajectory", atif_trajectory),
+        ("mode", mode),
+        ("mcp_servers", tuple(mcp_servers) if mcp_servers else None),
+    ):
+        if value is not None:
+            judge_kwargs[key] = value
+    try:
+        judge = AgentJudge(
+            agent=resolve_agent(context, agent, agent_env=AGENT_NAME_ENV, default=DEFAULT_AGENT),
+            model=resolve_model(context, model),
+            **judge_kwargs,
+        )
+    except Exception as exc:  # noqa: BLE001 - 槽值非法属配置问题,按 unjudged
+        return {"unjudged": True, "reason": f"agentic 槽值非法：{exc}"}
+
+    result = await arun_agent(
+        judge,
+        criteria_objs,
+        weights,
+        workspace=str(root),
+        system_prompt=system_prompt,
+    )
+    all_scores = [
+        {
+            "id": c.id or score.name or f"criterion_{index}",
+            "score": score.value,
+            "reasoning": score.reasoning,
+            "warnings": list(result.warnings),
+            "weight": weights[index],
+            "optional": c.optional,
+        }
+        for index, (c, score) in enumerate(zip(criteria_objs, result.scores))
+    ]
+    return {
+        "scores": all_scores,
+        "output": result.output,
+        "warnings": list(result.warnings),
+        "judge_logs": list(result.judge_logs),
+    }

tf_rewardkit-1.6.4.dev0/src/rewardkit/workflow/component/llm_judge.py

@@ -0,0 +1,145 @@
+"""判分组件 ``llm_judge@1``：LLM judge 的图内入口（args 传参薄封装）。
+
+不读任何清单文件：评分项与调用配置全部由图 ``args`` 传入，一个 node
+一次 ``arun_llm`` 调用。
+
+- ``criteria`` 必填槽为评分项表的数组（元素字段对齐存量 judge TOML 的
+  ``[[criterion]]`` 表：``description`` 必填，``type`` / ``points`` /
+  ``levels`` / ``min`` / ``max`` / ``files`` / ``negate`` / ``optional`` /
+  ``name`` / ``id`` / ``weight`` 选填、缺省同存量配置）；条目 ``weight``
+  透传为 ``arun_llm`` 的 weights 入参与分数信封的 ``weight`` 键；
+- 其余选填槽与 ``LLMJudge`` 字段逐字同名（``model`` / ``files`` /
+  ``mode`` / ``timeout`` / ``reasoning_effort`` / ``reference`` /
+  ``atif_trajectory``），另加 ``system_prompt``；槽不写即不传该字段，
+  落 ``LLMJudge`` 自身缺省（mode=batched、timeout=300 等）；槽值非法
+  （如 mode 不在枚举内）按 unjudged 处置；
+- model 覆盖链：``context.model_name``（CLI --model）> ``REWARDKIT_MODEL``
+  环境变量 > ``REWARDKIT_JUDGE`` > args ``model`` > 组件缺省；
+- ``files`` 槽为组级共享（单路径字符串或路径列表），相对路径一律按
+  workspace 根解析（``arun_llm`` 直读路径相对进程 CWD，而 workspace 是
+  独立注入的）；条目级 ``files`` 仅 individual 模式生效（存量 arun_llm
+  语义）；batched 下被忽略——组件在该条目的 warnings 里记录提示，不
+  静默丢失；
+- 产出 ``{"scores": [{"id", "score", "reasoning", "warnings", "weight",
+  "optional"}, ...], "output", "warnings"}``（``score`` 键是交分信封
+  ``value`` 的别名，可直接标记 ``is_score_node``；``optional`` 进图级
+  聚合）；任一次调用异常 → Failure（error_kind 为 llm-error）；
+  ``criteria`` 槽非法 → unjudged。
+
+模型调用经 ``context.model``（ModelService）的 ``acompletion`` 注入
+arun_llm——占 ``max_concurrent_workflow`` 名额；注入的替身无
+acompletion 时回退 litellm 直调。
+
+本组件不 import prefect（分层约束）。
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from rewardkit.judges import arun_llm
+from rewardkit.models import LLMJudge
+from rewardkit.workflow.component._judge_args import (
+    JUDGE_NAME_ENV,
+    parse_criteria_entries,
+    resolve_arg_files,
+    resolve_model,
+)
+from rewardkit.workflow.component.registry import InputSlot, component
+from rewardkit.workflow.errors import Failure
+
+DEFAULT_JUDGE_MODEL = "anthropic/claude-sonnet-5"
+"""judge 模型缺省（与 JudgeConfig.judge 的缺省一致，作文档锚点）。"""
+
+
+@component(
+    name="llm_judge",
+    version=1,
+    outputs=["verdict"],
+    inputs=[
+        InputSlot(
+            name="criteria",
+            accepted_types=("array",),
+            description="评分项表的数组，元素字段对齐存量 [[criterion]] 表",
+        ),
+        InputSlot(name="model", required=False, accepted_types=("string",)),
+        InputSlot(name="files", required=False, accepted_types=("string", "array")),
+        InputSlot(name="mode", required=False, accepted_types=("string",)),
+        InputSlot(name="timeout", required=False, accepted_types=("int",)),
+        InputSlot(name="reasoning_effort", required=False, accepted_types=("string",)),
+        InputSlot(name="reference", required=False, accepted_types=("string",)),
+        InputSlot(name="atif_trajectory", required=False, accepted_types=("string",)),
+        InputSlot(name="system_prompt", required=False, accepted_types=("string",)),
+    ],
+)
+async def llm_judge(
+    context,
+    criteria,
+    model=None,
+    files=None,
+    mode=None,
+    timeout=None,
+    reasoning_effort=None,
+    reference=None,
+    atif_trajectory=None,
+    system_prompt=None,
+) -> Any:
+    """``arun_llm`` 的薄封装：评分项与调用配置全来自 args（见模块 docstring）。"""
+    root = context.workspace.root
+    criteria_objs, weights, reason = parse_criteria_entries(criteria, root)
+    if reason:
+        return {"unjudged": True, "reason": reason}
+
+    judge_kwargs: dict[str, Any] = {}
+    for key, value in (
+        ("mode", mode),
+        ("timeout", timeout),
+        ("reasoning_effort", reasoning_effort),
+        ("reference", reference),
+        ("atif_trajectory", atif_trajectory),
+    ):
+        if value is not None:
+            judge_kwargs[key] = value
+    try:
+        judge = LLMJudge(
+            model=resolve_model(context, model, extra_envs=(JUDGE_NAME_ENV,), default=DEFAULT_JUDGE_MODEL),
+            files=resolve_arg_files(files, root),
+            **judge_kwargs,
+        )
+    except Exception as exc:  # noqa: BLE001 - 槽值非法属配置问题,按 unjudged
+        return {"unjudged": True, "reason": f"llm_judge 槽值非法：{exc}"}
+
+    per_item_warns: list[list[str]] = []
+    for c in criteria_objs:
+        warns_for_item: list[str] = []
+        if judge.mode == "batched" and c.files:
+            warns_for_item.append(
+                f"mode=batched：条目 {c.id or c.name} 的 files 被忽略"
+                "（组级 files 生效；条目级 files 需 mode=individual）"
+            )
+        per_item_warns.append(warns_for_item)
+
+    completion = getattr(context.model, "acompletion", None)
+    try:
+        scores, output, warns = await arun_llm(
+            judge, criteria_objs, weights, system_prompt, completion=completion
+        )
+    except Exception as exc:  # noqa: BLE001 - 判分调用失败 = 节点失败
+        return Failure(
+            root_cause_node_id=context.node_id,
+            message=f"llm_judge 调用失败：{exc}",
+            error_kind="llm-error",
+            chain=(context.node_id,),
+        )
+    all_scores = [
+        {
+            "id": c.id or score.name or f"criterion_{index}",
+            "score": score.value,
+            "reasoning": score.reasoning,
+            "warnings": list(warns) + per_item_warns[index],
+            "weight": weights[index],
+            "optional": c.optional,
+        }
+        for index, (c, score) in enumerate(zip(criteria_objs, scores))
+    ]
+    return {"scores": all_scores, "output": output, "warnings": list(warns)}

{tf_rewardkit-1.6.3 → tf_rewardkit-1.6.4.dev0}/src/rewardkit/workflow/executor.py

@@ -56,7 +56,6 @@ from rewardkit.workflow.attribution import (
     is_unjudged,
     make_unjudged,
 )
-from rewardkit.workflow.component._source import clear_source_cache
 from rewardkit.workflow.component.registry import (
     ComponentSpec,
     ensure_builtin_components,
@@ -503,9 +502,6 @@ def run(
        并参与并发上限的优先级合并。
     """
     ensure_builtin_components()
-    # 每次 run() 取数据源的新鲜快照：清空判分组件共享的 tests/*.toml
-    # 解析缓存，使进程内缓存不跨运行泄漏。
-    clear_source_cache()
     rc = dict(run_context or {})
     workflow_limit, tag_limits = _effective_concurrency(plan, rc)
     if model is None: