agent-tune-kit 2.1.1__tar.gz → 2.2.0__tar.gz

{agent_tune_kit-2.1.1 → agent_tune_kit-2.2.0}/.codex-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "agent-tune-kit",
-  "version": "2.1.1",
+  "version": "2.2.0",
   "description": "Local Codex plugin for iterative Agent tuning with guided Skills, reusable runner templates, versioned results, and static validation.",
   "author": {
     "name": "hustyichi",

{agent_tune_kit-2.1.1 → agent_tune_kit-2.2.0}/.gitignore

@@ -36,4 +36,5 @@ node_modules/
 .pnpm-store/
 
 # Agent tune kit generated outputs
-.atk/
+.atk/
+!templates/.atk/

{agent_tune_kit-2.1.1 → agent_tune_kit-2.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: agent-tune-kit
-Version: 2.1.1
+Version: 2.2.0
 Summary: Local Codex plugin for iterative Agent tuning with guided Skills, reusable runner templates, versioned results, and static validation.
 Project-URL: Homepage, https://github.com/hustyichi/agent-tune-kit
 Project-URL: Repository, https://github.com/hustyichi/agent-tune-kit
@@ -14,6 +14,12 @@ Description-Content-Type: text/markdown
 
 # Agent Tune Kit
 
+Ground-truth reviews, judge calibration and assessment provenance are documented in
+[the trust contract](docs/truth-and-assessment.md). Existing datasets remain readable, not automatically certified.
+`atk dataset inspect/update` supports content-bound confirmations; `atk results calibrate` checks reviewed outputs
+without executing the Agent or allocating an evaluation version. Rejudgments preserve history and comparisons expose
+changes to their criteria and judge.
+
 English | [简体中文](README.md)
 
 [![PyPI](https://img.shields.io/pypi/v/agent-tune-kit.svg)](https://pypi.org/project/agent-tune-kit/)
@@ -207,4 +213,25 @@ atk visualize dataset --no-open
 atk visualize failures --no-open
 ```
 
-These commands run from the Agent project. New project adapters delegate to the shared engine; existing full runners remain executable. Dataset fields, IDs, result paths, unknown verdicts, logs and offline review pages retain their contracts. Use `--help` on each command for its options.
+These commands run from the Agent project. New project adapters delegate to the shared engine; legacy full runners require migration before writing alongside new evidence Runs. Dataset fields, IDs, result paths, unknown verdicts, logs and offline review pages retain their contracts. Use `--help` on each command for its options.
+
+
+## 本地执行证据 / Local execution evidence
+
+V2 adapters declare `ATK_ADAPTER_VERSION = 2` and `EVIDENCE.retain(role, value)`.
+Each execution saves independent input/output, context, structured events and artifacts beneath
+`.atk/results/vN/executions/<id>/`; shared Cases map multiple selected samples to one directory.
+Retention runs before persistence. Existing V1 adapters remain explicit; no historical migration is automatic.
+
+```sh
+atk results inspect --json
+atk results evidence --version v1 --atk-id 1 --json
+atk results evidence --execution-dir ./copied-execution --json
+atk results import-evidence --source ./native-run --adapter ./importer.py --json
+```
+
+`inspect` separates latest Run from the evaluation target. Evidence-only imports do not displace the latter.
+Diagnosis and reports bind `--version`; changed evidence invalidates dependent judgments. Import adapters are
+trusted Python code, not a sandbox. No Agent call, remote fetch, automatic cleanup or post-seal supplementation
+is performed. Declared capture coverage is distinct from successful persistence and business verdicts.
+See [execution evidence plan](docs/execution-evidence-and-tracing-plan.md) for interfaces, source limitations, recovery and validation.

{agent_tune_kit-2.1.1 → agent_tune_kit-2.2.0}/README.en.md

@@ -1,5 +1,11 @@
 # Agent Tune Kit
 
+Ground-truth reviews, judge calibration and assessment provenance are documented in
+[the trust contract](docs/truth-and-assessment.md). Existing datasets remain readable, not automatically certified.
+`atk dataset inspect/update` supports content-bound confirmations; `atk results calibrate` checks reviewed outputs
+without executing the Agent or allocating an evaluation version. Rejudgments preserve history and comparisons expose
+changes to their criteria and judge.
+
 English | [简体中文](README.md)
 
 [![PyPI](https://img.shields.io/pypi/v/agent-tune-kit.svg)](https://pypi.org/project/agent-tune-kit/)
@@ -193,4 +199,25 @@ atk visualize dataset --no-open
 atk visualize failures --no-open
 ```
 
-These commands run from the Agent project. New project adapters delegate to the shared engine; existing full runners remain executable. Dataset fields, IDs, result paths, unknown verdicts, logs and offline review pages retain their contracts. Use `--help` on each command for its options.
+These commands run from the Agent project. New project adapters delegate to the shared engine; legacy full runners require migration before writing alongside new evidence Runs. Dataset fields, IDs, result paths, unknown verdicts, logs and offline review pages retain their contracts. Use `--help` on each command for its options.
+
+
+## 本地执行证据 / Local execution evidence
+
+V2 adapters declare `ATK_ADAPTER_VERSION = 2` and `EVIDENCE.retain(role, value)`.
+Each execution saves independent input/output, context, structured events and artifacts beneath
+`.atk/results/vN/executions/<id>/`; shared Cases map multiple selected samples to one directory.
+Retention runs before persistence. Existing V1 adapters remain explicit; no historical migration is automatic.
+
+```sh
+atk results inspect --json
+atk results evidence --version v1 --atk-id 1 --json
+atk results evidence --execution-dir ./copied-execution --json
+atk results import-evidence --source ./native-run --adapter ./importer.py --json
+```
+
+`inspect` separates latest Run from the evaluation target. Evidence-only imports do not displace the latter.
+Diagnosis and reports bind `--version`; changed evidence invalidates dependent judgments. Import adapters are
+trusted Python code, not a sandbox. No Agent call, remote fetch, automatic cleanup or post-seal supplementation
+is performed. Declared capture coverage is distinct from successful persistence and business verdicts.
+See [execution evidence plan](docs/execution-evidence-and-tracing-plan.md) for interfaces, source limitations, recovery and validation.

{agent_tune_kit-2.1.1 → agent_tune_kit-2.2.0}/README.md

@@ -1,5 +1,10 @@
 # Agent Tune Kit
 
+真值复核、裁判校准和判定追溯见[可信度契约](docs/truth-and-assessment.md)：数据集可按答案字段记录来源和
+有效确认，输入/答案/标准变化会使确认过期；`atk dataset update --review-path dataset_review.csv` 可只导入
+确认而不改答案。`atk results calibrate` 用已复核输出检查裁判，不调用 Agent 或创建评测轮次。
+判定可绑定标准与裁判快照，重判保存历史，比较显式列出可比性差异。旧数据可读，但不自动标为已确认。
+
 简体中文 | [English](README.en.md)
 
 [![PyPI](https://img.shields.io/pypi/v/agent-tune-kit.svg)](https://pypi.org/project/agent-tune-kit/)
@@ -176,7 +181,7 @@ ATK 可选读取本地私有的 `.atk/context.md`，用于保存用户沟通后
 ## 能力评估
 
 - [Ground truth 与评分方案支持评估（2026-10-25）](docs/ground-truth-and-grading-assessment-2026-10-25.md)：基于当前源码与 44 项相关测试，梳理数据准备、人工校准和判定支持，区分小规模调优可用性与可信评分体系的缺口；未重新验收业务调优效果。
-- [执行现场与调用轨迹增强方案（2026-10-25）](docs/execution-evidence-and-tracing-plan.md)：基于模块化核心，设计完整同进程采集、运行快照、日志与调用接入、证据读取及兼容迁移；范围已确认，尚未实现。
+- [执行项现场与调用轨迹改造实现方案（2026-10-26）](docs/execution-evidence-and-tracing-plan.md)：明确每项独立现场、V2 托管采集及外部 runner 本地证据导入，结合 Magic 双 Case 实践补齐来源身份、文件变更与验收；首版不做封存后补采，按 P0–P5 实施，详见方案中的验收状态。
 
 兼容一次性安装命令 `uvx --from agent-tune-kit atk install`；日常使用 Skill 需要 PATH 中持续可用的 `atk` 命令。
 
@@ -198,4 +203,25 @@ atk visualize dataset --no-open
 atk visualize failures --no-open
 ```
 
-新生成的项目适配文件使用统一执行引擎，已有完整 runner 保留兼容执行路径。数据字段、ID、结果目录、未判定状态、日志和离线浏览页遵循原有约定。各命令参数见对应 `--help`。
+新生成的项目适配文件使用统一执行引擎，旧完整 runner 与新证据现场共存时需先满足写入保护，否则启动前提示迁移。数据字段、ID、结果目录、未判定状态、日志和离线浏览页遵循原有约定。各命令参数见对应 `--help`。
+
+
+## 本地执行证据 / Local execution evidence
+
+V2 adapters declare `ATK_ADAPTER_VERSION = 2` and `EVIDENCE.retain(role, value)`.
+Each execution saves independent input/output, context, structured events and artifacts beneath
+`.atk/results/vN/executions/<id>/`; shared Cases map multiple selected samples to one directory.
+Retention runs before persistence. Existing V1 adapters remain explicit; no historical migration is automatic.
+
+```sh
+atk results inspect --json
+atk results evidence --version v1 --atk-id 1 --json
+atk results evidence --execution-dir ./copied-execution --json
+atk results import-evidence --source ./native-run --adapter ./importer.py --json
+```
+
+`inspect` separates latest Run from the evaluation target. Evidence-only imports do not displace the latter.
+Diagnosis and reports bind `--version`; changed evidence invalidates dependent judgments. Import adapters are
+trusted Python code, not a sandbox. No Agent call, remote fetch, automatic cleanup or post-seal supplementation
+is performed. Declared capture coverage is distinct from successful persistence and business verdicts.
+See [execution evidence plan](docs/execution-evidence-and-tracing-plan.md) for interfaces, source limitations, recovery and validation.

{agent_tune_kit-2.1.1 → agent_tune_kit-2.2.0}/agent_tune_kit/__init__.py

@@ -1,3 +1,3 @@
 """Agent Tune Kit package."""
 
-__version__ = "2.1.1"
+__version__ = "2.2.0"

{agent_tune_kit-2.1.1 → agent_tune_kit-2.2.0}/agent_tune_kit/cli/dataset.py

@@ -6,6 +6,7 @@ from agent_tune_kit.workspace import DATASET_PATH
 def add_options(parser):
     parser.add_argument("--dataset-path", default=str(DATASET_PATH))
     parser.add_argument("--field", default="ground_truth")
+    parser.add_argument("--input-fields", nargs="+")
     parser.add_argument("--home-dir")
     mode = parser.add_mutually_exclusive_group()
     mode.add_argument("--review-path")
@@ -27,7 +28,7 @@ def register(subparsers):
     add_options(commands.add_parser("inspect"))
     update = commands.add_parser("update")
     add_options(update)
-    update.add_argument("--updates-path", required=True)
+    update.add_argument("--updates-path")
 
 
 def execute(args):
@@ -39,6 +40,7 @@ def execute(args):
         review_path=args.review_path,
         field=args.field,
         home_dir=args.home_dir,
+        input_fields=args.input_fields,
     )
     if args.action == "inspect":
         return datasets.inspect_dataset(**options)
@@ -61,6 +63,7 @@ def legacy_main(argv=None):
         review_path=args.review_path,
         field=args.field,
         home_dir=args.home_dir,
+        input_fields=args.input_fields,
     )
     try:
         if args.dump_context or not args.updates_path:

agent_tune_kit-2.2.0/agent_tune_kit/cli/results.py

@@ -0,0 +1,99 @@
+from agent_tune_kit import results
+from agent_tune_kit.execution import run_rule
+
+
+def register(subparsers):
+    commands = subparsers.add_parser("results", help="inspect and judge versioned results").add_subparsers(
+        dest="action", required=True
+    )
+    calibration = commands.add_parser(
+        "calibrate", help="check a judge against reviewed outputs without running the Agent"
+    )
+    calibration.add_argument("--samples-path", required=True)
+    calibration.add_argument("--output-path", required=True)
+    mode = calibration.add_mutually_exclusive_group(required=True)
+    mode.add_argument("--rule-path")
+    mode.add_argument("--judgments-path")
+    calibration.add_argument("--judge-id")
+    calibration.add_argument("--instructions")
+    calibration.add_argument("--python")
+    calibration.add_argument("--json", action="store_true")
+    for name in ["inspect", "run-rule", "apply-judgments", "compare", "evidence", "import-evidence"]:
+        parser = commands.add_parser(name)
+        parser.add_argument("--results-dir", default=".atk/results")
+        parser.add_argument("--json", action="store_true")
+        if name != "import-evidence":
+            parser.add_argument("--version")
+        if name == "run-rule":
+            parser.add_argument("--rule-path", default=".atk/runner/failure_rule.py")
+            parser.add_argument("--python")
+        elif name == "apply-judgments":
+            parser.add_argument("--judgments-path", required=True)
+            parser.add_argument("--source-results")
+        elif name == "compare":
+            parser.add_argument("--ids", nargs="+", required=True)
+        elif name == "import-evidence":
+            parser.add_argument("--source", required=True)
+            parser.add_argument("--adapter", required=True, help="explicit trusted Python adapter (not sandboxed)")
+        elif name == "evidence":
+            parser.add_argument("--execution-dir")
+            parser.add_argument("--atk-id")
+            parser.add_argument("--execution-id")
+            parser.add_argument("--event-id")
+            parser.add_argument("--artifact-id")
+            parser.add_argument("--after-seq", type=int)
+            parser.add_argument("--offset", type=int, default=0)
+            parser.add_argument("--limit", type=int)
+
+
+def execute(args):
+    if args.action == "calibrate":
+        from agent_tune_kit.execution.calibration import launch_calibration
+
+        return launch_calibration(
+            **{
+                key: getattr(args, key)
+                for key in (
+                    "samples_path",
+                    "output_path",
+                    "rule_path",
+                    "judgments_path",
+                    "judge_id",
+                    "instructions",
+                    "python",
+                )
+            }
+        )
+    if args.action == "import-evidence":
+        from agent_tune_kit.results.evidence import import_evidence
+
+        return import_evidence(args.source, args.adapter, args.results_dir)
+    if args.action == "evidence":
+        from agent_tune_kit.results.evidence import query_evidence
+
+        return query_evidence(
+            **{
+                key: getattr(args, key)
+                for key in [
+                    "results_dir",
+                    "version",
+                    "execution_dir",
+                    "atk_id",
+                    "execution_id",
+                    "event_id",
+                    "artifact_id",
+                    "after_seq",
+                    "offset",
+                    "limit",
+                ]
+            }
+        )
+    if args.action == "inspect":
+        return results.inspect_results(args.results_dir, version=args.version)
+    if args.action == "compare":
+        return results.compare_targets(args.ids, args.results_dir, version=args.version)
+    if args.action == "apply-judgments":
+        return results.apply_judgments(
+            args.judgments_path, source_results=args.source_results, results_dir=args.results_dir, version=args.version
+        )
+    return run_rule(rule_path=args.rule_path, results_dir=args.results_dir, python=args.python, version=args.version)

{agent_tune_kit-2.1.1 → agent_tune_kit-2.2.0}/agent_tune_kit/cli/visualize.py

@@ -16,6 +16,7 @@ def add_options(parser, kind):
         parser.add_argument("--dataset-path", default=".atk/datasets/dataset.csv")
     else:
         parser.add_argument("--results-dir", default=".atk/results")
+        parser.add_argument("--version")
         parser.add_argument("--no-report", action="store_true")
         parser.add_argument("--no-history", action="store_true")
 
@@ -32,7 +33,9 @@ def execute(args):
     options = dict(overwrite=args.overwrite, open_browser=args.open_browser)
     if args.action == "dataset":
         return generate_dataset(args.dataset_path, **options)
-    return generate_failures(args.results_dir, no_report=args.no_report, no_history=args.no_history, **options)
+    return generate_failures(
+        args.results_dir, no_report=args.no_report, no_history=args.no_history, version=args.version, **options
+    )
 
 
 def legacy_main(kind, argv=None):

{agent_tune_kit-2.1.1 → agent_tune_kit-2.2.0}/agent_tune_kit/datasets/prepare.py

@@ -4,6 +4,7 @@ from agent_tune_kit.errors import UserActionRequired
 from agent_tune_kit.utils.csv_io import parse_csv, write_csv_atomic
 from agent_tune_kit.workspace import DATASET_PATH
 
+from .truth import TRUTH_FIELD
 from .validation import validate_dataset_rows
 
 
@@ -15,8 +16,10 @@ def prepare_dataset(source, dataset_path=DATASET_PATH, *, append=False, overwrit
         raise UserActionRequired("Append requires an existing canonical dataset.")
     if append:
         validate_dataset_rows(old_fields, old_rows)
-        if set(fields) - {"atk_id"} != set(old_fields) - {"atk_id"}:
+        if set(fields) - {"atk_id", TRUTH_FIELD} != set(old_fields) - {"atk_id", TRUTH_FIELD}:
             raise UserActionRequired("Append fields differ; confirm the field mapping before appending.")
+        if TRUTH_FIELD in fields and TRUTH_FIELD not in old_fields:
+            old_fields = old_fields + [TRUTH_FIELD]
     if "atk_id" not in fields:
         fields = fields + ["atk_id"]
         start = max((int(row["atk_id"]) for row in old_rows), default=0) + 1 if append else 1

agent_tune_kit-2.2.0/agent_tune_kit/datasets/truth.py

@@ -0,0 +1,179 @@
+"""Content-bound review declarations; integrity never implies domain correctness."""
+
+import hashlib
+import json
+from datetime import UTC, datetime
+from pathlib import Path
+
+from agent_tune_kit.errors import UserActionRequired
+
+TRUTH_FIELD = "atk_truth"
+REVIEW_STATES = {"pending", "confirmed", "disputed"}
+STANDARD_SECTIONS = {"Agent Behavior Standard", "Ground Truth Standard"}
+INPUT_CANDIDATES = ("input", "query", "question", "prompt", "task", "user_input", "instruction")
+
+
+def digest(value):
+    return hashlib.sha256(
+        json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode()
+    ).hexdigest()
+
+
+def standard_snapshot(path):
+    path = Path(path)
+    text = path.read_text(encoding="utf-8") if path.is_file() else ""
+    if path.name == "context.md":
+        lines, active = [], False
+        for line in text.splitlines():
+            if line.startswith("## "):
+                active = line[3:].strip() in STANDARD_SECTIONS
+            if active:
+                lines.append(line.rstrip())
+        text = "\n".join(lines).strip()
+    return {"text": text, "sha256": digest(text)}
+
+
+def validate_inputs(row, field, inputs):
+    if field not in row or field in {TRUTH_FIELD, "atk_id"}:
+        raise UserActionRequired(f"Unknown or reserved answer field: {field}")
+    if (
+        not isinstance(inputs, list)
+        or not inputs
+        or any(not isinstance(k, str) for k in inputs)
+        or len(set(inputs)) != len(inputs)
+        or any(k not in row or k in {field, TRUTH_FIELD, "atk_id"} for k in inputs)
+    ):
+        raise UserActionRequired(
+            "Review requires explicit, unique business input fields, excluding answer and ATK metadata."
+        )
+
+
+def subject_digest(row, field, inputs, standard, *, source="", evidence=""):
+    validate_inputs(row, field, inputs)
+    return digest(
+        {
+            "atk_id": row.get("atk_id"),
+            "field": field,
+            "answer": row[field],
+            "inputs": {k: row[k] for k in inputs},
+            "standard": standard["sha256"],
+            "source": source,
+            "evidence": evidence,
+        }
+    )
+
+
+def truth_records(row):
+    raw = row.get(TRUTH_FIELD, "")
+    if not raw:
+        return {}
+    try:
+        data = json.loads(raw)
+        if data.get("schema") != 1 or not isinstance(data.get("fields"), dict):
+            raise ValueError("Unknown truth schema")
+        for field, record in data["fields"].items():
+            validate_inputs(row, field, record.get("input_fields"))
+            standard = record["standard"]
+            if (
+                record["status"] not in REVIEW_STATES
+                or not isinstance(standard["text"], str)
+                or standard["sha256"] != digest(standard["text"])
+                or not isinstance(record["subject_sha256"], str)
+                or len(record["subject_sha256"]) != 64
+            ):
+                raise ValueError("Invalid review binding")
+            if record["status"] == "confirmed" and any(
+                not isinstance(record.get(k), str) or not record[k].strip()
+                for k in ("source", "evidence", "reviewer", "reviewed_at")
+            ):
+                raise ValueError("Confirmed reviews require source, evidence, reviewer and time")
+        return data["fields"]
+    except (ValueError, TypeError, AttributeError, KeyError) as exc:
+        raise UserActionRequired(f"Invalid atk_truth for {row.get('atk_id')}: {exc}") from exc
+
+
+def truth_status(row, field, *, standard=None):
+    record = truth_records(row).get(field)
+    if record is None:
+        return {"field": field, "status": "unreviewed", "reason": "No review recorded"}
+    current = standard if standard is not None else record["standard"]
+    stale = (
+        subject_digest(
+            row,
+            field,
+            record["input_fields"],
+            current,
+            source=record.get("source", ""),
+            evidence=record.get("evidence", ""),
+        )
+        != record["subject_sha256"]
+    )
+    return {
+        "field": field,
+        "status": "stale" if stale else record["status"],
+        "reason": "Input, answer or relevant standard changed" if stale else "Recorded review declaration",
+        "record": record,
+    }
+
+
+def review_context(path, rows, field, input_fields=None):
+    path = Path(path)
+    standard = standard_snapshot(path.parent.parent / "context.md")
+    inputs = input_fields or next(([k] for k in INPUT_CANDIDATES if rows and k in rows[0]), [])
+    truth = []
+    for row in rows:
+        state = truth_status(row, field, standard=standard)
+        state["atk_id"] = row.get("atk_id", "")
+        state["input_fields"] = input_fields or state.get("record", {}).get("input_fields", inputs)
+        truth.append(state)
+    return {
+        "dataset_sha256": hashlib.sha256(path.read_bytes()).hexdigest(),
+        "dataset_scope": digest(str(path.resolve())),
+        "standard": standard,
+        "truth": truth,
+        "truth_counts": {
+            s: sum(r["status"] == s for r in truth) for s in ("unreviewed", "pending", "confirmed", "disputed", "stale")
+        },
+    }
+
+
+def validate_review_binding(review, context):
+    for key in ("dataset_sha256", "dataset_scope"):
+        if review.get(key) != context[key]:
+            raise UserActionRequired("Review is stale or belongs to another dataset; inspect the current data again.")
+    if review.get("standard_sha256") != context["standard"]["sha256"]:
+        raise UserActionRequired("Review standard changed; inspect the current standard again.")
+
+
+def apply_review(row, review, standard, *, answer_changed=False):
+    field = review.get("field", "")
+    try:
+        inputs = json.loads(review.get("input_fields", "[]"))
+    except (ValueError, TypeError) as exc:
+        raise UserActionRequired("input_fields must be a JSON array.") from exc
+    validate_inputs(row, field, inputs)
+    status = review.get("review_status")
+    if status not in REVIEW_STATES:
+        raise UserActionRequired("review_status must be pending, confirmed or disputed.")
+    if status == "confirmed" and (
+        not row[field].strip() or any(not review.get(k, "").strip() for k in ("source", "evidence", "reviewer"))
+    ):
+        raise UserActionRequired("Confirmation requires a non-empty answer, source, evidence and reviewer.")
+    records = truth_records(row)
+    record = {
+        "status": "pending" if answer_changed else status,
+        "input_fields": inputs,
+        "source": review.get("source", ""),
+        "evidence": review.get("evidence", ""),
+        "reviewer": review.get("reviewer", ""),
+        "standard": standard,
+        "subject_sha256": subject_digest(
+            row, field, inputs, standard, source=review.get("source", ""), evidence=review.get("evidence", "")
+        ),
+    }
+    old = records.get(field, {})
+    # Reapplying the same declaration does not manufacture a new review event.
+    record["reviewed_at"] = old.get("reviewed_at") if all(old.get(k) == v for k, v in record.items()) else None
+    record["reviewed_at"] = record["reviewed_at"] or datetime.now(UTC).isoformat()
+    records[field] = record
+    row[TRUTH_FIELD] = json.dumps({"schema": 1, "fields": records}, ensure_ascii=False, sort_keys=True)

{agent_tune_kit-2.1.1 → agent_tune_kit-2.2.0}/agent_tune_kit/datasets/updates.py

@@ -6,6 +6,7 @@ from agent_tune_kit.utils.csv_io import parse_csv
 from agent_tune_kit.utils.csv_io import write_csv_atomic as write_dataset_atomic
 from agent_tune_kit.workspace import DATASET_PATH
 
+from .truth import TRUTH_FIELD, apply_review, review_context, validate_review_binding
 from .validation import find_duplicates, parse_dataset_csv
 
 REVIEW_FILENAME = "dataset_review.csv"
@@ -18,7 +19,7 @@ def parse_review_csv(path: Path, dataset_rows: list[dict[str, str]]) -> tuple[li
     missing = [field for field in REVIEW_REQUIRED_FIELDS if field not in fieldnames]
     if missing:
         raise UserActionRequired(f"{path.as_posix()} is missing required review fields: {', '.join(missing)}")
-    review_rows = [row for row in rows if (row.get("review_feedback") or "").strip()]
+    review_rows = [row for row in rows if (row.get("review_feedback") or "").strip() or row.get("review_status")]
     if not review_rows:
         raise UserActionRequired(f"{path.as_posix()} contains no non-empty review_feedback values.")
     dataset_ids = {row["atk_id"] for row in dataset_rows}
@@ -27,9 +28,9 @@ def parse_review_csv(path: Path, dataset_rows: list[dict[str, str]]) -> tuple[li
         raise UserActionRequired(
             f"{path.as_posix()} contains review atk_id values absent from dataset.csv: {', '.join(unknown)}"
         )
-    duplicates = find_duplicates((row.get("atk_id") or "").strip() for row in review_rows)
+    duplicates = find_duplicates((row.get("atk_id", ""), row.get("field", "")) for row in review_rows)
     if duplicates:
-        raise UserActionRequired(f"{path.as_posix()} contains duplicate review atk_id values: {', '.join(duplicates)}")
+        raise UserActionRequired(f"{path.as_posix()} contains duplicate review targets: {duplicates}")
     return fieldnames, review_rows
 
 
@@ -82,7 +83,11 @@ def parse_updates_csv(
     if not updates:
         raise UserActionRequired(f"{path.as_posix()} contains no non-empty {field} updates.")
     dataset_ids = {row["atk_id"] for row in dataset_rows}
-    review_ids = {row["atk_id"] for row in review_rows} if review_rows is not None else dataset_ids
+    review_ids = (
+        {row["atk_id"] for row in review_rows if not row.get("field") or row["field"] == field}
+        if review_rows is not None
+        else dataset_ids
+    )
     unknown = sorted({row["atk_id"] for row in updates if row["atk_id"] not in dataset_ids})
     if unknown:
         raise UserActionRequired(f"{path.as_posix()} contains unknown atk_id values: {', '.join(unknown)}")
@@ -144,15 +149,18 @@ def build_context(
     }
 
 
-def inspect_dataset(dataset_path=DATASET_PATH, *, direct=False, review_path=None, home_dir=None, field="ground_truth"):
+def inspect_dataset(
+    dataset_path=DATASET_PATH, *, direct=False, review_path=None, home_dir=None, field="ground_truth", input_fields=None
+):
     dataset_path = Path(dataset_path)
-    if not field.strip() or field == "atk_id":
+    if not field.strip() or field in {"atk_id", TRUTH_FIELD}:
         raise UserActionRequired("Answer field must be non-empty; cannot update atk_id.")
     if not dataset_path.is_file():
         raise UserActionRequired(f"Dataset {dataset_path} is missing; run atk-dataset or atk-init first.")
     _, rows = parse_dataset_csv(dataset_path)
+    trust = review_context(dataset_path, rows, field, input_fields)
     if direct:
-        return {"dataset_path": str(dataset_path), "field": field, "rows": rows}
+        return {"dataset_path": str(dataset_path), "field": field, "rows": rows, **trust}
     review = resolve_review_path(
         str(review_path) if review_path else None,
         dataset_path.parent,
@@ -160,24 +168,57 @@ def inspect_dataset(dataset_path=DATASET_PATH, *, direct=False, review_path=None
         rows,
     )
     _, feedback = parse_review_csv(review, rows)
-    return build_context(dataset_path, review, rows, feedback)
+    for row in feedback:
+        if row.get("review_status") or row.get("dataset_sha256"):
+            validate_review_binding(row, trust)
+    return {**build_context(dataset_path, review, rows, feedback), **trust}
 
 
 def update_dataset(
-    updates_path, dataset_path=DATASET_PATH, *, direct=False, review_path=None, home_dir=None, field="ground_truth"
+    updates_path=None,
+    dataset_path=DATASET_PATH,
+    *,
+    direct=False,
+    review_path=None,
+    home_dir=None,
+    field="ground_truth",
+    input_fields=None,
 ):
-    context = inspect_dataset(dataset_path, direct=direct, review_path=review_path, home_dir=home_dir, field=field)
+    context = inspect_dataset(
+        dataset_path, direct=direct, review_path=review_path, home_dir=home_dir, field=field, input_fields=input_fields
+    )
     fields, rows = parse_dataset_csv(Path(dataset_path))
+    before = [dict(row) for row in rows]
     feedback = None
     if not direct:
         _, feedback = parse_review_csv(Path(context["review_path"]), rows)
-    updates = parse_updates_csv(Path(updates_path), rows, feedback, field)
+    updates = parse_updates_csv(Path(updates_path), rows, feedback, field) if updates_path else []
+    if not updates and not any(r.get("review_status") for r in feedback or []):
+        raise UserActionRequired("Provide answer updates or explicit content-bound review decisions.")
     output_fields, output_rows, count = apply_updates(fields, rows, updates, field)
-    if count:
+    original = {row["atk_id"]: row for row in before}
+    by_id = {row["atk_id"]: row for row in output_rows}
+    for review in feedback or []:
+        if review.get("review_status"):
+            row = by_id[review["atk_id"]]
+            apply_review(
+                row,
+                review,
+                context["standard"],
+                answer_changed=original[row["atk_id"]].get(review.get("field")) != row.get(review.get("field")),
+            )
+            if TRUTH_FIELD not in output_fields:
+                output_fields.append(TRUTH_FIELD)
+    reviewed = sum(a.get(TRUTH_FIELD, "") != b.get(TRUTH_FIELD, "") for a, b in zip(before, output_rows, strict=True))
+    current = review_context(Path(dataset_path), before, field)
+    if any(current[k] != context[k] for k in ("dataset_sha256", "standard")):
+        raise UserActionRequired("Dataset or standard changed during review; nothing was written.")
+    if count or reviewed:
         write_dataset_atomic(Path(dataset_path), output_fields, output_rows)
     result = {
         "dataset": str(dataset_path),
         "updated_rows": count,
+        "reviewed_rows": reviewed,
         f"{field}_column": "existing" if field in fields else "appended",
     }
     if not direct:

{agent_tune_kit-2.1.1 → agent_tune_kit-2.2.0}/agent_tune_kit/datasets/validation.py

@@ -4,6 +4,8 @@ from typing import Any
 from agent_tune_kit.errors import UserActionRequired
 from agent_tune_kit.utils.csv_io import parse_csv
 
+from .truth import truth_records
+
 
 def validate_dataset_rows(fieldnames: list[str], rows: list[dict[str, str]]) -> None:
     if "atk_id" not in fieldnames:
@@ -11,6 +13,7 @@ def validate_dataset_rows(fieldnames: list[str], rows: list[dict[str, str]]) ->
     seen: set[str] = set()
     invalid: list[str] = []
     for row in rows:
+        truth_records(row)
         atk_id = (row.get("atk_id") or "").strip()
         if not atk_id or not atk_id.isdigit() or int(atk_id) <= 0 or atk_id in seen:
             invalid.append(atk_id or "<empty>")