agent-comparability 0.5.4__tar.gz → 0.5.5__tar.gz

{agent_comparability-0.5.4/src/agent_comparability.egg-info → agent_comparability-0.5.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agent-comparability
-Version: 0.5.4
+Version: 0.5.5
 Summary: Agent evaluation comparability semantics: spec + conformance suite + thin reference implementation
 License: Apache-2.0
 Project-URL: Homepage, https://github.com/Rethymus/agent-comparability
@@ -27,9 +27,9 @@ Dynamic: license-file
 **Experimental Specification & Conformance Suite** for AI agent evaluation comparability
 
 [![CI](https://github.com/Rethymus/agent-comparability/actions/workflows/ci.yml/badge.svg)](https://github.com/Rethymus/agent-comparability/actions/workflows/ci.yml)
-[![License: Apache-2.0](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](LICENSE)
-[![Python 3.10+](https://img.shields.io/badge/python-3.10%2B-3776AB?logo=python&logoColor=white)](pyproject.toml)
-[![PyPI](https://img.shields.io/pypi/v/agent-comparability)](https://pypi.org/project/agent-comparability/)
+[![License: Apache-2.0](https://img.shields.io/badge/License-Apache_2.0-blue.svg?style=flat-square)](LICENSE)
+[![Python 3.10+](https://img.shields.io/badge/python-3.10%2B-3776AB?logo=python&logoColor=white&style=flat-square)](pyproject.toml)
+[![PyPI](https://img.shields.io/pypi/v/agent-comparability?style=flat-square)](https://pypi.org/project/agent-comparability/)
 
 **English** | [简体中文](README.zh-CN.md)
 
@@ -187,6 +187,8 @@ number, "reward_status", ...optional}]}`); unknown record fields are ignored and
 recorded in `unknown_conditions` (SPEC §5). Add `--manifest out.manifest.json` to
 also emit an Evidence Manifest documenting artifact digests and evidence trust
 states (never auto-upgraded; see [docs/evidence-manifest.md](docs/evidence-manifest.md)).
+For four end-to-end walkthroughs (offline JSON, evidence manifest, single Inspect log,
+cross-log comparison), see [docs/usage-examples.md](docs/usage-examples.md).
 
 **Python API**:
 

{agent_comparability-0.5.4 → agent_comparability-0.5.5}/README.md

@@ -5,9 +5,9 @@
 **Experimental Specification & Conformance Suite** for AI agent evaluation comparability
 
 [![CI](https://github.com/Rethymus/agent-comparability/actions/workflows/ci.yml/badge.svg)](https://github.com/Rethymus/agent-comparability/actions/workflows/ci.yml)
-[![License: Apache-2.0](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](LICENSE)
-[![Python 3.10+](https://img.shields.io/badge/python-3.10%2B-3776AB?logo=python&logoColor=white)](pyproject.toml)
-[![PyPI](https://img.shields.io/pypi/v/agent-comparability)](https://pypi.org/project/agent-comparability/)
+[![License: Apache-2.0](https://img.shields.io/badge/License-Apache_2.0-blue.svg?style=flat-square)](LICENSE)
+[![Python 3.10+](https://img.shields.io/badge/python-3.10%2B-3776AB?logo=python&logoColor=white&style=flat-square)](pyproject.toml)
+[![PyPI](https://img.shields.io/pypi/v/agent-comparability?style=flat-square)](https://pypi.org/project/agent-comparability/)
 
 **English** | [简体中文](README.zh-CN.md)
 
@@ -165,6 +165,8 @@ number, "reward_status", ...optional}]}`); unknown record fields are ignored and
 recorded in `unknown_conditions` (SPEC §5). Add `--manifest out.manifest.json` to
 also emit an Evidence Manifest documenting artifact digests and evidence trust
 states (never auto-upgraded; see [docs/evidence-manifest.md](docs/evidence-manifest.md)).
+For four end-to-end walkthroughs (offline JSON, evidence manifest, single Inspect log,
+cross-log comparison), see [docs/usage-examples.md](docs/usage-examples.md).
 
 **Python API**:
 

{agent_comparability-0.5.4 → agent_comparability-0.5.5}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "agent-comparability"
-version = "0.5.4"
+version = "0.5.5"
 description = "Agent evaluation comparability semantics: spec + conformance suite + thin reference implementation"
 readme = "README.md"
 requires-python = ">=3.10"

{agent_comparability-0.5.4 → agent_comparability-0.5.5}/src/agent_comparability/__init__.py

@@ -1,3 +1,3 @@
 """Agent Comparability — cmp/0.4.0 参考实现。"""
 from .core import CDR, adjudicate, POLICY_VERSION
-__version__ = "0.5.4"
+__version__ = "0.5.5"

{agent_comparability-0.5.4 → agent_comparability-0.5.5}/src/agent_comparability/inspect_log.py

@@ -193,7 +193,8 @@ def main(argv=None) -> int:
                                           task_label=task_label, harness=harnesses[args.log.index(log_path)],
                                           score_map=score_map)
         except MappingError as e:
-            print(f"error: {e}", file=sys.stderr)
+            # PR #36 评审缺口：多日志模式下诊断必须可定位来源——路径前置，原消息不变
+            print(f"error: 日志 {log_path}: {e}", file=sys.stderr)
             return 2
         _fill_model(recs, hdr.eval.model)
         for rec in recs:

{agent_comparability-0.5.4 → agent_comparability-0.5.5/src/agent_comparability.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agent-comparability
-Version: 0.5.4
+Version: 0.5.5
 Summary: Agent evaluation comparability semantics: spec + conformance suite + thin reference implementation
 License: Apache-2.0
 Project-URL: Homepage, https://github.com/Rethymus/agent-comparability
@@ -27,9 +27,9 @@ Dynamic: license-file
 **Experimental Specification & Conformance Suite** for AI agent evaluation comparability
 
 [![CI](https://github.com/Rethymus/agent-comparability/actions/workflows/ci.yml/badge.svg)](https://github.com/Rethymus/agent-comparability/actions/workflows/ci.yml)
-[![License: Apache-2.0](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](LICENSE)
-[![Python 3.10+](https://img.shields.io/badge/python-3.10%2B-3776AB?logo=python&logoColor=white)](pyproject.toml)
-[![PyPI](https://img.shields.io/pypi/v/agent-comparability)](https://pypi.org/project/agent-comparability/)
+[![License: Apache-2.0](https://img.shields.io/badge/License-Apache_2.0-blue.svg?style=flat-square)](LICENSE)
+[![Python 3.10+](https://img.shields.io/badge/python-3.10%2B-3776AB?logo=python&logoColor=white&style=flat-square)](pyproject.toml)
+[![PyPI](https://img.shields.io/pypi/v/agent-comparability?style=flat-square)](https://pypi.org/project/agent-comparability/)
 
 **English** | [简体中文](README.zh-CN.md)
 
@@ -187,6 +187,8 @@ number, "reward_status", ...optional}]}`); unknown record fields are ignored and
 recorded in `unknown_conditions` (SPEC §5). Add `--manifest out.manifest.json` to
 also emit an Evidence Manifest documenting artifact digests and evidence trust
 states (never auto-upgraded; see [docs/evidence-manifest.md](docs/evidence-manifest.md)).
+For four end-to-end walkthroughs (offline JSON, evidence manifest, single Inspect log,
+cross-log comparison), see [docs/usage-examples.md](docs/usage-examples.md).
 
 **Python API**:
 

{agent_comparability-0.5.4 → agent_comparability-0.5.5}/src/agent_comparability.egg-info/SOURCES.txt

@@ -23,5 +23,6 @@ tests/test_evidence.py
 tests/test_gate_regressions.py
 tests/test_inspect_log.py
 tests/test_properties.py
+tests/test_properties_04.py
 tests/test_scenarios_001.py
 tests/test_smoke.py

{agent_comparability-0.5.4 → agent_comparability-0.5.5}/tests/test_inspect_log.py

@@ -350,3 +350,78 @@ def test_real_full_read_trajectory_observed(tmp_path):
                   "--task-label", "ep", "--save-case", str(c2))
     case2 = json.loads(c2.read_text(encoding="utf-8"))
     assert all("trajectory_available" not in x for x in case2["records"])
+
+
+# ---------- 边界加固：epochs × 多日志合并，与跨日志映射失败定位 ----------
+
+def _var_log_targets(model, pairs, tag, ids=None):
+    """按需生成变体真实日志：自定义 (input, target) 对以控制 exact 的 C/I 取值；
+    ids 可选，用于让失败样本在转换器诊断中可定位（同 _var_log，不做缓存）。"""
+    from inspect_ai import Task, eval
+    from inspect_ai.dataset import Sample
+    from inspect_ai.scorer import exact
+    from inspect_ai.solver import generate
+    import tempfile, pathlib
+    d = pathlib.Path(tempfile.mkdtemp(prefix=f"var-{tag}-"))
+    samples = [Sample(input=i, target=t) if sid is None else Sample(input=i, target=t, id=sid)
+               for (i, t), sid in zip(pairs, ids or [None] * len(pairs))]
+    task = Task(dataset=samples, solver=[generate()], scorer=exact())
+    logs = eval(task, model=model, log_dir=str(d / "logs"))
+    assert logs[0].status == "success"
+    return str(logs[0].location)
+
+
+def test_multilog_epochs_log_plus_single_run_merge(tmp_path):
+    """多日志 × epochs 组合：epochs=2 日志 + 单样本日志（同 task-label）合并裁决与 repetition_count。
+
+    防回归：(1) 同任务同模型同 harness 的跨日志合并仍是合法 score_comparison（ADMISSIBLE）；
+    (2) repetition_count 只按来源日志内统计——epochs=2 日志的记录带 2，单样本日志的记录
+    不得编造，更不得跨日志按 task_id 重算（错误重算会得出 3）；(3) 逐记录制品指向保持
+    来源忠实（manifest 与 case 记录按序对齐）。
+    """
+    c = tmp_path / "case.json"
+    m = tmp_path / "m.json"
+    r = run_conv(_epoch_log(), _var_log("mockllm/model", ("solo-Q1",), "solo"),
+                 "--claim", "score_comparison", "--status", "verified",
+                 "--score-map", "C=1.0,I=0.0", "--task-label", "merge-ep",
+                 "--save-case", str(c), "--manifest", str(m))
+    assert r.returncode == 0, r.stderr
+    cdr = json.loads(r.stdout)
+    assert cdr["admissibility"] == "ADMISSIBLE"       # 同任务+同模型+harness 合并=合法比较
+    assert cdr["policy_version"] == "cmp/0.4.0"
+    case = json.loads(c.read_text(encoding="utf-8"))
+    assert len(case["records"]) == 3                  # 1 样本×2 epoch + 1 样本×1
+    assert {x["task_id"] for x in case["records"]} == {"merge-ep#1"}  # task-label 断言生效
+    assert [x.get("repetition_count") for x in case["records"]] == [2, 2, None]
+    # ↑ 前两条来自 epochs=2 日志（源日志内计 2）；第三条来自单样本日志，不得跨日志重算为 3
+    from agent_comparability.evidence import verify_manifest
+    man = json.loads(m.read_text(encoding="utf-8"))
+    assert verify_manifest(man) == []
+    refs = [e["artifact_ref"] for e in man["records"]]
+    assert refs[0] == refs[1] != refs[2]              # epoch 对同源日志，第三条来自另一日志
+    assert any("multilog_merge" in w for w in man["mapping_warnings"])
+
+
+def test_multilog_second_log_unmapped_letter_exit2(tmp_path):
+    """第二份日志出现未映射字母分数 → exit 2，诊断可定位到该日志的失败样本。
+
+    防回归：多日志循环吞掉或笼统化逐样本映射错误——MappingError 必须带样本级定位，
+    操作者才能定位是哪份日志的哪个样本。诊断三要素（PR #36 评审缺口修复后）：
+    来源日志路径（路径级定位）+ 失败样本 id（此处为第二份日志独有的显式 id）+
+    未映射取值与补救开关。
+    """
+    from inspect_ai.model._providers.mockllm import MockLLM
+    a = _var_log_targets("mockllm/model", [("q1?", "deliberately-wrong")], "umap-a")
+    # 日志 A：exact 对 mockllm 错答 → I（在映射表内）
+    b = _var_log_targets("mockllm/model", [("q2?", MockLLM.default_output)],
+                         "umap-b", ids=["second-log-only"])
+    # 日志 B：target=mockllm 默认输出 → exact 得 C（映射表故意只含 I，缺 C）
+    sanity = run_conv(a, "--claim", "score_comparison", "--status", "verified",
+                      "--score-map", "I=0.0", "--task-label", "umap")
+    assert sanity.returncode == 0, sanity.stderr      # 对照：日志 A 单独可过——失败确属日志 B 内容
+    r = run_conv(a, b, "--claim", "score_comparison", "--status", "verified",
+                 "--score-map", "I=0.0", "--task-label", "umap")
+    assert r.returncode == 2                          # 未映射字母 → 程序级失败（exit 2）
+    assert b in r.stderr                              # 定位到来源日志路径（多日志哪份出错）
+    assert "second-log-only" in r.stderr              # 定位到第二份日志中的失败样本
+    assert "'C'" in r.stderr and "score-map" in r.stderr  # 指明未映射取值与补救开关

agent_comparability-0.5.5/tests/test_properties_04.py

@@ -0,0 +1,130 @@
+"""cmp/0.4.0 单变量对称性属性测试——D 冻结对（D1/D2）的随机构造泛化。
+
+与 tests/test_properties.py 同风格：随机构造合法记录组，断言 0.4 语义不变式
+（非为数量造例）。语义锚点：SPEC §3.3 判定优先级、§4 行 1-5、
+conformance/cases-0.4.json（D 集，#27 方向 B，先于实现冻结）。
+覆盖维度是 0.3 时代 test_properties.py 未含的对称性与优先级组合。
+"""
+import random
+
+import pytest
+
+from agent_comparability import adjudicate
+
+STATUSES = ["verified", "unresolved_zero", "other"]
+
+
+def rec(t, h, m, r, s, **kw):
+    d = {"task_id": t, "harness": h, "model": m, "reward": r, "reward_status": s}
+    d.update(kw)
+    return d
+
+
+@pytest.mark.parametrize("seed", range(50))
+def test_model_only_variation_stays_admissible_with_same_model_info(seed):
+    """SPEC §4 行 2：仅模型不同（同任务同 harness、值∈[0,1]、状态不混同）→ 恒 ADMISSIBLE。
+
+    防回归：(a) 0.3.1 语义回潮——same_model 被重新列为 score_comparison 的
+    INADMISSIBLE 硬触发（D1 冻结预期的随机化泛化）；(b) same_model 信息项被静默
+    抹去（裁决必须如实记录 violated_conditions）；(c) 单变量判定误伤
+    （模型-only 必须判 satisfied single_variable_isolation）。
+    """
+    rng = random.Random(seed)
+    n = rng.randint(2, 5)
+    k = rng.randint(2, min(n, 4))  # ≥2 个不同模型、≤模型池大小，且全部在组内出现
+    models = rng.sample(["model-a", "model-b", "model-c", "model-d"], k)
+    status = rng.choice(STATUSES)  # 状态不混同：全组单一 reward_status
+    recs = [rec("t", "h1", models[i % k], rng.random(), status) for i in range(n)]
+    cdr = adjudicate({"id": "p04-model-only", "records": recs}, "score_comparison")
+    assert cdr.admissibility == "ADMISSIBLE"          # 防 same_model 回潮为硬触发（D1）
+    assert "same_model" in cdr.violated_conditions    # 信息项必须如实记录，不得静默丢失
+    assert "single_variable_isolation" in cdr.satisfied_conditions  # 模型-only=单变量
+    assert any(c.startswith("continuous_score_preserved")
+               for c in cdr.satisfied_conditions)     # 值∈[0,1] 走隐式默认尺度
+    assert cdr.policy_version == "cmp/0.4.0"          # 防版本漂移静默改判
+
+
+@pytest.mark.parametrize("seed", range(50))
+def test_model_and_harness_confound_always_inadmissible(seed):
+    """SPEC §4 行 3：模型与 harness 同时不同（同任务）→ 恒 INADMISSIBLE。
+
+    防回归：0.4 对称性只把 same_model 移出硬触发——混淆检测
+    （single_variable_isolation）必须仍硬触发；防“对称性放松”被误推广为
+    “多变量同变也放行”。混同零语义组同样恒拒（§3.3 优先级 2 先于 4）。
+    """
+    rng = random.Random(seed)
+    n = rng.randint(2, 5)
+    combos = [("h1", "m1"), ("h2", "m2"), ("h1", "m2"), ("h2", "m1")]
+    recs = [rec("t", "h1", "m1", rng.random(), rng.choice(STATUSES)),
+            rec("t", "h2", "m2", rng.random(), rng.choice(STATUSES))]  # 保底双变量
+    recs += [rec("t", *rng.choice(combos), rng.random(), rng.choice(STATUSES))
+             for _ in range(n - 2)]
+    cdr = adjudicate({"id": "p04-confound", "records": recs}, "score_comparison")
+    assert cdr.admissibility == "INADMISSIBLE"        # 混淆恒拒（§4 行 3，对称性不覆盖多变量）
+    assert "single_variable_isolation" in cdr.violated_conditions  # 防混淆检测被移除
+    assert "same_model" in cdr.violated_conditions    # 与 harness 同变=混淆，非单变量
+
+
+@pytest.mark.parametrize("seed", range(50))
+def test_causal_claim_rejected_under_any_0_4_construction(seed):
+    """D1 冻结对因果半边：单次运行记录组的因果主张恒 INADMISSIBLE。
+
+    防回归：0.4 对称性重构为因果开洞——单变量、缺轨迹、越界值等任何 0.4 构造
+    都不得使因果主张获准（SPEC §3.3：禁止单次运行支持因果）。同时钉住分支顺序
+    契约：因果分支先于缺轨迹 INCONCLUSIVE 裁决。
+    """
+    rng = random.Random(seed)
+    recs = [rec(rng.choice(["t1", "t2"]), rng.choice(["h1", "h2"]),
+                rng.choice(["m1", "m2"]), rng.random() * 1.5 - 0.25,
+                rng.choice(STATUSES)) for _ in range(rng.randint(1, 4))]
+    if rng.random() < 0.5:  # 随机混入缺轨迹记录（因果仍须拒，而非降级 INCONCLUSIVE）
+        recs[rng.randrange(len(recs))]["trajectory_available"] = False
+    cdr = adjudicate({"id": "p04-causal", "records": recs}, "causal")
+    assert cdr.admissibility == "INADMISSIBLE"        # 因果恒拒：任何单次运行构造均不豁免
+    assert any("causal_attribution" in u
+               for u in cdr.unsupported_claims)       # 拒绝理由必须可审计
+
+
+@pytest.mark.parametrize("seed", range(50))
+def test_missing_trace_priority_dominates_all_hard_triggers(seed):
+    """§3.3 score_comparison 优先级 1：缺轨迹 INCONCLUSIVE 压过全部条件冲突。
+
+    防回归：优先级倒置——任务/混淆/尺度冲突（INADMISSIBLE）或零语义（PARTIAL）
+    抢先于证据缺失裁决；同时防“转录保留”被删（unknown_conditions 的 retained 项）。
+    构造保证任务、模型、harness 全部同时变化且全体缺轨迹（全硬触发在场）。
+    """
+    rng = random.Random(seed)
+    recs = [rec("t1", "h1", "m1", rng.random() * 2.0, rng.choice(STATUSES),
+                trajectory_available=False),
+            rec("t2", "h2", "m2", rng.random() * 2.0, rng.choice(STATUSES),
+                trajectory_available=False)]
+    recs += [rec(rng.choice(["t1", "t2"]), rng.choice(["h1", "h2"]),
+                 rng.choice(["m1", "m2"]), rng.random() * 2.0,
+                 rng.choice(STATUSES), trajectory_available=False)
+             for _ in range(rng.randint(0, 2))]
+    cdr = adjudicate({"id": "p04-trace-priority", "records": recs}, "score_comparison")
+    assert cdr.admissibility == "INCONCLUSIVE"        # 缺轨迹恒第一优先（全硬触发在场）
+    assert "trace_available" in cdr.violated_conditions
+    viol = {v.split("(")[0] for v in cdr.violated_conditions}  # 登记不得被优先级抹除
+    assert {"same_task_id", "same_model", "single_variable_isolation"} <= viol
+    assert any("retained" in u for u in cdr.unknown_conditions)  # 评分保留不删
+
+
+@pytest.mark.parametrize("seed", range(50))
+def test_scale_gate_not_relaxed_by_model_only_symmetry(seed):
+    """D2 冻结点的随机化泛化：模型-only + 任一记录越界未声明尺度 → 恒 INADMISSIBLE。
+
+    防回归：0.4 把 same_model 移出硬触发时放松 score_scale_consistent——尺度把关
+    必须对模型-only 组仍独立硬触发（D2 冻结预期）；状态不混同以隔离冲突源。
+    """
+    rng = random.Random(seed)
+    n = rng.randint(2, 4)
+    status = rng.choice(STATUSES)  # 状态不混同：尺度是唯一冲突源
+    recs = [rec("t", "h1", "model-a", rng.random() * 2.0, status),
+            rec("t", "h1", "model-b", 1.0 + rng.random(), status)]  # 必有越界记录
+    recs += [rec("t", "h1", rng.choice(["model-a", "model-b"]),
+                 rng.random() * 2.0, status) for _ in range(n - 2)]
+    cdr = adjudicate({"id": "p04-scale", "records": recs}, "score_comparison")
+    assert cdr.admissibility == "INADMISSIBLE"        # 尺度把关未被对称性放松（D2）
+    assert "score_scale_consistent" in cdr.violated_conditions
+    assert "same_model" in cdr.violated_conditions    # 模型-only 语义仍如实在场