agentic-dataset-conformance 0.1.0__py3-none-any.whl

agentic_dataset_conformance/__init__.py

@@ -0,0 +1,68 @@
+"""The portable conformance suite for the agentic-dataset contract.
+
+Fifteen normative assertions, AD-001 .. AD-015, as language-neutral executable
+vectors plus a runner that evaluates them against any implementation through a
+four-method interface. It imports no implementation, including the reference
+one, and the vectors travel with it.
+
+    from agentic_dataset_conformance import load_suite, run
+    report = run(my_subject)
+    print(report.passed, [f.assertion for f in report.failures])
+
+The vectors and worlds are the normative artifact and are dedicated to the
+public domain under CC0-1.0; the software around them is Apache-2.0. Copy the
+vectors into a Rust, Go or TypeScript project and write your own runner --
+that is what they are for, and `export_vectors()` exists to make it one call.
+
+See `verbs.md` beside this module for the control-verb vocabulary a subject
+must understand, and `interface.py` for what it must expose.
+"""
+
+from __future__ import annotations
+
+import shutil
+from importlib import resources
+from pathlib import Path
+
+from .interface import ConformanceSubject, Observation, Scope
+from .runner import (
+    INVARIANTS,
+    AssertionResult,
+    SubjectReport,
+    Vector,
+    VectorSuite,
+    load_suite,
+    run,
+)
+from .toy import ToyImplementation
+
+__version__ = "0.1.0"
+
+#: The fifteen assertion identifiers, in order.
+ASSERTIONS = tuple(f"AD-{i:03d}" for i in range(1, 16))
+
+__all__ = [
+    "ASSERTIONS", "AssertionResult", "ConformanceSubject", "INVARIANTS",
+    "Observation", "Scope", "SubjectReport", "ToyImplementation", "Vector",
+    "VectorSuite", "__version__", "export_vectors", "load_suite", "run",
+    "vectors_path",
+]
+
+
+def vectors_path():
+    """The packaged normative data as a traversable resource."""
+    return resources.files(__package__) / "data"
+
+
+def export_vectors(destination: str | Path) -> Path:
+    """Copy the normative worlds and vectors out, for use anywhere.
+
+    They are CC0-1.0: no attribution required, no conditions. Vendor them into
+    another language's repository and write a runner there.
+    """
+    destination = Path(destination)
+    destination.mkdir(parents=True, exist_ok=True)
+    with resources.as_file(vectors_path()) as data:
+        for sub in ("worlds", "vectors"):
+            shutil.copytree(data / sub, destination / sub, dirs_exist_ok=True)
+    return destination

agentic_dataset_conformance/__main__.py

@@ -0,0 +1,4 @@
+from .cli import main
+
+if __name__ == "__main__":
+    raise SystemExit(main())

agentic_dataset_conformance/cli.py

@@ -0,0 +1,169 @@
+"""Command line for the portable conformance suite.
+
+    agentic-dataset-conformance run                     # against the built-in toy
+    agentic-dataset-conformance run --subject mod:make  # against yours
+    agentic-dataset-conformance run --matrix            # mutation characterisation
+    agentic-dataset-conformance vectors --list
+    agentic-dataset-conformance vectors --export ./vectors
+
+`--subject` takes `module:attribute`. The attribute may be a subject, a
+callable returning one, or a callable returning several. Nothing about
+resolving it is specific to any implementation, which is the point: a
+conforming implementation in another package is tested by naming it.
+"""
+
+from __future__ import annotations
+
+import argparse
+import importlib
+import json
+import sys
+from pathlib import Path
+
+from . import ASSERTIONS, export_vectors
+from .runner import load_suite, run
+
+
+def _resolve(spec: str) -> list:
+    module_name, _, attribute = spec.partition(":")
+    if not attribute:
+        raise SystemExit(f"--subject expects module:attribute, got {spec!r}")
+    sys.path.insert(0, str(Path.cwd()))
+    module = importlib.import_module(module_name)
+    target = getattr(module, attribute)
+    found = target() if callable(target) else target
+    return list(found) if isinstance(found, (list, tuple)) else [found]
+
+
+def _subjects(specs: list[str]) -> list:
+    if not specs:
+        from .toy import ToyImplementation
+
+        return [ToyImplementation()]
+    return [s for spec in specs for s in _resolve(spec)]
+
+
+def _matrix(rows: list[tuple[str, str, bool, list[str]]]) -> str:
+    labels = [f"M{i:02d}" for i in range(1, len(rows) + 1)]
+    out = ["        " + " ".join(labels), "        " + " ".join("---" for _ in labels)]
+    for assertion in ASSERTIONS:
+        cells = []
+        for _, target, ok, caught in rows:
+            if assertion == target:
+                cells.append(" T " if ok else " ! ")
+            elif assertion in caught:
+                cells.append(" x ")
+            else:
+                cells.append(" . ")
+        detected = sum(1 for c in cells if c.strip() in ("T", "x"))
+        out.append(f"{assertion}  {' '.join(cells)}   {detected}")
+    out.append("")
+    for label, (name, target, ok, _) in zip(labels, rows):
+        out.append(f"{label}  {target}  {name.removeprefix('mutant:')}"
+                   + ("" if ok else "   ** NOT CAUGHT BY ITS TARGET **"))
+    caught_n = sum(1 for _, _, ok, _ in rows if ok)
+    covered = {t for _, t, _, _ in rows}
+    out += [
+        "",
+        f"target detection : {caught_n}/{len(rows)} mutants caught by their intended assertion",
+        f"cross-detection  : {sum(len(c) for _, _, _, c in rows) / len(rows):.1f} "
+        "assertions per mutant on average",
+        f"coverage         : {len(covered)}/15 assertions have a mutant of their own",
+        "",
+        "T = caught by its target assertion   x = caught redundantly",
+        ". = not detected                     ! = target failed to catch it",
+    ]
+    return "\n".join(out)
+
+
+def _run(args: argparse.Namespace) -> int:
+    suite = load_suite(args.vectors)
+    reports = [run(s, suite) for s in _subjects(args.subject)]
+    failed = any(not r.passed for r in reports)
+
+    rows: list[tuple[str, str, bool, list[str]]] = []
+    if args.mutants or args.matrix:
+        from .mutations import TARGETS
+
+        for cls, target in TARGETS.items():
+            caught = [f.assertion for f in run(cls(), suite).failures]
+            rows.append((cls.name, target, target in caught, caught))
+        failed = failed or any(not ok for _, _, ok, _ in rows)
+
+    if args.json:
+        print(json.dumps({
+            "vectors": len(suite.vectors),
+            "subjects": [r.to_dict() for r in reports],
+            "mutants": [{"mutant": n, "target": t, "caught": ok, "caught_by": by}
+                        for n, t, ok, by in rows],
+        }, indent=2))
+        return 1 if failed else 0
+
+    width = max((len(r.subject) for r in reports), default=10)
+    print(f"{len(suite.vectors)} vectors, {sum(len(v.steps) for v in suite.vectors)} steps, "
+          f"{len(reports)} subject(s)\n")
+    print(f"{'SUBJECT':<{width}}  RESULT  ASSERTIONS  OBSERVATIONS")
+    for report in reports:
+        passed = len(report.results) - len(report.failures)
+        print(f"{report.subject:<{width}}  {'PASS' if report.passed else 'FAIL':<6}  "
+              f"{passed:>6}/{len(report.results)}  {report.observations:>12}")
+        for failure in report.failures:
+            print(f"    {failure.assertion}: {failure.detail}")
+    if rows and args.matrix:
+        print()
+        print(_matrix(rows))
+    elif rows:
+        print(f"\n{'MUTANT':<42} {'TARGET':<8} CAUGHT BY")
+        for name, target, ok, caught in rows:
+            print(f"{name:<42} {target:<8} {'' if ok else 'MISSED '}"
+                  f"{','.join(caught) or 'nothing'}")
+    return 1 if failed else 0
+
+
+def _vectors(args: argparse.Namespace) -> int:
+    if args.export:
+        where = export_vectors(args.export)
+        print(f"normative worlds and vectors written to {where} (CC0-1.0, "
+              "no attribution required)")
+        return 0
+    suite = load_suite(args.vectors)
+    print(f"{len(suite.vectors)} vectors, "
+          f"{sum(len(v.steps) for v in suite.vectors)} steps\n")
+    for vector in suite.vectors:
+        print(f"{vector.assertion}  {vector.name:<44} {len(vector.steps):>3} steps")
+        print(f"          rules out: {vector.rules_out}")
+    return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="agentic-dataset-conformance",
+        description="Run AD-001..AD-015 against any implementation.",
+    )
+    sub = parser.add_subparsers(dest="command")
+
+    r = sub.add_parser("run", help="evaluate a subject against the vectors")
+    r.add_argument("--subject", action="append", default=[], metavar="MODULE:ATTR",
+                   help="an implementation to test; repeatable. Defaults to the "
+                        "built-in toy subject.")
+    r.add_argument("--mutants", action="store_true",
+                   help="also check that broken variants are caught")
+    r.add_argument("--matrix", action="store_true",
+                   help="print the mutation detection matrix (implies --mutants)")
+    r.add_argument("--vectors", metavar="DIR", default=None,
+                   help="use vectors from DIR instead of the packaged ones")
+    r.add_argument("--json", action="store_true", help="machine-readable output")
+    r.set_defaults(func=_run)
+
+    v = sub.add_parser("vectors", help="list or export the normative vectors")
+    v.add_argument("--export", metavar="DIR", default=None,
+                   help="copy the CC0 worlds and vectors into DIR")
+    v.add_argument("--vectors", metavar="DIR", default=None,
+                   help="list vectors from DIR instead of the packaged ones")
+    v.set_defaults(func=_vectors)
+
+    args = parser.parse_args(argv)
+    if args.command is None:
+        parser.print_help()
+        return 0
+    return args.func(args)

agentic_dataset_conformance/data/LICENSE

@@ -0,0 +1,13 @@
+The files in this directory -- the normative worlds and conformance vectors --
+are dedicated to the public domain under CC0 1.0 Universal.
+
+    SPDX-License-Identifier: CC0-1.0
+    Full text: ../../../LICENSES/CC0-1.0.txt
+               https://creativecommons.org/publicdomain/zero/1.0/
+
+No attribution is required and no conditions are attached. They are meant to be
+copied unchanged into implementations in other languages, and every condition
+would be friction against the only thing they exist for.
+
+The software in the parent package is Apache-2.0. The distribution as a whole
+is therefore "Apache-2.0 AND CC0-1.0".

agentic_dataset_conformance/data/vectors/ad-001-descriptor-valid.json

@@ -0,0 +1,28 @@
+{
+  "assertion": "AD-001",
+  "rules_out": "a dataset participating in admission without a well-formed contract",
+  "steps": [
+    {
+      "op": "register_descriptor",
+      "descriptor": {
+        "dataset": "broken-dataset",
+        "version": "1",
+        "description": "no revision, no capabilities"
+      }
+    },
+    {
+      "op": "request",
+      "principal": "process_engineer",
+      "text": "search broken",
+      "dataset": "broken-dataset",
+      "capability": "search",
+      "expect": {
+        "decision": "REFUSED",
+        "reason": "DESCRIPTOR_INVALID",
+        "granted": false,
+        "executed": false
+      }
+    }
+  ],
+  "world": "reference"
+}

agentic_dataset_conformance/data/vectors/ad-002-capability-registered.json

@@ -0,0 +1,122 @@
+{
+  "assertion": "AD-002",
+  "rules_out": "an executable action with no capability metadata behind it",
+  "capability_surface_matches_descriptors": true,
+  "steps": [
+    {
+      "op": "register_descriptor",
+      "descriptor": {
+        "dataset": "purification-batches",
+        "version": "2026.08.31",
+        "revision": "s3-etag-4c1f9a",
+        "schema_version": "3",
+        "description": "Process and analytical data describing purification batches: recovery, yield, step conditions and in-process controls for downstream chromatography.",
+        "schemas": [
+          "batch_record",
+          "in_process_control",
+          "step_condition"
+        ],
+        "capabilities": [
+          {
+            "name": "search",
+            "description": "find batches by free text over batch records",
+            "effect": "read",
+            "sensitivity": "internal",
+            "policy": null,
+            "arguments": [
+              "query"
+            ]
+          },
+          {
+            "name": "compare_batches",
+            "description": "compare recovery and step conditions across two batches",
+            "effect": "read",
+            "sensitivity": "internal",
+            "policy": "BPD-DATA-014",
+            "arguments": [
+              "batch_a",
+              "batch_b"
+            ]
+          },
+          {
+            "name": "calculate_yield",
+            "description": "compute step yield and cumulative recovery for a batch",
+            "effect": "compute",
+            "sensitivity": "internal",
+            "policy": null,
+            "arguments": [
+              "batch_id"
+            ]
+          },
+          {
+            "name": "detect_outliers",
+            "description": "flag anomalous batches against the historical distribution",
+            "effect": "compute",
+            "sensitivity": "confidential",
+            "policy": "BPD-DATA-021",
+            "arguments": [
+              "metric"
+            ]
+          },
+          {
+            "name": "phantom_export",
+            "effect": "read",
+            "sensitivity": "internal",
+            "description": "advertised with nothing behind it",
+            "policy": null,
+            "arguments": []
+          }
+        ],
+        "prohibited": [
+          "delete_source",
+          "overwrite_batch_record",
+          "bypass_governance",
+          "expose_restricted_identifiers"
+        ],
+        "policies": [
+          "BPD-DATA-014",
+          "BPD-DATA-021"
+        ],
+        "provenance": {
+          "system": "volume",
+          "source": "s3"
+        },
+        "freshness": {
+          "maximum_age_s": 86400
+        },
+        "age_s": 3600,
+        "quality": {
+          "required_fields": [
+            "batch_ids",
+            "recovery"
+          ]
+        },
+        "retention": {
+          "observations": 1000
+        },
+        "endpoints": {
+          "mcp": "stdio://purification"
+        }
+      }
+    },
+    {
+      "op": "grant",
+      "principal": "process_engineer",
+      "dataset": "purification-batches",
+      "capability": "phantom_export"
+    },
+    {
+      "op": "request",
+      "principal": "process_engineer",
+      "text": "export everything",
+      "dataset": "purification-batches",
+      "capability": "phantom_export",
+      "expect": {
+        "executed": false,
+        "result_present": false,
+        "error_contains": "not a registered capability"
+      }
+    }
+  ],
+  "world": "reference"
+}

agentic_dataset_conformance/data/vectors/ad-003-grant-required-for-execution.json

@@ -0,0 +1,30 @@
+{
+  "assertion": "AD-003",
+  "rules_out": "execution reachable without an authorization artifact",
+  "note": "the general form is the cross-vector invariant executed => granted",
+  "steps": [
+    {
+      "op": "request",
+      "principal": "process_engineer",
+      "text": "Compare the recovery of batches B001 and B002",
+      "expect": {
+        "decision": "GRANTED",
+        "granted": true,
+        "executed": true
+      }
+    },
+    {
+      "op": "request",
+      "principal": "process_engineer",
+      "text": "Calculate the yield for batch B003",
+      "grant_ttl_s": -1,
+      "expect": {
+        "decision": "GRANTED",
+        "executed": false,
+        "result_present": false,
+        "error_contains": "expired"
+      }
+    }
+  ],
+  "world": "reference"
+}

agentic_dataset_conformance/data/vectors/ad-004-refusal-has-no-grant.json

@@ -0,0 +1,75 @@
+{
+  "assertion": "AD-004",
+  "rules_out": "a refusal that still mints authority",
+  "steps": [
+    {
+      "op": "request",
+      "principal": "process_engineer",
+      "text": "delete the source",
+      "dataset": "purification-batches",
+      "capability": "delete_source",
+      "prohibited": true,
+      "expect": {
+        "decision": "REFUSED",
+        "reason": "PROHIBITED_OPERATION",
+        "policy_id": "AD-POL-004",
+        "granted": false,
+        "executed": false
+      }
+    },
+    {
+      "op": "request",
+      "principal": "external_auditor",
+      "text": "Compare the recovery of batches B001 and B002",
+      "dataset": "purification-batches",
+      "capability": "compare_batches",
+      "expect": {
+        "decision": "REFUSED",
+        "reason": "INSUFFICIENT_PRIVILEGE",
+        "granted": false,
+        "executed": false
+      }
+    },
+    {
+      "op": "request",
+      "principal": "analyst",
+      "text": "detect outliers in recovery",
+      "dataset": "purification-batches",
+      "capability": "detect_outliers",
+      "expect": {
+        "decision": "REFUSED",
+        "granted": false,
+        "executed": false
+      }
+    },
+    {
+      "op": "request",
+      "principal": "process_engineer",
+      "text": "Compare the recovery of batches B001 and B002",
+      "dataset": "purification-batches",
+      "capability": "compare_batches",
+      "expected_schema_version": "99",
+      "expect": {
+        "decision": "REFUSED",
+        "reason": "SCHEMA_VERSION_MISMATCH",
+        "granted": false,
+        "executed": false
+      }
+    },
+    {
+      "op": "request",
+      "principal": "process_engineer",
+      "text": "search chromatography runs",
+      "dataset": "chromatography-results",
+      "capability": "search",
+      "freshness": 60,
+      "expect": {
+        "decision": "REFUSED",
+        "reason": "FRESHNESS_UNSATISFIABLE",
+        "granted": false,
+        "executed": false
+      }
+    }
+  ],
+  "world": "reference"
+}

agentic_dataset_conformance/data/vectors/ad-005-indeterminate-has-no-grant.json

@@ -0,0 +1,41 @@
+{
+  "assertion": "AD-005",
+  "rules_out": "unknown authority becoming permission",
+  "steps": [
+    {
+      "op": "request",
+      "principal": "process_engineer",
+      "text": "Compare the recovery of batches B001 and B002",
+      "evaluator": {
+        "reachable": false,
+        "latency_s": 0.0
+      },
+      "expect": {
+        "decision": "INDETERMINATE",
+        "reason": "EVALUATOR_UNAVAILABLE",
+        "policy_id": null,
+        "rationale_present": true,
+        "granted": false,
+        "executed": false
+      }
+    },
+    {
+      "op": "request",
+      "principal": "process_engineer",
+      "text": "Compare the recovery of batches B001 and B002",
+      "evaluator": {
+        "reachable": true,
+        "latency_s": 5.0
+      },
+      "expect": {
+        "decision": "INDETERMINATE",
+        "reason": "EVALUATOR_TIMEOUT",
+        "policy_id": null,
+        "rationale_present": true,
+        "granted": false,
+        "executed": false
+      }
+    }
+  ],
+  "world": "reference"
+}

agentic_dataset_conformance/data/vectors/ad-006-unknown-capability-denied.json

@@ -0,0 +1,58 @@
+{
+  "assertion": "AD-006",
+  "rules_out": "default-allow on an unregistered tool",
+  "steps": [
+    {
+      "op": "request",
+      "principal": "process_engineer",
+      "text": "do the thing",
+      "dataset": "purification-batches",
+      "capability": "query_database",
+      "expect": {
+        "decision": "REFUSED",
+        "reason": "UNREGISTERED_CAPABILITY",
+        "granted": false,
+        "executed": false
+      }
+    },
+    {
+      "op": "request",
+      "principal": "process_engineer",
+      "text": "do the thing",
+      "dataset": "purification-batches",
+      "capability": "exfiltrate",
+      "expect": {
+        "decision": "REFUSED",
+        "reason": "UNREGISTERED_CAPABILITY",
+        "granted": false,
+        "executed": false
+      }
+    },
+    {
+      "op": "request",
+      "principal": "process_engineer",
+      "text": "do the thing",
+      "dataset": "purification-batches",
+      "capability": "search_all",
+      "expect": {
+        "decision": "REFUSED",
+        "reason": "UNREGISTERED_CAPABILITY",
+        "granted": false,
+        "executed": false
+      }
+    },
+    {
+      "op": "request",
+      "principal": "process_engineer",
+      "text": "do the thing",
+      "dataset": "purification-batches",
+      "expect": {
+        "decision": "REFUSED",
+        "reason": "UNREGISTERED_CAPABILITY",
+        "granted": false,
+        "executed": false
+      }
+    }
+  ],
+  "world": "reference"
+}

agentic_dataset_conformance/data/vectors/ad-007-authorization-scope-preserved.json

@@ -0,0 +1,36 @@
+{
+  "assertion": "AD-007",
+  "rules_out": "scope widening between admission and execution",
+  "note": "the general form is the cross-vector invariant grant_scope covers executed_scope",
+  "steps": [
+    {
+      "op": "request",
+      "principal": "process_engineer",
+      "text": "Compare the recovery of batches B001 and B002",
+      "expect": {
+        "decision": "GRANTED",
+        "granted": true
+      }
+    },
+    {
+      "op": "delegate",
+      "channel": "a2a",
+      "dataset": "purification-batches",
+      "capability": "compare_batches",
+      "scope": {
+        "principal_class": "process-engineer",
+        "dataset": "purification-batches",
+        "capabilities": [
+          "compare_batches",
+          "detect_outliers"
+        ],
+        "max_sensitivity": "restricted"
+      },
+      "expect": {
+        "executed": false,
+        "error_contains": "widen"
+      }
+    }
+  ],
+  "world": "reference"
+}