docassert 0.10.1__tar.gz → 0.12.0__tar.gz

{docassert-0.10.1/docassert.egg-info → docassert-0.12.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: docassert
-Version: 0.10.1
+Version: 0.12.0
 Summary: Unit testing for business documents — validate structured Markdown docs against a configurable audit standard.
 Author: C4G Enterprises Inc.
 License: Apache-2.0

{docassert-0.10.1 → docassert-0.12.0}/docassert/__init__.py

@@ -5,4 +5,4 @@ standard: deterministic structural checks that gate a merge, plus optional
 AI-graded semantic checks that advise.
 """
 
-__version__ = "0.10.1"
+__version__ = "0.12.0"

{docassert-0.10.1 → docassert-0.12.0}/docassert/_data/consistency.yaml

@@ -39,7 +39,18 @@ coverage:
 # (0 = no cap).
 alignment_limit: 25
 
+# Rules match in order; the first matching rule grades a link. Typed rules
+# (child_type/parent_type) come before the generic ones.
 alignment:
+  - relation: traces
+    child_type: PR
+    parent_type: BR
+    prompt: >
+      The PARENT is a business outcome or KPI; the CHILD is a product
+      mechanism tracing to it. Judge whether delivering the child would
+      plausibly and materially contribute to achieving the parent outcome.
+      The child need not restate the metric; penalise only mechanisms that
+      would not meaningfully move the parent outcome, or that contradict it.
   - relation: traces
     prompt: >
       Judge whether the CHILD requirement genuinely refines or implements the
@@ -47,8 +58,11 @@ alignment:
       related. Penalise links that don't actually follow from the parent.
   - relation: verifies
     prompt: >
-      Judge whether the acceptance criterion actually verifies the requirement
-      it targets: testable, and covering the requirement's intent.
+      Judge whether the acceptance criterion concretely and testably verifies
+      some aspect of the requirement it targets. Several criteria may share
+      one requirement, each covering a slice, so do not penalise a criterion
+      for leaving other clauses to its siblings. Penalise criteria that are
+      untestable, verify nothing the requirement states, or contradict it.
   - relation: tests
     prompt: >
       Judge whether the test case actually exercises the acceptance criterion

{docassert-0.10.1 → docassert-0.12.0}/docassert/consistency.py

@@ -18,6 +18,7 @@ from pathlib import Path
 import yaml
 
 from . import config as config_mod
+from . import semantic
 from .graph import build_graph
 from .models import CheckResult
 from .semantic import run_alignment
@@ -134,37 +135,64 @@ def check_profile_completeness(documents_dir: str | Path = "documents") -> Check
 # ── semantic (advisory) ────────────────────────────────────────────────────
 # Each alignment edge costs one API call, so a large graph could otherwise run
 # away on cost. Cap per run; tune with `alignment_limit` in consistency.yaml
-# (0 disables the cap).
+# (0 disables the cap). The cap budgets API calls, not links: grades already
+# in the semantic cache replay for free, so successive runs that persist
+# `.docassert-cache` walk the whole graph `alignment_limit` links at a time.
 DEFAULT_ALIGNMENT_LIMIT = 25
 
 
 def run_alignment_checks(graph, config) -> list[CheckResult]:
     edges = []  # (prompt, parent, child, relation)
+    seen: set[tuple[str, str, str]] = set()
+    # Rules match in order and the first matching rule grades a link, so a
+    # typed rule (child_type/parent_type) placed before the generic one can
+    # give e.g. PR->BR traces a contribution rubric while other traces keep
+    # the refinement rubric.
     for rule in config.get("alignment", []):
         relation, prompt = rule["relation"], rule.get("prompt", "").strip()
+        child_type = rule.get("child_type")
+        parent_type = rule.get("parent_type")
         for child in graph.all_items():
+            if child_type and child.type != child_type:
+                continue
             for target in child.targets(relation):
                 parent = graph.canonical(target)
-                if parent is not None:
-                    edges.append((prompt, parent, child, relation))
+                if parent is None:
+                    continue
+                if parent_type and parent.type != parent_type:
+                    continue
+                key = (child.id, relation, parent.id)
+                if key in seen:
+                    continue
+                seen.add(key)
+                edges.append((prompt, parent, child, relation))
 
     if not edges:
         return []
 
-    limit = int(config.get("alignment_limit", DEFAULT_ALIGNMENT_LIMIT) or 0)
-    note: CheckResult | None = None
-    if limit and len(edges) > limit:
-        note = CheckResult(
-            "alignment-limit", True, False,
-            f"graded {limit} of {len(edges)} link(s) — raise `alignment_limit` "
-            f"in consistency.yaml to grade more per run",
-            kind="semantic", score=None)
-        edges = edges[:limit]
-
     if not os.environ.get("ANTHROPIC_API_KEY"):
         return [CheckResult("alignment", True, False,
                             f"skipped — no ANTHROPIC_API_KEY ({len(edges)} link(s) to grade)",
                             kind="semantic", score=None)]
+
+    limit = int(config.get("alignment_limit", DEFAULT_ALIGNMENT_LIMIT) or 0)
+    note: CheckResult | None = None
+    if limit:
+        # Spend the budget on cache misses; cached grades replay for free.
+        fresh = [e for e in edges
+                 if not semantic.is_cached(e[0], semantic.alignment_content(
+                     e[1].text, e[2].text))]
+        if len(fresh) > limit:
+            over_budget = set(id(e) for e in fresh[limit:])
+            edges = [e for e in edges if id(e) not in over_budget]
+            note = CheckResult(
+                "alignment-limit", True, False,
+                f"graded {limit} of {len(fresh)} ungraded link(s) "
+                f"({len(edges) - limit} from cache) — raise `alignment_limit` in "
+                f"consistency.yaml, or persist `.docassert-cache` between runs "
+                f"to walk the rest",
+                kind="semantic", score=None)
+
     results = [run_alignment(f"align:{c.id}-{rel}-{p.id}", prompt, p.text, c.text)
                for prompt, p, c, rel in edges]
     if note is not None:

{docassert-0.10.1 → docassert-0.12.0}/docassert/semantic.py

@@ -9,12 +9,15 @@ from __future__ import annotations
 import hashlib
 import json
 import os
+import re
 from pathlib import Path
 
 from .models import CheckResult, Document
 
-DEFAULT_MODEL = os.environ.get("DOCUNIT_MODEL", "claude-sonnet-5")
-CACHE_DIR = Path(os.environ.get("DOCUNIT_CACHE", ".docassert-cache"))
+DEFAULT_MODEL = (os.environ.get("DOCASSERT_MODEL")
+                 or os.environ.get("DOCUNIT_MODEL", "claude-sonnet-5"))
+CACHE_DIR = Path(os.environ.get("DOCASSERT_CACHE")
+                 or os.environ.get("DOCUNIT_CACHE", ".docassert-cache"))
 
 _SYSTEM = (
     "You are a meticulous document auditor. You are given one audit criterion "
@@ -53,6 +56,31 @@ def _cache_put(key: str, value: dict) -> None:
         pass  # caching is best-effort
 
 
+def _parse_grade(text: str) -> dict:
+    """Parse the model's JSON grade, tolerating prose wrappers and truncation.
+
+    Responses put score and pass before the free-text rationale, so a reply
+    that ran out of tokens mid-rationale still carries a usable grade.
+    """
+    start, end = text.find("{"), text.rfind("}")
+    if start != -1 and end > start:
+        try:
+            return json.loads(text[start:end + 1])
+        except json.JSONDecodeError:
+            pass
+    m = re.search(r'"score"\s*:\s*([0-9.]+)', text)
+    if m:
+        grade: dict = {"score": float(m.group(1))}
+        pm = re.search(r'"pass"\s*:\s*(true|false)', text)
+        if pm:
+            grade["pass"] = pm.group(1) == "true"
+        rm = re.search(r'"rationale"\s*:\s*"(.*)', text, re.DOTALL)
+        if rm:
+            grade["rationale"] = rm.group(1).rstrip('"').strip() + " [truncated]"
+        return grade
+    raise ValueError(f"no JSON in model response: {text[:120]!r}")
+
+
 def _grade(prompt: str, content: str, model: str) -> dict:
     """Call the Anthropic API and return the parsed JSON grade."""
     import anthropic  # imported lazily so structural-only runs need no dependency
@@ -62,7 +90,7 @@ def _grade(prompt: str, content: str, model: str) -> dict:
     # not set it. Grading is kept consistent by the strict JSON rubric instead.
     message = client.messages.create(
         model=model,
-        max_tokens=400,
+        max_tokens=800,
         system=_SYSTEM,
         messages=[{
             "role": "user",
@@ -71,11 +99,7 @@ def _grade(prompt: str, content: str, model: str) -> dict:
     )
     text = "".join(getattr(block, "text", "") for block in message.content
                    if getattr(block, "type", None) == "text").strip()
-    # tolerate models that wrap JSON in prose or fences
-    start, end = text.find("{"), text.rfind("}")
-    if start == -1 or end == -1:
-        raise ValueError(f"no JSON in model response: {text[:120]!r}")
-    return json.loads(text[start:end + 1])
+    return _parse_grade(text)
 
 
 def advisory(check_id: str, prompt: str, content: str,
@@ -117,8 +141,17 @@ def run_semantic(doc: Document, spec: dict, content: str) -> CheckResult:
     )
 
 
+def alignment_content(parent_text: str, child_text: str) -> str:
+    return f"PARENT:\n{parent_text}\n\nCHILD:\n{child_text}"
+
+
+def is_cached(prompt: str, content: str, model: str = DEFAULT_MODEL) -> bool:
+    """True when this grade can be replayed without an API call."""
+    return _cache_get(_cache_key(model, prompt, content)) is not None
+
+
 def run_alignment(check_id: str, prompt: str, parent_text: str,
                   child_text: str, threshold: float = 0.7) -> CheckResult:
     """Advisory: does the child item genuinely fulfil the parent it links to?"""
-    content = f"PARENT:\n{parent_text}\n\nCHILD:\n{child_text}"
-    return advisory(check_id, prompt, content, threshold=threshold)
+    return advisory(check_id, prompt, alignment_content(parent_text, child_text),
+                    threshold=threshold)

{docassert-0.10.1 → docassert-0.12.0/docassert.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: docassert
-Version: 0.10.1
+Version: 0.12.0
 Summary: Unit testing for business documents — validate structured Markdown docs against a configurable audit standard.
 Author: C4G Enterprises Inc.
 License: Apache-2.0

{docassert-0.10.1 → docassert-0.12.0}/docassert.egg-info/SOURCES.txt

@@ -113,5 +113,6 @@ tests/test_kinds_reporting.py
 tests/test_profiles.py
 tests/test_projects.py
 tests/test_scaffold.py
+tests/test_semantic.py
 tests/test_status.py
 tests/test_structural.py

docassert-0.12.0/tests/test_semantic.py

@@ -0,0 +1,176 @@
+"""Tests for the advisory grading layer: parsing, caching, and the API budget."""
+import json
+
+import pytest
+
+from docassert import consistency, semantic
+from docassert.models import Item
+from docassert.semantic import _parse_grade
+
+
+# ── _parse_grade ─────────────────────────────────────────────────────────────
+def test_parse_grade_clean_json():
+    g = _parse_grade('{"score": 0.9, "pass": true, "rationale": "solid"}')
+    assert g == {"score": 0.9, "pass": True, "rationale": "solid"}
+
+
+def test_parse_grade_tolerates_prose_and_fences():
+    g = _parse_grade('Here you go:\n```json\n{"score": 0.4, "pass": false, "rationale": "weak"}\n```')
+    assert g["score"] == 0.4 and g["pass"] is False
+
+
+def test_parse_grade_recovers_truncated_response():
+    # The exact failure shape seen live: max_tokens cut the rationale short.
+    text = '{"score": 0.35, "pass": false, "rationale": "The child only refines the \'2 seconds\' timing aspect (backend fingerprint r'
+    g = _parse_grade(text)
+    assert g["score"] == 0.35
+    assert g["pass"] is False
+    assert g["rationale"].endswith("[truncated]")
+
+
+def test_parse_grade_truncated_without_pass_leaves_it_to_threshold():
+    g = _parse_grade('{"score": 0.8, "ratio')
+    assert g["score"] == 0.8 and "pass" not in g
+
+
+def test_parse_grade_raises_on_garbage():
+    with pytest.raises(ValueError):
+        _parse_grade("I cannot grade this.")
+
+
+# ── cache replay ─────────────────────────────────────────────────────────────
+def _use_tmp_cache(tmp_path, monkeypatch):
+    monkeypatch.setattr(semantic, "CACHE_DIR", tmp_path / "cache")
+    monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key")
+
+
+def test_advisory_caches_grades(tmp_path, monkeypatch):
+    _use_tmp_cache(tmp_path, monkeypatch)
+    calls = []
+    monkeypatch.setattr(semantic, "_grade", lambda p, c, m: (
+        calls.append(1) or {"score": 1.0, "pass": True, "rationale": "ok"}))
+    for _ in range(2):
+        r = semantic.advisory("chk", "prompt", "content")
+        assert r.passed and r.score == 1.0
+    assert len(calls) == 1
+
+
+def test_advisory_does_not_cache_failures(tmp_path, monkeypatch):
+    _use_tmp_cache(tmp_path, monkeypatch)
+    calls = []
+
+    def boom(p, c, m):
+        calls.append(1)
+        raise ValueError("no JSON in model response: ''")
+
+    monkeypatch.setattr(semantic, "_grade", boom)
+    for _ in range(2):
+        r = semantic.advisory("chk", "prompt", "content")
+        assert "unavailable" in r.detail
+    assert len(calls) == 2  # transient errors retry on the next run
+
+
+# ── the alignment budget spends on cache misses ──────────────────────────────
+def _alignment_graph(n):
+    from docassert.graph import Graph
+    g = Graph()
+    g.add(Item("TST-PR-001", "TST", "PR", "the parent requirement", {},
+               "d.md", "prd", "approved", "S"))
+    for i in range(1, n + 1):
+        g.add(Item(f"TST-US-{i:03d}", "TST", "US", f"story {i}",
+                   {"traces": ["TST-PR-001"]}, "d.md", "user-story",
+                   "approved", "S"))
+    return g
+
+
+def test_alignment_budget_walks_the_graph_across_runs(tmp_path, monkeypatch):
+    _use_tmp_cache(tmp_path, monkeypatch)
+    calls = []
+    monkeypatch.setattr(semantic, "_grade", lambda p, c, m: (
+        calls.append(c) or {"score": 1.0, "pass": True, "rationale": "ok"}))
+    graph = _alignment_graph(5)
+    config = {"alignment": [{"relation": "traces", "prompt": "aligned?"}],
+              "alignment_limit": 2}
+
+    first = consistency.run_alignment_checks(graph, config)
+    assert len(calls) == 2                       # budget spent on misses
+    assert any(r.check_id == "alignment-limit" and "2 of 5" in r.detail
+               for r in first)
+
+    second = consistency.run_alignment_checks(graph, config)
+    assert len(calls) == 4                       # next two misses, not repeats
+    assert len(set(map(json.dumps, calls))) == 4
+    graded = [r for r in second if r.check_id.startswith("align:")]
+    assert len(graded) == 4                      # 2 cached replays + 2 new
+
+    third = consistency.run_alignment_checks(graph, config)
+    assert len(calls) == 5                       # the last miss
+    assert not any(r.check_id == "alignment-limit" for r in third)
+    graded = [r for r in third if r.check_id.startswith("align:")]
+    assert len(graded) == 5                      # full coverage reached
+
+
+def test_alignment_limit_zero_grades_everything(tmp_path, monkeypatch):
+    _use_tmp_cache(tmp_path, monkeypatch)
+    calls = []
+    monkeypatch.setattr(semantic, "_grade", lambda p, c, m: (
+        calls.append(1) or {"score": 1.0, "pass": True, "rationale": "ok"}))
+    graph = _alignment_graph(4)
+    config = {"alignment": [{"relation": "traces", "prompt": "aligned?"}],
+              "alignment_limit": 0}
+    consistency.run_alignment_checks(graph, config)
+    assert len(calls) == 4
+
+
+# ── typed rules: first match grades the link ─────────────────────────────────
+def _typed_graph():
+    from docassert.graph import Graph
+    g = Graph()
+    g.add(Item("TST-BR-001", "TST", "BR", "the business outcome", {},
+               "d.md", "brd", "approved", "S"))
+    g.add(Item("TST-PR-001", "TST", "PR", "the product mechanism",
+               {"traces": ["TST-BR-001"]}, "d.md", "prd", "approved", "S"))
+    g.add(Item("TST-US-001", "TST", "US", "the story",
+               {"traces": ["TST-PR-001"]}, "d.md", "user-story",
+               "approved", "S"))
+    return g
+
+
+TYPED_CONFIG = {"alignment": [
+    {"relation": "traces", "child_type": "PR", "parent_type": "BR",
+     "prompt": "contribution rubric"},
+    {"relation": "traces", "prompt": "refinement rubric"},
+], "alignment_limit": 0}
+
+
+def test_typed_rule_grades_matching_links_first(tmp_path, monkeypatch):
+    _use_tmp_cache(tmp_path, monkeypatch)
+    prompts = []
+    monkeypatch.setattr(semantic, "_grade", lambda p, c, m: (
+        prompts.append(p) or {"score": 1.0, "pass": True, "rationale": "ok"}))
+    results = consistency.run_alignment_checks(_typed_graph(), TYPED_CONFIG)
+    graded = {r.check_id for r in results if r.check_id.startswith("align:")}
+    assert graded == {"align:TST-PR-001-traces-TST-BR-001",
+                      "align:TST-US-001-traces-TST-PR-001"}
+    assert sorted(prompts) == ["contribution rubric", "refinement rubric"]
+
+
+def test_typed_rule_does_not_double_grade(tmp_path, monkeypatch):
+    _use_tmp_cache(tmp_path, monkeypatch)
+    calls = []
+    monkeypatch.setattr(semantic, "_grade", lambda p, c, m: (
+        calls.append((p, c)) or {"score": 1.0, "pass": True, "rationale": "ok"}))
+    consistency.run_alignment_checks(_typed_graph(), TYPED_CONFIG)
+    assert len(calls) == 2  # one grade per link, despite two matching relations
+
+
+def test_packaged_defaults_carry_typed_rules():
+    from docassert import config as config_mod
+    cfg = config_mod.read_consistency_config()
+    rules = cfg.get("alignment", [])
+    typed = [r for r in rules if r.get("child_type") == "PR"
+             and r.get("parent_type") == "BR"]
+    assert typed and rules.index(typed[0]) < len(rules) - 1
+    assert "contribute" in typed[0]["prompt"]
+    verifies = [r for r in rules if r["relation"] == "verifies"][0]
+    assert "slice" in verifies["prompt"]