docassert 0.10.1__tar.gz → 0.11.0__tar.gz

{docassert-0.10.1/docassert.egg-info → docassert-0.11.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: docassert
-Version: 0.10.1
+Version: 0.11.0
 Summary: Unit testing for business documents — validate structured Markdown docs against a configurable audit standard.
 Author: C4G Enterprises Inc.
 License: Apache-2.0

{docassert-0.10.1 → docassert-0.11.0}/docassert/__init__.py

@@ -5,4 +5,4 @@ standard: deterministic structural checks that gate a merge, plus optional
 AI-graded semantic checks that advise.
 """
 
-__version__ = "0.10.1"
+__version__ = "0.11.0"

{docassert-0.10.1 → docassert-0.11.0}/docassert/consistency.py

@@ -18,6 +18,7 @@ from pathlib import Path
 import yaml
 
 from . import config as config_mod
+from . import semantic
 from .graph import build_graph
 from .models import CheckResult
 from .semantic import run_alignment
@@ -134,7 +135,9 @@ def check_profile_completeness(documents_dir: str | Path = "documents") -> Check
 # ── semantic (advisory) ────────────────────────────────────────────────────
 # Each alignment edge costs one API call, so a large graph could otherwise run
 # away on cost. Cap per run; tune with `alignment_limit` in consistency.yaml
-# (0 disables the cap).
+# (0 disables the cap). The cap budgets API calls, not links: grades already
+# in the semantic cache replay for free, so successive runs that persist
+# `.docassert-cache` walk the whole graph `alignment_limit` links at a time.
 DEFAULT_ALIGNMENT_LIMIT = 25
 
 
@@ -151,20 +154,29 @@ def run_alignment_checks(graph, config) -> list[CheckResult]:
     if not edges:
         return []
 
-    limit = int(config.get("alignment_limit", DEFAULT_ALIGNMENT_LIMIT) or 0)
-    note: CheckResult | None = None
-    if limit and len(edges) > limit:
-        note = CheckResult(
-            "alignment-limit", True, False,
-            f"graded {limit} of {len(edges)} link(s) — raise `alignment_limit` "
-            f"in consistency.yaml to grade more per run",
-            kind="semantic", score=None)
-        edges = edges[:limit]
-
     if not os.environ.get("ANTHROPIC_API_KEY"):
         return [CheckResult("alignment", True, False,
                             f"skipped — no ANTHROPIC_API_KEY ({len(edges)} link(s) to grade)",
                             kind="semantic", score=None)]
+
+    limit = int(config.get("alignment_limit", DEFAULT_ALIGNMENT_LIMIT) or 0)
+    note: CheckResult | None = None
+    if limit:
+        # Spend the budget on cache misses; cached grades replay for free.
+        fresh = [e for e in edges
+                 if not semantic.is_cached(e[0], semantic.alignment_content(
+                     e[1].text, e[2].text))]
+        if len(fresh) > limit:
+            over_budget = set(id(e) for e in fresh[limit:])
+            edges = [e for e in edges if id(e) not in over_budget]
+            note = CheckResult(
+                "alignment-limit", True, False,
+                f"graded {limit} of {len(fresh)} ungraded link(s) "
+                f"({len(edges) - limit} from cache) — raise `alignment_limit` in "
+                f"consistency.yaml, or persist `.docassert-cache` between runs "
+                f"to walk the rest",
+                kind="semantic", score=None)
+
     results = [run_alignment(f"align:{c.id}-{rel}-{p.id}", prompt, p.text, c.text)
                for prompt, p, c, rel in edges]
     if note is not None:

{docassert-0.10.1 → docassert-0.11.0}/docassert/semantic.py

@@ -9,12 +9,15 @@ from __future__ import annotations
 import hashlib
 import json
 import os
+import re
 from pathlib import Path
 
 from .models import CheckResult, Document
 
-DEFAULT_MODEL = os.environ.get("DOCUNIT_MODEL", "claude-sonnet-5")
-CACHE_DIR = Path(os.environ.get("DOCUNIT_CACHE", ".docassert-cache"))
+DEFAULT_MODEL = (os.environ.get("DOCASSERT_MODEL")
+                 or os.environ.get("DOCUNIT_MODEL", "claude-sonnet-5"))
+CACHE_DIR = Path(os.environ.get("DOCASSERT_CACHE")
+                 or os.environ.get("DOCUNIT_CACHE", ".docassert-cache"))
 
 _SYSTEM = (
     "You are a meticulous document auditor. You are given one audit criterion "
@@ -53,6 +56,31 @@ def _cache_put(key: str, value: dict) -> None:
         pass  # caching is best-effort
 
 
+def _parse_grade(text: str) -> dict:
+    """Parse the model's JSON grade, tolerating prose wrappers and truncation.
+
+    Responses put score and pass before the free-text rationale, so a reply
+    that ran out of tokens mid-rationale still carries a usable grade.
+    """
+    start, end = text.find("{"), text.rfind("}")
+    if start != -1 and end > start:
+        try:
+            return json.loads(text[start:end + 1])
+        except json.JSONDecodeError:
+            pass
+    m = re.search(r'"score"\s*:\s*([0-9.]+)', text)
+    if m:
+        grade: dict = {"score": float(m.group(1))}
+        pm = re.search(r'"pass"\s*:\s*(true|false)', text)
+        if pm:
+            grade["pass"] = pm.group(1) == "true"
+        rm = re.search(r'"rationale"\s*:\s*"(.*)', text, re.DOTALL)
+        if rm:
+            grade["rationale"] = rm.group(1).rstrip('"').strip() + " [truncated]"
+        return grade
+    raise ValueError(f"no JSON in model response: {text[:120]!r}")
+
+
 def _grade(prompt: str, content: str, model: str) -> dict:
     """Call the Anthropic API and return the parsed JSON grade."""
     import anthropic  # imported lazily so structural-only runs need no dependency
@@ -62,7 +90,7 @@ def _grade(prompt: str, content: str, model: str) -> dict:
     # not set it. Grading is kept consistent by the strict JSON rubric instead.
     message = client.messages.create(
         model=model,
-        max_tokens=400,
+        max_tokens=800,
         system=_SYSTEM,
         messages=[{
             "role": "user",
@@ -71,11 +99,7 @@ def _grade(prompt: str, content: str, model: str) -> dict:
     )
     text = "".join(getattr(block, "text", "") for block in message.content
                    if getattr(block, "type", None) == "text").strip()
-    # tolerate models that wrap JSON in prose or fences
-    start, end = text.find("{"), text.rfind("}")
-    if start == -1 or end == -1:
-        raise ValueError(f"no JSON in model response: {text[:120]!r}")
-    return json.loads(text[start:end + 1])
+    return _parse_grade(text)
 
 
 def advisory(check_id: str, prompt: str, content: str,
@@ -117,8 +141,17 @@ def run_semantic(doc: Document, spec: dict, content: str) -> CheckResult:
     )
 
 
+def alignment_content(parent_text: str, child_text: str) -> str:
+    return f"PARENT:\n{parent_text}\n\nCHILD:\n{child_text}"
+
+
+def is_cached(prompt: str, content: str, model: str = DEFAULT_MODEL) -> bool:
+    """True when this grade can be replayed without an API call."""
+    return _cache_get(_cache_key(model, prompt, content)) is not None
+
+
 def run_alignment(check_id: str, prompt: str, parent_text: str,
                   child_text: str, threshold: float = 0.7) -> CheckResult:
     """Advisory: does the child item genuinely fulfil the parent it links to?"""
-    content = f"PARENT:\n{parent_text}\n\nCHILD:\n{child_text}"
-    return advisory(check_id, prompt, content, threshold=threshold)
+    return advisory(check_id, prompt, alignment_content(parent_text, child_text),
+                    threshold=threshold)

{docassert-0.10.1 → docassert-0.11.0/docassert.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: docassert
-Version: 0.10.1
+Version: 0.11.0
 Summary: Unit testing for business documents — validate structured Markdown docs against a configurable audit standard.
 Author: C4G Enterprises Inc.
 License: Apache-2.0

{docassert-0.10.1 → docassert-0.11.0}/docassert.egg-info/SOURCES.txt

@@ -113,5 +113,6 @@ tests/test_kinds_reporting.py
 tests/test_profiles.py
 tests/test_projects.py
 tests/test_scaffold.py
+tests/test_semantic.py
 tests/test_status.py
 tests/test_structural.py

docassert-0.11.0/tests/test_semantic.py

@@ -0,0 +1,122 @@
+"""Tests for the advisory grading layer: parsing, caching, and the API budget."""
+import json
+
+import pytest
+
+from docassert import consistency, semantic
+from docassert.models import Item
+from docassert.semantic import _parse_grade
+
+
+# ── _parse_grade ─────────────────────────────────────────────────────────────
+def test_parse_grade_clean_json():
+    g = _parse_grade('{"score": 0.9, "pass": true, "rationale": "solid"}')
+    assert g == {"score": 0.9, "pass": True, "rationale": "solid"}
+
+
+def test_parse_grade_tolerates_prose_and_fences():
+    g = _parse_grade('Here you go:\n```json\n{"score": 0.4, "pass": false, "rationale": "weak"}\n```')
+    assert g["score"] == 0.4 and g["pass"] is False
+
+
+def test_parse_grade_recovers_truncated_response():
+    # The exact failure shape seen live: max_tokens cut the rationale short.
+    text = '{"score": 0.35, "pass": false, "rationale": "The child only refines the \'2 seconds\' timing aspect (backend fingerprint r'
+    g = _parse_grade(text)
+    assert g["score"] == 0.35
+    assert g["pass"] is False
+    assert g["rationale"].endswith("[truncated]")
+
+
+def test_parse_grade_truncated_without_pass_leaves_it_to_threshold():
+    g = _parse_grade('{"score": 0.8, "ratio')
+    assert g["score"] == 0.8 and "pass" not in g
+
+
+def test_parse_grade_raises_on_garbage():
+    with pytest.raises(ValueError):
+        _parse_grade("I cannot grade this.")
+
+
+# ── cache replay ─────────────────────────────────────────────────────────────
+def _use_tmp_cache(tmp_path, monkeypatch):
+    monkeypatch.setattr(semantic, "CACHE_DIR", tmp_path / "cache")
+    monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key")
+
+
+def test_advisory_caches_grades(tmp_path, monkeypatch):
+    _use_tmp_cache(tmp_path, monkeypatch)
+    calls = []
+    monkeypatch.setattr(semantic, "_grade", lambda p, c, m: (
+        calls.append(1) or {"score": 1.0, "pass": True, "rationale": "ok"}))
+    for _ in range(2):
+        r = semantic.advisory("chk", "prompt", "content")
+        assert r.passed and r.score == 1.0
+    assert len(calls) == 1
+
+
+def test_advisory_does_not_cache_failures(tmp_path, monkeypatch):
+    _use_tmp_cache(tmp_path, monkeypatch)
+    calls = []
+
+    def boom(p, c, m):
+        calls.append(1)
+        raise ValueError("no JSON in model response: ''")
+
+    monkeypatch.setattr(semantic, "_grade", boom)
+    for _ in range(2):
+        r = semantic.advisory("chk", "prompt", "content")
+        assert "unavailable" in r.detail
+    assert len(calls) == 2  # transient errors retry on the next run
+
+
+# ── the alignment budget spends on cache misses ──────────────────────────────
+def _alignment_graph(n):
+    from docassert.graph import Graph
+    g = Graph()
+    g.add(Item("TST-PR-001", "TST", "PR", "the parent requirement", {},
+               "d.md", "prd", "approved", "S"))
+    for i in range(1, n + 1):
+        g.add(Item(f"TST-US-{i:03d}", "TST", "US", f"story {i}",
+                   {"traces": ["TST-PR-001"]}, "d.md", "user-story",
+                   "approved", "S"))
+    return g
+
+
+def test_alignment_budget_walks_the_graph_across_runs(tmp_path, monkeypatch):
+    _use_tmp_cache(tmp_path, monkeypatch)
+    calls = []
+    monkeypatch.setattr(semantic, "_grade", lambda p, c, m: (
+        calls.append(c) or {"score": 1.0, "pass": True, "rationale": "ok"}))
+    graph = _alignment_graph(5)
+    config = {"alignment": [{"relation": "traces", "prompt": "aligned?"}],
+              "alignment_limit": 2}
+
+    first = consistency.run_alignment_checks(graph, config)
+    assert len(calls) == 2                       # budget spent on misses
+    assert any(r.check_id == "alignment-limit" and "2 of 5" in r.detail
+               for r in first)
+
+    second = consistency.run_alignment_checks(graph, config)
+    assert len(calls) == 4                       # next two misses, not repeats
+    assert len(set(map(json.dumps, calls))) == 4
+    graded = [r for r in second if r.check_id.startswith("align:")]
+    assert len(graded) == 4                      # 2 cached replays + 2 new
+
+    third = consistency.run_alignment_checks(graph, config)
+    assert len(calls) == 5                       # the last miss
+    assert not any(r.check_id == "alignment-limit" for r in third)
+    graded = [r for r in third if r.check_id.startswith("align:")]
+    assert len(graded) == 5                      # full coverage reached
+
+
+def test_alignment_limit_zero_grades_everything(tmp_path, monkeypatch):
+    _use_tmp_cache(tmp_path, monkeypatch)
+    calls = []
+    monkeypatch.setattr(semantic, "_grade", lambda p, c, m: (
+        calls.append(1) or {"score": 1.0, "pass": True, "rationale": "ok"}))
+    graph = _alignment_graph(4)
+    config = {"alignment": [{"relation": "traces", "prompt": "aligned?"}],
+              "alignment_limit": 0}
+    consistency.run_alignment_checks(graph, config)
+    assert len(calls) == 4