backspec 0.0.1__py3-none-any.whl

backspec/modules/extraction/use_cases.py

@@ -0,0 +1,228 @@
+"""The pipeline: discover, triage, observe and verify, orient, assign, synthesise, assess, report.
+
+The program drives. A model is asked to read and to write statements; structure, citations and
+merging are done here and in the modules this composes.
+"""
+from __future__ import annotations
+
+import collections
+from dataclasses import dataclass, field
+from typing import Callable
+
+from backspec.kernel.llm import LLMError
+from backspec.kernel.prompts import PROMPT_VERSION
+from backspec.kernel.usage import Spend
+from backspec.modules.capabilities.facade import PLUMBING, UNASSIGNED, CapabilitiesFacade, Capability
+from backspec.modules.discovery.facade import DiscoveryFacade
+from backspec.modules.evidence.facade import EvidenceFacade
+from backspec.modules.lint.facade import lint_findings
+from backspec.modules.report.facade import Diagnostics, ExtractionResult, Run, StageUsage, build_coverage, earlier_evidence, earlier_triage
+from backspec.modules.requirements.facade import Requirement, RequirementsFacade
+from backspec.modules.review.facade import ReviewFacade, ReviewSummary
+from backspec.modules.triage.facade import DEFAULT_AUDIT, TriageFacade
+
+TOOL_VERSION = "0.0.1"
+Progress = Callable[[str], None]
+
+
+@dataclass
+class Settings:
+    include: list[str] = field(default_factory=list)
+    exclude: list[str] = field(default_factory=list)
+    default_excludes: bool = True
+    assess_support: bool = True
+    gate: bool = False   # stop once the capabilities are settled, before assignment and synthesis
+    triage: bool = True            # a cheap first look; files surely of no relevance are not read in full
+    triage_audit: float = DEFAULT_AUDIT   # share of the skipped files read anyway; 0 is off, 1.0 reads them all
+
+
+def run(root: str, provider: str, settings: Settings, discovery: DiscoveryFacade, evidence: EvidenceFacade,
+        capabilities: CapabilitiesFacade, requirements: RequirementsFacade,
+        usage: Callable[[], tuple[int, int]] = lambda: (0, 0), progress: Progress = lambda _: None,
+        review: ReviewFacade | None = None, before: ExtractionResult | None = None,
+        reuse: bool = True, spend: Callable[[], Spend] = Spend, triage: TriageFacade | None = None) -> ExtractionResult:
+    """`before` is the previous result, if there is one. Requirement ids are carried forward from
+    it, and with `reuse` so is everything whose inputs have not changed: observations of unchanged
+    files, the capabilities, where each observation was placed, and the requirements of any
+    capability whose observations are the same. A model is asked only about what is new."""
+    # An unusable capabilities file is refused before any file is read or model called.
+    from_file = capabilities.load()
+    kept = before if reuse else None
+    found = discovery.discover(settings.include, settings.exclude, settings.default_excludes)
+    progress(f"discovered {len(found.files)} files to read, {len(found.left)} left out")
+
+    triaged = None
+    if settings.triage and triage is not None:
+        verdicts = earlier_triage(kept, triage.fingerprint()) if kept else None
+        triaged = triage.assess(list(found.files), verdicts, settings.triage_audit)
+        progress(f"triaged {len(found.files)} files: {len(triaged.skipped)} skipped, "
+                 f"{len(found.files) - len(triaged.skipped)} to read, {len(triaged.audited)} skipped files "
+                 f"read anyway as an audit; {triaged.reused_files} verdicts reused, "
+                 f"{len(triaged.unresolved)} without a verdict (read), in {triaged.batches} batches")
+    to_read = triaged.to_read(list(found.files)) if triaged else list(found.files)
+
+    earlier = earlier_evidence(kept, evidence.fingerprint()) if kept else None
+    observed = evidence.observe(to_read, earlier)
+    progress(f"observed {len(observed.observations)} verified, {len(observed.rejected)} rejected, "
+             f"in {observed.batches} batches; {observed.reused_files} unchanged files reused, "
+             f"{observed.partly_read_files} changed files read in part")
+
+    if triaged is not None and triaged.audited:
+        found_by_audit = {o.path for o in observed.observations} & set(triaged.audited)
+        progress(f"triage audit: {len(found_by_audit)} of {len(triaged.audited)} audited files had a verified observation")
+
+    caps: list[Capability] = []
+    assignments: dict[str, str] = {}
+    source = "oriented"
+    orient_failed = ""
+    if from_file is not None:
+        caps, source = from_file, "file"
+        progress(f"using {len(caps)} capabilities from {capabilities.location}")
+    elif kept and any(c.id not in (PLUMBING, UNASSIGNED) for c in kept.capabilities):
+        caps, source = [c for c in kept.capabilities if c.id != UNASSIGNED], "previous"
+        progress(f"using {len(caps)} capabilities from the previous result")
+    elif observed.observations:
+        try:
+            caps = capabilities.orient(observed.observations)
+            progress(f"oriented into {len(caps)} capabilities")
+            if settings.gate:
+                capabilities.save(caps)
+                progress(f"capabilities written to {capabilities.location}")
+        except LLMError as error:   # nothing can be placed without capabilities: record it and stop
+            orient_failed = str(error)[:300]
+
+    unresolved = dict(triaged.unresolved) if triaged else {}
+
+    def describe(status: str, **more: object) -> Run:
+        calls, cached = usage()
+        used = spend()
+        return Run(status=status, tool_version=TOOL_VERSION, prompt_version=PROMPT_VERSION, provider=provider,
+                   root=root, calls=calls, cached_calls=cached, capabilities_source=source,
+                   input_tokens=used.input_tokens, output_tokens=used.output_tokens,
+                   tokens_estimated=used.estimated, cost_usd=used.cost_usd, budget_reached=used.budget_reached,
+                   usage_by_stage={s: StageUsage(**u) for s, u in used.by_stage.items()},
+                   observe_fingerprint=evidence.fingerprint(), reused_files=observed.reused_files, **_analysis_counts(observed),
+                   partly_read_files=observed.partly_read_files, synthesis_fingerprint=requirements.fingerprint(),
+                   triage_fingerprint=triage.fingerprint() if triaged is not None and triage else "", **more)
+
+    all_observations = sorted(observed.observations, key=lambda o: (o.path, o.lines[0], o.id))
+    if orient_failed:
+        progress(f"orient failed: {orient_failed}")
+        return ExtractionResult(
+            run=describe("partial"), capabilities=[], requirements=[], observations=all_observations,
+            assignments={}, coverage=build_coverage(found, observed, [], {}),
+            diagnostics=Diagnostics(rejected=observed.rejected, failed_stages={"orient": orient_failed}))
+    if settings.gate:
+        # Stopped on purpose: the run is `partial` and says where it stopped.
+        progress("gate: stopping before assignment and synthesis; edit the file, then run again")
+        gated = ExtractionResult(
+            run=describe("partial", stopped_after="capabilities"), capabilities=caps, requirements=[],
+            observations=all_observations, assignments={}, coverage=build_coverage(found, observed, [], {}, triaged),
+            diagnostics=Diagnostics(rejected=observed.rejected, triage_unresolved=unresolved))
+        gated.diagnostics.lint = lint_findings(gated)
+        return gated
+
+    if observed.observations:
+        assignments = capabilities.assign(observed.observations, caps, kept.assignments if kept else None)
+        if UNASSIGNED in assignments.values():
+            caps.append(Capability(id=UNASSIGNED, name="Unassigned",
+                                   summary="Observations that were not placed in a capability."))
+
+    by_capability = collections.defaultdict(list)
+    for observation in observed.observations:
+        by_capability[assignments[observation.id]].append(observation)
+    product = [c for c in caps if c.id != PLUMBING]
+
+    # A capability whose observations are exactly those the previous result had keeps its requirements.
+    reused: list[Requirement] = []
+    reused_capabilities: list[str] = []
+    current = {o.id: o for o in observed.observations}
+    open_questions: dict[str, list[str]] = {}
+    if kept and kept.run.stopped_after is None and kept.run.synthesis_fingerprint == requirements.fingerprint():
+        placed_before = collections.defaultdict(set)
+        for observation_id, capability_id in kept.assignments.items():
+            placed_before[capability_id].add(observation_id)
+        made_before = collections.defaultdict(list)
+        for requirement in [*kept.requirements, *kept.hidden]:
+            made_before[requirement.capability].append(requirement)
+        unsettled = {*kept.diagnostics.failed_capabilities, *kept.diagnostics.unconsolidated}
+        for capability in product:
+            now = {o.id for o in by_capability.get(capability.id, [])}
+            if now and now == placed_before.get(capability.id) and capability.id not in unsettled:
+                reused_capabilities.append(capability.id)
+                # review is applied afresh below, so a decision withdrawn since then is withdrawn here too
+                reused += [_refreshed(r, current) for r in made_before[capability.id]]
+                if capability.id in kept.diagnostics.open_questions:
+                    open_questions[capability.id] = kept.diagnostics.open_questions[capability.id]
+    synthesis = requirements.synthesise([c for c in product if c.id not in reused_capabilities], by_capability)
+    open_questions.update(synthesis.open_questions)
+    progress(f"synthesised {len(synthesis.requirements)} requirements; "
+             f"{len(reused)} kept from {len(reused_capabilities)} unchanged capabilities")
+
+    final = [*reused, *synthesis.requirements]
+    if settings.assess_support:
+        unjudged = [r for r in final if r.support == "not_assessed"]
+        if unjudged:
+            judged = {r.id: r for r in requirements.assess(unjudged, {f.path: f.lines for f in found.files})}
+            final = [judged.get(r.id, r) for r in final]
+            counts = collections.Counter(r.support for r in final)
+            progress("assessed support: " + ", ".join(f"{n} {s}" for s, n in sorted(counts.items())))
+    review_summary: ReviewSummary | None = None
+    hidden: list[Requirement] = []
+    previous = [*before.requirements, *before.hidden] if before else None
+    if review is not None and review.applies(previous):
+        reviewed = review.apply(final, previous)
+        final, review_summary, hidden = reviewed.requirements, reviewed.summary, reviewed.hidden
+        progress(f"review: {review_summary.carried} ids carried, {review_summary.rejected_hidden} rejected hidden, "
+                 f"{review_summary.needs_review} need review, {len(review_summary.stale_entries)} stale entries")
+    final = sorted(final, key=lambda r: (r.capability, r.id))
+
+    complete = not observed.failed and not synthesis.failed and not spend().budget_reached
+    result = ExtractionResult(
+        run=describe("complete" if complete else "partial", reused_capabilities=sorted(reused_capabilities)),
+        capabilities=caps, requirements=final, hidden=sorted(hidden, key=lambda r: (r.capability, r.id)),
+        observations=all_observations, assignments=assignments,
+        coverage=build_coverage(found, observed, [*final, *hidden], assignments, triaged),
+        diagnostics=Diagnostics(rejected=observed.rejected, failed_capabilities=synthesis.failed,
+                                unconsolidated=synthesis.unconsolidated,
+                                dropped_scenarios=synthesis.dropped_scenarios, open_questions=open_questions,
+                                review=review_summary, triage_unresolved=unresolved))
+    result.diagnostics.lint = lint_findings(result)
+    return result
+
+
+def _analysis_counts(observed) -> dict[str, int]:
+    modes = list(observed.analysis.values())
+    return {"syntax_files": modes.count("syntax"), "text_files": modes.count("text")}
+
+
+def _refreshed(requirement: Requirement, current: dict) -> Requirement:
+    """A kept requirement with its evidence pointing at where the cited code is now. The lines
+    may have moved since it was written; the cited text has not changed."""
+    def moved(evidence: list) -> list:
+        return [e.model_copy(update={"path": current[e.observation].path, "lines": current[e.observation].lines,
+                                     "quote": current[e.observation].quote,
+                                     "symbol": getattr(current[e.observation], "symbol", None)})
+                if e.observation in current else e for e in evidence]
+    scenarios = [s.model_copy(update={"evidence": moved(s.evidence)}) for s in requirement.scenarios]
+    return requirement.model_copy(update={"review": "unreviewed", "evidence": moved(requirement.evidence),
+                                          "scenarios": scenarios})
+
+
+def orient_only(settings: Settings, discovery: DiscoveryFacade, evidence: EvidenceFacade,
+                capabilities: CapabilitiesFacade, force: bool = False,
+                progress: Progress = lambda _: None) -> list[Capability]:
+    """Discover, observe, orient and write the capabilities file; nothing after that.
+    Refuses to replace an existing file unless `force`, before any model is called."""
+    if capabilities.exists() and not force:
+        raise FileExistsError(f"{capabilities.location} already exists; pass --force to replace it")
+    found = discovery.discover(settings.include, settings.exclude, settings.default_excludes)
+    progress(f"discovered {len(found.files)} files to read, {len(found.left)} left out")
+    observed = evidence.observe(list(found.files))
+    progress(f"observed {len(observed.observations)} verified, {len(observed.rejected)} rejected")
+    if not observed.observations:
+        raise ValueError("no observations were verified, so there is nothing to orient from; no file was written")
+    caps = capabilities.orient(observed.observations)
+    capabilities.save(caps)
+    progress(f"capabilities written to {capabilities.location}")
+    return caps

backspec/modules/lint/domain.py

@@ -0,0 +1,152 @@
+"""Lint: deterministic checks on a result. No model is involved.
+
+A finding is an `error` when the result contradicts itself (a citation that points at nothing, a
+duplicate id) and a `warning` when it is merely suspect (a vague or compound statement). The
+checks read a result only through the small protocols below, so this module needs nothing of the
+report that holds the result.
+"""
+from __future__ import annotations
+
+from typing import Literal, Protocol, Sequence
+
+from pydantic import BaseModel, Field
+
+from backspec.kernel.text import squash
+
+Severity = Literal["error", "warning"]
+MIN_STATEMENT = 25
+PLUMBING, UNASSIGNED = "plumbing", "unassigned"
+
+
+class Finding(BaseModel):
+    code: str
+    severity: Severity
+    ids: list[str] = Field(default_factory=list)   # the requirement, capability or observation ids concerned
+    message: str
+
+
+class LintReport(BaseModel):
+    findings: list[Finding] = Field(default_factory=list)
+
+    @property
+    def errors(self) -> list[Finding]:
+        return [f for f in self.findings if f.severity == "error"]
+
+    @property
+    def warnings(self) -> list[Finding]:
+        return [f for f in self.findings if f.severity == "warning"]
+
+    def failed(self, strict: bool = False) -> bool:
+        return bool(self.errors or (strict and self.warnings))
+
+
+class _Named(Protocol):
+    id: str
+
+
+class _Evidence(Protocol):
+    observation: str
+    role: str
+    path: str
+    lines: tuple[int, int]
+
+
+class _Requirement(Protocol):
+    id: str
+    capability: str
+    statement: str
+    support: str
+    evidence: Sequence[_Evidence]
+
+
+class _Observation(Protocol):
+    id: str
+    path: str
+    lines: tuple[int, int]
+
+
+class _Run(Protocol):
+    stopped_after: str | None
+
+
+class Linted(Protocol):
+    run: _Run
+    capabilities: Sequence[_Named]
+    requirements: Sequence[_Requirement]
+    observations: Sequence[_Observation]
+
+
+def normalise(statement: str) -> str:
+    return squash(statement).lower()
+
+
+def joins_obligations(statement: str) -> bool:
+    return "; " in statement or " and also " in statement or statement.count(" and ") > 2
+
+
+def check(result: Linted) -> list[Finding]:
+    found: list[Finding] = []
+    capability_ids = {c.id for c in result.capabilities}
+    observations = {o.id: o for o in result.observations}
+
+    def add(code: str, severity: Severity, ids: list[str], message: str) -> None:
+        found.append(Finding(code=code, severity=severity, ids=ids, message=message))
+
+    seen: dict[str, int] = {}
+    for requirement in result.requirements:
+        seen[requirement.id] = seen.get(requirement.id, 0) + 1
+    for requirement_id, count in seen.items():
+        if count > 1:
+            add("duplicate_requirement_id", "error", [requirement_id], f"{count} requirements share the id {requirement_id}")
+
+    for r in result.requirements:
+        if r.capability not in capability_ids:
+            add("unknown_capability", "error", [r.id, r.capability],
+                f"{r.id} names the capability {r.capability!r}, which the result does not have")
+        if not r.evidence:
+            add("no_evidence", "error", [r.id], f"{r.id} has no evidence")
+        for e in r.evidence:
+            seen_observation = observations.get(e.observation)
+            if seen_observation is None:
+                add("unknown_observation", "error", [r.id, e.observation],
+                    f"{r.id} cites {e.observation}, which is not among the observations")
+            elif (e.path, tuple(e.lines)) != (seen_observation.path, tuple(seen_observation.lines)):
+                add("evidence_mismatch", "error", [r.id, e.observation],
+                    f"{r.id}: evidence says {e.path}:{e.lines[0]}-{e.lines[1]} but {e.observation} is "
+                    f"{seen_observation.path}:{seen_observation.lines[0]}-{seen_observation.lines[1]}")
+
+    if result.run.stopped_after is None:
+        used = {r.capability for r in result.requirements}
+        for capability in result.capabilities:
+            if capability.id not in (PLUMBING, UNASSIGNED) and capability.id not in used:
+                add("capability_without_requirements", "warning", [capability.id],
+                    f"capability {capability.id} has no requirements")
+
+    by_statement: dict[tuple[str, str], list[str]] = {}
+    for r in result.requirements:
+        by_statement.setdefault((r.capability, normalise(r.statement)), []).append(r.id)
+    for (capability, _), ids in by_statement.items():
+        if len(ids) > 1:
+            add("duplicate_statement", "warning", ids, f"{', '.join(ids)} say the same thing in {capability}")
+
+    for r in result.requirements:
+        if len(r.statement.strip()) < MIN_STATEMENT:
+            add("short_statement", "warning", [r.id], f"{r.id}: the statement is shorter than {MIN_STATEMENT} characters")
+        if joins_obligations(r.statement):
+            add("compound_statement", "warning", [r.id], f"{r.id}: the statement probably joins several obligations; consider splitting it")
+        if r.evidence and all(e.role == "context" for e in r.evidence):
+            add("context_only_evidence", "warning", [r.id], f"{r.id}: all of its evidence is context, none establishes it")
+        if r.support in ("insufficient", "conflicting"):
+            add("weak_support", "warning", [r.id], f"{r.id}: the support check found it {r.support}")
+    return found
+
+
+def summary(report: LintReport) -> str:
+    """Findings grouped by severity, then a one-line count."""
+    lines: list[str] = []
+    for title, items in (("errors", report.errors), ("warnings", report.warnings)):
+        if items:
+            lines.append(f"{title}:")
+            lines += [f"  {f.code}  {f.message}" for f in items]
+    lines.append(f"{len(report.errors)} errors, {len(report.warnings)} warnings")
+    return "\n".join(lines)

backspec/modules/lint/facade.py

@@ -0,0 +1,7 @@
+"""Lint, as the rest of backspec sees it."""
+from __future__ import annotations
+
+from backspec.modules.lint.domain import Finding, Linted, LintReport, Severity, summary
+from backspec.modules.lint.use_cases import lint, lint_findings
+
+__all__ = ["Finding", "Linted", "LintReport", "Severity", "lint", "lint_findings", "summary"]

backspec/modules/lint/use_cases.py

@@ -0,0 +1,16 @@
+"""Run the checks on a result."""
+from __future__ import annotations
+
+from backspec.modules.lint.domain import Finding, Linted, LintReport, check
+
+
+def lint(result: Linted) -> LintReport:
+    return LintReport(findings=check(result))
+
+
+def lint_findings(result: Linted) -> list[Finding]:
+    """The findings for a run. Lint is an aid: whatever goes wrong inside it never fails a run."""
+    try:
+        return check(result)
+    except Exception:
+        return []

backspec/modules/report/domain.py

@@ -0,0 +1,128 @@
+"""The per-file ledger. Every file found ends in exactly one state, with a reason.
+
+This is processing coverage: what was read. It says nothing about whether the requirements in
+a file were found.
+"""
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+from typing import Literal
+
+from pydantic import BaseModel
+
+State = Literal["excluded", "skipped", "extracted", "no_requirements", "candidates_rejected", "failed",
+               "triaged_out"]
+
+
+class FileCoverage(BaseModel):
+    path: str
+    state: State
+    reason: str = ""
+    observations: int = 0
+    rejected: int = 0
+    lines: int = 0   # length of the file as it was read, for files that were read
+    cited_by: int = 0   # requirements that cite this file
+    source_hash: str = ""   # the file as it was read, for files that were read
+    analysis: Literal["syntax", "text"] = "text"   # how a file that was read was analysed
+    line_digest: str = ""   # a short digest per line, so a later run can tell which lines changed
+    triage_role: str = ""        # the role triage gave the file; empty if it was not triaged
+    triage_relevance: str = ""   # high, medium, low, none or unknown
+    triage_skipped: bool = False   # triage judged it surely irrelevant, so it was not read unless audited
+    triage_audited: bool = False   # skipped by triage but read anyway, as part of the audit
+    found_by_audit: bool = False   # audited, and a verified observation came of it
+
+
+def file_state(observations: int, rejected: int, failed: str | None, nothing_reason: str | None) -> tuple[State, str]:
+    """The state of a file that was sent to the observe stage."""
+    if failed:
+        return "failed", failed
+    if observations:
+        return "extracted", ""
+    if rejected:
+        return "candidates_rejected", f"{rejected} observation(s) failed the citation check"
+    return "no_requirements", nothing_reason or "the model reported nothing for this file"
+
+
+@dataclass(frozen=True)
+class RenderOutput:
+    """What a renderer returns: one text document, or a set of files (relative path -> text)."""
+    text: str | None = None
+    files: dict[str, str] = field(default_factory=dict)
+
+    @property
+    def is_tree(self) -> bool:
+        return self.text is None
+
+
+TITLE_MAX = 60
+_CLAUSE_END = re.compile(r"[,;:]\s|\s[-\u2013\u2014]\s|[.!?]\s")
+_TRAILING = " \t\n.,;:!?-\u2013\u2014"
+_MODAL = re.compile(r"\b(SHALL|MUST)\b")
+_LOWER_MODAL = re.compile(r"\b(shall|must)\b", re.I)
+
+
+def one_line(text: str) -> str:
+    """The text on one line, runs of white space made single spaces."""
+    return " ".join(text.split())
+
+
+def requirement_title(statement: str) -> str:
+    """A short title from a statement: its first clause, cut at a word to at most TITLE_MAX
+    characters, with no trailing punctuation."""
+    text = one_line(statement)
+    match = _CLAUSE_END.search(text)
+    clause = text[:match.start()] if match else text
+    clause = clause.strip(_TRAILING) or text.strip(_TRAILING) or text
+    if len(clause) > TITLE_MAX:
+        cut = clause[:TITLE_MAX + 1]
+        clause = cut[:cut.rfind(" ")] if " " in cut[1:] else cut[:TITLE_MAX]
+        clause = clause.strip(_TRAILING) or clause
+    return clause
+
+
+def with_shall(statement: str) -> str:
+    """The statement rewritten only as far as needed to contain SHALL or MUST. One that has
+    them is unchanged; one with a lower-case shall or must has it upper-cased; any other is
+    wrapped, never reworded."""
+    text = one_line(statement)
+    if _MODAL.search(text):
+        return text
+    if _LOWER_MODAL.search(text):
+        return _LOWER_MODAL.sub(lambda m: m.group(1).upper(), text)
+    return f"The system SHALL ensure: {text}"
+WORTH_A_LOOK_LINES = 80
+TRUNCATE_AT = 30
+
+
+class CoverageReport(BaseModel):
+    """A saved result's coverage, read for people: what happened to each file and what to look at."""
+    counts: dict[str, int]
+    files: dict[str, list[FileCoverage]]   # state -> files, every state but `excluded`
+    unused_observations: dict[str, int]    # capability -> observations no requirement cites
+    worth_a_look: list[FileCoverage]
+
+
+def worth_looking_at(entry: FileCoverage) -> bool:
+    """A file that was read, is long, and yielded nothing or was never cited."""
+    return bool(entry.source_hash) and entry.lines > WORTH_A_LOOK_LINES and (
+        entry.observations == 0 or entry.cited_by == 0)
+
+
+def _clip(lines: list[str], show_all: bool) -> list[str]:
+    if show_all or len(lines) <= TRUNCATE_AT:
+        return lines
+    return lines[:TRUNCATE_AT] + [f"  ... and {len(lines) - TRUNCATE_AT} more (--all shows every line)"]
+
+
+def coverage_text(report: CoverageReport, show_all: bool = False) -> str:
+    out = ["files per state: " + (", ".join(f"{n} {s}" for s, n in sorted(report.counts.items())) or "none")]
+    for state, entries in report.files.items():
+        out += ["", f"{state} ({len(entries)}):"]
+        out += _clip([f"  {e.path}" + (f"  ({e.reason})" if e.reason else "") for e in entries], show_all)
+    out += ["", "unused observations per capability:"]
+    out += _clip([f"  {c}: {n}" for c, n in report.unused_observations.items()] or ["  none"], show_all)
+    out += ["", "worth a look (read, over %d lines, no observation or cited by no requirement):" % WORTH_A_LOOK_LINES]
+    out += _clip([f"  {e.path}  ({e.lines} lines, {e.observations} observations, cited by {e.cited_by})"
+                  for e in report.worth_a_look] or ["  none"], show_all)
+    return "\n".join(out)

backspec/modules/report/dto.py

@@ -0,0 +1,92 @@
+"""The result of a run: the one document backspec writes. JSON is its canonical form."""
+from __future__ import annotations
+
+from typing import Literal
+
+from pydantic import BaseModel, Field
+
+from backspec.modules.capabilities.facade import Capability
+from backspec.modules.evidence.facade import Observation, Rejected
+from backspec.modules.lint.facade import Finding
+from backspec.modules.report.domain import FileCoverage
+from backspec.modules.requirements.facade import Requirement
+from backspec.modules.review.facade import ReviewSummary
+
+SCHEMA_VERSION = "0.1"
+
+
+class StageUsage(BaseModel):
+    calls: int = 0    # replies the model produced
+    cached: int = 0   # replies served from the cache: they cost nothing
+    input_tokens: int = 0
+    output_tokens: int = 0
+
+
+class Run(BaseModel):
+    status: Literal["complete", "partial"]
+    tool_version: str
+    prompt_version: str
+    provider: str
+    root: str
+    calls: int = 0
+    cached_calls: int = 0
+    capabilities_source: Literal["file", "oriented", "previous"] = "oriented"
+    observe_fingerprint: str = ""   # how files were observed; a later run reuses observations only if it matches
+    reused_files: int = 0           # files unchanged since the previous result, not read again
+    partly_read_files: int = 0      # files changed since the previous result: only the changed parts were read
+    syntax_files: int = 0           # files read with symbols from a parser
+    text_files: int = 0             # files read as plain text
+    synthesis_fingerprint: str = ""  # how requirements were written and judged
+    triage_fingerprint: str = ""    # how files were triaged; a later run reuses verdicts only if it matches
+    reused_capabilities: list[str] = Field(default_factory=list)   # evidence unchanged, requirements kept
+    stopped_after: Literal["capabilities"] | None = None   # set when a gated run stopped early
+    input_tokens: int = 0
+    output_tokens: int = 0
+    tokens_estimated: bool = False   # true if any call's tokens were estimated, not reported
+    cost_usd: float | None = None    # the backend's own cost figure, summed, when it reports one
+    usage_by_stage: dict[str, StageUsage] = Field(default_factory=dict)
+    budget_reached: bool = False     # a call was not sent because the token budget was reached
+
+
+class TriageAudit(BaseModel):
+    """Whether triage threw real evidence away: some skipped files were read anyway."""
+    skipped: int = 0     # files triage skipped
+    audited: int = 0     # of those, read anyway and read without failure
+    yielded: int = 0     # audited files that had a verified observation
+    yielded_paths: list[str] = Field(default_factory=list)
+    failed_paths: list[str] = Field(default_factory=list)   # audited files whose read failed; not counted in `audited`
+    miss_rate_interval: tuple[float, float] | None = None   # 95% Clopper-Pearson for yielded / audited; null if none audited
+
+
+class Coverage(BaseModel):
+    summary: dict[str, int]
+    files: list[FileCoverage]
+    triage_audit: TriageAudit | None = None   # set when triage ran
+    unused_observations: dict[str, int] = Field(default_factory=dict)   # capability -> count
+
+
+class Diagnostics(BaseModel):
+    rejected: list[Rejected] = Field(default_factory=list)
+    failed_capabilities: dict[str, str] = Field(default_factory=dict)
+    unconsolidated: dict[str, str] = Field(default_factory=dict)   # capability -> why its merge pass failed
+    dropped_scenarios: int = 0   # scenarios the model wrote that were left out (incomplete or uncited)
+    open_questions: dict[str, list[str]] = Field(default_factory=dict)
+    triage_unresolved: dict[str, str] = Field(default_factory=dict)   # path -> why triage gave no verdict; the file was read
+    review: ReviewSummary | None = None   # set when ids were carried forward or a review file applied
+    lint: list[Finding] = Field(default_factory=list)   # deterministic checks on the finished result
+    failed_stages: dict[str, str] = Field(default_factory=dict)   # a stage the whole run depends on -> why it failed
+
+
+class ExtractionResult(BaseModel):
+    schema_version: str = SCHEMA_VERSION
+    run: Run
+    capabilities: list[Capability]
+    requirements: list[Requirement]
+    hidden: list[Requirement] = Field(default_factory=list)   # rejected in review: kept here, left out of the catalogue
+    observations: list[Observation]
+    assignments: dict[str, str] = Field(default_factory=dict)   # observation id -> capability id
+    coverage: Coverage
+    diagnostics: Diagnostics = Field(default_factory=Diagnostics)
+
+    def to_json(self) -> str:
+        return self.model_dump_json(indent=1)

backspec/modules/report/facade.py

@@ -0,0 +1,11 @@
+"""The report, as the rest of backspec sees it."""
+from __future__ import annotations
+
+from backspec.modules.report.domain import CoverageReport, FileCoverage, RenderOutput, coverage_text
+from backspec.modules.report.dto import Coverage, Diagnostics, ExtractionResult, Run, StageUsage
+from backspec.modules.report.renderers import UnknownFormat, formats, render
+from backspec.modules.report.use_cases import build_coverage, coverage_report, earlier_evidence, earlier_triage, to_markdown
+
+__all__ = ["Coverage", "CoverageReport", "Diagnostics", "ExtractionResult", "FileCoverage", "RenderOutput", "Run", "StageUsage",
+           "UnknownFormat", "build_coverage", "coverage_report", "coverage_text", "earlier_evidence", "earlier_triage", "formats",
+           "render", "to_markdown"]