lorecraft 0.1.0__py3-none-any.whl

lorecraft/__init__.py

@@ -0,0 +1,5 @@
+"""Lorecraft: checks for a repository's agent-facing documentation."""
+
+from .metadata import __version__
+
+__all__: list[str] = ['__version__']

lorecraft/__main__.py

@@ -0,0 +1,6 @@
+"""Entry point for `python -m lorecraft`, equivalent to the `lorecraft` console script."""
+
+from .cli import main
+
+if __name__ == '__main__':
+    main()

lorecraft/checks/__init__.py

@@ -0,0 +1,35 @@
+"""Document checks: each validates one aspect of the documents a workspace model lists.
+
+A check is pure over the one part of a document it reads: the header check over the frontmatter node, the
+structure check over the headings, the budget check over the token count. The ``Database`` caches the model,
+the frontmatter, the parse trees and the token counts of one snapshot, and every check reads through it, so each
+part is computed once whichever checks read it; ``run`` asks the database for the part the check reads and hands
+only that to the check. A check returns violations, which name no document; ``run`` files them under the
+document's report, which locates them as findings. Nothing here prints: the ``check`` commands own the output
+and the exit codes.
+"""
+
+from .budget import BudgetCheckResult, validate_budget
+from .database import Database
+from .header import HeaderCheckResult, validate_header
+from .reporting import Finding, Violation, format_finding
+from .run import CheckRun, DocumentReport, run_budget, run_header, run_structure
+from .structure import StructureCheckResult, validate_structure
+
+__all__ = [
+    'Database',
+    'Finding',
+    'Violation',
+    'format_finding',
+    'HeaderCheckResult',
+    'validate_header',
+    'StructureCheckResult',
+    'validate_structure',
+    'BudgetCheckResult',
+    'validate_budget',
+    'DocumentReport',
+    'CheckRun',
+    'run_header',
+    'run_structure',
+    'run_budget',
+]

lorecraft/checks/budget.py

@@ -0,0 +1,54 @@
+"""Check one document's whole-file token count against the token budgets that govern it.
+
+The check is pure: it takes the already validated structure aspects that govern a document and the document's
+token count, and returns violations. It reads the count and nothing else, so it is handed that number rather than
+the text, and not the document's path: the run that called it attaches that. The budget is the global ``tokens``
+key of a structure specification, but it is a check of its own: the structure check reads a document's parse tree,
+and this one reads its raw text, frontmatter, code and tables included, since that is what loading it costs an
+agent.
+
+Each aspect is applied on its own. A namespace specification's budget does not replace the corpus one, so a
+document governed by both must fit both, and a namespace can only tighten the corpus budget.
+"""
+
+from dataclasses import dataclass
+from typing import Final
+
+from lorecraft.project.schemas import StructureAspect
+from lorecraft.project.syntax import LineNumber
+
+from .reporting import Violation
+
+_FIRST_LINE: Final[LineNumber] = LineNumber(1)
+"""Where a budget violation is reported: it concerns the whole file, not one line of it."""
+
+
+@dataclass(frozen=True, slots=True)
+class BudgetCheckResult:
+    """What the budget check found in one document.
+
+    Attributes:
+        violations: One per aspect whose budget the document exceeds, in aspect order; empty when it fits all.
+    """
+
+    violations: tuple[Violation, ...]
+
+
+def validate_budget(aspects: tuple[StructureAspect, ...], *, token_count: int) -> BudgetCheckResult:
+    """Check one document's token count against the budget of each structure aspect that sets one.
+
+    Every violation's message ends by naming the structure specification file that sets the budget, as the header
+    check names its schema: the number is in that file, not in the prose.
+
+    Args:
+        aspects: Applied each on its own; an aspect without a ``tokens`` budget is skipped. Empty means the
+            document is ungoverned, which yields no violations.
+        token_count: The ``o200k_base`` tokens in the document's whole file, as ``count_tokens`` counts them.
+    """
+    violations: list[Violation] = []
+    for aspect in aspects:
+        if aspect.tokens is None or token_count <= aspect.tokens:
+            continue
+        message = f'{token_count} tokens; the budget is {aspect.tokens} (per {aspect.path.name})'
+        violations.append(Violation(line=_FIRST_LINE, rule='budget.tokens', message=message, spec=aspect.path))
+    return BudgetCheckResult(violations=tuple(violations))

lorecraft/checks/database.py

@@ -0,0 +1,124 @@
+"""One frozen state of a workspace: its snapshot, and the cached data every check reads through.
+
+The layering follows the IntelliJ Platform's. The ``Snapshot`` plays the virtual file system: every byte a
+check can see, and never changed once built. Above it sit two kinds of cached data, each computed on first
+use and kept for as long as the database lives (pattern-memoization):
+
+- ``model()``: the workspace model, like the IDE's project model. It reads the structure of the snapshot, the
+  specifications and the corpus directories, and no document's contents.
+- ``frontmatter(ref)``: one document's frontmatter node, like a stub: the part of a file the IDE reads without
+  building its full syntax tree. It reads that document's bytes and nothing else.
+- ``parse(ref)``: one document's parse tree, like a PSI file or a per-file index entry. It reads that
+  document's bytes and nothing else.
+- ``tokens(ref)``: what one document's whole file costs an agent that loads it, like another per-file index
+  entry: counted from the raw text, frontmatter and code included, without a parse. It reads that document's
+  bytes and nothing else.
+
+Checks are plain functions of a database and a ref, like an inspection run over one file. Each asks for the
+cheapest query that holds what it reads, so a check that needs only the frontmatter never pays for the full
+parse, and every check reading the same query shares one computation of it. Their results are not cached; a
+check runs again every time.
+
+Nothing here records what a cached value read, so no dependency is tracked. Invalidation is written by hand instead,
+the way the IDE drops per-file index entries on a file change event and resets structural caches on a project model
+change. Reserved, not implemented: ``advance(snapshot) -> Database``, the next state. It would ``diff`` the two
+snapshots and carry over each cached value the change set leaves valid: the frontmatter, the parse and the token
+count of every document whose bytes did not change, and the model unless an entry under ``docs/`` was added or
+deleted or a specification changed. That rule holds only while the frontmatter, the parse and the token count each
+read their own document and the model reads no document, so keep them that way: data drawn from several documents
+belongs in a new cache with its own rule.
+"""
+
+from lorecraft.project.document import DocumentRef
+from lorecraft.project.document import Repository as DocumentRepository
+from lorecraft.project.syntax import FrontmatterNode, ParsedDocument, count_tokens, parse_document, parse_frontmatter
+from lorecraft.project.workspace import WorkspaceModel, load_model
+from lorecraft.vfs import Snapshot, VirtualFileSystem
+
+
+class Database:
+    """The workspace model, the frontmatter, the parse trees and the token counts of one snapshot, each cached for
+    its lifetime."""
+
+    def __init__(self, snapshot: Snapshot) -> None:
+        """Index the snapshot for reading; performs no I/O and computes nothing yet."""
+        self._fs = VirtualFileSystem(snapshot)
+        self._documents = DocumentRepository(self._fs)
+        # `None` until the first `model()` call; a loaded model is never `None`, so the two cannot be confused.
+        self._model: WorkspaceModel | None = None
+        self._frontmatters: dict[DocumentRef, FrontmatterNode] = {}
+        self._parses: dict[DocumentRef, ParsedDocument] = {}
+        self._token_counts: dict[DocumentRef, int] = {}
+
+    def model(self) -> WorkspaceModel:
+        """The workspace model the snapshot declares, loaded on the first call.
+
+        A load that fails is not cached, so each call raises the same error again.
+
+        Raises:
+            ListSpecsError: If the specification directory cannot be listed.
+            ListCorpusDirectoriesError: If docs/ cannot be listed.
+            ListDocumentsError: If a corpus directory cannot be listed.
+            GetHeaderSchemaError: If any header schema cannot be read or decoded.
+            InvalidHeaderSchemaError: If any header schema is not a well-formed JSON Schema.
+            GetStructureSchemaError: If any structure specification cannot be read.
+            InvalidStructureSchemaError: If any structure specification is not JSON in the dialect, or states no
+                usable rules.
+        """
+        if self._model is None:
+            self._model = load_model(self._fs)
+        return self._model
+
+    def frontmatter(self, ref: DocumentRef) -> FrontmatterNode:
+        """The frontmatter of one document, parsed from the snapshot on the first call for its ref.
+
+        Cached apart from ``parse(ref)`` and never read from it, so the answer does not depend on which of the two
+        was asked first; ``parse_frontmatter`` guarantees the two agree.
+
+        A document that cannot be read is not cached, so each call raises the same error again.
+
+        Raises:
+            DocumentDecodeError: If the document's bytes are not UTF-8.
+            GetDocumentError: If the snapshot holds no regular file at the document's path.
+        """
+        decoded = self._frontmatters.get(ref)
+        if decoded is None:
+            text = self._documents.get_document(ref).text
+            decoded = parse_frontmatter(text)
+            self._frontmatters[ref] = decoded
+        return decoded
+
+    def parse(self, ref: DocumentRef) -> ParsedDocument:
+        """The parse tree of one document, parsed from the snapshot on the first call for its ref.
+
+        A document that cannot be read is not cached, so each call raises the same error again.
+
+        Raises:
+            DocumentDecodeError: If the document's bytes are not UTF-8.
+            GetDocumentError: If the snapshot holds no regular file at the document's path.
+        """
+        parsed = self._parses.get(ref)
+        if parsed is None:
+            text = self._documents.get_document(ref).text
+            parsed = parse_document(text)
+            self._parses[ref] = parsed
+        return parsed
+
+    def tokens(self, ref: DocumentRef) -> int:
+        """The tokens in one document's whole file, counted from the snapshot on the first call for its ref.
+
+        Cached apart from ``parse(ref)`` and never read from it: the count needs the raw text, not the tree, so
+        a check that needs only one of the two never pays for the other.
+
+        A document that cannot be read is not cached, so each call raises the same error again.
+
+        Raises:
+            DocumentDecodeError: If the document's bytes are not UTF-8.
+            GetDocumentError: If the snapshot holds no regular file at the document's path.
+        """
+        count = self._token_counts.get(ref)
+        if count is None:
+            text = self._documents.get_document(ref).text
+            count = count_tokens(text)
+            self._token_counts[ref] = count
+        return count

lorecraft/checks/header.py

@@ -0,0 +1,122 @@
+"""Validate one document's frontmatter against the header schemas that govern it.
+
+The check is pure: it takes the already decoded, already validated header schemas that govern a document, the
+document's frontmatter node, and the filename and corpus the frontmatter is checked against, and returns violations.
+It reads those and nothing else, so it is handed that node rather than the whole parse tree, and not the document's
+path: the run that called it attaches that. Reading and parsing the document, choosing the schemas and deciding what
+a decode failure means all happen above it, in ``checks.run`` and the database.
+"""
+
+from dataclasses import dataclass
+from typing import Final
+
+from jsonschema import Draft202012Validator
+from jsonschema.exceptions import ValidationError
+
+from lorecraft.project.aspect import AspectFilename
+from lorecraft.project.corpus import CorpusName
+from lorecraft.project.schemas import HeaderAspect
+from lorecraft.project.syntax import (
+    Frontmatter,
+    FrontmatterNode,
+    InvalidYamlFrontmatter,
+    LineNumber,
+    MissingFrontmatter,
+    NonMappingFrontmatter,
+)
+
+from .reporting import Violation
+
+_FIRST_LINE: Final[LineNumber] = LineNumber(1)
+"""Where a violation with no more precise position is reported: a missing block, a missing key."""
+
+
+@dataclass(frozen=True, slots=True)
+class HeaderCheckResult:
+    """What the header check found in one document.
+
+    Attributes:
+        violations: In the order the check finds them; empty when the document conforms.
+    """
+
+    violations: tuple[Violation, ...]
+
+
+def validate_header(
+    schemas: tuple[HeaderAspect, ...], *, frontmatter: FrontmatterNode, filename: AspectFilename, corpus: CorpusName
+) -> HeaderCheckResult:
+    """Check one document's frontmatter against the header schemas that govern it. Pure: raises nothing.
+
+    Args:
+        schemas: Applied in order; each violation names the schema's path. Empty means the document is
+            ungoverned, which yields no violations.
+        filename: The document's filename, which the frontmatter ``name`` must equal (rule
+            ``frontmatter.name-matches-filename``).
+        corpus: The document's corpus, which namespaces the rule of every schema violation.
+    """
+    if not schemas:
+        return HeaderCheckResult(violations=())
+
+    if isinstance(frontmatter, MissingFrontmatter):
+        return _one_violation('frontmatter.missing', 'no `---` delimited frontmatter block')
+    if isinstance(frontmatter, InvalidYamlFrontmatter):
+        return _one_violation('frontmatter.unparseable', f'frontmatter is not valid YAML: {frontmatter.detail}')
+    if isinstance(frontmatter, NonMappingFrontmatter):
+        return _one_violation('frontmatter.unparseable', 'frontmatter is not a YAML mapping')
+
+    violations: list[Violation] = []
+    expected_name = str(filename)
+    name = frontmatter.data.get('name')
+    if name != expected_name:
+        violations.append(
+            Violation(
+                line=_key_line(frontmatter, 'name'),
+                rule='frontmatter.name-matches-filename',
+                message=f'`name` is {name!r}; expected {expected_name!r}',
+            )
+        )
+
+    rule_namespace = str(corpus)
+    for aspect in schemas:
+        validator = Draft202012Validator(aspect.schema)
+        errors = sorted(
+            validator.iter_errors(frontmatter.data),
+            key=lambda error: (tuple(str(part) for part in error.path), error.message),
+        )
+        for error in errors:
+            for field in _violated_fields(error):
+                rule = f'{rule_namespace}.{field}' if field else f'{rule_namespace}.frontmatter'
+                violations.append(
+                    Violation(
+                        line=_key_line(frontmatter, field) if field else _FIRST_LINE,
+                        rule=rule,
+                        message=f'{error.message} (per {aspect.path})',
+                        spec=aspect.path,
+                    )
+                )
+
+    return HeaderCheckResult(violations=tuple(violations))
+
+
+def _one_violation(rule: str, message: str) -> HeaderCheckResult:
+    """The result of a document whose frontmatter is unusable: one violation on its first line."""
+    return HeaderCheckResult(violations=(Violation(line=_FIRST_LINE, rule=rule, message=message),))
+
+
+def _key_line(frontmatter: Frontmatter, key: str) -> LineNumber:
+    """The line a top-level key is written on, or line 1 when the frontmatter does not have it."""
+    line = frontmatter.key_line(key)
+    if line is None:
+        return _FIRST_LINE
+    return line
+
+
+def _violated_fields(error: ValidationError) -> list[str]:
+    """Return the top-level fields implicated by one schema error."""
+    if error.path:
+        return [str(error.path[0])]
+
+    if error.validator == 'required' and isinstance(error.instance, dict):
+        return [str(field) for field in error.validator_value if field not in error.instance]
+
+    return ['']

lorecraft/checks/reporting.py

@@ -0,0 +1,59 @@
+"""Shared findings and output formatting for document checks.
+
+A check returns violations: where in the document a rule is broken, but not which document, since a check is a
+pure function of what it reads and never needs the document's path. The run that checked a document knows which
+it was, and ``Finding.at`` joins the two into a finding, the located form the output prints.
+"""
+
+from dataclasses import dataclass
+from typing import Self
+
+from lorecraft.project.syntax import LineNumber
+from lorecraft.vfs import RootRelativePath
+
+
+@dataclass(frozen=True, slots=True)
+class Violation:
+    """One broken rule in one document, without the document's path.
+
+    Attributes:
+        line: Where the violation is reported; line 1 when it concerns the whole document rather than one line.
+        rule: Stable identifier for the violated rule.
+        message: Human-readable explanation of the violation.
+        spec: The specification file that states the broken rule, or None, the default, for a rule the check
+            itself holds, such as a document that is not UTF-8.
+    """
+
+    line: LineNumber
+    rule: str
+    message: str
+    spec: RootRelativePath | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class Finding:
+    """One validation finding, located in a repository document; frozen, so findings compare and hash by value.
+
+    Attributes:
+        path: The affected document; turned into text only where a finding is printed or serialised.
+        line: The line containing the finding.
+        rule: Stable identifier for the violated rule.
+        message: Human-readable explanation of the finding.
+        spec: The specification file that states the broken rule, or None for a rule the check itself holds.
+    """
+
+    path: RootRelativePath
+    line: LineNumber
+    rule: str
+    message: str
+    spec: RootRelativePath | None = None
+
+    @classmethod
+    def at(cls, path: RootRelativePath, violation: Violation) -> Self:
+        """The finding a violation is, in the document at ``path``."""
+        return cls(path=path, line=violation.line, rule=violation.rule, message=violation.message, spec=violation.spec)
+
+
+def format_finding(finding: Finding) -> str:
+    """Format one finding for text output: ``<path>:<line>: [<rule>] <message>``."""
+    return f'{finding.path}:{finding.line}: [{finding.rule}] {finding.message}'

lorecraft/checks/run.py

@@ -0,0 +1,230 @@
+"""Run a check over documents of one database.
+
+A run asks the database for everything it reads: the model decides which aspects govern each document, and
+the part of the document the check reads is what the pure check validates — the frontmatter for the header
+check, the parse tree's headings for the structure check, the whole file's token count for the budget check.
+Selecting which documents to check is the caller's business: a run checks the refs it is handed, in the order
+given, and reads only the governed ones. Every check reports in the same shape, so the ``check`` commands print
+every run the same way.
+"""
+
+from dataclasses import dataclass
+
+from lorecraft.project.document import DocumentDecodeError, DocumentRef
+from lorecraft.project.schemas import StructureAspect
+from lorecraft.project.syntax import FrontmatterNode, LineNumber, ParsedDocument
+
+from .budget import validate_budget
+from .database import Database
+from .header import validate_header
+from .reporting import Finding, Violation
+from .structure import validate_structure
+
+
+@dataclass(frozen=True, slots=True)
+class DocumentReport:
+    """The outcome of checking one selected document.
+
+    Attributes:
+        ref: The document the report is about; its path is the report path.
+        governed: False when no specification governs the document for the check's aspect; the document was
+            then never parsed.
+        violations: What the check found, without the document's path; empty for an ungoverned document.
+    """
+
+    ref: DocumentRef
+    governed: bool
+    violations: tuple[Violation, ...]
+
+    def findings(self) -> tuple[Finding, ...]:
+        """Every violation, located in the report's document; empty exactly when the document is clean."""
+        return tuple(Finding.at(self.ref.path, violation) for violation in self.violations)
+
+
+@dataclass(frozen=True, slots=True)
+class CheckRun:
+    """One pass of one check over the selected documents: what the text and JSON printers consume.
+
+    Attributes:
+        reports: One per selected document, in the order the refs were given.
+    """
+
+    reports: tuple[DocumentReport, ...]
+
+    def findings(self) -> tuple[Finding, ...]:
+        """Every finding of every report, in report order; empty exactly when the run is clean."""
+        findings: list[Finding] = []
+        for report in self.reports:
+            findings.extend(report.findings())
+        return tuple(findings)
+
+
+def run_header(database: Database, refs: tuple[DocumentRef, ...]) -> CheckRun:
+    """Check each ref against the header schemas that govern it, in the order given.
+
+    A governed document that is not UTF-8 carries the single violation ``frontmatter.undecodable`` at line 1.
+
+    Args:
+        database: The snapshot state the refs come from; its model decides which schemas govern each document.
+        refs: The documents to check; each must be one the database's model lists.
+
+    Raises:
+        GetDocumentError: If a governed document is missing from the snapshot; a decode failure is a finding.
+        ListSpecsError: If the model is not loaded yet and the specification directory cannot be listed.
+        ListCorpusDirectoriesError: If the model is not loaded yet and docs/ cannot be listed.
+        ListDocumentsError: If the model is not loaded yet and a corpus directory cannot be listed.
+        GetHeaderSchemaError: If the model is not loaded yet and a header schema cannot be read or decoded.
+        InvalidHeaderSchemaError: If the model is not loaded yet and a header schema is malformed.
+        GetStructureSchemaError: If the model is not loaded yet and a structure specification cannot be read.
+        InvalidStructureSchemaError: If the model is not loaded yet and a structure specification is malformed.
+    """
+    reports: list[DocumentReport] = []
+    for ref in refs:
+        aspects = database.model().governance(ref).header_schemas()
+        if not aspects:
+            reports.append(DocumentReport(ref, governed=False, violations=()))
+            continue
+        frontmatter = _frontmatter(database, ref)
+        if frontmatter is None:
+            reports.append(DocumentReport(ref, governed=True, violations=(_undecodable('frontmatter'),)))
+            continue
+        result = validate_header(aspects, frontmatter=frontmatter, filename=ref.filename, corpus=ref.corpus)
+        reports.append(DocumentReport(ref, governed=True, violations=result.violations))
+    return CheckRun(reports=tuple(reports))
+
+
+def run_structure(database: Database, refs: tuple[DocumentRef, ...]) -> CheckRun:
+    """Check each ref against the structure specifications that govern it, in the order given.
+
+    A governed document that is not UTF-8 carries the single violation ``structure.undecodable`` at line 1.
+
+    Args:
+        database: The snapshot state the refs come from; its model decides which specifications govern each
+            document.
+        refs: The documents to check; each must be one the database's model lists.
+
+    Raises:
+        GetDocumentError: If a governed document is missing from the snapshot; a decode failure is a finding.
+        ListSpecsError: If the model is not loaded yet and the specification directory cannot be listed.
+        ListCorpusDirectoriesError: If the model is not loaded yet and docs/ cannot be listed.
+        ListDocumentsError: If the model is not loaded yet and a corpus directory cannot be listed.
+        GetHeaderSchemaError: If the model is not loaded yet and a header schema cannot be read or decoded.
+        InvalidHeaderSchemaError: If the model is not loaded yet and a header schema is malformed.
+        GetStructureSchemaError: If the model is not loaded yet and a structure specification cannot be read.
+        InvalidStructureSchemaError: If the model is not loaded yet and a structure specification is malformed.
+    """
+    reports: list[DocumentReport] = []
+    for ref in refs:
+        aspects = database.model().governance(ref).structure_specs()
+        if not aspects:
+            reports.append(DocumentReport(ref, governed=False, violations=()))
+            continue
+        document = _parse(database, ref)
+        if document is None:
+            reports.append(DocumentReport(ref, governed=True, violations=(_undecodable('structure'),)))
+            continue
+        result = validate_structure(aspects, headings=document.headings)
+        reports.append(DocumentReport(ref, governed=True, violations=result.violations))
+    return CheckRun(reports=tuple(reports))
+
+
+def run_budget(database: Database, refs: tuple[DocumentRef, ...]) -> CheckRun:
+    """Check each ref's whole-file token count against the budgets that govern it, in the order given.
+
+    A document is governed when at least one of its structure specifications sets a ``tokens`` budget; one whose
+    specifications set none is ungoverned, and its text is never read. A governed document that is not UTF-8
+    carries the single violation ``budget.undecodable`` at line 1.
+
+    Args:
+        database: The snapshot state the refs come from; its model decides which specifications govern each
+            document.
+        refs: The documents to check; each must be one the database's model lists.
+
+    Raises:
+        GetDocumentError: If a governed document is missing from the snapshot; a decode failure is a finding.
+        ListSpecsError: If the model is not loaded yet and the specification directory cannot be listed.
+        ListCorpusDirectoriesError: If the model is not loaded yet and docs/ cannot be listed.
+        ListDocumentsError: If the model is not loaded yet and a corpus directory cannot be listed.
+        GetHeaderSchemaError: If the model is not loaded yet and a header schema cannot be read or decoded.
+        InvalidHeaderSchemaError: If the model is not loaded yet and a header schema is malformed.
+        GetStructureSchemaError: If the model is not loaded yet and a structure specification cannot be read.
+        InvalidStructureSchemaError: If the model is not loaded yet and a structure specification is malformed.
+    """
+    reports: list[DocumentReport] = []
+    for ref in refs:
+        aspects = _budgeted(database.model().governance(ref).structure_specs())
+        if not aspects:
+            reports.append(DocumentReport(ref, governed=False, violations=()))
+            continue
+        token_count = _tokens(database, ref)
+        if token_count is None:
+            reports.append(DocumentReport(ref, governed=True, violations=(_undecodable('budget'),)))
+            continue
+        result = validate_budget(aspects, token_count=token_count)
+        reports.append(DocumentReport(ref, governed=True, violations=result.violations))
+    return CheckRun(reports=tuple(reports))
+
+
+def _budgeted(aspects: tuple[StructureAspect, ...]) -> tuple[StructureAspect, ...]:
+    """The structure aspects that set a ``tokens`` budget, in the order given."""
+    budgeted: list[StructureAspect] = []
+    for aspect in aspects:
+        if aspect.tokens is not None:
+            budgeted.append(aspect)
+    return tuple(budgeted)
+
+
+def _frontmatter(database: Database, ref: DocumentRef) -> FrontmatterNode | None:
+    """The document's frontmatter node, or ``None`` when its bytes are not UTF-8.
+
+    Returns:
+        The frontmatter node. ``None`` is the degraded return ``_parse`` documents, for the same reason.
+
+    Raises:
+        GetDocumentError: If the document is missing from the snapshot; a decode failure is not raised.
+    """
+    try:
+        return database.frontmatter(ref)
+    except DocumentDecodeError:
+        return None
+
+
+def _parse(database: Database, ref: DocumentRef) -> ParsedDocument | None:
+    """The document's parse tree, or ``None`` when its bytes are not UTF-8.
+
+    Returns:
+        The parse tree. ``None`` is the degraded return for bytes that are present but not UTF-8: those are on
+        the same side of the line as invalid YAML, since the document is wrong, so the caller reports a finding
+        rather than taking the exit-2 path an unreadable file takes.
+
+    Raises:
+        GetDocumentError: If the document is missing from the snapshot; a decode failure is not raised.
+    """
+    try:
+        return database.parse(ref)
+    except DocumentDecodeError:
+        return None
+
+
+def _tokens(database: Database, ref: DocumentRef) -> int | None:
+    """The tokens in the document's whole file, or ``None`` when its bytes are not UTF-8.
+
+    Returns:
+        The token count. ``None`` is the degraded return ``_parse`` documents, for the same reason.
+
+    Raises:
+        GetDocumentError: If the document is missing from the snapshot; a decode failure is not raised.
+    """
+    try:
+        return database.tokens(ref)
+    except DocumentDecodeError:
+        return None
+
+
+def _undecodable(rule_namespace: str) -> Violation:
+    """The violation a governed document that is not UTF-8 carries instead of the check's own."""
+    return Violation(
+        line=LineNumber(1),
+        rule=f'{rule_namespace}.undecodable',
+        message='document is not valid UTF-8',
+    )