primarysource 0.1.0__py3-none-any.whl

primarysource/pdfcorpus/spec.py

@@ -0,0 +1,37 @@
+"""What a fetched-document collection tells the shared skeleton about itself."""
+
+from __future__ import annotations
+
+import os
+from collections.abc import Callable, Mapping
+from dataclasses import dataclass
+from pathlib import Path
+
+
+@dataclass(frozen=True)
+class CorpusSpec:
+    """One collection's names and hooks.
+
+    `env_var` overrides the corpus directory (the old `NISTLIB_CORPUS`), else `default_dir()` gives it.
+    `read_defaults(root)` returns `{publication name: doc_id}` for bare names with no revision, and `alias(key)`
+    turns a normalised reference into such a name (NIST: "nist sp 800-53" -> "800-53"). Both are optional.
+
+    `fetch_command` and `index_command` are what a human runs when the corpus or its search index is missing or
+    stale; they become an error's `action:` line. With no `index_command` the fetch command is named, since a fetch
+    also builds the index.
+    """
+
+    name: str
+    server_name: str
+    env_var: str
+    fetch_command: str
+    default_dir: Callable[[], Path]
+    instructions: str
+    cite_example: str
+    read_defaults: Callable[[Path], Mapping[str, str]] | None = None
+    alias: Callable[[str], str] | None = None
+    index_command: str = ""
+
+    def corpus_dir(self) -> Path:
+        env = os.environ.get(self.env_var)
+        return Path(env) if env else self.default_dir()

primarysource/pdfcorpus/split.py

@@ -0,0 +1,391 @@
+"""The numbered-heading splitter shared by the fetched-PDF collections (nist, usb): pdftotext text in, one
+`Section` per heading out (ROADMAP `cl.segment_numbered`).
+
+It was usblib's splitter, the better of the two copies, chosen by measurement (docs/validation.md): against the
+PDF bookmarks it beat nistlib's copy on NIST's own documents (median agreement 0.93 against 0.60) and on USB's.
+A heading is accepted when it is the next child of a parent already seen, or when the document's own contents
+lists it (`_admitted_by_contents`), which stops one missed heading from dropping every later sibling.
+"""
+
+from __future__ import annotations
+
+import re
+from collections import Counter
+from collections.abc import Iterable
+
+from primarysource.pdfcorpus.models import Section
+
+# "Appendix A. Title" -> id "A";  "A.4.1. Title" -> id "A.4.1";  "3.2.4. Title" or "3.2.4 Title" -> id "3.2.4".
+# The title may follow the id without a space ("A.5.1.1.1GENERAL ...") when pdftotext loses the gap.
+# Also the colon and dash forms NIST uses ("Appendix A: Title", sp800-38c line 580).
+_APPENDIX = re.compile(r"^\f?\s*(?:Appendix|APPENDIX)\s+(?P<id>[A-Z])\s*[.:—–-]\s*(?P<heading>\S.*)$")
+# A running page header glued in front of a heading by pdftotext -layout (NIST's DOI line or an underscore
+# rule, sp800-218 line 295). Stripped before matching; the line keeps its number.
+_RUNNING_HEADER = re.compile(
+    r"^(\f?)\s*(?:This publication is available free of charge from:\s*\S+|_{20,})\s{2,}(?=\S)"
+)
+_LETTERED = re.compile(r"^\f?\s*(?P<id>[A-Z](?:\.\d+)+)\.?\s*(?P<heading>[A-Z].*)$")
+_NUMBERED = re.compile(r"^\f?\s*(?P<id>\d+(?:\.\d+)*)(?P<dot>\.?)\s*(?P<heading>[A-Z].*)$")
+_CHAPTER = re.compile(r"^\f?\s*Chapter\s+(?P<id>\d+)\.?\s+(?P<heading>[A-Z].*)$")
+_CHAPTER_NUMBER_ONLY = re.compile(r"^\f?\s*Chapter\s+(?P<id>\d+)\.?\s*$")
+# "Chapter 1: Introduction". The title must be single-spaced to the end of the line: a running page header
+# pads the title out to a column ("Chapter 1: Introduction     Revision 1.1  3/3/2024") and is not a heading.
+# (usblib d15f16d: SuperSpeed electrical compliance, 2 wrong sections to 4 right ones.)
+_CHAPTER_COLON = re.compile(r"^\f?\s*Chapter\s+(?P<id>\d+):\s+(?P<heading>[A-Z]\S*(?: \S+)*)\s*$")
+_LEADER = re.compile(r"\.{4,}|(?:\. ){4,}")
+# A contents row without a dot leader: "1 Introduction        15", a page number after a run of spaces.
+_CONTENTS_ROW = re.compile(r"^\f?\s*(?P<id>\d+)\s+(?P<heading>[A-Z].*?[A-Za-z)])\s{3,}\d+\s*$")
+_MAX_HEADING = 120
+_MIN_RUNNING_REPEATS = 3
+# The foreign headings `_retract_foreign_tail` may drop at once (USB's case was 3: chapters 3, 4 and 4.1).
+_MAX_FOREIGN_TAIL = 3
+
+
+_MARGIN_NUMBER = re.compile(r"^(\f?\s*)\d{1,4}\s{2,}(?=\d+(?:\.\d+)*\.?\s+[A-Z])")
+
+
+def _strip_margin(line: str) -> str:
+    """Drop the margin line number that some specifications print before a dotted heading ("119  2.2.3 Title")."""
+    m = _MARGIN_NUMBER.match(line)
+    return line if m is None else m.group(1) + line[m.end() :]
+
+
+def _heading_at(lines: list[str], index: int) -> tuple[str, str] | None:
+    """Heading at ``lines[index]``: ``Chapter N. Title``, ``Chapter N`` with the title next, or a numbered line."""
+    line = _strip_margin(lines[index])
+    m = _CHAPTER.match(line) or _CHAPTER_COLON.match(line)
+    if m is not None:
+        return _accept_title(m.group("id"), m.group("heading"))
+    m = _CHAPTER_NUMBER_ONLY.match(line)
+    if m is not None:
+        for following in lines[index + 1 : index + 3]:
+            if following.strip():
+                return _accept_title(m.group("id"), following.strip())
+        return None
+    return _match_heading(line)
+
+
+def _accept_title(sec_id: str, title: str) -> tuple[str, str] | None:
+    """A chapter title is accepted when it reads as a title: capitalised, short, and not a table-of-contents row."""
+    title = title.strip()
+    if _LEADER.search(title) or len(title) > _MAX_HEADING or title.endswith("."):
+        return None
+    if not title[:1].isupper():
+        return None
+    return sec_id, title
+
+
+def _match_heading(line: str) -> tuple[str, str] | None:
+    """Return (id, heading) when ``line`` looks like a section heading, else None."""
+    for pattern in (_APPENDIX, _LETTERED, _NUMBERED):
+        m = pattern.match(line)
+        if m is None:
+            continue
+        heading = m.group("heading").strip()
+        # A bare chapter number ("3 CCID Functional ...") is a heading only when it reads as one: a table row
+        # has runs of spaces between columns, and a wrapped note has an unclosed bracket.
+        bare_chapter = pattern is _NUMBERED and not m.group("dot") and "." not in m.group("id")
+        if bare_chapter and ("  " in heading or heading.count("(") != heading.count(")")):
+            return None
+        if _LEADER.search(heading) or len(heading) > _MAX_HEADING or heading.endswith("."):
+            return None  # table-of-contents row or running prose
+        return m.group("id"), heading
+    return None
+
+
+def _ordinal(part: str) -> int:
+    return int(part) if part.isdigit() else ord(part) - ord("A") + 1
+
+
+def _canon(sec_id: str) -> str:
+    """``03.01`` and ``3.1`` are one id for counting (sp800-171r3 numbers its requirements ``03.01.01``)."""
+    return ".".join(str(int(part)) if part.isdigit() else part for part in sec_id.split("."))
+
+
+def _parent_key(sec_id: str) -> str:
+    """Key of the counter ``sec_id`` belongs to: its parent id, or one of two top-level sequences."""
+    parent, _, _ = sec_id.rpartition(".")
+    if parent:
+        return _canon(parent)
+    return "#appendix" if sec_id.isalpha() else "#chapter"
+
+
+def _is_next(sec_id: str, last_child: dict[str, int]) -> bool:
+    """True when ``sec_id`` is the next child of a parent already seen (``1`` and ``A`` open the top levels)."""
+    key = _parent_key(sec_id)
+    if key not in last_child and not key.startswith("#"):
+        return False
+    return _ordinal(sec_id.rpartition(".")[2]) == last_child.get(key, 0) + 1
+
+
+def _running_titles(hits: Iterable[tuple[str, str]]) -> set[str]:
+    """Titles that follow a bare number on many lines: page footers and headers such as ``5   Revision 1.0``.
+
+    A chapter title is not repeated behind three different chapter numbers, but a footer is repeated behind
+    every page number, and the next page number would otherwise pass as the next chapter. A title repeated
+    behind one number only (``2. IAD Use Model Example`` at the top of every page of chapter 2) is that
+    chapter's running header, not a footer, and is left alone (usblib 2ff1feb).
+    """
+    counts: Counter[str] = Counter()
+    numbers: dict[str, set[str]] = {}
+    for sec_id, heading in hits:
+        if "." not in sec_id and sec_id.isdigit():
+            counts[heading] += 1
+            numbers.setdefault(heading, set()).add(sec_id)
+    return {
+        heading for heading, count in counts.items() if count >= _MIN_RUNNING_REPEATS and len(numbers[heading]) >= 2
+    }
+
+
+def _counters(found: list[tuple[str, str, int]]) -> dict[str, int]:
+    """Rebuild the per-parent counters from the headings accepted so far."""
+    last_child: dict[str, int] = {}
+    for sec_id, _, _ in found:
+        _advance(last_child, sec_id)
+    return last_child
+
+
+def _advance(last_child: dict[str, int], sec_id: str) -> None:
+    """Record ``sec_id`` as the latest child of its parent. Its own number, not a count: a heading admitted by
+    the contents after a gap (``_admitted_by_contents``) moves the counter past the gap."""
+    key = _parent_key(sec_id)
+    last_child[key] = max(last_child.get(key, 0), _ordinal(sec_id.rpartition(".")[2]))
+    last_child.setdefault(_canon(sec_id), 0)
+
+
+def _admitted_by_contents(
+    sec_id: str, heading: str, last_child: dict[str, int], toc: set[tuple[str, str]], toc_ids: set[str]
+) -> bool:
+    """A heading the sequence rule rejected, but which the document's own contents lists under the same number.
+
+    The sequence rule is what keeps list items and cross-references out, and it is also why one missed heading
+    used to drop every later sibling and child: the next one is never "next" (docs/validation.md, usb: 1,485 of
+    1,749 misses). The contents is an independent list of the real headings, so a heading it names may skip a
+    gap. Forward only, so a contents row met again later cannot rewind the counters, and its parent must be
+    known (seen, or itself in the contents).
+    """
+    if sec_id not in toc_ids:
+        return False
+    title = _normal(heading)
+    if not any(
+        toc_id == sec_id and (toc_title == title or toc_title.startswith(title) or title.startswith(toc_title))
+        for toc_id, toc_title in toc
+        if len(toc_title) >= 3
+    ):
+        return False
+    key = _parent_key(sec_id)
+    if not key.startswith("#") and key not in last_child and key not in {_canon(t) for t in toc_ids}:
+        return False
+    return _ordinal(sec_id.rpartition(".")[2]) > last_child.get(key, 0)
+
+
+def _retract_premature_chapters(found: list[tuple[str, str, int]], sec_id: str) -> bool:
+    """Drop trailing chapters that ``sec_id`` proves were false: a list item or footnote read as ``8 Title``.
+
+    When 7.3 turns up after a chapter 8, the 8 cannot have been a heading, or 7.3 would sit under the
+    wrong chapter. Only bare chapter numbers are retracted, and only the unbroken run just before ``sec_id``.
+    """
+    top = _ordinal(sec_id.partition(".")[0]) if sec_id[0].isdigit() else None
+    if top is None:
+        return False
+    dropped = False
+    while found and "." not in found[-1][0] and found[-1][0].isdigit() and int(found[-1][0]) > top:
+        found.pop()
+        dropped = True
+    return dropped
+
+
+def _number_after_appendix(sec_id: str, last_child: dict[str, int], toc_ids: set[str]) -> bool:
+    """Appendices follow the chapters, so a bare number after one is a list item (usblib d15f16d: USB PD's
+    revision-history item "11. VCONN Ownership" was read as chapter 11) unless the contents lists it as a
+    chapter. Without that exception a stray early "Appendix A" line (a cross-reference in the front matter)
+    blocked every chapter after it: NIST sp800-204d fell from 0.97 to 0.05 against its bookmarks."""
+    return sec_id.isdigit() and "#appendix" in last_child and _canon(sec_id) not in {_canon(t) for t in toc_ids}
+
+
+def _retract_foreign_tail(found: list[tuple[str, str, int]], sec_id: str) -> bool:
+    """Drop the headings accepted after ``X.(n-1)``'s subtree when ``X.n`` shows they were not inside ``X``.
+
+    A compliance test prints another specification's chapter headers inside its own assertion table, so a
+    ``3``, ``4`` and ``4.1`` can be accepted between ``2.3`` and ``2.4``. When ``2.4`` arrives as the next child
+    of ``2``, those headings cannot belong to ``2``'s outline. Nothing is dropped when any of them is a
+    descendant of ``X``, so an ordinary subsection is never retracted by its sibling (usblib d15f16d). Nor when
+    the tail is longer than ``_MAX_FOREIGN_TAIL``: then ``X.n`` is more likely a cross-reference deep in a later
+    chapter than the end of a short foreign table, and retracting would drop real chapters (measured on NIST:
+    unlimited, the median agreement fell from 0.929 to 0.880).
+    Ids are compared in their counting form, so ``03.01.02`` finds ``03.01.01``.
+    """
+    parent, _, last = sec_id.rpartition(".")
+    if not parent or not last.isdigit() or int(last) < 2:
+        return False
+    parent = _canon(parent)
+    previous = f"{parent}.{int(last) - 1}"
+    position = next((i for i in range(len(found) - 1, -1, -1) if _canon(found[i][0]) == previous), None)
+    if position is None:
+        return False
+    start = position + 1
+    while start < len(found) and _canon(found[start][0]).startswith(previous + "."):
+        start += 1
+    tail = found[start:]
+    if not tail or len(tail) > _MAX_FOREIGN_TAIL or any(_canon(entry[0]).startswith(parent + ".") for entry in tail):
+        return False
+    del found[start:]
+    return True
+
+
+def _retract_unexplained_chapters(found: list[tuple[str, str, int]], chapter_id: str, explicit: set[int]) -> bool:
+    """Drop trailing bare numbers at or above ``chapter_id`` that were not written as "Chapter N" lines.
+
+    An explicit "Chapter 4. Title" line proves that an earlier bare "4. A Source May ..." list item, which
+    arrived first, was not a heading. Only the unbroken run of such bare numbers at the end is dropped.
+    """
+    number = int(chapter_id)
+    dropped = False
+    while (
+        found
+        and "." not in found[-1][0]
+        and found[-1][0].isdigit()
+        and int(found[-1][0]) >= number
+        and found[-1][2] not in explicit
+    ):
+        found.pop()
+        dropped = True
+    return dropped
+
+
+def _contents_entries(lines: list[str]) -> set[tuple[str, str]]:
+    """(id, normalised title) of every numbered row of a table of contents.
+
+    A row either ends in a dot leader, or is a number and title followed by a page number after a run of
+    spaces, as in the HID Usage Tables contents ("1 Introduction      15").
+    """
+    entries: set[tuple[str, str]] = set()
+    for line in lines:
+        leader = _LEADER.search(line)
+        if leader is not None:
+            m = _NUMBERED.match(line[: leader.start()].rstrip() or "x")
+            if m is not None:
+                entries.add((m.group("id"), _normal(m.group("heading"))))
+            continue
+        m = _CONTENTS_ROW.match(line)
+        if m is not None:
+            entries.add((m.group("id"), _normal(m.group("heading"))))
+    return entries
+
+
+def _normal(title: str) -> str:
+    return " ".join(title.split()).casefold()
+
+
+def _drop_childless_chapters(
+    found: list[tuple[str, str, int]], contents: set[tuple[str, str]]
+) -> list[tuple[str, str, int]]:
+    """Drop runs of two or more consecutive bare chapters that own no ``N.x`` subsection.
+
+    Numbered steps in a test procedure ("4 Verify that the link is up", "5 Reset the port") read exactly like
+    chapters but come in runs with no subsection between them. A lone chapter without subsections, such as
+    "2 Normative References" between two structured chapters, is kept, and so is any chapter that the document's
+    own table of contents lists with the same title. A document in which no chapter has a subsection is left
+    alone, since nothing then separates its chapters from its steps.
+    """
+    with_children = {sec_id.partition(".")[0] for sec_id, _, _ in found if "." in sec_id}
+    if not with_children:
+        return found
+
+    def bare(entry: tuple[str, str, int]) -> bool:
+        return entry[0].isdigit() and entry[0] not in with_children and (entry[0], _normal(entry[1])) not in contents
+
+    kept: list[tuple[str, str, int]] = []
+    run: list[tuple[str, str, int]] = []
+    for entry in [*found, None]:
+        if entry is not None and bare(entry):
+            run.append(entry)
+            continue
+        if len(run) < 2:
+            kept.extend(run)
+        run = []
+        if entry is not None:
+            kept.append(entry)
+    return kept
+
+
+def _reads_as_prose(title: str) -> bool:
+    """True when a heading carries running text or a table row: a sentence break, a mid-line colon, or columns.
+
+    A label that ends in a colon ("Host Key(s):") is a heading, not running text, so it does not count here.
+    """
+    return ": " in title or ". " in title or "   " in title
+
+
+def _shallow_chapters(toc_ids: set[str]) -> set[str]:
+    """Chapters the contents list without any subsection, in a contents that does list subsections elsewhere.
+
+    When some chapters have their subsections listed and a chapter does not, its subsections are either absent
+    or were misread as headings, and a heading under such a chapter must then also read as running text.
+    """
+    deep = {sec_id.partition(".")[0] for sec_id in toc_ids if "." in sec_id}
+    chapters = {sec_id for sec_id in toc_ids if "." not in sec_id}
+    return chapters - deep if deep else set()
+
+
+def split_sections(text: str) -> tuple[Section, ...]:
+    """Find numbered headings in ``text`` and return one Section per heading.
+
+    Lines are numbered from 1 and split on newline only, so form feeds do not shift the numbering. Sections
+    do not nest: each runs from its heading to the line before the next heading (the last runs to the end).
+    A heading is accepted only when it is the next child of a parent already seen (3.2.4 after 3.2.3 inside
+    3.2; 2 after 1; B after A). That keeps numbered list items and cross-references that look like headings
+    from being read as sections, and it also means one missed heading drops its later siblings.
+    """
+    lines = [_RUNNING_HEADER.sub(r"\1", line) for line in text.split("\n")]
+    toc = _contents_entries(lines)
+    toc_ids = {sec_id for sec_id, _ in toc}
+    shallow = _shallow_chapters(toc_ids)
+    if lines and lines[-1] == "":
+        lines.pop()
+    hits = [(number, _heading_at(lines, number - 1)) for number in range(1, len(lines) + 1)]
+    # A "Chapter N" header repeats on every page of its chapter; it is never a footer, so it is exempt
+    # from the running-title filter and does not count towards it.
+    chapter_lines = {
+        number
+        for number, line in enumerate(lines, start=1)
+        if _CHAPTER.match(line) or _CHAPTER_COLON.match(line) or _CHAPTER_NUMBER_ONLY.match(line)
+    }
+    running = _running_titles(hit for number, hit in hits if hit is not None and number not in chapter_lines)
+    found: list[tuple[str, str, int]] = []
+    last_child: dict[str, int] = {}
+    for number, hit in hits:
+        if hit is None:
+            continue
+        sec_id, heading = hit
+        if "." not in sec_id and heading in running and number not in chapter_lines:
+            continue
+        if (
+            "." not in sec_id
+            and number in chapter_lines
+            and _retract_unexplained_chapters(found, sec_id, chapter_lines)
+        ):
+            last_child = _counters(found)
+        if (
+            "." in sec_id
+            and sec_id.partition(".")[0] in shallow
+            and sec_id not in toc_ids
+            and _reads_as_prose(heading)
+        ):
+            continue  # a cross-reference or table cell under a chapter the contents shows without subsections
+        if _number_after_appendix(sec_id, last_child, toc_ids):
+            continue
+        if not _is_next(sec_id, last_child) and not _admitted_by_contents(sec_id, heading, last_child, toc, toc_ids):
+            continue
+        if "." in sec_id and _retract_foreign_tail(found, sec_id):
+            last_child = _counters(found)
+        if "." in sec_id and _retract_premature_chapters(found, sec_id):
+            last_child = _counters(found)
+        _advance(last_child, sec_id)
+        found.append((sec_id, heading, number))
+    found = _drop_childless_chapters(found, toc)
+    sections: list[Section] = []
+    for index, (sec_id, heading, start) in enumerate(found):
+        end = found[index + 1][2] - 1 if index + 1 < len(found) else len(lines)
+        sections.append(Section(id=sec_id, heading=heading, start_line=start, end_line=end))
+    return tuple(sections)

primarysource/pdfcorpus/tools.py

@@ -0,0 +1,244 @@
+"""The five tools as plain functions returning `render()` text. ``ToolError`` marks a caller mistake.
+
+Every error is a `LibError`: the message says what is wrong, `hint` names the tool call that helps next (with the
+collection's prefix, `nist_outline('sp800-73pt2-5')`), and `action` the fetch or index command when a human must
+act. A search that matches nothing is text saying so, not an error.
+
+Cursors (PLAN.md §5.11): a read stops at its token budget and its `next:` is a ref, `<doc_id>:<first>-<last>`, that
+`read` takes back as its `doc`; search and outline are lists whose `next:` is `start=<n>`.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+from primarysource import budget, index
+from primarysource.errors import LibError
+from primarysource.pdfcorpus import fts
+from primarysource.pdfcorpus.corpus import Corpus
+from primarysource.pdfcorpus.models import Document, Section
+from primarysource.refs import RefGrammar
+from primarysource.render import next_start, render
+
+# Document ids are file-safe slugs (`sp800-73pt2-5`, `sp800-53r5-oscal`), matched without regard to case.
+# Section ids are dotted numbers, appendix letters and control ids (`3.2.4`, `A.1`, `AC-2(1)`).
+GRAMMAR = RefGrammar(r"[A-Za-z0-9][A-Za-z0-9._-]*", r"[0-9A-Za-z][0-9A-Za-z.()_-]*")
+
+REF_FORMS = "use <doc_id>§<section> or <doc_id>:<first>-<last>"
+READ_TOKENS = 2000
+OUTLINE_ROWS = 200
+
+
+class ToolError(LibError):
+    """A bad argument or unknown reference; `hint` says what to call instead."""
+
+
+def _norm(s: str) -> str:
+    return " ".join(s.lower().split())
+
+
+def resolve_doc(corpus: Corpus, doc: str) -> Document:
+    """Find a document by ``doc_id`` or by ``number`` (case/space-insensitive).
+
+    Failing that, a bare name with no revision resolves through the corpus's defaults (NIST: "NIST SP 800-53"
+    -> the current final revision), when the collection supplies them. An explicit id or number always wins.
+    """
+    key = _norm(doc)
+    for d in corpus.documents:
+        if _norm(d.doc_id) == key or _norm(d.number) == key:
+            return d
+    if corpus.defaults:
+        wanted = corpus.defaults.get(corpus.alias(key) if corpus.alias else key)
+        if wanted is not None:
+            for d in corpus.documents:
+                if d.doc_id == wanted:
+                    return d
+    known = ", ".join(d.doc_id for d in corpus.documents) or "none"
+    raise ToolError(f"unknown document {doc!r} (known doc_ids: {known})", f"{corpus.tool('search')}('<words>')")
+
+
+def resolve_section(corpus: Corpus, document: Document, section: str) -> Section:
+    sid = section.strip().lstrip("§").strip()
+    for s in document.sections:
+        if s.id == sid:
+            return s
+    raise ToolError(f"{document.doc_id} has no section {sid!r}", f"{corpus.tool('outline')}({document.doc_id!r})")
+
+
+def parse_lines(spec: str) -> tuple[int, int]:
+    m = re.fullmatch(r"\s*(\d+)\s*(?:[-:]\s*(\d+))?\s*", spec)
+    if not m:
+        raise ToolError(f"cannot parse lines={spec!r}", "give 'START-END', e.g. lines='120-160'")
+    start = int(m.group(1))
+    return start, int(m.group(2) or start)
+
+
+@dataclass(frozen=True, slots=True)
+class Target:
+    """What a read or cite names: a document and a section, a line range, or neither."""
+
+    document: Document
+    section: Section | None = None
+    lines: tuple[int, int] | None = None
+
+
+def resolve(corpus: Corpus, doc: str, section: str | None = None, lines: str | None = None) -> Target:
+    """`doc` is a doc_id, a number, or a ref: `<doc_id>§<section>` or `<doc_id>:<first>-<last>` (a read's `next:`).
+
+    A ref's section or range and an explicit `section`/`lines` argument may not both be given.
+    """
+    if section is not None and lines is not None:
+        raise ToolError("give one of section or lines, not both", "")
+    ref = None
+    if "§" in doc or "::" in doc or ":" in doc:
+        ref = GRAMMAR.parse(doc)
+        if ref.para is not None or ref.anchor is not None:
+            raise ToolError(f"{doc!r}: this collection has no ¶ or # refs", REF_FORMS)
+        if ref.doc is None:
+            raise ToolError(f"{doc!r} names no document", REF_FORMS)
+        if section is not None or lines is not None:
+            raise ToolError(f"{doc!r} already names a section or lines; drop the section/lines argument", "")
+    document = resolve_doc(corpus, ref.doc if ref is not None and ref.doc else doc)
+    if ref is not None and ref.section is not None:
+        section = ref.section
+    if section is not None:
+        return Target(document, section=resolve_section(corpus, document, section))
+    span = ref.lines if ref is not None else (parse_lines(lines) if lines is not None else None)
+    return Target(document, lines=span)
+
+
+def _numbered(lines: list[str], start: int, end: int, width: int) -> list[str]:
+    return [f"{n:>{width}}: {lines[n - 1]}" for n in range(start, end + 1)]
+
+
+def _section_lines(document: Document) -> dict[str, Section]:
+    return {s.id: s for s in document.sections}
+
+
+def search(corpus: Corpus, query: str, doc: str | None = None, limit: int = 8, start: int = 0) -> str:
+    """Sections best matching `query` from the FTS5 index, `limit` from the 0-based `start`."""
+    if not index.query_terms(query):
+        raise ToolError("the query has no words to search for", f"{corpus.tool('search')}('<words>')")
+    if limit < 1:
+        raise ToolError("limit must be at least 1", "")
+    if start < 0:
+        raise ToolError("start is a 0-based offset", "pass the start=<n> of a next: line")
+    only = resolve_doc(corpus, doc).doc_id if doc else None
+    conn = fts.open_index(corpus)
+    try:
+        result = fts.search(conn, query, doc_id=only, limit=limit, start=start)
+    finally:
+        conn.close()
+    if not result.hits:
+        if start:
+            return render(f"No more passages for {query!r} after {start}.", {})
+        return render(f"No passages match {query!r}.", {})
+    documents = {d.doc_id: d for d in corpus.documents}
+    out = [f"Passages {start + 1}-{start + len(result.hits)} for {query!r}, best first:"]
+    for hit in result.hits:
+        document = documents.get(hit.doc_id)
+        section = _section_lines(document).get(hit.section_id) if document else None
+        where = f"lines {section.start_line}-{section.end_line}, " if section else ""
+        snippet = " ".join(hit.snippet.split())
+        out.append(f"\n{hit.doc_id}§{hit.section_id}  {hit.title}  ({where}score {hit.score:.3g})\n  {snippet}")
+    footer = {
+        "next": next_start(result.next_start),
+        "relaxed": "no section holds every query word; these hold some" if result.relaxed else "",
+    }
+    return render("\n".join(out), footer)
+
+
+def read(
+    corpus: Corpus,
+    doc: str,
+    section: str | None = None,
+    lines: str | None = None,
+    max_tokens: int = READ_TOKENS,
+) -> str:
+    """Numbered lines of a section or a line range, cut at whole lines within `max_tokens`."""
+    if max_tokens < 1:
+        raise ToolError("max_tokens must be at least 1", "")
+    target = resolve(corpus, doc, section, lines)
+    d = target.document
+    if target.section is not None:
+        s = target.section
+        start, end, label = s.start_line, s.end_line, f"{d.doc_id}§{s.id} {s.heading}"
+    elif target.lines is not None:
+        start, end = target.lines
+        label = d.doc_id
+    else:
+        raise ToolError(
+            f"say what to read in {d.doc_id}: a section or a line range",
+            f"{corpus.tool('outline')}({d.doc_id!r}) lists sections; then {corpus.tool('read')}('{d.doc_id}§<id>')",
+        )
+    if start < 1 or end < start:
+        raise ToolError(f"invalid line range {start}-{end}", "lines are 1-based and START <= END")
+    text = corpus.text_lines(d)
+    if end > len(text):
+        raise ToolError(f"line range {start}-{end} is past the end of {d.doc_id} ({len(text)} lines)", "")
+    width = len(str(end))
+    taken: list[str] = []
+    used = 0
+    for row in _numbered(text, start, end, width):
+        cost = budget.tokens(row + "\n")
+        if taken and used + cost > max_tokens:
+            break
+        taken.append(row)
+        used += cost
+    last = start + len(taken) - 1
+    shown = f"lines {start}-{last}" + ("" if last == end else f" of {start}-{end}")
+    cite = f"{d.number} §{target.section.id}" if target.section is not None else f"{d.number}, lines {start}-{last}"
+    footer = {
+        "cite": cite,
+        "next": f"{d.doc_id}:{last + 1}-{end}" if last < end else "",
+        "warning": f"status is {d.status!r}, not 'final'" if d.status.lower() != "final" else "",
+    }
+    return render(f"{label} ({shown})\n" + "\n".join(taken), footer)
+
+
+def outline(corpus: Corpus, doc: str, start: int = 0, limit: int = OUTLINE_ROWS) -> str:
+    d = resolve_doc(corpus, doc)
+    if limit < 1:
+        raise ToolError("limit must be at least 1", "")
+    if start < 0:
+        raise ToolError("start is a 0-based offset", "pass the start=<n> of a next: line")
+    if not d.sections:
+        return render(f"{d.doc_id}: the index lists no sections for this document.", {})
+    rows = d.sections[start : start + limit]
+    out = [f"{d.number}, {d.title}"]
+    for s in rows:
+        indent = "  " * s.id.count(".")
+        out.append(f"{indent}{s.id}  {s.heading}  (lines {s.start_line}-{s.end_line})")
+    more = start + limit < len(d.sections)
+    return render("\n".join(out), {"next": next_start(start + limit if more else None)})
+
+
+def info(corpus: Corpus, doc: str) -> str:
+    d = resolve_doc(corpus, doc)
+    out = [
+        f"doc_id: {d.doc_id}",
+        f"number: {d.number}",
+        f"title: {d.title}",
+        f"date: {d.date}",
+        f"status: {d.status}",
+        f"url: {d.url}",
+        f"sha256: {d.sha256}",
+        f"sections: {len(d.sections)}",
+    ]
+    warning = ""
+    if d.status.lower() != "final":
+        warning = f"status is {d.status!r}, not 'final'. Check for a current version before relying on it."
+    return render("\n".join(out), {"warning": warning})
+
+
+def cite(corpus: Corpus, doc: str, section: str | None = None) -> str:
+    target = resolve(corpus, doc, section)
+    d = target.document
+    if target.section is not None:
+        s = target.section
+        return render(f"{d.number} §{s.id}\n{d.url}\nHeading: {s.heading}", {})
+    if target.lines is not None:
+        first, last = target.lines
+        return render(f"{d.number}, lines {first}-{last}\n{d.url}", {})
+    return render(f"{d.number}\n{d.url}", {})