primarysource 0.1.0__py3-none-any.whl

primarysource/pkgdocs/serving.py

@@ -0,0 +1,158 @@
+"""The MCP server and the command line for a pkgdocs collection, built from its `Spec`.
+
+Tools carry the collection's prefix (`gitdocs_read`, `pep_search`), answer in text with the fixed footer, and turn a
+`LibError` into an MCP tool error (`isError`) whose text names the remedy. Every tool reads installed files only;
+none fetches or writes (`primarysource <collection> import` is the one command that does, and it is not a tool).
+`mcp` is imported only when a server is built, so the readers and the importer need nothing but the standard
+library.
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from collections.abc import Sequence
+from typing import Any
+
+from primarysource import __version__
+from primarysource.errors import LibError
+from primarysource.pkgdocs.runtime import Library, Spec
+from primarysource.scaffold import add_tools
+
+DEFAULT_MAX_TOKENS = 2000
+
+
+def build_server(spec: Spec, library: Library | None = None) -> Any:
+    """An `MCPServer` with read, outline, search, cite, docs and status tools for `spec`."""
+    from mcp.server.mcpserver import MCPServer
+    from mcp.types import ToolAnnotations
+
+    lib = library or Library(spec)
+    examples = "; ".join(spec.example_refs)
+
+    def read(ref: str, max_tokens: int = DEFAULT_MAX_TOKENS) -> str:
+        return lib.read(ref, max_tokens=max_tokens)
+
+    def outline(doc: str, start: int = 0, max_tokens: int = 4000) -> str:
+        return lib.outline(doc, start=start, max_tokens=max_tokens)
+
+    def search(query: str, doc_prefix: str = "", limit: int = 10, start: int = 0) -> str:
+        return lib.search(query, limit=limit, start=start, doc_prefix=doc_prefix)
+
+    def cite(ref: str) -> str:
+        return lib.cite(ref)
+
+    def docs(prefix: str = "", limit: int = 200, start: int = 0) -> str:
+        return lib.docs(prefix, limit=limit, start=start)
+
+    def status() -> str:
+        return lib.status()
+
+    read.__doc__ = (
+        f"Read {spec.title} text by ref, cut at block boundaries within max_tokens. A 'next: <ref>' footer line is "
+        "the ref that reads the next page: pass it back as ref. '¶n' reads from the nth block. The cite: line is the "
+        "citation.\n\n"
+        f"ref: '<doc>[@<commit>]§<section>[#<anchor>][¶<n>]'; the section is an id, an anchor, a heading or a unique "
+        f"part of one. Examples: {examples}"
+    )
+    outline.__doc__ = (
+        "List a document's sections, each with the ref that reads it. doc: a document id (see docs). A "
+        "'next: start=N' footer line means call again with start=N (a 0-based offset)."
+    )
+    search.__doc__ = (
+        f"Find sections of {spec.title} by words (all of them, else some, which the relaxed: line says). "
+        "One line per "
+        "hit: '<ref>: <document> / <section> -- <snippet>'. doc_prefix narrows to document ids starting with it. "
+        "A 'next: start=N' footer line means call again with start=N (a 0-based offset)."
+    )
+    cite.__doc__ = (
+        "The citation for a ref (document, section, edition, commit, source file, licence) and its span hash."
+    )
+    docs.__doc__ = (
+        "List document ids and titles, optionally those whose id starts with prefix. A 'next: start=N' footer line "
+        "means call again with start=N (a 0-based offset)."
+    )
+    status.__doc__ = "What this collection pins, what is imported, and its licence."
+
+    server = MCPServer(f"primarysource-{spec.name}", version=__version__, instructions=spec.instructions)
+    annotations = ToolAnnotations(
+        read_only_hint=True, idempotent_hint=True, destructive_hint=False, open_world_hint=False
+    )
+    add_tools(server, (read, outline, search, cite, docs, status), prefix=spec.prefix, annotations=annotations)
+    return server
+
+
+def _parser(spec: Spec) -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(prog=f"primarysource {spec.name}", description=spec.title)
+    commands = parser.add_subparsers(dest="command", required=True)
+    imp = commands.add_parser("import", help="fetch the pinned edition (one request per second) and install it")
+    imp.add_argument("--tree", help="only this tree")
+    imp.add_argument("--commit", help="a pinned edition other than the current one: a label or commit prefix")
+    imp.add_argument("--archive", help="read this downloaded archive instead of fetching (needs --tree)")
+    imp.add_argument("--allow-unpinned", action="store_true", help="trust the digest of this one fetch")
+    commands.add_parser("status", help="what is pinned and imported, and the licence")
+    docs = commands.add_parser("docs", help="list documents")
+    docs.add_argument("prefix", nargs="?", default="")
+    docs.add_argument("--limit", type=int, default=200)
+    docs.add_argument("--start", type=int, default=0)
+    outline = commands.add_parser("outline", help="a document's sections with their refs")
+    outline.add_argument("doc")
+    outline.add_argument("--start", type=int, default=0)
+    outline.add_argument("--max-tokens", type=int, default=4000)
+    read = commands.add_parser("read", help="read by ref")
+    read.add_argument("ref")
+    read.add_argument("--max-tokens", type=int, default=DEFAULT_MAX_TOKENS)
+    search = commands.add_parser("search", help="find sections by words")
+    search.add_argument("query")
+    search.add_argument("--doc-prefix", default="")
+    search.add_argument("--limit", type=int, default=10)
+    search.add_argument("--start", type=int, default=0)
+    commands.add_parser("cite", help="the citation for a ref").add_argument("ref")
+    return parser
+
+
+def _write(text: str) -> None:
+    """UTF-8 to stdout whatever the console's code page is: refs contain `§` and `¶`."""
+    stream = getattr(sys.stdout, "buffer", None)
+    if stream is not None:
+        sys.stdout.flush()
+        stream.write(text.encode("utf-8"))
+        stream.flush()
+    else:
+        sys.stdout.write(text)
+
+
+def run(spec: Spec, argv: Sequence[str], library: Library | None = None) -> int:
+    """`primarysource <collection> ...`: exit 0, or 1 with the error and its hint on stderr."""
+    args = _parser(spec).parse_args(list(argv))
+    lib = library or Library(spec)
+    try:
+        if args.command == "import":
+            for report in lib.import_edition(
+                args.tree, args.commit, args.archive, allow_unpinned=args.allow_unpinned
+            ):
+                skipped = ", ".join(f"{k}: {v}" for k, v in sorted(report.skipped.items())) or "none"
+                print(
+                    f"{report.tree} {report.label} {report.commit[:12]}: {report.docs} documents, {report.sections} "
+                    f"sections, {report.units} blocks, {report.warnings} reader warnings\n"
+                    f"  files sha256 {report.files_sha256} ({'matches the pin' if report.pinned else 'NOT pinned'})\n"
+                    f"  skipped: {skipped}\n  installed at {report.path}"
+                )
+            return 0
+        if args.command == "status":
+            text = lib.status()
+        elif args.command == "docs":
+            text = lib.docs(args.prefix, limit=args.limit, start=args.start)
+        elif args.command == "outline":
+            text = lib.outline(args.doc, start=args.start, max_tokens=args.max_tokens)
+        elif args.command == "read":
+            text = lib.read(args.ref, max_tokens=args.max_tokens)
+        elif args.command == "search":
+            text = lib.search(args.query, limit=args.limit, start=args.start, doc_prefix=args.doc_prefix)
+        else:
+            text = lib.cite(args.ref)
+        _write(text if text.endswith("\n") else text + "\n")
+        return 0
+    except LibError as exc:
+        print(exc.as_text(), file=sys.stderr)
+        return 1

primarysource/pkgdocs/store.py

@@ -0,0 +1,167 @@
+"""An installed edition: parsed documents and a search index under `$PRIMARYSOURCE_HOME/<collection>/`.
+
+    <collection>/<tree>/<commit>/edition.json    what was imported, from where, under which digest
+                                 index.json       the documents (id, title, file, meta) and chapter aliases
+                                 docs/<n>.json    one parsed document
+                                 index.sqlite     the full-text index
+
+An edition is written into a staging directory beside its destination and renamed into place, so a reader sees
+the old edition or the new one. What is stored is the parsed sections, not the archive and not the repository;
+nothing in this module or its caller writes under the source tree.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import secrets
+import shutil
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from primarysource.errors import LibError
+from primarysource.pkgdocs import index
+from primarysource.pkgdocs.model import Doc
+
+RECORD = "edition.json"
+CATALOGUE = "index.json"
+SEARCH = "index.sqlite"
+_COMMIT = frozenset("0123456789abcdef")
+
+
+@dataclass(frozen=True, slots=True)
+class Entry:
+    """A document as listed in the edition's catalogue."""
+
+    id: str
+    title: str
+    file: str
+    sections: int
+    meta: Mapping[str, str]
+
+
+class Edition:
+    """One installed edition, opened for reading."""
+
+    def __init__(self, path: Path) -> None:
+        self.path = path
+        try:
+            self.record: dict[str, Any] = json.loads((path / RECORD).read_text(encoding="utf-8"))
+            listing = json.loads((path / CATALOGUE).read_text(encoding="utf-8"))
+            self.entries: dict[str, Entry] = {
+                str(e["id"]): Entry(
+                    str(e["id"]), str(e["title"]), str(e["file"]), int(e["sections"]), dict(e["meta"])
+                )
+                for e in listing["docs"]
+            }
+            self.aliases: dict[str, str] = {str(k): str(v) for k, v in listing["aliases"].items()}
+        except (OSError, ValueError, KeyError, TypeError) as exc:
+            raise LibError(f"the edition at {path} is damaged ({exc})", "", "import it again") from exc
+
+    @property
+    def commit(self) -> str:
+        return str(self.record["commit"])
+
+    @property
+    def label(self) -> str:
+        return str(self.record.get("label", ""))
+
+    def load(self, doc_id: str) -> Doc:
+        entry = self.entries.get(doc_id)
+        if entry is None:
+            raise LibError(f"no document {doc_id!r} in this edition", "list the documents first")
+        try:
+            return Doc.from_json(json.loads((self.path / "docs" / entry.file).read_text(encoding="utf-8")))
+        except (OSError, ValueError) as exc:
+            raise LibError(f"document {doc_id!r} is unreadable ({exc})", "", "import the edition again") from exc
+
+    @property
+    def index_path(self) -> Path:
+        return self.path / SEARCH
+
+
+def write_edition(
+    base: Path,
+    commit: str,
+    record: Mapping[str, Any],
+    docs: Sequence[Doc],
+    aliases: Mapping[str, str] | None = None,
+) -> Path:
+    """Install `docs` as `<base>/<commit>`, replacing an earlier install of the same commit. Returns the path."""
+    if not commit or not set(commit) <= _COMMIT:
+        raise LibError(f"{commit!r} is not a commit id", "a pin names a full lowercase hex commit")
+    base.mkdir(parents=True, exist_ok=True)
+    staging = base / f".{commit}.part-{secrets.token_hex(4)}"
+    staging.mkdir()
+    try:
+        (staging / "docs").mkdir()
+        listing: list[dict[str, Any]] = []
+        for number, doc in enumerate(docs, start=1):
+            name = f"{number:05d}.json"
+            (staging / "docs" / name).write_text(json.dumps(doc.to_json(), ensure_ascii=False), encoding="utf-8")
+            listing.append(
+                {
+                    "id": doc.id,
+                    "title": doc.title,
+                    "file": name,
+                    "sections": len(doc.sections),
+                    "meta": dict(doc.meta),
+                }
+            )
+        (staging / CATALOGUE).write_text(
+            json.dumps({"docs": listing, "aliases": dict(aliases or {})}, ensure_ascii=False, indent=0),
+            encoding="utf-8",
+        )
+        sections = index.build(staging / SEARCH, docs)
+        (staging / RECORD).write_text(
+            json.dumps({**record, "commit": commit, "docs": len(docs), "sections": sections}, indent=2) + "\n",
+            encoding="utf-8",
+        )
+        target = base / commit
+        old = base / f".{commit}.old-{secrets.token_hex(4)}"
+        if target.exists():
+            os.replace(target, old)
+        try:
+            os.replace(staging, target)
+        except BaseException:
+            if old.exists():
+                os.replace(old, target)
+            raise
+        shutil.rmtree(old, ignore_errors=True)
+        return target
+    finally:
+        shutil.rmtree(staging, ignore_errors=True)
+
+
+def installed(base: Path) -> list[Edition]:
+    """Every readable edition under `base`, newest import first. A damaged one is skipped, not hidden by
+    raising: `find` reports the damage when it is the one asked for."""
+    found: list[Edition] = []
+    if not base.is_dir():
+        return found
+    for child in sorted(base.iterdir()):
+        if child.is_dir() and not child.name.startswith(".") and (child / RECORD).is_file():
+            try:
+                found.append(Edition(child))
+            except LibError:
+                continue
+    found.sort(key=lambda e: str(e.record.get("imported", "")), reverse=True)
+    return found
+
+
+def find(base: Path, commit: str | None, what: str, hint: str) -> Edition:
+    """The edition whose commit starts with `commit`, or the newest import when `commit` is None."""
+    editions = installed(base)
+    if not editions:
+        raise LibError(f"{what} is not imported (nothing under {base})", hint)
+    if commit is None:
+        return editions[0]
+    matches = [e for e in editions if e.commit.startswith(commit.lower())]
+    if len(matches) == 1:
+        return matches[0]
+    have = ", ".join(e.commit[:12] for e in editions)
+    if not matches:
+        raise LibError(f"no imported edition of {what} has a commit starting {commit!r} (have: {have})", hint)
+    raise LibError(f"{commit!r} matches more than one edition of {what} ({have})", "give more of the commit")

primarysource/pms/__init__.py

@@ -0,0 +1,16 @@
+"""`pms`: the Gentoo Package Manager Specification, a pinned EAPI release, read from its LaTeX source.
+Served under `pms_`.
+
+Stdlib only. `mcp` is imported by `server.py` when a server is built; `primarysource pms ...` runs `cli.py`.
+`docs/pms.md` is the description; `catalogue.py` holds the pin, the licence and the build.
+"""
+
+from __future__ import annotations
+
+from primarysource.pkgdocs.runtime import make_collection
+from primarysource.pms.catalogue import SPEC
+
+COLLECTION = make_collection(SPEC, "primarysource.pms")
+"""Refs: `pms:pms§Ebuild-defined variables`; see `pkgdocs.runtime`."""
+
+__all__ = ["COLLECTION", "SPEC"]

primarysource/pms/catalogue.py

@@ -0,0 +1,88 @@
+"""The `pms` collection: the Gentoo Package Manager Specification, read from its LaTeX source.
+
+Source, placement and pin (read from the repository on 2026-11-13, see `docs/pms.md`):
+
+- the tree is the root of gentoo/pms at the commit of the tag `eapi-9-approved-2026-06-14` (the approved EAPI 9
+  text), fetched as a GitHub tarball (77 KB). `pms.tex` includes the chapters in order; the build reads them from
+  the archive's own files, never from disk, into one document, `pms`, numbered as the typeset specification is
+  (chapters 1, 2, ..., then A, B, ... after `\\appendix`);
+- the licence is **CC-BY-SA-4.0**, from the title page of `pms.tex`: "released under the Creative Commons
+  Attribution-ShareAlike 4.0 International licence". (Group F's notes said 3.0, read on the EAPI 3 page; the
+  current source says 4.0, and that is what is recorded.) Placement is **layer, with attribution**: every
+  citation carries the credit, and the licence line travels with the layer;
+- GLEPs, the other half of the original brief, are not here: gitweb.gentoo.org answered HTTP 502 to the
+  snapshot request on 2026-11-13 and the project has no GitHub mirror (`gentoo/glep` and `projg2/glep` are not
+  found). See ROADMAP row `col.gentoo_glep`.
+"""
+
+from __future__ import annotations
+
+from collections import Counter
+from collections.abc import Mapping
+
+from primarysource.errors import LibError
+from primarysource.pkgdocs.fetch import Source
+from primarysource.pkgdocs.readers.latex import read_latex
+from primarysource.pkgdocs.runtime import Build, Licence, Pin, Spec, TreeSpec, matching
+
+COMMIT = "2cee9c6bd86851738e8ba1874ac622cfcec70969"
+
+PIN = Pin(
+    commit=COMMIT,
+    label="eapi-9-approved-2026-06-14",
+    date="2026-06-14",
+    files_sha256="c302537c3d04c28a69b3e4fae19247f168913a29beff587e2e26d6af08d62502",
+    sources=(
+        Source(f"https://codeload.github.com/gentoo/pms/tar.gz/{COMMIT}", strip=1),
+        Source(f"https://github.com/gentoo/pms/archive/{COMMIT}.tar.gz", strip=1, hosts=("codeload.github.com",)),
+    ),
+)
+
+LICENCE = Licence(
+    spdx="CC-BY-SA-4.0",
+    redistribute="layer",
+    attribution="(c) 2007-2026 Stephen Bennett, Ciaran McCreesh and others; contributions are owned by their authors",
+    evidence="the title page of pms.tex in gentoo/pms " + COMMIT[:12],
+    credit="CC BY-SA 4.0, Stephen Bennett, Ciaran McCreesh and others",
+)
+
+TITLE = "Package Manager Specification"
+
+
+def build(files: Mapping[str, bytes], pin: Pin) -> Build:
+    skipped: Counter[str] = Counter()
+    if "pms.tex" not in files:
+        raise LibError("pms.tex is not in the archive", "the repository layout changed; report the collection")
+    try:
+        doc = read_latex(files, "pms.tex", "pms", f"{TITLE} ({pin.label})")
+    except LibError:
+        skipped["not UTF-8 text"] += 1
+        raise
+    aliases = {s.id.split(".")[0]: "pms" for s in doc.sections if s.id != "0"}
+    aliases["pms-" + pin.label] = "pms"
+    return Build((doc.with_meta(path="pms.tex", licence="CC-BY-SA-4.0"),), aliases, skipped)
+
+
+SPEC = Spec(
+    name="pms",
+    title="Gentoo Package Manager Specification",
+    licence=LICENCE,
+    trees=(
+        TreeSpec(
+            name="pms",
+            title="gentoo/pms (LaTeX)",
+            pins=(PIN,),
+            include=matching(r"[A-Za-z0-9._-]+\.tex"),
+            build=build,
+        ),
+    ),
+    instructions=(
+        "The Gentoo Package Manager Specification (PMS), one document, 'pms', numbered as the typeset text is: "
+        "chapters 1-14, then appendices A, B, ... Loop: pms_search('<words>') -> pms_outline('pms') -> "
+        "pms_read('<ref>'). Section numbers work as refs without a document: 'pms:8.2.4'. A section is an id, "
+        "an anchor (the LaTeX label, e.g. ch:ebuild-vars), a heading or a unique part of one. Text is reduced "
+        "from LaTeX: \\ref numbers are resolved, markup is dropped, and the warning: line lists the commands "
+        "that were dropped. Every citation carries the CC BY-SA 4.0 credit."
+    ),
+    example_refs=("pms:8.2.4", "pms:pms§Ebuild-defined variables", "pms:pms§ch:ebuild-vars¶2"),
+)

primarysource/pms/cli.py

@@ -0,0 +1,12 @@
+"""`primarysource pms import|status|docs|outline|read|search|cite`."""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+from primarysource.pkgdocs.serving import run
+from primarysource.pms.catalogue import SPEC
+
+
+def main(argv: Sequence[str]) -> int:
+    return run(SPEC, argv)

primarysource/pms/server.py

@@ -0,0 +1,8 @@
+"""The MCP server for `pms` (`primarysource mcp --collection pms`). Imports `mcp` when this module loads."""
+
+from __future__ import annotations
+
+from primarysource.pkgdocs.serving import build_server
+from primarysource.pms.catalogue import SPEC
+
+server = build_server(SPEC)

primarysource/readers/__init__.py

@@ -0,0 +1,10 @@
+"""The core readers (PLAN.md §4). Each takes bytes or text and never unpacks an archive to disk.
+
+- `html`: HTML to readable markdown blocks, zimlore's converter plus a `Profile` (`cl.reader_html`).
+- `markdown`: a strict Markdown subset into headings and blocks (`cl.reader_markdown`).
+- `pdf`: a bookmarked PDF into sections of sentences, `pypdf` imported only when a PDF is read
+  (`cl.reader_pdf_pypdf`); `bookmarks` cuts the page texts at the bookmarks (`cl.segment_bookmarks`).
+- `xml`: the one hardened XML parser: no DTDs, no entities, no network (`cl.reader_xml_hardened`).
+
+Standard library only, apart from `pypdf`, which is an ingest-time extra (PLAN.md §5.13).
+"""

primarysource/readers/bookmarks.py

@@ -0,0 +1,192 @@
+"""Cut a bookmarked document's page texts into numbered sections of sentences (`cl.segment_bookmarks`).
+
+Backend-independent: it takes the page texts and the flattened bookmark outline that a reader produced
+(`primarysource.readers.pdf`) and never opens a file. Lifted from the ecma collection's private reader
+(`primarysource/ecma/_pdf.py`), whose tests moved with it.
+
+- a bookmark whose title starts with a clause number (`5.3 Separator characters`, `A.2.1 Data Types`) or
+  `Annex A` gets that number as its id (`5.3`, `A.2.1`, `A`), an `Appendix I` bookmark gets `Appendix-I`, any
+  other (`Bibliography`) its slug;
+- page furniture (a header or footer line that repeats on many pages, once digits and Roman numerals are
+  ignored) is dropped before anything is cut;
+- a section's text runs from its heading to the next bookmark's heading, subsections excluded, and is split
+  into sentences, because a PDF gives sentences and not paragraphs (`Unit.SENTENCE`).
+"""
+
+from __future__ import annotations
+
+import re
+import unicodedata
+from collections import Counter
+from typing import Any
+
+_NUMBERED = re.compile(r"^(?P<id>\d+(?:\.\d+)*)\.?\s+(?P<title>\S.*)$")
+_LETTERED = re.compile(r"^(?P<id>[A-Z](?:\.\d+)+)\.?\s+(?P<title>\S.*)$")
+_ANNEX = re.compile(r"^(?:Annex|Appendix)\s+(?P<id>[A-Z0-9]+(?:\.\d+)*)\b[.:]?\s*(?P<title>.*)$", re.IGNORECASE)
+_DIGITS = re.compile(r"\d+(?:\s+of\s+\d+)?")
+_ROMAN = re.compile(r"^(?:[ivxlcdm]+\s+)|(?:\s+[ivxlcdm]+)$", re.IGNORECASE)
+_SENTENCE_END = re.compile(r"(?<=[.!?])\s+(?=[A-Z0-9(\[\"'•])")
+_ABBREV = ("e.g.", "i.e.", "etc.", "vs.", "cf.", "No.", "Fig.", "Sec.", "approx.")
+
+Outline = list[tuple[str, int, int]]
+"""A bookmark tree flattened to (title, 0-based page, depth), in document order."""
+
+
+def slug(text: str) -> str:
+    folded = unicodedata.normalize("NFKC", text).lower()
+    return re.sub(r"[^a-z0-9]+", "-", folded).strip("-") or "section"
+
+
+def split_sentences(text: str) -> list[str]:
+    """Sentences of already-flowed text: split after . ! ? before a capital, digit or bracket, not after the
+    abbreviations in `_ABBREV`."""
+    flowed = " ".join(text.split())
+    if not flowed:
+        return []
+    parts = _SENTENCE_END.split(flowed)
+    out: list[str] = []
+    for part in parts:
+        if out and out[-1].endswith(_ABBREV):
+            out[-1] = f"{out[-1]} {part}"
+        else:
+            out.append(part)
+    return out
+
+
+def furniture_key(line: str) -> str:
+    """A line with its digits and leading or trailing Roman numerals folded away, so a running header or footer
+    gives the same key on every page."""
+    key = _DIGITS.sub("#", line.strip())
+    key = _ROMAN.sub("", key).strip()
+    return re.sub(r"\s+", " ", key)
+
+
+def heading_id(title: str) -> tuple[str, str]:
+    """The section id and the title text a bookmark title gives (see the module docstring)."""
+    for pattern in (_NUMBERED, _LETTERED, _ANNEX):
+        match = pattern.match(title)
+        if match:
+            ident = match.group("id").upper() if pattern is not _NUMBERED else match.group("id")
+            if pattern is _ANNEX and title.lower().startswith("appendix"):
+                ident = f"Appendix-{ident}"
+            return ident, match.group("title").strip() or title
+    return slug(title), title
+
+
+_TRACKING_MARK = re.compile(r"^\s*[-`,]{8,}")
+"""A distributor's tracking string printed before the first line of a page (``--`,``,``,,,`---1 Scope``)."""
+
+
+def clean_pages(pages: list[str], *, keep_spacing: bool = False) -> list[list[str]]:
+    """Each page's non-empty lines with page furniture dropped: a line whose `furniture_key` occurs on at least a
+    quarter of the pages (at least 3) is a running header or footer. Whitespace is collapsed unless `keep_spacing`
+    (layout text, whose column gaps the numbered-heading splitter reads); a distributor's tracking string at the
+    start of a line is removed either way."""
+
+    def tidy(raw: str) -> str:
+        line = _TRACKING_MARK.sub("", raw)
+        return line.rstrip() if keep_spacing else " ".join(line.split())
+
+    per_page = [[ln for ln in (tidy(raw) for raw in text.splitlines()) if ln.strip()] for text in pages]
+    seen: Counter[str] = Counter()
+    for lines in per_page:
+        seen.update({furniture_key(" ".join(ln.split())) for ln in lines})
+    limit = max(3, len(pages) // 4)
+    return [[ln for ln in lines if seen[furniture_key(" ".join(ln.split()))] < limit] for lines in per_page]
+
+
+def cut_numbered(pages: list[str]) -> tuple[list[dict[str, Any]], int]:
+    """Cut page texts into sections at numbered headings, for a PDF without bookmarks (older ISO editions, for
+    one). Furniture is dropped as in `cut`, then the shared numbered-heading splitter
+    (`primarysource.pdfcorpus.split.split_sections`, `cl.segment_numbered`) finds the headings. Text before the
+    first heading becomes a `front-matter` section. Returns the sections in `cut`'s shape and how many headings
+    were found."""
+    from primarysource.pdfcorpus.split import split_sections
+
+    lines: list[str] = []
+    page_of: list[int] = []
+    for number, page_lines in enumerate(clean_pages(pages, keep_spacing=True)):
+        lines.extend(page_lines)
+        page_of.extend([number] * len(page_lines))
+    found = split_sections("\n".join(lines))
+    sections: list[dict[str, Any]] = []
+    first = found[0].start_line - 1 if found else len(lines)
+    if first > 0:
+        sections.append(
+            {
+                "id": "front-matter",
+                "title": "Front matter",
+                "depth": 0,
+                "page": 1,
+                "text": "\n".join(split_sentences(" ".join(lines[:first]))),
+            }
+        )
+    used: Counter[str] = Counter()
+    for section in found:
+        ident = section.id
+        used[ident] += 1
+        if used[ident] > 1:
+            ident = f"{ident}-{used[ident] - 1}"
+        body = lines[section.start_line : section.end_line]
+        sections.append(
+            {
+                "id": ident,
+                "title": " ".join(section.heading.split()),
+                "depth": section.id.count("."),
+                "page": page_of[section.start_line - 1] + 1,
+                "text": "\n".join(split_sentences(" ".join(body))),
+            }
+        )
+    return sections, len(found)
+
+
+def cut(pages: list[str], outline: Outline) -> tuple[list[dict[str, Any]], int]:
+    """Cut page texts into sections at the bookmarked headings. Returns the sections
+    (`{"id", "title", "depth", "page", "text"}`, `page` 1-based, `text` the sentences joined by newlines) and how
+    many headings were found as a line of text (the rest fall back to the start of their page)."""
+    count = len(pages)
+    cleaned = ["\n".join(lines) for lines in clean_pages(pages)]
+    starts: list[int] = []
+    joined = ""
+    for text in cleaned:
+        starts.append(len(joined))
+        joined += text + "\n"
+
+    spots: list[tuple[int, int]] = []  # (start, end) of each heading's line in `joined`, never moving backwards
+    cursor = 0
+    located = 0
+    for heading, page, _ in outline:
+        ident, text_title = heading_id(heading)
+        numbered = _NUMBERED.match(heading) or _LETTERED.match(heading)
+        words = r"\s+".join(re.escape(w) for w in (text_title if numbered else heading).split())
+        number = re.escape(ident) if numbered else ""
+        pattern = re.compile(rf"(?m)^{number}\.?\s*{words}" if number else rf"(?m)^{words}")
+        first = min(page, count - 1)
+        base = max(starts[first], cursor)
+        found = pattern.search(joined, base, starts[first + 1] if first + 1 < count else len(joined))
+        if found is None:
+            found = pattern.search(joined, base)
+        if found is not None:
+            located += 1
+        spot = (found.start(), found.end()) if found else (base, base)
+        spots.append(spot)
+        cursor = max(cursor, spot[0])
+    sections: list[dict[str, Any]] = []
+    used: Counter[str] = Counter()
+    for n, (heading, page, depth) in enumerate(outline):
+        ident, text_title = heading_id(heading)
+        used[ident] += 1
+        if used[ident] > 1:
+            ident = f"{ident}-{used[ident] - 1}"
+        begin = spots[n][1]
+        stop = spots[n + 1][0] if n + 1 < len(outline) else len(joined)
+        sections.append(
+            {
+                "id": ident,
+                "title": text_title,
+                "depth": depth,
+                "page": page + 1,
+                "text": "\n".join(split_sentences(joined[begin : max(begin, stop)])),
+            }
+        )
+    return sections, located