primarysource 0.1.0__py3-none-any.whl

primarysource/bbf/catalogue.py

@@ -0,0 +1,137 @@
+"""The Broadband Forum catalogue: data-model XML pinned to a commit, and Technical Reports pinned by sha256.
+
+Two kinds of source, two licences, two placements (`terms`):
+- `models`: XML files of https://github.com/BroadbandForum/cwmp-data-models at one pinned commit. Each file
+  carries a BSD-3-Clause-style notice (redistribution with notice permitted), so placement may be an image layer.
+- `reports`: Technical Report PDFs (TR-069 Amendment 6 Corrigendum 1 so far). The Forum's terms of use allow
+  use for implementing and distribution only as a whole copy with the notice, and forbid modification or
+  derivative works, so they are fetched by the user into a volume.
+
+Metadata only, never text. `catalogue.json` is package data.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import dataclass
+from importlib import resources
+from typing import Any
+
+from primarysource.errors import LibError
+
+DOC_ID = re.compile(r"[a-z][a-z0-9]*(?:-[a-z0-9]+)*")
+COMMIT = re.compile(r"[0-9a-f]{40}")
+SHA256 = re.compile(r"[0-9a-f]{64}")
+HOSTS = ("raw.githubusercontent.com", "www.broadband-forum.org")
+
+
+@dataclass(frozen=True, slots=True)
+class Terms:
+    licence: str
+    placement: str
+    terms_url: str
+    terms_read: str
+    summary: str
+
+
+@dataclass(frozen=True, slots=True)
+class Model:
+    """One data-model XML file at the pinned commit."""
+
+    id: str
+    path: str
+    title: str
+    size: int
+    sha256: str
+
+
+@dataclass(frozen=True, slots=True)
+class Report:
+    """One Technical Report PDF."""
+
+    id: str
+    designation: str
+    title: str
+    issue: str
+    page_url: str
+    source_url: str
+    size: int
+    sha256: str
+
+
+@dataclass(frozen=True, slots=True)
+class Catalogue:
+    repo: str
+    commit: str
+    raw_base: str
+    model_terms: Terms
+    report_terms: Terms
+    models: tuple[Model, ...]
+    reports: tuple[Report, ...]
+
+    def model(self, doc_id: str) -> Model:
+        for found in self.models:
+            if found.id == doc_id:
+                return found
+        raise LibError(f"no BBF data model {doc_id!r} in the catalogue", "bbf_docs lists what is catalogued")
+
+    def report(self, doc_id: str) -> Report:
+        for found in self.reports:
+            if found.id == doc_id:
+                return found
+        raise LibError(f"no BBF report {doc_id!r} in the catalogue", "bbf_docs lists what is catalogued")
+
+    def model_url(self, model: Model) -> str:
+        return f"{self.raw_base}{self.commit}/{model.path}"
+
+
+def _terms(raw: dict[str, Any]) -> Terms:
+    terms = Terms(raw["licence"], raw["placement"], raw["terms_url"], raw["terms_read"], raw["summary"])
+    if terms.placement not in ("layer", "volume"):
+        raise ValueError("terms.placement must be 'layer' or 'volume'")
+    return terms
+
+
+def parse_catalogue(raw: dict[str, Any]) -> Catalogue:
+    """Validate a catalogue document. Raises `ValueError`: a malformed catalogue is a packaging bug."""
+    if not COMMIT.fullmatch(raw["commit"]):
+        raise ValueError("catalogue commit must be 40 lowercase hex digits")
+    models = tuple(Model(m["id"], m["path"], m["title"], int(m["size"]), m["sha256"]) for m in raw["models"])
+    reports = tuple(
+        Report(
+            r["id"],
+            r["designation"],
+            r["title"],
+            r["issue"],
+            r["page_url"],
+            r["source_url"],
+            int(r["size"]),
+            r["sha256"],
+        )
+        for r in raw["reports"]
+    )
+    entries: list[Model | Report] = [*models, *reports]
+    for entry in entries:
+        if not DOC_ID.fullmatch(entry.id) or not SHA256.fullmatch(entry.sha256):
+            raise ValueError(f"catalogue entry {entry.id!r} is malformed")
+    for model in models:
+        if model.path != f"{model.id}.xml":
+            raise ValueError(f"catalogue entry {model.id!r}: path must be <id>.xml")
+    ids = [e.id for e in entries]
+    if len(set(ids)) != len(ids):
+        raise ValueError("duplicate ids in the catalogue")
+    return Catalogue(
+        raw["repo"],
+        raw["commit"],
+        raw["raw_base"],
+        _terms(raw["terms"]["models"]),
+        _terms(raw["terms"]["reports"]),
+        models,
+        reports,
+    )
+
+
+def catalogue() -> Catalogue:
+    text = resources.files("primarysource.bbf").joinpath("catalogue.json").read_text(encoding="utf-8")
+    return parse_catalogue(json.loads(text))

primarysource/bbf/cli.py

@@ -0,0 +1,79 @@
+"""`primarysource bbf ...`: the collection's own commands.
+
+primarysource bbf import-models [ID ...]
+primarysource bbf import-report <ID> [--file PDF]
+primarysource bbf docs [FILTER]
+primarysource bbf outline <doc> [--under PATH] [--limit N] [--start N]
+primarysource bbf read <ref> [--max-tokens N]
+primarysource bbf search <query> [--doc ID] [--limit N] [--start N]
+primarysource bbf cite <ref>
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from collections.abc import Sequence
+from pathlib import Path
+
+from primarysource.bbf import importer, library
+from primarysource.errors import LibError
+
+
+def _parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(prog="primarysource bbf", description="Broadband Forum data models and reports")
+    commands = parser.add_subparsers(dest="command", required=True)
+    models = commands.add_parser("import-models", help="fetch and verify data-model XML (one request per second)")
+    models.add_argument("ids", nargs="*", help="file stems to import (default: every catalogued file)")
+    report = commands.add_parser(
+        "import-report", help="fetch (or read a file), verify and extract a Technical Report"
+    )
+    report.add_argument("id", help="e.g. tr-069-1-6-1")
+    report.add_argument("--file", type=Path, help="the PDF you downloaded yourself, instead of fetching it")
+    commands.add_parser("docs", help="the catalogue and what is imported").add_argument(
+        "filter", nargs="?", default=""
+    )
+    outline = commands.add_parser("outline", help="clauses of a report, or nodes of a data model")
+    outline.add_argument("doc")
+    outline.add_argument("--under", default="")
+    outline.add_argument("--limit", type=int, default=200)
+    outline.add_argument("--start", type=int, default=0)
+    read = commands.add_parser("read", help="read by ref")
+    read.add_argument("ref")
+    read.add_argument("--max-tokens", type=int, default=2000)
+    search = commands.add_parser("search", help="find units containing words")
+    search.add_argument("query")
+    search.add_argument("--doc")
+    search.add_argument("--limit", type=int, default=10)
+    search.add_argument("--start", type=int, default=0)
+    commands.add_parser("cite", help="the citation for a ref").add_argument("ref")
+    return parser
+
+
+def _out(text: str) -> int:
+    sys.stdout.write(text if text.endswith("\n") else text + "\n")
+    return 0
+
+
+def main(argv: Sequence[str]) -> int:
+    args = _parser().parse_args(list(argv))
+    home = library.default_home()
+    try:
+        if args.command == "import-models":
+            result = importer.import_models(home, args.ids)
+            return _out(f"imported {len(result.ids)} data-model files, {result.requests} request(s), into {home}")
+        if args.command == "import-report":
+            result = importer.import_report(home, args.id, file=args.file)
+            return _out(f"imported {args.id}, {result.requests} request(s), into {home}")
+        if args.command == "docs":
+            return _out(library.docs(args.filter))
+        if args.command == "outline":
+            return _out(library.outline(args.doc, under=args.under, limit=args.limit, start=args.start))
+        if args.command == "read":
+            return _out(library.read(args.ref, max_tokens=args.max_tokens))
+        if args.command == "search":
+            return _out(library.search(args.query, doc=args.doc, limit=args.limit, start=args.start))
+        return _out(library.cite(args.ref))
+    except LibError as exc:
+        print(exc.as_text(), file=sys.stderr)
+        return 1

primarysource/bbf/datamodel.py

@@ -0,0 +1,281 @@
+"""Read a Broadband Forum data-model XML file (CWMP and USP `dm:document`) into addressable nodes.
+
+A node is an object, a parameter, a command, an event, a profile, a data type or a model. Its address (the part
+of a ref after `§`) is the full path: `Device.IP.Interface.{i}` for an object (written without its trailing dot;
+`Device.IP.Interface.{i}.` also resolves), `Device.IP.Interface.{i}.Enable` for a parameter,
+`Device.Reboot()` for a command, `Device.Boot!` for an event, `Baseline:1` for a profile, `IPAddress` for a data
+type, `Root:2.22` for a model. A command's arguments are `Device.Reboot().Input.Delay` and
+`Device.Reboot().Output.X`.
+
+`¶n` counts the paragraphs of a node's description, in order. The description is kept as the file gives it: BBF
+template macros (`{{param}}`, `{{enum}}`, `{{numentries}}`, `{{object}}`) are NOT expanded; they are the
+publisher's own markup and expanding them would be this code's interpretation, not the standard's text.
+A profile's units are its description paragraphs and then one line per object or parameter it requires.
+"""
+
+from __future__ import annotations
+
+import re
+import xml.etree.ElementTree as ET  # noqa: S405 - elements come from the hardened parser in `_xml`
+from dataclasses import dataclass, field
+
+from primarysource.bbf import _xml
+from primarysource.errors import LibError
+
+KINDS = ("model", "object", "parameter", "command", "event", "profile", "datatype")
+MAX_NODES = 400_000
+MAX_ENUM = 200
+_PARAGRAPH_BREAK = re.compile(r"\n[ \t]*\n")
+_TITLE = re.compile(rb"<!--\s*\n?\s*([^\n]+)")
+
+
+@dataclass(frozen=True, slots=True)
+class Node:
+    """One addressable item. `props` is the ordered attribute summary shown in the header line."""
+
+    kind: str
+    id: str
+    props: tuple[tuple[str, str], ...] = ()
+    paragraphs: tuple[str, ...] = ()
+    component: str = ""
+
+    def header(self) -> str:
+        shown = " ".join(f"{k}={v}" for k, v in self.props if v != "")
+        text = f"{self.kind} {self.id}" + (f"  [{shown}]" if shown else "")
+        return text + (f"  (component {self.component})" if self.component else "")
+
+
+@dataclass(frozen=True, slots=True)
+class DataModel:
+    """A parsed file. `nodes` are in document order; `imports` are the files it imports."""
+
+    title: str
+    spec: str
+    file: str
+    imports: tuple[str, ...]
+    nodes: tuple[Node, ...]
+    index: dict[str, int] = field(default_factory=dict)
+
+    def find(self, wanted: str) -> tuple[Node, str]:
+        """The node `wanted` names and a warning ("" if none): exact id, then without a trailing dot, then with
+        case ignored. Raises `LibError` with close matches when nothing fits."""
+        text = wanted.strip()
+        for candidate in (text, text.rstrip(".") if text.endswith(".") else text):
+            if candidate in self.index:
+                return self._answer(candidate)
+        low = text.rstrip(".").casefold()
+        for key in self.index:
+            if key.casefold() == low:
+                return self._answer(key)
+        import difflib
+
+        close = difflib.get_close_matches(text.rstrip("."), list(self.index), n=3, cutoff=0.6)
+        hint = f"did you mean: {', '.join(close)}" if close else "bbf_outline lists the nodes of a data model"
+        raise LibError(f"no node {text!r} in {self.file or 'this file'}", hint)
+
+    def _answer(self, key: str) -> tuple[Node, str]:
+        same = [n for n in self.nodes if n.id == key]
+        warning = f"{len(same)} definitions of {key}; this is the first" if len(same) > 1 else ""
+        return self.nodes[self.index[key]], warning
+
+    def children(self, node: Node) -> tuple[Node, ...]:
+        """The parameters, commands and events directly under an object."""
+        if node.kind != "object":
+            return ()
+        prefix = node.id + "."
+        out: list[Node] = []
+        for other in self.nodes:
+            if other.kind in ("parameter", "command", "event") and other.id.startswith(prefix):
+                rest = other.id[len(prefix) :]
+                if "." not in rest.replace("().", "()#"):
+                    out.append(other)
+        return tuple(out)
+
+
+def _paragraphs(element: ET.Element | None) -> tuple[str, ...]:
+    """The text of a `<description>`: blank-line-separated paragraphs, whitespace collapsed inside each."""
+    if element is None:
+        return ()
+    text = "".join(element.itertext()).strip("\n")
+    return tuple(" ".join(p.split()) for p in _PARAGRAPH_BREAK.split(text) if p.strip())
+
+
+def _syntax(parameter: ET.Element) -> list[tuple[str, str]]:
+    """The type and restrictions of a parameter as header props."""
+    syntax = parameter.find("syntax")
+    if syntax is None:
+        return []
+    props: list[tuple[str, str]] = []
+    is_list = syntax.find("list")
+    holder = is_list if is_list is not None else syntax  # a list holds the element type, and its own bounds
+    type_el = next(
+        (c for c in holder if c.tag not in ("list", "default", "description", "hidden", "command", "size")), None
+    )
+    scope = type_el if type_el is not None else syntax
+    name = ""
+    if type_el is not None:
+        name = type_el.get("ref") or type_el.tag
+        extras = {
+            k: v
+            for k, v in type_el.attrib.items()
+            if k in ("refType", "targetParent", "targetType", "targetParentScope")
+        }
+        if extras:
+            name += "(" + ",".join(f"{k}={v}" for k, v in sorted(extras.items())) + ")"
+    bounds: list[str] = []
+    for restriction in [*scope, *(list(is_list.findall("size")) if is_list is not None else [])]:
+        if restriction.tag == "size":
+            bounds.append(f"size={restriction.get('minLength', '')}..{restriction.get('maxLength', '')}")
+        elif restriction.tag == "range":
+            bounds.append("range=" + restriction.get("minInclusive", "") + ".." + restriction.get("maxInclusive", ""))
+        elif restriction.tag == "pattern" and restriction.get("value"):
+            bounds.append(f"pattern={restriction.get('value')}")
+        elif restriction.tag == "units":
+            bounds.append(f"units={restriction.get('value', '')}")
+    if is_list is not None:
+        name = f"list of {name}" if name else "list"
+        bounds = [f"items={is_list.get('minItems', '')}..{is_list.get('maxItems', '')}", *bounds]
+    props.append(("type", name + (f"({', '.join(bounds)})" if bounds else "")))
+    values = [e.get("value", "") for e in scope.iter("enumeration")][:MAX_ENUM]
+    if values:
+        props.append(("enum", "|".join(values)))
+    default = syntax.find("default")
+    if default is not None and default.get("value") is not None:
+        props.append(("default", repr(default.get("value"))))
+    return props
+
+
+_COMMON = (
+    "access",
+    "version",
+    "status",
+    "mandatory",
+    "activeNotify",
+    "forcedInform",
+    "async",
+    "minEntries",
+    "maxEntries",
+)
+
+
+def _props(element: ET.Element, extra: list[tuple[str, str]] | None = None) -> tuple[tuple[str, str], ...]:
+    found = [(k, element.get(k, "")) for k in _COMMON if element.get(k) is not None]
+    return tuple(found + (extra or []))
+
+
+def parse(data: bytes) -> DataModel:
+    """Parse one file. Raises `LibError` if it is not a well-formed BBF data-model document."""
+    root = _xml.parse(data)
+    if root.tag != "document":
+        raise LibError(
+            f"the root element is <{root.tag}>, not a data-model document", "this is not a BBF data-model file"
+        )
+    title_match = _TITLE.search(data[:4096])
+    title = title_match.group(1).decode("utf-8", errors="replace").strip() if title_match else ""
+    nodes: list[Node] = []
+
+    def add(node: Node) -> None:
+        if len(nodes) >= MAX_NODES:
+            raise LibError("the data model has too many nodes", "")
+        nodes.append(node)
+
+    def walk_object(obj: ET.Element, component: str) -> None:
+        path = (obj.get("name") or obj.get("ref") or "").strip()
+        if not path:
+            return
+        oid = path.rstrip(".")
+        keys = [p.get("ref", "") for k in obj.findall("uniqueKey") for p in k.findall("parameter")]
+        extra = [("uniqueKey", ",".join(keys))] if keys else []
+        if obj.get("numEntriesParameter"):
+            extra.append(("numEntriesParameter", obj.get("numEntriesParameter", "")))
+        add(Node("object", oid, _props(obj, extra), _paragraphs(obj.find("description")), component))
+        for child in obj:
+            if child.tag == "parameter" and child.get("name"):
+                pid = f"{oid}.{child.get('name')}"
+                add(
+                    Node(
+                        "parameter",
+                        pid,
+                        _props(child, _syntax(child)),
+                        _paragraphs(child.find("description")),
+                        component,
+                    )
+                )
+            elif child.tag in ("command", "event") and child.get("name"):
+                name = child.get("name", "")
+                cid = f"{oid}.{name}" if child.tag == "event" or name.endswith("()") else f"{oid}.{name}()"
+                add(Node(child.tag, cid, _props(child), _paragraphs(child.find("description")), component))
+                for part in ("input", "output"):
+                    for arg in child.findall(f"{part}/parameter"):
+                        if arg.get("name"):
+                            aid = f"{cid}.{part.capitalize()}.{arg.get('name')}"
+                            add(
+                                Node(
+                                    "parameter",
+                                    aid,
+                                    _props(arg, _syntax(arg)),
+                                    _paragraphs(arg.find("description")),
+                                    component,
+                                )
+                            )
+                if child.tag == "event":
+                    for arg in child.findall("parameter"):
+                        if arg.get("name"):
+                            add(
+                                Node(
+                                    "parameter",
+                                    f"{cid}.{arg.get('name')}",
+                                    _props(arg, _syntax(arg)),
+                                    _paragraphs(arg.find("description")),
+                                    component,
+                                )
+                            )
+
+    def walk_profile(profile: ET.Element, component: str) -> None:
+        name = profile.get("name")
+        if not name:
+            return
+        lines: list[str] = []
+        for member in profile:
+            if member.tag == "object" and member.get("ref"):
+                lines.append(f"object {member.get('ref')} requires {member.get('requirement', '')}")
+                lines.extend(
+                    f"parameter {member.get('ref')}{p.get('ref')} requires {p.get('requirement', '')}"
+                    for p in member.findall("parameter")
+                    if p.get("ref")
+                )
+        extra = [(k, profile.get(k, "")) for k in ("base", "extends") if profile.get(k)]
+        add(Node("profile", name, tuple(extra), (*_paragraphs(profile.find("description")), *lines), component))
+
+    def walk_container(container: ET.Element, component: str) -> None:
+        for child in container:
+            if child.tag == "object":
+                walk_object(child, component)
+            elif child.tag == "profile":
+                walk_profile(child, component)
+
+    imports: list[str] = []
+    for child in root:
+        if child.tag == "import" and child.get("file"):
+            imports.append(child.get("file", ""))
+        elif child.tag == "dataType" and child.get("name"):
+            add(
+                Node(
+                    "datatype",
+                    child.get("name", ""),
+                    _props(child, _syntax(child)),
+                    _paragraphs(child.find("description")),
+                )
+            )
+        elif child.tag == "component":
+            walk_container(child, child.get("name", ""))
+        elif child.tag == "model" and child.get("name"):
+            refs = [c.get("ref", "") for c in child.findall("component")]
+            extra = [("components", ",".join(refs))] if refs else []
+            extra += [(k, child.get(k, "")) for k in ("base", "isService") if child.get(k)]
+            add(Node("model", child.get("name", ""), tuple(extra), _paragraphs(child.find("description"))))
+            walk_container(child, "")
+    index: dict[str, int] = {}
+    for position, node in enumerate(nodes):
+        index.setdefault(node.id, position)
+    return DataModel(title, root.get("spec", ""), root.get("file", ""), tuple(imports), tuple(nodes), index)

primarysource/bbf/importer.py

@@ -0,0 +1,151 @@
+"""Import Broadband Forum data models and Technical Reports into `$PRIMARYSOURCE_HOME/bbf/`.
+
+- `import_models`: each XML file is fetched from `raw.githubusercontent.com/BroadbandForum/cwmp-data-models/<commit>/`
+  (one request per second), checked against its pinned sha256, parsed once with the hardened parser (a file that
+  does not parse is not installed) and written to `models/<id>.xml`.
+- `import_report`: a Technical Report PDF is fetched from broadband-forum.org (or taken from a file the user
+  downloaded), checked against its sha256, stored as `reports/<id>.pdf`, and read once into `reports/<id>.json`
+  with the private PDF reader. Reports go on a volume: the Forum's terms do not allow redistribution of
+  derivatives, and this code only writes to the user's own home.
+
+`install.json` is written last, after every file it names. Run as `primarysource bbf import-models [ID ...]` and
+`primarysource bbf import-report <ID> [--file PDF]`.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from collections.abc import Sequence
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+from primarysource.bbf import _xml, datamodel
+from primarysource.bbf.catalogue import DOC_ID, HOSTS, Catalogue, Model
+from primarysource.bbf.catalogue import catalogue as packaged_catalogue
+from primarysource.c2sp._fetch import Fetcher, Transport, sha256, verify
+from primarysource.ecma import _pdf
+from primarysource.errors import LibError
+from primarysource.home import atomic_write
+
+INSTALL_RECORD = "install.json"
+MAX_PDF = 64 * 1024 * 1024
+
+
+@dataclass(frozen=True, slots=True)
+class ImportResult:
+    ids: tuple[str, ...]
+    requests: int
+
+
+def read_record(home: Path) -> dict[str, Any]:
+    """The install record, or an empty one. Always has `models` and `reports` mappings."""
+    try:
+        loaded = json.loads((home / INSTALL_RECORD).read_text(encoding="utf-8"))
+    except (OSError, ValueError):
+        loaded = {}
+    if not isinstance(loaded, dict):
+        loaded = {}
+    for key in ("models", "reports"):
+        if not isinstance(loaded.get(key), dict):
+            loaded[key] = {}
+    return loaded
+
+
+def _write_record(home: Path, record: dict[str, Any]) -> None:
+    record["collection"] = "bbf"
+    atomic_write(home / INSTALL_RECORD, (json.dumps(record, indent=2) + "\n").encode("utf-8"))
+
+
+def import_models(
+    home: Path,
+    only: Sequence[str] = (),
+    *,
+    cat: Catalogue | None = None,
+    transport: Transport | None = None,
+    fetcher: Fetcher | None = None,
+) -> ImportResult:
+    """Fetch, verify, parse and install the catalogued data-model files (all, or those named in `only`)."""
+    cat = cat or packaged_catalogue()
+    chosen = [cat.model(name) for name in only] if only else list(cat.models)
+    fetcher = fetcher or Fetcher(HOSTS, transport)
+    record = read_record(home)
+    done: list[str] = []
+    for model in chosen:
+        data = fetcher.get(cat.model_url(model), min(_xml.MAX_BYTES, max(model.size * 2, 4096)))
+        verify(data, model.sha256, model.id)
+        datamodel.parse(data)
+        atomic_write(home / "models" / model.path, data)
+        record["models"][model.id] = model.sha256
+        done.append(model.id)
+    record["commit"] = cat.commit
+    record["models_licence"] = cat.model_terms.licence
+    record["imported"] = datetime.now(UTC).isoformat(timespec="seconds")
+    _write_record(home, record)
+    return ImportResult(tuple(done), fetcher.requests)
+
+
+def import_report(
+    home: Path,
+    report_id: str,
+    *,
+    file: Path | None = None,
+    cat: Catalogue | None = None,
+    transport: Transport | None = None,
+    fetcher: Fetcher | None = None,
+) -> ImportResult:
+    """Fetch (or read `file`), verify, store and extract one Technical Report PDF."""
+    cat = cat or packaged_catalogue()
+    report = cat.report(report_id)
+    requests = 0
+    if file is not None:
+        try:
+            data = file.read_bytes()
+        except OSError as exc:
+            raise LibError(f"cannot read {file}: {exc}", "pass the PDF you downloaded from the Forum") from exc
+        if len(data) > MAX_PDF:
+            raise LibError(f"{file} is larger than {MAX_PDF} bytes", "")
+    else:
+        fetcher = fetcher or Fetcher(HOSTS, transport)
+        data = fetcher.get(report.source_url, MAX_PDF)
+        requests = fetcher.requests
+    verify(data, report.sha256, report.designation)
+    if not data.startswith(b"%PDF-"):
+        raise LibError(f"{report.designation}: not a PDF", "the pinned file is a PDF; nothing was written")
+    folder = home / "reports"
+    folder.mkdir(parents=True, exist_ok=True)
+    part = folder / f"{report.id}.pdf.part"
+    part.write_bytes(data)
+    try:
+        extraction = _pdf.extract(data, part)
+    except BaseException:
+        part.unlink(missing_ok=True)
+        raise
+    os.replace(part, folder / f"{report.id}.pdf")
+    extraction.update(
+        {"title": report.title, "designation": report.designation, "edition": report.issue, "sha256": report.sha256}
+    )
+    atomic_write(folder / f"{report.id}.json", (json.dumps(extraction, ensure_ascii=False) + "\n").encode("utf-8"))
+    record = read_record(home)
+    record["reports"][report.id] = {
+        "sha256": report.sha256,
+        "extractor": extraction["extractor"],
+        "imported": datetime.now(UTC).isoformat(timespec="seconds"),
+    }
+    _write_record(home, record)
+    return ImportResult((report.id,), requests)
+
+
+def pin_models(raw_base: str, commit: str, names: Sequence[str], fetcher: Fetcher) -> list[Model]:
+    """Maintainer step: fetch `names` (file stems) at `commit` and return catalogue entries with size and sha256.
+    The result is what goes in `catalogue.json`; it is a review step, never run by the importer."""
+    pinned: list[Model] = []
+    for name in names:
+        if not DOC_ID.fullmatch(name):
+            raise LibError(f"not a data-model file stem: {name!r}", "")
+        data = fetcher.get(f"{raw_base}{commit}/{name}.xml", _xml.MAX_BYTES)
+        parsed = datamodel.parse(data)
+        pinned.append(Model(name, f"{name}.xml", parsed.title or name, len(data), sha256(data)))
+    return pinned