primarysource 0.1.0__py3-none-any.whl

primarysource/nist/ingest/prose_layouts.py

@@ -0,0 +1,236 @@
+"""Section splitting for NIST prose documents whose headings the generic splitter does not recognise.
+
+``extract.split_sections`` reads headings that start a line.  Several families of NIST PDF do not satisfy that:
+
+``numbered``       ``1 Introduction`` / ``1. Introduction`` chapters.  ``pdftotext`` often glues the chapter heading
+                   after the page's running header (``This publication is available free of charge from: <doi>  1
+                   Introduction``), which is why the generic splitter sees no ``1``.  Sub-headings (``1.1 Purpose``)
+                   are indented but otherwise ordinary.
+``numbered_zero``  ``1.0 Introduction`` chapters (id ``1``), sub-headings ``3.1 ...``.
+``section_colon``  ``SECTION 1: INTRODUCTION`` chapters, ``1.1 Background`` sub-headings.
+``chapter_word``   ``CHAPTER ONE`` chapters whose title is on the next line, all-caps ``1.1 TITLE`` sub-headings.
+
+Every layout also reads appendices: ``Appendix A—Title``, ``Appendix A: Title``, ``APPENDIX A: TITLE``, a bare
+``APPENDIX A`` with the title on the next line, and ``Appendix A Title`` when it opens a page.
+
+The acceptance rule is the generic one: a heading counts only when it is the next child of a parent already seen,
+so cross-references, footnotes and repeated page headers cannot become sections.  Documents with no numbered
+headings at all (the annual reports, and the newer-format publications with unnumbered headings) are not split:
+``detect_layout`` returns None for them, because any section id would have to be invented.
+"""
+
+from __future__ import annotations
+
+import re
+
+from primarysource.nist.models import Section
+
+LAYOUTS: tuple[str, ...] = (
+    "numbered",
+    "numbered_zero",
+    "section_colon",
+    "chapter_word",
+)
+
+# pdftotext puts the running header and the first line of the page on one line.
+_RUNNING_HEADER = re.compile(
+    r"^\f?\s*This publication is available free of charge from:\s*\S+\s*(?P<rest>.*)$",
+    re.IGNORECASE,
+)
+_LEADER = re.compile(r"\.{4,}|(?:\. ){4,}")
+_MAX_HEADING = 120
+
+_CHAPTER = re.compile(r"^(?P<id>\d+)\.?\s+(?P<heading>[A-Z].*)$")
+_CHAPTER_ZERO = re.compile(r"^(?P<id>\d+)\.0\s+(?P<heading>[A-Z].*)$")
+_SECTION_COLON = re.compile(r"^SECTION\s+(?P<id>\d+)\s*[:.]\s*(?P<heading>\S.*)$")
+_CHAPTER_WORD = re.compile(
+    r"^CHAPTER\s+(?P<word>ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN|ELEVEN|TWELVE)\b\s*(?P<rest>.*)$"
+)
+_WORDS = "ONE TWO THREE FOUR FIVE SIX SEVEN EIGHT NINE TEN ELEVEN TWELVE".split()
+_SUB = re.compile(r"^(?P<id>(?:\d+|[A-Z])(?:\.\d+)+)\.?\s+(?P<heading>[A-Z0-9].*)$")
+_APPENDIX_SEP = re.compile(
+    r"^(?:Appendix|APPENDIX)\s+(?P<id>[A-Z])\s*[—–:.\-]+\s*(?P<heading>\S.*)$"
+)
+_APPENDIX_OPEN = re.compile(
+    r"^(?:Appendix|APPENDIX)\s+(?P<id>[A-Z])\s+(?P<heading>[A-Z].*)$"
+)
+_APPENDIX_BARE = re.compile(r"^APPENDIX\s+(?P<id>[A-Z])$")
+
+# A hit is (id, heading, number of extra lines consumed after the heading line).
+_Hit = tuple[str, str, int]
+
+
+def _clean_heading(heading: str) -> str | None:
+    heading = heading.strip()
+    if not heading or _LEADER.search(heading) or len(heading) > _MAX_HEADING:
+        return None
+    if heading.endswith(".") or heading.count("(") != heading.count(")"):
+        return None  # table-of-contents row or running prose
+    if re.search(r"\s{3,}PAGE\s+\d+$", heading):
+        return None  # running page header, not a heading
+    return heading
+
+
+def _next_text(lines: list[str], index: int) -> str | None:
+    """The next non-blank line after ``index``, stripped, or None when it is not close by."""
+    for look in range(index + 1, min(index + 4, len(lines))):
+        stripped = lines[look].strip()
+        if stripped:
+            return stripped
+    return None
+
+
+def _appendix_hit(
+    text: str, opens_page: bool, lines: list[str], index: int
+) -> _Hit | None:
+    m = _APPENDIX_SEP.match(text)
+    if m is None and opens_page:
+        m = _APPENDIX_OPEN.match(text)
+    if m is not None:
+        heading = _clean_heading(m.group("heading"))
+        return (m.group("id"), heading, 0) if heading else None
+    m = _APPENDIX_BARE.match(text)
+    if m is not None:
+        following = _next_text(lines, index)
+        heading = _clean_heading(following) if following else None
+        return (m.group("id"), heading, 1) if heading else None
+    return None
+
+
+def _top_level(layout: str, text: str, lines: list[str], index: int) -> _Hit | None:
+    if layout == "chapter_word":
+        m = _CHAPTER_WORD.match(text)
+        if m is None:
+            return None
+        number = str(_WORDS.index(m.group("word")) + 1)
+        rest = m.group("rest").strip()
+        if rest and not rest.startswith("PAGE"):
+            heading = _clean_heading(rest)
+            return (number, heading, 0) if heading else None
+        if rest:
+            return None  # "CHAPTER TWO   PAGE 6" running header
+        following = _next_text(lines, index)
+        heading = _clean_heading(following) if following else None
+        return (number, heading, 1) if heading else None
+    pattern = {
+        "numbered": _CHAPTER,
+        "numbered_zero": _CHAPTER_ZERO,
+        "section_colon": _SECTION_COLON,
+    }[layout]
+    m = pattern.match(text)
+    if m is None:
+        return None
+    heading = _clean_heading(m.group("heading"))
+    if heading is None:
+        return None
+    if layout == "numbered" and "  " in heading:
+        return None  # a table row: columns are separated by runs of spaces
+    return m.group("id"), heading, 0
+
+
+def _match(
+    layout: str, line: str, lines: list[str], index: int
+) -> tuple[_Hit, bool] | None:
+    """Match one line: the hit, and whether the line opens a page (heading glued after the running header)."""
+    opens_page = False
+    header = _RUNNING_HEADER.match(line)
+    if header is not None:
+        opens_page = True
+        line = header.group("rest")
+    text = line.strip().lstrip("\f").strip()
+    if not text:
+        return None
+    hit = _appendix_hit(text, opens_page, lines, index)
+    if hit is None:
+        hit = _top_level(layout, text, lines, index)
+    if hit is not None:
+        return hit, opens_page
+    m = _SUB.match(text)
+    if m is None:
+        return None
+    heading = _clean_heading(m.group("heading"))
+    return ((m.group("id"), heading, 0), opens_page) if heading else None
+
+
+def _ordinal(part: str) -> int:
+    return int(part) if part.isdigit() else ord(part) - ord("A") + 1
+
+
+def _parent_key(sec_id: str) -> str:
+    parent, _, _ = sec_id.rpartition(".")
+    if parent:
+        return parent
+    return "#appendix" if sec_id.isalpha() else "#chapter"
+
+
+def _is_next(sec_id: str, last_child: dict[str, int]) -> bool:
+    key = _parent_key(sec_id)
+    if key not in last_child and not key.startswith("#"):
+        return False
+    return _ordinal(sec_id.rpartition(".")[2]) == last_child.get(key, 0) + 1
+
+
+def split_for_layout(text: str, layout: str) -> tuple[Section, ...]:
+    """Split ``text`` into sections using the heading conventions of ``layout`` (one of ``LAYOUTS``).
+
+    Lines are numbered from 1 and split on newline only.  Sections do not nest: each runs from its heading line
+    to the line before the next heading, the last to the end of the text.
+    """
+    if layout not in LAYOUTS:
+        raise ValueError(
+            f"unknown layout {layout!r}; expected one of {', '.join(LAYOUTS)}"
+        )
+    lines = text.split("\n")
+    if lines and lines[-1] == "":
+        lines.pop()
+    matches = [
+        (index, m)
+        for index, line in enumerate(lines)
+        if (m := _match(layout, line, lines, index))
+    ]
+    # A bare chapter number ("3 Administrators should consult ...") also starts a footnote.  Where a chapter
+    # number has a candidate at the top of a page, behind the running header, that one is the heading and the
+    # others are dropped; otherwise a footnote met first would take the chapter's place in the sibling order.
+    page_openers = {
+        hit[0]
+        for _, (hit, opens) in matches
+        if opens and layout == "numbered" and hit[0].isdigit()
+    }
+    found: list[tuple[str, str, int]] = []
+    last_child: dict[str, int] = {}
+    for index, (hit, opens_page) in matches:
+        sec_id, heading, _ = hit
+        if layout == "numbered" and sec_id in page_openers and not opens_page:
+            continue
+        if not _is_next(sec_id, last_child):
+            continue
+        key = _parent_key(sec_id)
+        last_child[key] = last_child.get(key, 0) + 1
+        last_child.setdefault(sec_id, 0)
+        found.append((sec_id, heading, index + 1))
+    sections: list[Section] = []
+    for position, (sec_id, heading, start) in enumerate(found):
+        end = found[position + 1][2] - 1 if position + 1 < len(found) else len(lines)
+        sections.append(
+            Section(id=sec_id, heading=heading, start_line=start, end_line=end)
+        )
+    return tuple(sections)
+
+
+MIN_SECTIONS = 3
+
+
+def detect_layout(text: str) -> str | None:
+    """Name the layout that finds the most sections in ``text``, or None when none finds ``MIN_SECTIONS``.
+
+    The layouts are mutually exclusive in practice: each one's chapters are the only parents the others'
+    sub-headings would accept, so a document written one way yields nothing under the others.  A document with
+    no numbered headings (an annual report, an unnumbered newer-format publication) yields None.
+    """
+    best: str | None = None
+    best_count = MIN_SECTIONS - 1
+    for layout in LAYOUTS:
+        count = len(split_for_layout(text, layout))
+        if count > best_count:
+            best, best_count = layout, count
+    return best

primarysource/nist/ingest/sources.py

@@ -0,0 +1,94 @@
+"""Parser for data/sources.toml, the list of publications ``nistlib-fetch`` may download."""
+
+from __future__ import annotations
+
+import re
+import tomllib
+from dataclasses import dataclass
+from pathlib import Path
+
+_DOC_ID = re.compile(r"^[a-z0-9][a-z0-9._-]*$")
+_DATE = re.compile(r"^\d{4}-(0[1-9]|1[0-2])$")
+_SHA256 = re.compile(r"^[0-9a-f]{64}$")
+_REQUIRED = ("doc_id", "url", "number", "title", "date", "status")
+_KNOWN = (*_REQUIRED, "sha256", "splitter")
+_SPLITTERS = ("headings", "controls")
+
+
+class SourcesError(ValueError):
+    """Raised when sources.toml is missing, malformed, or inconsistent."""
+
+
+@dataclass(frozen=True, slots=True)
+class Source:
+    """One entry of sources.toml. ``sha256`` is empty until recorded."""
+
+    doc_id: str
+    url: str
+    number: str
+    title: str
+    date: str
+    status: str
+    sha256: str = ""
+    splitter: str = (
+        "headings"  # "headings" for prose, "controls" for control catalogues
+    )
+
+
+def parse_sources(text: str) -> tuple[Source, ...]:
+    """Parse the text of sources.toml: an array of ``[[source]]`` tables."""
+    try:
+        data = tomllib.loads(text)
+    except tomllib.TOMLDecodeError as error:
+        raise SourcesError(f"sources.toml is not valid TOML: {error}") from error
+    entries = data.get("source")
+    if not isinstance(entries, list) or not entries:
+        raise SourcesError("sources.toml has no [[source]] entries")
+    sources: list[Source] = []
+    seen: set[str] = set()
+    for position, entry in enumerate(entries, start=1):
+        label = f"source #{position}"
+        if not isinstance(entry, dict):
+            raise SourcesError(f"{label}: not a table")
+        missing = [
+            key
+            for key in _REQUIRED
+            if not isinstance(entry.get(key), str) or not entry[key].strip()
+        ]
+        if missing:
+            raise SourcesError(
+                f"{label}: missing or empty field(s): {', '.join(missing)}"
+            )
+        unknown = sorted(set(entry) - set(_KNOWN))
+        if unknown:
+            raise SourcesError(f"{label}: unknown field(s): {', '.join(unknown)}")
+        doc_id = entry["doc_id"]
+        label = f"source {doc_id!r}"
+        if not _DOC_ID.match(doc_id):
+            raise SourcesError(f"{label}: doc_id must be a lowercase file-safe slug")
+        if doc_id in seen:
+            raise SourcesError(f"{label}: duplicate doc_id")
+        seen.add(doc_id)
+        if not entry["url"].startswith("https://"):
+            raise SourcesError(f"{label}: url must be https")
+        if not _DATE.match(entry["date"]):
+            raise SourcesError(f"{label}: date must be YYYY-MM, got {entry['date']!r}")
+        digest = entry.get("sha256", "")
+        if digest and not _SHA256.match(digest):
+            raise SourcesError(f"{label}: sha256 must be 64 lowercase hex digits")
+        splitter = entry.get("splitter", "headings")
+        if splitter not in _SPLITTERS:
+            raise SourcesError(
+                f"{label}: splitter must be one of {_SPLITTERS}, got {splitter!r}"
+            )
+        sources.append(Source(**{key: entry[key] for key in _KNOWN if key in entry}))
+    return tuple(sources)
+
+
+def load_sources(path: Path) -> tuple[Source, ...]:
+    """Read and parse ``path``."""
+    try:
+        text = path.read_text(encoding="utf-8")
+    except OSError as error:
+        raise SourcesError(f"cannot read {path}: {error}") from error
+    return parse_sources(text)

primarysource/nist/ingest/status.py

@@ -0,0 +1,255 @@
+"""Publication status of each corpus document, read from the bulk data the library already holds.
+
+Three sources, each checked on its own:
+
+* the CSRC listing (``data/nist-catalog.json``, written by ``discover_nist``): the status NIST gives each
+  publication (Final, Draft, Withdrawn, ...), and whether it is current;
+* the OSCAL catalogue's own revision history (``metadata/revisions``): the versions NIST released and when;
+* the front matter of the PDF text: a draft carries "Draft" or "Initial Public Draft" on its first pages.
+
+A document's default status is the one NIST's listing gives its publication. The OSCAL history supplies the
+newest release of that publication, so a family can be resolved to its current final revision without a
+hand-written table.
+"""
+
+from __future__ import annotations
+
+from primarysource.nist.paths import PACKAGE_DATA, corpus_dir
+
+import json
+import re
+import sys
+import xml.etree.ElementTree as ET
+from collections.abc import Iterable
+from dataclasses import dataclass
+from pathlib import Path
+
+OSCAL_NS = "{http://csrc.nist.gov/ns/oscal/1.0}"
+_DOC_ID = re.compile(r"^sp(?P<series>\d+)-(?P<number>\d+)(?P<letter>(?:(?!r\d)[a-z])?)(?:r(?P<rev>\d+))?")
+_DRAFT = re.compile(r"\b(initial public draft|draft)\b", re.I)
+_FRONT_MATTER_CHARS = 4000
+# The main document of a publication, not a baseline profile of it ("sp800-53r5-high-oscal").
+_MAIN_CATALOGUE = re.compile(r"^sp\d+-\d+[a-z]?(r\d+)?(-oscal)?$")
+DEFAULTS_FILE = "defaults.json"
+DEFAULTS_SCHEMA = 1
+
+
+@dataclass(frozen=True)
+class Release:
+    version: str
+    last_modified: str  # YYYY-MM-DD
+
+
+@dataclass(frozen=True)
+class DocumentStatus:
+    doc_id: str
+    csrc_number: str | None  # e.g. "800-53 Rev. 5", None when the listing has no such publication
+    csrc_status: str | None  # "Final", "Draft", "Withdrawn", ... as NIST lists it
+    csrc_current: bool  # the listing marks it "final and current"
+    latest_release: Release | None  # newest OSCAL release, when the document is an OSCAL catalogue
+    pdf_marked_draft: bool | None  # None when there is no PDF text to read
+    oscal_status: str | None  # the catalogue's own publication-status prop, lowercased; None when absent
+    framework: str | None = None  # the catalogue's framework-identifier prop, lowercased ("csf")
+    variant: str | None = None  # a baseline profile's name ("low"), None for the main catalogue
+
+
+_VARIANT = re.compile(r"^sp\d+-\d+[a-z]?(?:r\d+)?-([a-z]+)-oscal$")
+
+
+def csrc_number_for(doc_id: str) -> str | None:
+    """The CSRC number a corpus id stands for: ``sp800-53r5-oscal`` -> ``800-53 Rev. 5``.
+
+    An id without a revision keeps no "Rev." part: ``sp800-218`` -> ``800-218``, ``sp800-218a`` -> ``800-218A``.
+    """
+    m = _DOC_ID.match(doc_id)
+    if m is None:
+        return None
+    base = f"{m.group('series')}-{m.group('number')}{(m.group('letter') or '').upper()}"
+    rev = m.group("rev")
+    return base if rev is None else f"{base} Rev. {rev}"
+
+
+def publication_key(csrc_number: str) -> str:
+    """The publication a CSRC number belongs to, without its revision: ``800-53 Rev. 5`` -> ``800-53``."""
+    return csrc_number.split(" Rev.")[0]
+
+
+def load_csrc(path: Path) -> dict[str, list[dict]]:
+    """Rows of ``nist-catalog.json`` grouped by their CSRC number."""
+    rows: dict[str, list[dict]] = {}
+    for row in json.loads(path.read_text(encoding="utf-8"))["entries"]:
+        rows.setdefault(row["number"], []).append(row)
+    return rows
+
+
+def csrc_state(rows: list[dict] | None) -> tuple[str | None, bool]:
+    """The status NIST gives a publication, and whether it is current.
+
+    Several rows can share a number (a publication and its earlier drafts); the one marked final is preferred.
+    """
+    if not rows:
+        return None, False
+    final = [r for r in rows if r["status"] == "Final"]
+    chosen = final[0] if final else rows[0]
+    return chosen["status"], chosen["reason"] == "final and current"
+
+
+def oscal_releases(root: ET.Element) -> list[Release]:
+    """Every release the catalogue's revision history lists, oldest first."""
+    revisions = root.find(f"{OSCAL_NS}metadata/{OSCAL_NS}revisions")
+    if revisions is None:
+        return []
+    out: list[Release] = []
+    for rev in revisions.findall(f"{OSCAL_NS}revision"):
+        version = (rev.findtext(f"{OSCAL_NS}version") or "").strip()
+        modified = (rev.findtext(f"{OSCAL_NS}last-modified") or "").strip()[:10]
+        if version and modified:
+            out.append(Release(version, modified))
+    return sorted(out, key=lambda r: r.last_modified)
+
+
+def oscal_prop(root: ET.Element, name: str) -> str | None:
+    """The value of the first prop the catalogue states under ``name``, lowercased."""
+    for prop in root.iter(f"{OSCAL_NS}prop"):
+        if prop.get("name") == name and prop.get("value"):
+            return prop.get("value", "").strip().lower()
+    return None
+
+
+def oscal_publication_status(root: ET.Element) -> str | None:
+    """The ``publication-status`` prop the catalogue states for itself, lowercased."""
+    return oscal_prop(root, "publication-status")
+
+
+def latest_release(root: ET.Element) -> Release | None:
+    releases = oscal_releases(root)
+    return releases[-1] if releases else None
+
+
+def pdf_marked_draft(text: str) -> bool:
+    """True when the front matter of a PDF's text says it is a draft."""
+    return _DRAFT.search(text[:_FRONT_MATTER_CHARS]) is not None
+
+
+def document_status(
+    doc_id: str,
+    csrc_rows: dict[str, list[dict]],
+    *,
+    oscal_path: Path | None = None,
+    pdf_text: str | None = None,
+) -> DocumentStatus:
+    number = csrc_number_for(doc_id)
+    status, current = csrc_state(csrc_rows.get(number) if number else None)
+    release = None
+    oscal_status = None
+    framework = None
+    if oscal_path is not None:
+        root = ET.parse(oscal_path).getroot()
+        release = latest_release(root)
+        oscal_status = oscal_publication_status(root)
+        framework = oscal_prop(root, "framework-identifier")
+    variant = _VARIANT.match(doc_id)
+    return DocumentStatus(
+        doc_id=doc_id,
+        csrc_number=number,
+        csrc_status=status,
+        csrc_current=current,
+        latest_release=release,
+        pdf_marked_draft=None if pdf_text is None else pdf_marked_draft(pdf_text),
+        oscal_status=oscal_status,
+        framework=framework,
+        variant=variant.group(1) if variant else None,
+    )
+
+
+def compute_defaults(statuses: Iterable[DocumentStatus]) -> dict[str, str]:
+    """Publication -> the doc_id of its default document, for the server to resolve a bare publication number.
+
+    A document with no CSRC number (CSF 2.0, a CSWP) is its own default when its catalogue states it final.
+    """
+    statuses = list(statuses)
+    mapping = {publication_key(s.csrc_number): s.doc_id for s in default_documents(statuses)}
+    # A baseline profile is named by its publication and its level: "800-53 low".
+    for s in statuses:
+        if s.csrc_number and s.csrc_current and s.variant:
+            mapping[f"{publication_key(s.csrc_number)} {s.variant}"] = s.doc_id
+    # A document with no CSRC number (CSF 2.0, a CSWP) is named by its framework, and is its own default.
+    for s in statuses:
+        if s.csrc_number is None and s.oscal_status == "final":
+            mapping.setdefault((s.framework or s.doc_id).lower(), s.doc_id)
+    return dict(sorted(mapping.items()))
+
+
+def write_defaults(root: Path, mapping: dict[str, str]) -> Path:
+    path = root / DEFAULTS_FILE
+    path.write_text(
+        json.dumps({"schema": DEFAULTS_SCHEMA, "defaults": mapping}, indent=2) + "\n",
+        encoding="utf-8",
+    )
+    return path
+
+
+def read_defaults(root: Path) -> dict[str, str]:
+    """The defaults map, or an empty map when none has been written yet."""
+    path = root / DEFAULTS_FILE
+    if not path.is_file():
+        return {}
+    raw = json.loads(path.read_text(encoding="utf-8"))
+    if raw.get("schema") != DEFAULTS_SCHEMA:
+        return {}
+    return dict(raw["defaults"])
+
+
+def statuses_for_corpus(root: Path, catalog: Path) -> list[DocumentStatus]:
+    """Status of every document in ``root/index.json``, from the CSRC listing, OSCAL and the PDF text."""
+    rows = load_csrc(catalog)
+    raw = json.loads((root / "index.json").read_text(encoding="utf-8"))
+    out: list[DocumentStatus] = []
+    for doc in raw["documents"]:
+        doc_id = doc["doc_id"]
+        text = root / doc["text_file"]
+        oscal = root / f"{doc_id}.xml"
+        out.append(
+            document_status(
+                doc_id,
+                rows,
+                oscal_path=oscal if oscal.is_file() else None,
+                pdf_text=text.read_text(encoding="utf-8") if text.is_file() else None,
+            )
+        )
+    return out
+
+
+def main(argv: list[str] | None = None) -> int:
+    """Write ``defaults.json`` for a corpus directory: ``python -m primarysource.nist.ingest.status [corpus] [catalog]``."""
+    args = list(sys.argv[1:] if argv is None else argv)
+    root = Path(args[0]) if args else corpus_dir()
+    catalog = Path(args[1]) if len(args) > 1 else PACKAGE_DATA / "nist-catalog.json"
+    statuses = statuses_for_corpus(root, catalog)
+    mapping = compute_defaults(statuses)
+    write_defaults(root, mapping)
+    for publication, doc_id in mapping.items():
+        print(f"{publication}: {doc_id}")
+    return 0
+
+
+def default_documents(statuses: Iterable[DocumentStatus]) -> list[DocumentStatus]:
+    """The documents that are the current final revision of their publication.
+
+    A publication qualifies when NIST's listing marks it final and current. Among the corpus documents of one
+    publication, the default is the newest by OSCAL release date, or the one without a release when none has one.
+    """
+    by_number: dict[str, list[DocumentStatus]] = {}
+    for s in statuses:
+        if s.csrc_number and s.csrc_current and _MAIN_CATALOGUE.match(s.doc_id):
+            by_number.setdefault(s.csrc_number, []).append(s)
+    defaults: list[DocumentStatus] = []
+    for group in by_number.values():
+        def newest(s: DocumentStatus) -> str:
+            return s.latest_release.last_modified if s.latest_release else ""
+        defaults.append(max(group, key=newest))
+    return sorted(defaults, key=lambda s: s.doc_id)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

primarysource/nist/models.py

@@ -0,0 +1,7 @@
+"""The nist collection's document model: the shared one (primarysource.pdfcorpus.models)."""
+
+from __future__ import annotations
+
+from primarysource.pdfcorpus.models import Document, Section
+
+__all__ = ["Document", "Section"]

primarysource/nist/paths.py

@@ -0,0 +1,33 @@
+"""Where the nist collection keeps its files (primarysource PLAN.md §5.8).
+
+The data directory holds `sources.toml` overrides and the fetched `corpus/`: `$NISTLIB_DATA` if set (the old
+setting), else `primarysource.home.collection_home("nist")`, which is `$PRIMARYSOURCE_HOME/nist` or the per-user
+`primarysource/nist`. Never the working directory and never beside the installed package. The tracked metadata
+(`sources.toml`, catalogues, digests) ships in the package under `data/`.
+"""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+from primarysource.home import collection_home
+
+PACKAGE_DATA = Path(__file__).resolve().parent / "data"
+
+
+def data_dir() -> Path:
+    env = os.environ.get("NISTLIB_DATA")
+    return Path(env) if env else collection_home("nist", legacy_app="nistlib")
+
+
+def corpus_dir() -> Path:
+    """`$NISTLIB_CORPUS` if set, else `corpus/` in the data directory."""
+    env = os.environ.get("NISTLIB_CORPUS")
+    return Path(env) if env else data_dir() / "corpus"
+
+
+def sources_file(data: Path, name: str = "sources.toml") -> Path:
+    """The data directory's own `sources.toml` if it has one, else the one shipped in the package."""
+    local = data / name
+    return local if local.is_file() else PACKAGE_DATA / name

primarysource/nist/server/__init__.py

@@ -0,0 +1,3 @@
+"""Read-only MCP server over the locally fetched NIST corpus. Makes no network calls."""
+
+from __future__ import annotations

primarysource/nist/server/corpus.py

@@ -0,0 +1,38 @@
+"""Load the nist corpus: the shared loader (primarysource.pdfcorpus.corpus) with the nist spec."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from primarysource.nist.spec import SPEC
+from primarysource.pdfcorpus import corpus as _shared
+from primarysource.pdfcorpus.corpus import SCHEMA_VERSION, CorpusError
+
+ENV_VAR = SPEC.env_var
+FETCH_COMMAND = SPEC.fetch_command
+__all__ = ["ENV_VAR", "FETCH_COMMAND", "SCHEMA_VERSION", "Corpus", "CorpusError", "default_corpus_dir", "load_corpus"]
+
+
+class Corpus(_shared.Corpus):
+    """A nist corpus: the shared one with this collection's hooks as defaults."""
+
+    def __init__(self, root: Path, documents: tuple[_shared.Document, ...], defaults: dict[str, str] | None = None) -> None:
+        super().__init__(
+            root,
+            documents,
+            defaults or {},
+            SPEC.alias,
+            fetch_command=SPEC.fetch_command,
+            index_command=SPEC.index_command,
+            prefix=f"{SPEC.name}_",
+        )
+
+
+def default_corpus_dir() -> Path:
+    """``$NISTLIB_CORPUS`` if set, else ``corpus/`` in the collection's data directory."""
+    return SPEC.corpus_dir()
+
+
+def load_corpus(root: Path | None = None) -> _shared.Corpus:
+    """Read and validate the index; raise ``CorpusError`` whose ``action`` is the fetch command."""
+    return _shared.load_corpus(SPEC, root)