primarysource 0.1.0__py3-none-any.whl

primarysource/nist/identify.py

@@ -0,0 +1,41 @@
+"""How the nist collection recognises a NIST publication in a supplied PDF (`registry.Collection.identify`).
+
+Covers read `NIST Special Publication 800-53` with `Revision 5` near it, or `FIPS PUB 201-3`; running headers
+and DOIs repeat `NIST SP 800-53r5`. The nist collection fetches its own pinned copies (`data/sources.toml`), so
+the command it suggests is the fetch.
+"""
+
+from __future__ import annotations
+
+import re
+
+from primarysource.identify import Candidate, Probe, candidates
+
+_SP = re.compile(
+    r"\bNIST\s+(?:Special\s+Publication|SP)\s+(?P<num>800-\d{1,3}(?:(?-i:[A-Z])(?!\d))?)"
+    r"(?:(?:\s*,?\s*(?:Revision|Rev\.?)\s*|r)(?P<rev>\d{1,2}))?",
+    re.IGNORECASE,
+)
+_FIPS = re.compile(
+    r"\b(?:FIPS|Federal\s+Information\s+Processing\s+Standards?\s+Publication)\s+(?:PUB\s+)?(?P<num>\d{2,3}(?:-\d{1,2})?)\b",
+    re.IGNORECASE,
+)
+
+
+def _sp(match: re.Match[str]) -> str:
+    rev = f" Rev. {match['rev']}" if match["rev"] else ""
+    return f"NIST SP {match['num'].upper()}{rev}"
+
+
+def _fips(match: re.Match[str]) -> str:
+    return f"NIST FIPS {match['num']}"
+
+
+def identify(probe: Probe) -> list[Candidate]:
+    def command(_label: str) -> str:
+        return "primarysource nist fetch  (nist fetches its own pinned copies listed in data/sources.toml)"
+
+    return [
+        *candidates(probe, "nist", _SP, _sp, command),
+        *candidates(probe, "nist", _FIPS, _fips, command),
+    ]

primarysource/nist/ingest/__init__.py

@@ -0,0 +1 @@
+"""Ingest side of nistlib: fetch the publications listed in data/sources.toml and index them."""

primarysource/nist/ingest/acquire_nist.py

@@ -0,0 +1,128 @@
+"""Download the in-scope PDFs named in data/nist-catalog.json into data/corpus/<doc_id>.pdf.
+
+Each download goes to a .part file and is renamed only when complete. The result must start with ``%PDF``.
+data/acquired-nist.json records url, path, size and sha256 per file. A file already present whose sha256
+matches the manifest is skipped. Requests are throttled to one per second with a generic User-Agent.
+Pass --dry-run to print the plan and the total size (from HEAD requests) before downloading anything.
+"""
+
+from __future__ import annotations
+
+from primarysource.nist.paths import PACKAGE_DATA
+
+import argparse
+import hashlib
+import json
+import sys
+import time
+import urllib.error
+import urllib.request
+from collections.abc import Sequence
+from pathlib import Path
+
+USER_AGENT = "nistlib-acquire/0.1 (documentation corpus)"
+DELAY = 1.0
+CHUNK = 1 << 20
+
+
+def _size(url: str) -> int | None:
+    req = urllib.request.Request(url, method="HEAD", headers={"User-Agent": USER_AGENT})
+    try:
+        with urllib.request.urlopen(req, timeout=60) as resp:
+            length = resp.headers.get("Content-Length")
+            return int(length) if length else None
+    except (urllib.error.URLError, TimeoutError):
+        return None
+    finally:
+        time.sleep(DELAY)
+
+
+def _sha256_file(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as f:
+        while chunk := f.read(CHUNK):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def _download(url: str, dest: Path) -> tuple[int, str]:
+    dest.parent.mkdir(parents=True, exist_ok=True)
+    part = dest.with_suffix(dest.suffix + ".part")
+    digest = hashlib.sha256()
+    size = 0
+    req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
+    try:
+        with urllib.request.urlopen(req, timeout=300) as resp, part.open("wb") as out:
+            while chunk := resp.read(CHUNK):
+                if size == 0 and not chunk.startswith(b"%PDF"):
+                    raise OSError("response does not start with %PDF")
+                out.write(chunk)
+                digest.update(chunk)
+                size += len(chunk)
+        if size == 0:
+            raise OSError("empty response")
+        part.replace(dest)
+    finally:
+        part.unlink(missing_ok=True)
+        time.sleep(DELAY)
+    return size, digest.hexdigest()
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    parser.add_argument("--catalog", type=Path, default=PACKAGE_DATA / "nist-catalog.json")
+    parser.add_argument("--data-dir", type=Path, default=Path("data"))
+    parser.add_argument("--dry-run", action="store_true")
+    args = parser.parse_args(argv)
+    catalog = json.loads(args.catalog.read_text(encoding="utf-8"))
+    items = [e for e in catalog["entries"] if e["in_scope"]]
+    if args.dry_run:
+        total = 0
+        for e in items:
+            size = _size(e["pdf_url"])
+            total += size or 0
+            print(f"{e['doc_id']:<20} {size or '?':>10}  {e['pdf_url']}")
+        print(f"{len(items)} files, {total / 1e9:.2f} GB known", file=sys.stderr)
+        return 0
+    manifest_path = args.data_dir / "acquired-nist.json"
+    manifest = (
+        json.loads(manifest_path.read_text(encoding="utf-8"))
+        if manifest_path.exists()
+        else {"schema": 1, "files": []}
+    )
+    by_id = {f["doc_id"]: f for f in manifest["files"]}
+    failed = 0
+    for e in items:
+        doc_id = e["doc_id"]
+        dest = args.data_dir / "corpus" / f"{doc_id}.pdf"
+        old = by_id.get(doc_id)
+        if old and dest.exists() and _sha256_file(dest) == old["sha256"]:
+            continue
+        try:
+            if (
+                dest.exists() and not old
+            ):  # a file placed earlier (the PIV set): record it, never overwrite
+                size, sha = dest.stat().st_size, _sha256_file(dest)
+            else:
+                size, sha = _download(e["pdf_url"], dest)
+        except (urllib.error.URLError, OSError, TimeoutError) as err:
+            print(f"FAILED {e['pdf_url']}: {err}", file=sys.stderr)
+            failed += 1
+            continue
+        record = {
+            "doc_id": doc_id,
+            "url": e["pdf_url"],
+            "path": f"corpus/{doc_id}.pdf",
+            "size": size,
+            "sha256": sha,
+        }
+        by_id[doc_id] = record
+        manifest["files"] = list(by_id.values())
+        manifest_path.write_text(json.dumps(manifest, indent=2), encoding="utf-8")
+        print(f"ok {doc_id} {size}", file=sys.stderr)
+    print(f"{len(items) - failed} ok, {failed} failed", file=sys.stderr)
+    return 1 if failed else 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

primarysource/nist/ingest/controls.py

@@ -0,0 +1,82 @@
+"""Section splitter for NIST control catalogues printed as PDF (SP 800-53 and its relatives).
+
+Control catalogues do not number their sections the way prose documents do. A control is written as its
+identifier followed by an upper-case title (``AC-2 ACCOUNT MANAGEMENT``), and its enhancements are written
+relative to the parent (``(1) ACCOUNT MANAGEMENT | AUTOMATED ...`` under ``AC-2``, which is ``ac-2.1``).
+This splitter recognises both, keeps the first occurrence of each identifier (later ones are the appendix
+summary tables), and accepts a control identifier only where it is anchored: at the start of a line, after a
+run of spaces, or after the page header. Prose that mentions a control in capitals is not a heading. An enhancement heading may also be glued to
+the end of the page header when it opens a page, and is accepted there for the same reason.
+
+Where the text has a ``-1`` control, indexing starts at the first one. The worked example of a control's structure in the introduction
+(Figure 1 prints AU-4 and its first enhancement) would otherwise take those two ids, and the real AU-4 would
+then be dropped as a repeat.
+
+Validated against the XML catalogue of SP 800-53 Rev 5 (see the -oscal entries): every identifier found is in
+the catalogue, with no false positives.
+"""
+
+from __future__ import annotations
+
+import re
+
+from primarysource.nist.models import Section
+
+_CONTROL = re.compile(r"([A-Z]{2})-(\d+)(?:\((\d+)\))?\s{1,6}([A-Z][A-Z ,;:\-/'—]{3,})")
+_CHILD = re.compile(
+    r"^(?:\s*|.*(?:doi\.org|NIST\.SP)\S*\s+)\((\d+)\)\s+([A-Z][A-Z ,;:\-/'—|&]{3,})"
+)
+_PAGE_HEADER = ("doi.org", "NIST.SP")
+
+
+def _anchored(line: str, start: int) -> bool:
+    before = line[:start]
+    return (
+        before.strip() == ""
+        or before.endswith("   ")
+        or any(h in before for h in _PAGE_HEADER)
+    )
+
+
+def _heading_at(line: str, parent: str | None) -> tuple[str, str, bool] | None:
+    """Return (control id, title, is_parent) when ``line`` opens a control, else None."""
+    for m in _CONTROL.finditer(line):
+        if _anchored(line, m.start()):
+            base = f"{m.group(1).lower()}-{m.group(2)}"
+            if m.group(3):
+                return f"{base}.{m.group(3)}", m.group(4).strip(), False
+            return base, m.group(4).strip(), True
+    child = _CHILD.match(line)
+    if child and parent:
+        return f"{parent}.{child.group(1)}", child.group(2).strip(), False
+    return None
+
+
+def split_control_sections(text: str) -> tuple[Section, ...]:
+    """One Section per control or enhancement, in document order, each running to the next heading."""
+    lines = text.split("\n")
+    if lines and lines[-1] == "":
+        lines.pop()
+    found: list[tuple[int, str, str, bool]] = []
+    parent: str | None = None
+    for number, line in enumerate(lines, start=1):
+        hit = _heading_at(line, parent)
+        if hit is None:
+            continue
+        if hit[2]:
+            parent = hit[0]
+        found.append((number, *hit))
+    first = next((i for i, f in enumerate(found) if f[3] and f[1].endswith("-1")), 0)
+    heads: list[tuple[int, str, str]] = []
+    seen: set[str] = set()
+    for number, control_id, title, _ in found[first:]:
+        if control_id not in seen:
+            seen.add(control_id)
+            heads.append((number, control_id, title))
+    sections: list[Section] = []
+    for index, (start, control_id, title) in enumerate(heads):
+        end = heads[index + 1][0] - 1 if index + 1 < len(heads) else len(lines)
+        sections.append(
+            Section(id=control_id, heading=title, start_line=start, end_line=end)
+        )
+    return tuple(sections)

primarysource/nist/ingest/discover_nist.py

@@ -0,0 +1,218 @@
+"""Build data/nist-catalog.json: every FIPS and SP 800 / SP 1800 publication CSRC lists, marked in or out.
+
+CSRC has no machine-readable feed, but its publications search is server-rendered and takes
+``ipp-lg=all``, so one request per series returns the whole listing (FIPS: 70 rows, SP: 841 rows).
+Candidates (listing status Final, FIPS or SP 800 or SP 1800) then get one detail-page request each,
+for the PDF link and any supersession notice. Requests are throttled to one per second with a generic
+User-Agent. Nothing is downloaded except HTML pages; PDFs are fetched by ``acquire_nist``.
+"""
+
+from __future__ import annotations
+
+from primarysource.nist.paths import PACKAGE_DATA
+
+import argparse
+import html
+import json
+import re
+import sys
+import time
+import urllib.error
+import urllib.request
+from collections.abc import Sequence
+from pathlib import Path
+
+BASE = "https://csrc.nist.gov"
+LISTING = (
+    BASE + "/publications/search?sortBy-lg=Number%20ASC&viewMode-lg=brief&ipp-lg=all"
+    "&series-lg={series}&status-lg=Final&status-lg=Draft&status-lg=Withdrawn"
+)
+USER_AGENT = "nistlib-discover/0.1 (documentation catalogue)"
+DELAY = 1.0
+ROW = re.compile(r'<tr id="result-\d+">(.*?)</tr>', re.S)
+
+
+def fetch_text(url: str) -> str | None:
+    req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
+    try:
+        with urllib.request.urlopen(req, timeout=120) as resp:
+            return resp.read().decode("utf-8", errors="replace")
+    except (urllib.error.URLError, TimeoutError) as e:
+        print(f"  {url}: {e}", file=sys.stderr)
+        return None
+    finally:
+        time.sleep(DELAY)
+
+
+def _clean(s: str) -> str:
+    return re.sub(r"\s+", " ", html.unescape(re.sub(r"<[^>]+>", " ", s))).strip()
+
+
+def parse_listing(page: str) -> list[dict]:
+    rows = []
+    for row in ROW.findall(page):
+
+        def g(pattern: str, row: str = row) -> str:
+            m = re.search(pattern, row, re.S)
+            return _clean(m.group(1)) if m else ""
+
+        rows.append(
+            {
+                "series": g(r'id="pub-series-\d+">(.*?)</td>'),
+                "number": g(r'id="pub-number-\d+">(.*?)</td>'),
+                "title": g(r'id="pub-title-link-\d+">(.*?)</a>'),
+                "status": g(r'id="pub-status-\d+">(.*?)</td>'),
+                "date": g(r'id="pub-release-date-\d+">(.*?)</td>'),
+                "href": g(r'href="(/pubs/[^"]+)"'),
+            }
+        )
+    return rows
+
+
+def parse_detail(page: str) -> dict:
+    text = _clean(
+        re.sub(r"<script.*?</script>|<style.*?</style>", "", page, flags=re.S)
+    )
+    pdf = re.search(r'href="([^"]+)"[^>]*id="pub-local-download-link"', page)
+    head = re.search(
+        r"Date Published:(.*?)(?:Planning Note|Author\(s\)|Abstract)", text
+    )
+    sup = re.search(
+        r"Superseded By:\s*(.*?)(?:Supersedes:|Planning Note|Author\(s\))", text
+    )
+    return {
+        "pdf_url": pdf.group(1) if pdf else "",
+        "date_published": _clean(head.group(1)) if head else "",
+        "superseded_by": _clean(sup.group(1)) if sup else "",
+    }
+
+
+def parse_href(href: str) -> dict | None:
+    """Split /pubs/<series>/<number>/... into the parts a doc_id and a revision order are built from."""
+    parts = href.removeprefix("/pubs/").removesuffix("/final").split("/")
+    if parts[0] == "fips":
+        num, _, rev = parts[1].partition("-")
+        return {
+            "base": f"fips{num}",
+            "rev": int(rev) if rev.isdigit() else 0,
+            "doc_id": f"fips{parts[1]}",
+        }
+    if parts[0] != "sp" or parts[1] not in {"800", "1800"}:
+        return None
+    if parts[1] == "1800":
+        return {"base": f"sp1800-{parts[2]}", "rev": 0, "doc_id": f"sp1800-{parts[2]}"}
+    base = f"sp800-{parts[2]}"
+    rev_tok = ""
+    rev = 0
+    sup = False
+    for tok in parts[3:]:
+        if re.fullmatch(r"[a-z]", tok) or re.fullmatch(r"(pt|v)\d+", tok):
+            base += tok
+        elif m := re.fullmatch(r"(r|ver)(\d+)", tok):
+            rev, rev_tok = int(m.group(2)), tok
+        elif tok.isdigit():
+            rev, rev_tok = int(tok), f"-{tok}"
+        elif tok == "sup":
+            sup = True
+        elif re.fullmatch(r"upd\d+", tok):
+            pass
+        else:
+            return None
+    if sup:
+        base += "-add"
+    return {"base": base, "rev": rev, "doc_id": base + rev_tok}
+
+
+def classify(entries: list[dict]) -> None:
+    """Set entry['in_scope'] and entry['reason'] for every entry, in place."""
+    best: dict[str, int] = {}
+    for e in entries:
+        p = e["parts"]
+        if e["candidate"] and p and not e["superseded_by"]:
+            best[p["base"]] = max(best.get(p["base"], 0), p["rev"])
+    for e in entries:
+        p = e["parts"]
+        if e["series"] not in {"FIPS", "SP 800", "SP 1800"}:
+            e["in_scope"], e["reason"] = False, f"outside scope: {e['series']} series"
+        elif e["status"] != "Final":
+            e["in_scope"], e["reason"] = False, f"status {e['status']}"
+        elif e["superseded_by"]:
+            e["in_scope"], e["reason"] = False, f"superseded by {e['superseded_by']}"
+        elif p and p["rev"] < best.get(p["base"], 0):
+            e["in_scope"], e["reason"] = (
+                False,
+                "older revision; a later final revision is listed",
+            )
+        elif not e["pdf_url"]:
+            e["in_scope"], e["reason"] = False, "no PDF link on CSRC page"
+        else:
+            e["in_scope"], e["reason"] = True, "final and current"
+
+
+def series_group(row: dict) -> str:
+    if row["series"] == "FIPS":
+        return "FIPS"
+    return (
+        f"SP {row['number'].split('-')[0]}" if row["series"] == "SP" else row["series"]
+    )
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    parser.add_argument("--out", type=Path, default=PACKAGE_DATA / "nist-catalog.json")
+    args = parser.parse_args(argv)
+    rows: list[dict] = []
+    for series in ("FIPS", "SP"):
+        page = fetch_text(LISTING.format(series=series))
+        if page is None:
+            print(f"cannot list {series}", file=sys.stderr)
+            return 1
+        found = parse_listing(page)
+        print(f"{series}: {len(found)} rows", file=sys.stderr)
+        rows += found
+    entries = []
+    for row in rows:
+        group = series_group(row)
+        parts = parse_href(row["href"])
+        candidate = (
+            row["status"] == "Final"
+            and group in {"FIPS", "SP 800", "SP 1800"}
+            and parts is not None
+        )
+        entry = {
+            "series": group,
+            "number": row["number"],
+            "title": row["title"],
+            "date": row["date"],
+            "status": row["status"],
+            "detail_url": BASE + row["href"],
+            "pdf_url": "",
+            "superseded_by": "",
+            "doc_id": parts["doc_id"] if parts else "",
+            "candidate": candidate,
+            "parts": parts,
+        }
+        if candidate:
+            page = fetch_text(entry["detail_url"])
+            if page is None:
+                print(f"detail failed: {entry['detail_url']}", file=sys.stderr)
+            else:
+                d = parse_detail(page)
+                entry.update(pdf_url=d["pdf_url"], superseded_by=d["superseded_by"])
+                if d["date_published"]:
+                    entry["date_published"] = d["date_published"]
+        entries.append(entry)
+    classify(entries)
+    for e in entries:
+        e.pop("candidate")
+        e.pop("parts")
+    args.out.parent.mkdir(parents=True, exist_ok=True)
+    doc = {"schema": 1, "source": LISTING.format(series="FIPS|SP"), "entries": entries}
+    args.out.write_text(json.dumps(doc, indent=2, ensure_ascii=False), encoding="utf-8")
+    kept = sum(1 for e in entries if e["in_scope"])
+    print(f"wrote {args.out}: {len(entries)} entries, {kept} in scope", file=sys.stderr)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

primarysource/nist/ingest/extract.py

@@ -0,0 +1,48 @@
+"""Text extraction with ``pdftotext -layout``. The section splitter is primarysource.pdfcorpus.split."""
+
+from __future__ import annotations
+
+import shutil
+import subprocess
+from pathlib import Path
+
+
+MISSING_MESSAGE = (
+    "pdftotext was not found on PATH. Install poppler-utils (on MSYS2: pacman -S mingw-w64-x86_64-poppler) "
+    "and make sure pdftotext is on PATH."
+)
+
+
+class ExtractError(RuntimeError):
+    """Raised when text extraction cannot run or fails."""
+
+
+def find_pdftotext() -> str:
+    """Return the path of pdftotext, or raise ExtractError saying how to get it."""
+    found = shutil.which("pdftotext")
+    if found is None:
+        raise ExtractError(MISSING_MESSAGE)
+    return found
+
+
+def extract_text(pdf: Path, *, timeout: float = 300.0) -> str:
+    """Run ``pdftotext -layout`` on ``pdf`` and return the text (pages separated by form feeds)."""
+    exe = find_pdftotext()
+    try:
+        result = subprocess.run(  # noqa: S603
+            [exe, "-layout", "-enc", "UTF-8", str(pdf), "-"],
+            capture_output=True,
+            timeout=timeout,
+            check=False,
+        )
+    except subprocess.TimeoutExpired as error:
+        raise ExtractError(f"pdftotext timed out after {timeout:.0f}s on {pdf}") from error
+    if result.returncode != 0:
+        detail = result.stderr.decode("utf-8", "replace").strip()
+        raise ExtractError(f"pdftotext failed on {pdf} (exit {result.returncode}): {detail}")
+    return result.stdout.decode("utf-8", "replace")
+
+
+# The splitter is shared (primarysource.pdfcorpus.split); it is re-exported here for the importers that have
+# always found it in this module.
+from primarysource.pdfcorpus.split import split_sections as split_sections  # noqa: E402