primarysource 0.1.0__py3-none-any.whl

primarysource/ieee/catalogue.py

@@ -0,0 +1,123 @@
+"""The catalogue of IEEE editions: designations, titles, sources and pins. Metadata only, never text.
+
+`catalogue.json` is package data. The pins it holds are also exposed as module constants (`POSIX_2024`,
+`POSIX_TERMS`, `IEEE802_TERMS`) so that the core's collection manifest (`cl.collection_manifest`, owned by the
+core) can lift them when it lands. Until then this module is the only reader of the file.
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from importlib import resources
+from typing import Any
+
+from primarysource.errors import LibError
+
+PLACEMENTS = ("layer", "volume", "none")
+
+
+@dataclass(frozen=True, slots=True)
+class Terms:
+    """What a collection's publisher allows, and therefore where its text may be placed."""
+
+    collection: str
+    licence: str
+    placement: str
+    terms_url: str
+    terms_read: str
+    summary: str
+
+
+@dataclass(frozen=True, slots=True)
+class Edition:
+    """One edition of one document. `sha256` is None until a pin exists (an 802 PDF the user has not supplied)."""
+
+    collection: str
+    id: str
+    designations: tuple[str, ...]
+    title: str
+    publisher: str
+    edition_date: str
+    page_url: str
+    source_url: str | None
+    archive: str | None
+    sha256: str | None
+    size: int | None
+    root: str | None = None
+
+
+def _text(entry: dict[str, Any], key: str) -> str:
+    value = entry.get(key)
+    if not isinstance(value, str) or not value:
+        raise ValueError(f"catalogue entry {entry.get('id')!r}: {key} must be a non-empty string")
+    return value
+
+
+def _optional(entry: dict[str, Any], key: str) -> str | None:
+    value = entry.get(key)
+    if value is not None and not isinstance(value, str):
+        raise ValueError(f"catalogue entry {entry.get('id')!r}: {key} must be a string or null")
+    return value
+
+
+def parse(raw: str) -> tuple[dict[str, Terms], dict[str, Edition]]:
+    """Validate the catalogue text. A malformed catalogue is a packaging bug, so it raises ValueError."""
+    data = json.loads(raw)
+    if data.get("schema") != 1:
+        raise ValueError("catalogue schema must be 1")
+    terms: dict[str, Terms] = {}
+    editions: dict[str, Edition] = {}
+    for collection, body in data["collections"].items():
+        placement = _text(body, "placement")
+        if placement not in PLACEMENTS:
+            raise ValueError(f"{collection}: placement {placement!r} is not one of {PLACEMENTS}")
+        terms[collection] = Terms(
+            collection,
+            _text(body, "licence"),
+            placement,
+            _text(body, "terms_url"),
+            _text(body, "terms_read"),
+            _text(body, "terms_summary"),
+        )
+        for entry in body["editions"]:
+            digest = _optional(entry, "sha256")
+            if digest is not None and (len(digest) != 64 or digest.strip("0123456789abcdef")):
+                raise ValueError(f"{entry.get('id')!r}: sha256 must be 64 lowercase hex digits")
+            size = entry.get("size")
+            if size is not None and (not isinstance(size, int) or size <= 0):
+                raise ValueError(f"{entry.get('id')!r}: size must be a positive integer or null")
+            edition = Edition(
+                collection,
+                _text(entry, "id"),
+                tuple(entry["designations"]),
+                _text(entry, "title"),
+                _text(entry, "publisher"),
+                _text(entry, "edition_date"),
+                _text(entry, "page_url"),
+                _optional(entry, "source_url"),
+                _optional(entry, "archive"),
+                digest,
+                size,
+                _optional(entry, "root"),
+            )
+            if edition.id in editions:
+                raise ValueError(f"duplicate edition id {edition.id!r}")
+            editions[edition.id] = edition
+    return terms, editions
+
+
+TERMS, EDITIONS = parse(resources.files("primarysource.ieee").joinpath("catalogue.json").read_text(encoding="utf-8"))
+
+POSIX_TERMS = TERMS["posix"]
+IEEE802_TERMS = TERMS["ieee802"]
+POSIX_2024 = EDITIONS["posix-2024"]
+
+
+def edition(collection: str, edition_id: str) -> Edition:
+    """The catalogued edition, or a LibError that lists the ones this collection knows."""
+    found = EDITIONS.get(edition_id)
+    if found is None or found.collection != collection:
+        known = ", ".join(sorted(e.id for e in EDITIONS.values() if e.collection == collection)) or "none"
+        raise LibError(f"{edition_id!r} is not in the {collection} catalogue (known: {known})", "")
+    return found

primarysource/ieee/cli.py

@@ -0,0 +1,79 @@
+"""`primarysource posix ...` and `primarysource ieee802 ...`: the collections' own commands.
+
+primarysource posix import <susv5.tgz>
+primarysource posix pages [volume]
+primarysource posix outline <page>
+primarysource posix read <ref> [--max-tokens N]
+primarysource posix grep <text> [--volume V] [--limit N] [--start N]
+primarysource ieee802 import <file.pdf> <designation>
+primarysource ieee802 outline <designation>
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from collections.abc import Sequence
+
+from primarysource.errors import LibError
+from primarysource.ieee import ieee802, importer, posix
+
+
+def _posix_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(prog="primarysource posix", description="IEEE Std 1003.1-2024 (POSIX)")
+    commands = parser.add_subparsers(dest="command", required=True)
+    commands.add_parser("import", help="verify and import the user's susv5.tgz").add_argument("archive")
+    commands.add_parser("pages", help="list page ids").add_argument("volume", nargs="?")
+    commands.add_parser("outline", help="a page's headings with refs").add_argument("page")
+    read = commands.add_parser("read", help="read by ref")
+    read.add_argument("ref")
+    read.add_argument("--max-tokens", type=int, default=2000)
+    grep = commands.add_parser("grep", help="find blocks containing text")
+    grep.add_argument("text")
+    grep.add_argument("--volume")
+    grep.add_argument("--limit", type=int, default=20)
+    grep.add_argument("--start", type=int, default=0)
+    return parser
+
+
+def _ieee802_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(prog="primarysource ieee802", description="IEEE 802 standards (GET Program)")
+    commands = parser.add_subparsers(dest="command", required=True)
+    imp = commands.add_parser("import", help="import a PDF you obtained yourself")
+    imp.add_argument("pdf")
+    imp.add_argument("designation")
+    commands.add_parser("outline", help="clauses from the PDF's bookmarks").add_argument("designation")
+    return parser
+
+
+def _run(text: str) -> int:
+    sys.stdout.write(text if text.endswith("\n") else text + "\n")
+    return 0
+
+
+def posix_main(argv: Sequence[str]) -> int:
+    args = _posix_parser().parse_args(list(argv))
+    try:
+        if args.command == "import":
+            return importer.main(["posix", args.archive])
+        if args.command == "pages":
+            return _run("\n".join(posix.pages(args.volume)))
+        if args.command == "outline":
+            return _run(posix.outline(args.page))
+        if args.command == "read":
+            return _run(posix.read(args.ref, max_tokens=args.max_tokens))
+        return _run(posix.grep(args.text, volume=args.volume, limit=args.limit, start=args.start))
+    except LibError as exc:
+        print(exc.as_text(), file=sys.stderr)
+        return 1
+
+
+def ieee802_main(argv: Sequence[str]) -> int:
+    args = _ieee802_parser().parse_args(list(argv))
+    try:
+        if args.command == "import":
+            return importer.main(["ieee802", args.pdf, args.designation])
+        return _run(ieee802.outline(args.designation))
+    except LibError as exc:
+        print(exc.as_text(), file=sys.stderr)
+        return 1

primarysource/ieee/identify.py

@@ -0,0 +1,24 @@
+"""How the ieee802 collection recognises an IEEE 802 standard in a supplied PDF (`registry.Collection.identify`).
+
+Covers and running headers read `IEEE Std 802.1Q-2022`. The importer takes the user's own PDF and the designation.
+"""
+
+from __future__ import annotations
+
+import re
+
+from primarysource.identify import Candidate, Probe, candidates
+
+_STD = re.compile(r"\bIEEE\s+Std\.?\s+(?P<designation>802(?:\.\d{1,2}[A-Za-z]{0,4})*-\d{4})\b")
+
+
+def _label(match: re.Match[str]) -> str:
+    return f"IEEE Std {match['designation']}"
+
+
+def identify(probe: Probe) -> list[Candidate]:
+    def command(label: str) -> str:
+        where = probe.path if probe.path is not None else probe.name
+        return f'primarysource ieee802 import "{where}" {label.removeprefix("IEEE Std ")}'
+
+    return candidates(probe, "ieee802", _STD, _label, command)

primarysource/ieee/ieee802.py

@@ -0,0 +1,47 @@
+"""Read what `importer.import_pdf` stored for an 802 standard: its clause outline from the PDF's bookmarks.
+
+Reading a clause's sentences (`802.1Q-2022§5.4¶2`) needs a clause segmenter over the extracted page text; that is
+the core's planned `cl.segment_bookmarks` (itulib's walker), so it is a `[ ]` row (`ieee.ieee802_read`), not code
+written twice here.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from primarysource.errors import LibError
+from primarysource.ieee import IEEE802
+from primarysource.ieee.catalogue import edition
+from primarysource.ieee.importer import INSTALL_RECORD, collection_home, normalise_designation
+from primarysource.refs import Ref
+from primarysource.render import render
+
+
+def installed(designation: str, home: Path | None = None) -> Path:
+    """The directory of an imported 802 standard, or a LibError that says how to import it."""
+    pinned = edition("ieee802", normalise_designation(designation))
+    root = collection_home("ieee802", home) / pinned.id
+    if not (root / INSTALL_RECORD).is_file():
+        raise LibError(
+            f"{pinned.id} is not imported",
+            "",
+            f"obtain the PDF yourself from {pinned.page_url} (IEEE GET Program terms, see docs/ieee.md), then run "
+            f"primarysource ieee802 import <file.pdf> {pinned.id}",
+        )
+    return root
+
+
+def outline(designation: str, *, home: Path | None = None) -> str:
+    """The PDF's bookmarks, indented by depth, with a ref for each numbered clause and its PDF page."""
+    root = installed(designation, home)
+    pinned = edition("ieee802", normalise_designation(designation))
+    entries = json.loads((root / "outline.json").read_text(encoding="utf-8"))
+    lines = []
+    for entry in entries:
+        ref = IEEE802.grammar.format(Ref(pinned.id, entry["clause"])) if entry["clause"] else ""
+        where = f"p. {entry['page']}" if entry["page"] else "no page"
+        parts = [entry["title"], ref, f"({where})"]
+        lines.append("  " * (entry["depth"] - 1) + "  ".join(part for part in parts if part))
+    body = "\n".join(lines) if lines else f"{pinned.id} has no bookmarks"
+    return render(body, {"cite": f"{pinned.designations[0]}, {pinned.title}", "unit": IEEE802.unit})

primarysource/ieee/importer.py

@@ -0,0 +1,305 @@
+"""Import a user-supplied IEEE document into `$PRIMARYSOURCE_HOME/<collection>/<edition>/`.
+
+Nothing here fetches. The user downloads the POSIX archive or an 802 PDF themselves, under the publisher's terms
+(`docs/ieee.md`), and hands the file over; a name, an email address, a login or a terms checkbox is a human act
+this code never performs.
+
+`import_posix` checks the archive against the pinned sha256 before reading a single member, then streams the
+tarball (`tarfile` mode `r|gz`, no seeking) and writes only the HTML pages, under bounds, into a staging directory
+that replaces the installed edition in one rename. This is an installation step: the readers in `posix.py` then
+read one page file at a time.
+
+`import_pdf` reads an 802 PDF with `pypdf` (the `[ieee]` extra) and stores its bookmarks and page text.
+
+Run as `python -m primarysource.ieee.importer posix <susv5.tgz>` or `... ieee802 <file.pdf> <designation>`.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import re
+import secrets
+import shutil
+import sys
+import tarfile
+from collections import Counter
+from collections.abc import Sequence
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+from pathlib import Path, PurePosixPath
+from typing import Any
+
+from primarysource.errors import LibError
+from primarysource.home import atomic_write
+from primarysource.home import collection_home as core_collection_home
+from primarysource.ieee import POSIX_PAGE
+from primarysource.ieee.catalogue import POSIX_2024, Edition, edition
+
+INSTALL_RECORD = "install.json"
+INSTALL_HINT = 'pip install "primarysource[ieee]"'
+_PAGE = re.compile(POSIX_PAGE)
+_CHUNK = 1 << 20
+
+
+@dataclass(frozen=True, slots=True)
+class Limits:
+    """Ceilings on what an import will accept. The real POSIX 2024 archive measures 4,962,222 bytes, 1,715
+    members, 1,616 HTML pages, 24,248,288 bytes of HTML, largest page 459,326 bytes."""
+
+    archive_bytes: int = 64_000_000
+    members: int = 10_000
+    member_bytes: int = 4_000_000
+    total_bytes: int = 200_000_000
+    pdf_bytes: int = 300_000_000
+
+
+@dataclass(frozen=True, slots=True)
+class Report:
+    """What an import wrote and what it skipped, by reason."""
+
+    edition: str
+    path: Path
+    pages: int
+    bytes: int
+    skipped: dict[str, int] = field(default_factory=dict)
+
+
+def collection_home(collection: str, home: Path | None = None) -> Path:
+    """The core's `home.collection_home` (`$PRIMARYSOURCE_HOME/<collection>`, PLAN.md §5.8), or
+    `<home>/<collection>` when a home is given (tests, the measurement script)."""
+    return home / collection if home is not None else core_collection_home(collection)
+
+
+def sha256_file(path: Path, limit: int) -> tuple[str, int]:
+    """Digest and size of a file, refusing one larger than `limit` without reading past it."""
+    digest = hashlib.sha256()
+    size = 0
+    with path.open("rb") as stream:
+        while chunk := stream.read(_CHUNK):
+            size += len(chunk)
+            if size > limit:
+                raise LibError(f"{path.name} is over {limit:,} bytes; this is not the expected file", "")
+            digest.update(chunk)
+    return digest.hexdigest(), size
+
+
+def _verify(path: Path, pinned: Edition, limit: int) -> tuple[str, int]:
+    if not path.is_file():
+        raise LibError(f"no file at {path}", f"download {pinned.source_url or pinned.page_url} first")
+    got, size = sha256_file(path, limit)
+    if pinned.sha256 is not None and got != pinned.sha256:
+        raise LibError(
+            f"{path.name} has sha256 {got}, not the {pinned.sha256} pinned for {pinned.id}: the file is damaged, "
+            "truncated, or a different edition or archive format",
+            f"download {pinned.source_url or pinned.page_url} again (docs/ieee.md says how)",
+        )
+    return got, size
+
+
+def _member_path(name: str) -> PurePosixPath:
+    """The member's path, or a LibError for one that could escape the install directory."""
+    if "\x00" in name or "\\" in name or name.startswith("/") or re.match(r"[A-Za-z]:", name):
+        raise LibError(f"refusing archive member {name!r}: absolute or unsafe path", "")
+    path = PurePosixPath(name)
+    if ".." in path.parts:
+        raise LibError(f"refusing archive member {name!r}: path traversal", "")
+    return path
+
+
+def _install(staging: Path, target: Path) -> None:
+    """Replace `target` with `staging` by renames, so a reader sees the old edition or the new one."""
+    old = target.with_name(f".{target.name}.old-{secrets.token_hex(4)}")
+    if target.exists():
+        os.replace(target, old)
+    try:
+        os.replace(staging, target)
+    except BaseException:
+        if old.exists():
+            os.replace(old, target)
+        raise
+    shutil.rmtree(old, ignore_errors=True)
+
+
+def import_posix(
+    archive: Path | str,
+    *,
+    home: Path | None = None,
+    pinned: Edition = POSIX_2024,
+    limits: Limits | None = None,
+) -> Report:
+    """Verify and install the POSIX HTML archive. Refuses, naming the cause, on a digest mismatch, an unsafe or
+    oversize member, or more members or bytes than `limits` allow; nothing is installed unless all of it passed."""
+    limits = limits or Limits()
+    path = Path(archive)
+    if pinned.sha256 is None or pinned.root is None:
+        raise LibError(f"{pinned.id} has no pinned archive", "")
+    digest, size = _verify(path, pinned, limits.archive_bytes)
+    base = collection_home(pinned.collection, home)
+    base.mkdir(parents=True, exist_ok=True)
+    staging = base / f".{pinned.id}.part-{secrets.token_hex(4)}"
+    staging.mkdir()
+    skipped: Counter[str] = Counter()
+    pages = 0
+    written = 0
+    try:
+        with path.open("rb") as raw, tarfile.open(fileobj=raw, mode="r|gz") as tar:
+            for count, member in enumerate(tar, start=1):
+                if count > limits.members:
+                    raise LibError(f"{path.name} has more than {limits.members:,} members", "")
+                name = _member_path(member.name)
+                if not member.isfile():
+                    skipped["not a regular file"] += 1
+                    continue
+                relative = str(name)
+                if not relative.startswith(pinned.root):
+                    skipped["outside " + pinned.root] += 1
+                    continue
+                relative = relative[len(pinned.root) :]
+                if not relative.endswith(".html"):
+                    skipped["not html"] += 1
+                    continue
+                if not _PAGE.fullmatch(relative[: -len(".html")]):
+                    skipped["no page id"] += 1
+                    continue
+                if member.size > limits.member_bytes:
+                    raise LibError(f"refusing {member.name!r}: {member.size:,} bytes is over the per-page limit", "")
+                stream = tar.extractfile(member)
+                data = stream.read(limits.member_bytes + 1) if stream is not None else b""
+                if len(data) > limits.member_bytes:
+                    raise LibError(f"refusing {member.name!r}: over the per-page limit", "")
+                written += len(data)
+                if written > limits.total_bytes:
+                    raise LibError(f"{path.name} unpacks to more than {limits.total_bytes:,} bytes of pages", "")
+                destination = staging.joinpath(*PurePosixPath(relative).parts)
+                if staging.resolve() not in destination.resolve().parents:
+                    raise LibError(f"refusing archive member {member.name!r}: resolves outside the install", "")
+                destination.parent.mkdir(parents=True, exist_ok=True)
+                destination.write_bytes(data)
+                pages += 1
+        record = {
+            "edition": pinned.id,
+            "archive": path.name,
+            "sha256": digest,
+            "size": size,
+            "pages": pages,
+            "bytes": written,
+            "skipped": dict(sorted(skipped.items())),
+            "imported": datetime.now(UTC).isoformat(timespec="seconds"),
+        }
+        (staging / INSTALL_RECORD).write_text(json.dumps(record, indent=2) + "\n", encoding="utf-8")
+        target = base / pinned.id
+        _install(staging, target)
+    except tarfile.TarError as exc:
+        raise LibError(f"{path.name} is not a readable gzip tar archive ({exc})", "") from exc
+    finally:
+        shutil.rmtree(staging, ignore_errors=True)
+    return Report(pinned.id, target, pages, written, dict(skipped))
+
+
+_CLAUSE = re.compile(r"\s*(?:Annex\s+)?(?P<id>[0-9]+(?:\.[0-9]+)*|[A-Z](?:\.[0-9]+)*)\b[\s.:)-]*(?P<title>.*)")
+
+
+def clause_of(title: str) -> str | None:
+    """The clause or annex number a bookmark title starts with (`5.4 Bridge`, `Annex A`), or None."""
+    match = _CLAUSE.match(title)
+    if match is None:
+        return None
+    ident = match["id"]
+    if len(ident) == 1 and ident.isalpha() and not title.lstrip().startswith(("Annex", ident + ".", ident + " ")):
+        return None
+    return ident
+
+
+def normalise_designation(text: str) -> str:
+    """`IEEE Std 802.1Q-2022` and `IEEE 802.1Q-2022` -> `802.1Q-2022`."""
+    return re.sub(r"^\s*IEEE\s+(?:Std\.?\s+)?", "", text).strip()
+
+
+def import_pdf(
+    pdf: Path | str,
+    designation: str,
+    *,
+    home: Path | None = None,
+    limits: Limits | None = None,
+) -> Report:
+    """Store the bookmarks and page text of a user-supplied 802 PDF. Refuses a designation the catalogue does not
+    list, a file that is not a PDF, a digest that differs from a pin, and an encrypted PDF (never decrypted)."""
+    limits = limits or Limits()
+    pinned = edition("ieee802", normalise_designation(designation))
+    path = Path(pdf)
+    if not path.is_file():
+        raise LibError(f"no file at {path}", f"download {pinned.id} from {pinned.page_url} yourself first")
+    with path.open("rb") as stream:
+        if stream.read(5) != b"%PDF-":
+            raise LibError(f"{path.name} is not a PDF", "")
+    digest, size = _verify(path, pinned, limits.pdf_bytes)
+    try:
+        import pypdf
+    except ModuleNotFoundError as exc:
+        raise LibError(f"importing an 802 PDF needs {exc.name!r}, which is not installed", INSTALL_HINT) from exc
+    try:
+        reader = pypdf.PdfReader(path)
+        if reader.is_encrypted:
+            raise LibError(f"{path.name} is encrypted; primarysource never decrypts a document", "")
+        outline = _outline(reader, reader.outline, 1)
+        pages = [page.extract_text() or "" for page in reader.pages]
+    except pypdf.errors.PyPdfError as exc:
+        raise LibError(f"{path.name} could not be read as a PDF ({exc})", "") from exc
+    target = collection_home("ieee802", home) / pinned.id
+    record: dict[str, Any] = {
+        "edition": pinned.id,
+        "file": path.name,
+        "sha256": digest,
+        "size": size,
+        "pinned": pinned.sha256 is not None,
+        "backend": f"pypdf {pypdf.__version__}",
+        "pages": len(pages),
+        "imported": datetime.now(UTC).isoformat(timespec="seconds"),
+    }
+    atomic_write(target / "outline.json", (json.dumps(outline, indent=1) + "\n").encode("utf-8"))
+    atomic_write(target / "pages.json", json.dumps(pages).encode("utf-8"))
+    atomic_write(target / INSTALL_RECORD, (json.dumps(record, indent=2) + "\n").encode("utf-8"))
+    return Report(pinned.id, target, len(pages), sum(len(p) for p in pages), {})
+
+
+def _outline(reader: Any, items: Sequence[Any], depth: int) -> list[dict[str, Any]]:
+    """Bookmarks in reading order: depth, title, clause number (or None) and 1-based page."""
+    found: list[dict[str, Any]] = []
+    if depth > 12:
+        return found
+    for item in items:
+        if isinstance(item, list):
+            found.extend(_outline(reader, item, depth + 1))
+            continue
+        title = str(getattr(item, "title", "") or "").strip()
+        try:
+            page = reader.get_destination_page_number(item) + 1
+        except Exception:  # a broken destination loses its page, not the import
+            page = None
+        found.append({"depth": depth, "title": title, "clause": clause_of(title), "page": page})
+    return found
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    args = list(sys.argv[1:] if argv is None else argv)
+    try:
+        if len(args) == 2 and args[0] == "posix":
+            report = import_posix(args[1])
+        elif len(args) == 3 and args[0] == "ieee802":
+            report = import_pdf(args[1], args[2])
+        else:
+            print("usage: python -m primarysource.ieee.importer posix <susv5.tgz>", file=sys.stderr)
+            print("       python -m primarysource.ieee.importer ieee802 <file.pdf> <designation>", file=sys.stderr)
+            return 2
+    except LibError as exc:
+        print(exc.as_text(), file=sys.stderr)
+        return 1
+    skipped = ", ".join(f"{reason}: {n}" for reason, n in sorted(report.skipped.items())) or "none"
+    print(f"{report.edition}: {report.pages} pages, {report.bytes:,} bytes -> {report.path} (skipped: {skipped})")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())