primarysource 0.1.0__py3-none-any.whl

primarysource/ecma/_pdf.py

@@ -0,0 +1,143 @@
+"""The bbf and ecma collections' PDF model: stored extractions, section lookup and sentence search.
+
+Reading a PDF moved to the core: `extract` is `primarysource.readers.pdf.extract` (`cl.reader_pdf_pypdf`) and the
+cutting into sections is `primarysource.readers.bookmarks` (`cl.segment_bookmarks`). They are re-exported here so
+the importers, the libraries and their tests are unchanged. The import runs once and its result is stored as
+JSON beside the PDF, so serving reads `PdfDocument.from_json` and never needs `pypdf`. A proposal for the core
+version of the rest is in `docs/ecma.md`.
+"""
+
+from __future__ import annotations
+
+import re
+import shutil  # noqa: F401 - tests patch `_pdf.shutil.which` to keep pdftotext out
+from collections.abc import Sequence
+
+from primarysource.errors import LibError
+from primarysource.readers.bookmarks import cut as _cut
+from primarysource.readers.bookmarks import heading_id as _heading_id
+from primarysource.readers.bookmarks import slug, split_sentences
+from primarysource.readers.pdf import INSTALL_HINT, MAX_BYTES, MAX_OUTLINE, MAX_PAGES, extract
+
+__all__ = [
+    "INSTALL_HINT",
+    "MAX_BYTES",
+    "MAX_OUTLINE",
+    "MAX_PAGES",
+    "MAX_QUERY_TERMS",
+    "PdfDocument",
+    "PdfSection",
+    "_cut",
+    "_heading_id",
+    "extract",
+    "query_terms",
+    "score_units",
+    "slug",
+    "snippet",
+    "split_sentences",
+]
+
+
+class PdfSection:
+    """One bookmarked section of an extracted PDF: `sentences` are its own text, subsections excluded."""
+
+    __slots__ = ("depth", "id", "page", "sentences", "title")
+
+    def __init__(self, ident: str, title: str, depth: int, page: int, sentences: tuple[str, ...]) -> None:
+        self.id, self.title, self.depth, self.page, self.sentences = ident, title, depth, page, sentences
+
+
+class PdfDocument:
+    """An extracted PDF as stored on disk: see `extract` for the fields."""
+
+    def __init__(self, title: str, pages: int, extractor: str, sections: tuple[PdfSection, ...]) -> None:
+        self.title, self.pages, self.extractor, self.sections = title, pages, extractor, sections
+
+    @classmethod
+    def from_json(cls, raw: object) -> PdfDocument:
+        """Validate a stored extraction. A malformed one is a `LibError`: the install is damaged or hostile."""
+        try:
+            if not isinstance(raw, dict):
+                raise TypeError("not an object")
+            sections = tuple(
+                PdfSection(
+                    str(entry["id"]),
+                    str(entry["title"]),
+                    int(entry["depth"]),
+                    int(entry["page"]),
+                    tuple(line for line in str(entry["text"]).split("\n") if line),
+                )
+                for entry in raw["sections"]
+            )
+            return cls(str(raw.get("title", "")), int(raw["pages"]), str(raw.get("extractor", "")), sections)
+        except (AssertionError, KeyError, TypeError, ValueError) as exc:
+            raise LibError("the stored extraction is malformed", "", "re-import the document") from exc
+
+    def find(self, wanted: str) -> tuple[int, str]:
+        """Index of the section `wanted` names (its id, else its title's slug, else its title ignoring case),
+        and a warning ("" if none). Raises `LibError` with close matches when nothing fits."""
+        text = wanted.strip()
+        for match in (
+            [i for i, s in enumerate(self.sections) if s.id == text],
+            [i for i, s in enumerate(self.sections) if s.id.lower() == text.lower()],
+            [i for i, s in enumerate(self.sections) if slug(s.title) == slug(text)],
+        ):
+            if match:
+                warning = ""
+                if len(match) > 1:
+                    warning = f"{len(match)} sections match {text!r}; this is the first"
+                return match[0], warning
+        import difflib
+
+        close = difflib.get_close_matches(text, [s.id for s in self.sections], n=3, cutoff=0.6)
+        raise LibError(f"no section {text!r}", f"did you mean: {', '.join(close)}" if close else "see the outline")
+
+    def span(self, index: int) -> tuple[str, ...]:
+        """The section's sentences, then each subsection as a heading unit (`§5.3.1 Title`) followed by its own
+        sentences, down to the next section at the same or a shallower depth."""
+        root = self.sections[index]
+        units = list(root.sentences)
+        for sub in self.sections[index + 1 :]:
+            if sub.depth <= root.depth:
+                break
+            units.append(f"§{sub.id} {sub.title}")
+            units.extend(sub.sentences)
+        return tuple(units)
+
+
+_TERM = re.compile(r"\w+", re.UNICODE)
+MAX_QUERY_TERMS = 16
+
+
+def query_terms(query: str) -> list[str]:
+    """The words of a search query, case-folded, at most `MAX_QUERY_TERMS` of at most 64 characters each."""
+    terms = [t for t in _TERM.findall(query.casefold()) if len(t) <= 64]
+    if not terms:
+        raise LibError("the query has no searchable words", "use words or identifiers, e.g. 'Inform retry'")
+    return terms[:MAX_QUERY_TERMS]
+
+
+def snippet(text: str, terms: list[str], width: int = 160) -> str:
+    """About `width` characters of `text` around the first query word."""
+    flat = " ".join(text.split())
+    low = flat.casefold()
+    at = min((i for i in (low.find(t) for t in terms) if i >= 0), default=0)
+    begin = max(0, at - 60)
+    return ("..." if begin else "") + flat[begin : begin + width] + ("..." if begin + width < len(flat) else "")
+
+
+def score_units(units: Sequence[str], terms: list[str], title: str, need_all: bool) -> list[tuple[float, int]]:
+    """(score, 1-based unit number) for each unit containing every term (or, if not `need_all`, at least half).
+    Score: term frequency, plus 3 per distinct term present, plus 2 per term in the section title."""
+    found: list[tuple[float, int]] = []
+    low_title = title.casefold()
+    for number, unit in enumerate(units, start=1):
+        if unit.startswith("§"):
+            continue  # a subsection heading unit is reached through its own section
+        words = _TERM.findall(unit.casefold())
+        counts = [words.count(t) for t in terms]
+        present = sum(1 for c in counts if c)
+        if present == 0 or present * 2 < len(terms) or (need_all and present < len(terms)):
+            continue
+        found.append((sum(counts) + 3 * present + sum(2 for t in terms if t in low_title), number))
+    return found

primarysource/ecma/catalogue.json

@@ -0,0 +1,22 @@
+{
+ "terms": {
+  "licence": "Ecma International text copyright policy (default notice)",
+  "placement": "volume",
+  "terms_url": "https://ecma-international.org/policies/by-ipr/ecma-text-copyright-policy/",
+  "terms_read": "2026-11-12",
+  "summary": "The notice printed in ECMA-427 allows copying and distribution of the whole or parts with the copyright notice and licence on every copy, and derivative works only as commentary, accessibility, translation or format change, or implementation; the content may not otherwise be modified. Ecma prefers that the standard is downloaded from its own site. This collection therefore stays on a user-fetched volume: the importer downloads the PDF from ecma-international.org and writes only to the user's own home. Software in the standard is BSD-licensed (the Software License section of the PDF)."
+ },
+ "editions": [
+  {
+   "number": "427",
+   "edition": 1,
+   "designation": "ECMA-427",
+   "title": "Package-URL (PURL) specification",
+   "edition_name": "1st edition, December 2025",
+   "page_url": "https://ecma-international.org/publications-and-standards/standards/ecma-427/",
+   "source_url": "https://ecma-international.org/wp-content/uploads/ECMA-427_1st_edition_december_2025.pdf",
+   "size": 2320437,
+   "sha256": "037180df99e7d7c26d910dc3adb3fa07c4fb787cc4bd29e43634de3802313825"
+  }
+ ]
+}

primarysource/ecma/catalogue.py

@@ -0,0 +1,100 @@
+"""The Ecma catalogue: which standards and editions, where to fetch them, and the sha256 of each. Metadata only.
+
+`catalogue.json` is package data. Starting with ECMA-427 (Package-URL, 1st edition, December 2025). An edition is
+identified by the standard's number and its edition number; a ref names the number (`ecma:427`) and optionally the
+edition (`ecma:427@1`); with no edition the highest imported one is read.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import dataclass
+from importlib import resources
+from typing import Any
+
+from primarysource.errors import LibError
+
+NUMBER = re.compile(r"[0-9]{1,4}")
+SHA256 = re.compile(r"[0-9a-f]{64}")
+HOSTS = ("ecma-international.org", "www.ecma-international.org")
+
+
+@dataclass(frozen=True, slots=True)
+class Terms:
+    licence: str
+    placement: str
+    terms_url: str
+    terms_read: str
+    summary: str
+
+
+@dataclass(frozen=True, slots=True)
+class Edition:
+    """One edition of one Ecma standard, with its pinned PDF."""
+
+    number: str
+    edition: int
+    designation: str
+    title: str
+    edition_name: str
+    page_url: str
+    source_url: str
+    size: int
+    sha256: str
+
+    @property
+    def key(self) -> str:
+        """The file stem under the install directory: `427-1`."""
+        return f"{self.number}-{self.edition}"
+
+
+@dataclass(frozen=True, slots=True)
+class Catalogue:
+    terms: Terms
+    editions: tuple[Edition, ...]
+
+    def edition(self, number: str, edition: int | None = None) -> Edition:
+        """The catalogued edition of `number` (the highest if `edition` is None)."""
+        have = [e for e in self.editions if e.number == number and (edition is None or e.edition == edition)]
+        if not have:
+            known = ", ".join(f"{e.number}@{e.edition}" for e in self.editions)
+            raise LibError(
+                f"no Ecma standard {number}{'@' + str(edition) if edition else ''} in the catalogue",
+                f"known: {known}",
+            )
+        return max(have, key=lambda e: e.edition)
+
+
+def parse_catalogue(raw: dict[str, Any]) -> Catalogue:
+    """Validate a catalogue document. Raises `ValueError`: a malformed catalogue is a packaging bug."""
+    terms = raw["terms"]
+    if terms["placement"] not in ("layer", "volume"):
+        raise ValueError("terms.placement must be 'layer' or 'volume'")
+    editions: list[Edition] = []
+    for entry in raw["editions"]:
+        found = Edition(
+            str(entry["number"]),
+            int(entry["edition"]),
+            entry["designation"],
+            entry["title"],
+            entry["edition_name"],
+            entry["page_url"],
+            entry["source_url"],
+            int(entry["size"]),
+            entry["sha256"],
+        )
+        if not NUMBER.fullmatch(found.number) or not SHA256.fullmatch(found.sha256) or found.edition < 1:
+            raise ValueError(f"catalogue entry {found.designation!r} is malformed")
+        editions.append(found)
+    if len({e.key for e in editions}) != len(editions):
+        raise ValueError("duplicate editions in the catalogue")
+    return Catalogue(
+        Terms(terms["licence"], terms["placement"], terms["terms_url"], terms["terms_read"], terms["summary"]),
+        tuple(editions),
+    )
+
+
+def catalogue() -> Catalogue:
+    text = resources.files("primarysource.ecma").joinpath("catalogue.json").read_text(encoding="utf-8")
+    return parse_catalogue(json.loads(text))

primarysource/ecma/cli.py

@@ -0,0 +1,67 @@
+"""`primarysource ecma ...`: the collection's own commands.
+
+primarysource ecma import [NUMBER] [--edition N] [--file PDF]
+primarysource ecma docs
+primarysource ecma outline <doc>
+primarysource ecma read <ref> [--max-tokens N]
+primarysource ecma search <query> [--doc NUMBER] [--limit N] [--start N]
+primarysource ecma cite <ref>
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from collections.abc import Sequence
+from pathlib import Path
+
+from primarysource.ecma import importer, library
+from primarysource.errors import LibError
+
+
+def _parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(prog="primarysource ecma", description="Ecma International standards")
+    commands = parser.add_subparsers(dest="command", required=True)
+    imp = commands.add_parser("import", help="fetch (or read a file), verify and index one standard")
+    imp.add_argument("number", nargs="?", default="427", help="the standard's number (default 427)")
+    imp.add_argument("--edition", type=int, help="edition number (default: the newest catalogued)")
+    imp.add_argument("--file", type=Path, help="the PDF you downloaded yourself, instead of fetching it")
+    commands.add_parser("docs", help="the catalogue and what is imported")
+    commands.add_parser("outline", help="a standard's clauses with refs").add_argument("doc")
+    read = commands.add_parser("read", help="read by ref")
+    read.add_argument("ref")
+    read.add_argument("--max-tokens", type=int, default=2000)
+    search = commands.add_parser("search", help="find sentences containing words")
+    search.add_argument("query")
+    search.add_argument("--doc")
+    search.add_argument("--limit", type=int, default=10)
+    search.add_argument("--start", type=int, default=0)
+    commands.add_parser("cite", help="the citation for a ref").add_argument("ref")
+    return parser
+
+
+def _out(text: str) -> int:
+    sys.stdout.write(text if text.endswith("\n") else text + "\n")
+    return 0
+
+
+def main(argv: Sequence[str]) -> int:
+    args = _parser().parse_args(list(argv))
+    try:
+        if args.command == "import":
+            result = importer.import_edition(library.default_home(), args.number, args.edition, file=args.file)
+            return _out(
+                f"imported {result.key}: {result.sections} sections, {result.requests} request(s), {result.pdf}"
+            )
+        if args.command == "docs":
+            return _out(library.docs())
+        if args.command == "outline":
+            return _out(library.outline(args.doc))
+        if args.command == "read":
+            return _out(library.read(args.ref, max_tokens=args.max_tokens))
+        if args.command == "search":
+            return _out(library.search(args.query, doc=args.doc, limit=args.limit, start=args.start))
+        return _out(library.cite(args.ref))
+    except LibError as exc:
+        print(exc.as_text(), file=sys.stderr)
+        return 1

primarysource/ecma/identify.py

@@ -0,0 +1,25 @@
+"""How the ecma collection recognises an Ecma standard in a supplied PDF (`registry.Collection.identify`).
+
+Covers and running headers read `ECMA-427` (`Standard ECMA-427`); Ecma's download is `ECMA-427_1st_edition.pdf`.
+The importer takes the file the user downloaded (`--file`) and checks it against the catalogue's pinned digest.
+"""
+
+from __future__ import annotations
+
+import re
+
+from primarysource.identify import Candidate, Probe, candidates
+
+_ECMA = re.compile(r"\bECMA-(?P<num>\d{1,4})\b")
+
+
+def _label(match: re.Match[str]) -> str:
+    return f"ECMA-{match['num']}"
+
+
+def identify(probe: Probe) -> list[Candidate]:
+    def command(label: str) -> str:
+        where = probe.path if probe.path is not None else probe.name
+        return f'primarysource ecma import {label.removeprefix("ECMA-")} --file "{where}"'
+
+    return candidates(probe, "ecma", _ECMA, _label, command)

primarysource/ecma/importer.py

@@ -0,0 +1,108 @@
+"""Import an Ecma standard into `$PRIMARYSOURCE_HOME/ecma/`.
+
+The PDF is fetched from ecma-international.org (one request) or taken from a file the user downloaded, checked
+against the catalogue's pinned sha256 before anything is parsed or written, stored as `<number>-<edition>.pdf`,
+and read once with `_pdf.extract` into `<number>-<edition>.json`, which is what the readers use. `install.json`
+records the pin, the extractor and the time. Placement is a volume: this is the user's own copy, downloaded under
+Ecma's terms (see `catalogue.json`), and is not for an image layer.
+
+Run as `primarysource ecma import [NUMBER] [--file PDF]`.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+from primarysource.c2sp._fetch import Fetcher, Transport, verify
+from primarysource.ecma import _pdf
+from primarysource.ecma.catalogue import HOSTS, Catalogue, Edition
+from primarysource.ecma.catalogue import catalogue as packaged_catalogue
+from primarysource.errors import LibError
+from primarysource.home import atomic_write
+
+INSTALL_RECORD = "install.json"
+MAX_PDF = 64 * 1024 * 1024
+
+
+@dataclass(frozen=True, slots=True)
+class ImportResult:
+    key: str
+    pdf: Path
+    extraction: Path
+    sections: int
+    requests: int
+
+
+def read_record(home: Path) -> dict[str, Any]:
+    path = home / INSTALL_RECORD
+    try:
+        loaded = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, ValueError):
+        return {"editions": {}}
+    return loaded if isinstance(loaded, dict) and isinstance(loaded.get("editions"), dict) else {"editions": {}}
+
+
+def import_edition(
+    home: Path,
+    number: str,
+    edition: int | None = None,
+    *,
+    file: Path | None = None,
+    cat: Catalogue | None = None,
+    transport: Transport | None = None,
+    fetcher: Fetcher | None = None,
+) -> ImportResult:
+    """Fetch (or read `file`), verify, store and extract one edition. Nothing is written unless the sha256
+    matches the pin and the PDF reads."""
+    cat = cat or packaged_catalogue()
+    found: Edition = cat.edition(number, edition)
+    requests = 0
+    if file is not None:
+        try:
+            data = file.read_bytes()
+        except OSError as exc:
+            raise LibError(f"cannot read {file}: {exc}", "pass the PDF you downloaded from the publisher") from exc
+        if len(data) > MAX_PDF:
+            raise LibError(f"{file} is larger than {MAX_PDF} bytes", "")
+    else:
+        fetcher = fetcher or Fetcher(HOSTS, transport)
+        data = fetcher.get(found.source_url, MAX_PDF)
+        requests = fetcher.requests
+    verify(data, found.sha256, found.designation)
+    if not data.startswith(b"%PDF-"):
+        raise LibError(f"{found.designation}: not a PDF", "the pinned file is a PDF; nothing was written")
+    pdf_path = home / f"{found.key}.pdf"
+    home.mkdir(parents=True, exist_ok=True)
+    part = home / f"{found.key}.pdf.part"
+    part.write_bytes(data)
+    try:
+        extraction = _pdf.extract(data, part)
+    except BaseException:
+        part.unlink(missing_ok=True)
+        raise
+    os.replace(part, pdf_path)
+    extraction.update(
+        {
+            "title": found.title,
+            "designation": found.designation,
+            "edition": found.edition_name,
+            "page_url": found.page_url,
+            "sha256": found.sha256,
+        }
+    )
+    json_path = home / f"{found.key}.json"
+    atomic_write(json_path, (json.dumps(extraction, ensure_ascii=False) + "\n").encode("utf-8"))
+    record = read_record(home)
+    record["collection"] = "ecma"
+    record["editions"][found.key] = {
+        "sha256": found.sha256,
+        "extractor": extraction["extractor"],
+        "imported": datetime.now(UTC).isoformat(timespec="seconds"),
+    }
+    atomic_write(home / INSTALL_RECORD, (json.dumps(record, indent=2) + "\n").encode("utf-8"))
+    return ImportResult(found.key, pdf_path, json_path, len(extraction["sections"]), requests)