primarysource 0.1.0__py3-none-any.whl

primarysource/iso/identify.py

@@ -0,0 +1,100 @@
+"""How the iso collection recognises an ISO publication in a supplied PDF (`registry.Collection.identify`).
+
+An ISO cover prints `Reference number` and then the designation (`ISO/IEC 7816-3:2006(E)`), and every later page
+repeats the designation as a running header, so the cover and the header outweigh whatever the file is called.
+The PDF's title metadata is often useless (`Microsoft Word - C038770e.doc`) and is only one piece of evidence.
+"""
+
+from __future__ import annotations
+
+import re
+
+from primarysource.identify import Candidate, Probe, candidates, points
+from primarysource.iso.designation import _PATTERN, Designation, _from_match
+
+_REFERENCE = re.compile(r"Reference\s+number\s+(?P<designation>ISO\S*(?:\s+\S+){0,3})", re.IGNORECASE)
+REFERENCE_POINTS = 3
+
+
+def _label(match: re.Match[str]) -> str | None:
+    return _from_match(match).label
+
+
+def designation_of(label: str) -> Designation:
+    match = _PATTERN.fullmatch(label)
+    if match is None:  # every label this module produces parses; a foreign one is a bug
+        raise ValueError(label)
+    return _from_match(match)
+
+
+_EDITION = re.compile(r"\bedition\b", re.IGNORECASE)
+_DATE = re.compile(r"\d{4}-\d{2}-\d{2}")
+_STOP = re.compile(r"Reference\s+number|\bPartie\b|[àâçéèêëîïôûùüÿœ]")
+_JUNK_TITLE = re.compile(r"^\s*$|^(?:Microsoft Word|untitled)|\.(?:docx?|pdf|indd|fm)\b", re.IGNORECASE)
+
+
+def cover_title(probe: Probe) -> str:
+    """The English title printed on an ISO cover: the lines after the edition line, up to the French title or
+    the `Reference number` line. Empty when the cover does not have that shape."""
+    if not probe.pages:
+        return ""
+    lines = [" ".join(line.split()) for line in probe.pages[0].splitlines() if line.strip()]
+    start = next((i + 1 for i, line in enumerate(lines) if _EDITION.search(line)), None)
+    if start is None:
+        return ""
+    title: list[str] = []
+    for line in lines[start:]:
+        if _STOP.search(line):
+            break
+        if not _DATE.fullmatch(line):
+            title.append(line)
+    return " ".join(title)
+
+
+def useful_title(title: str, designation: str) -> bool:
+    """False for title metadata that names a file or a word processor (`Microsoft Word - C038770e.doc`) or that is
+    only the designation again."""
+    return not _JUNK_TITLE.search(title) and title.strip() != designation
+
+
+def _merge(found: list[Candidate]) -> list[Candidate]:
+    """One candidate per document: `ISO 7816-3:2006` from a file name and `ISO/IEC 7816-3:2006` from the cover
+    are the same document (`7816-3@2006`), spelled by the surest evidence, with the evidence pooled."""
+    by_doc: dict[str, list[Candidate]] = {}
+    for candidate in found:
+        by_doc.setdefault(designation_of(candidate.designation).doc, []).append(candidate)
+    merged: list[Candidate] = []
+    for group in by_doc.values():
+        best = max(group, key=lambda c: c.score)
+        evidence = tuple(dict.fromkeys(e for c in group for e in c.evidence))
+        merged.append(Candidate(best.collection, best.designation, points(evidence), evidence, best.command))
+    return merged
+
+
+def identify(probe: Probe) -> list[Candidate]:
+    """ISO designations in the probe, the cover's `Reference number` line counting 3 more."""
+
+    def command(label: str) -> str:
+        where = f'"{probe.path}"' if probe.path is not None else f'"{probe.name}"'
+        return f'primarysource iso import {where} --designation "{label}"'
+
+    found = _merge(candidates(probe, "iso", _PATTERN, _label, command))
+    if not probe.pages:
+        return found
+    referenced = set()
+    for match in _REFERENCE.finditer(probe.pages[0]):
+        inner = _PATTERN.search(match["designation"])
+        if inner is not None:
+            referenced.add(_from_match(inner).label)
+    out: list[Candidate] = []
+    for candidate in found:
+        if candidate.designation in referenced:
+            candidate = Candidate(
+                candidate.collection,
+                candidate.designation,
+                candidate.score + REFERENCE_POINTS,
+                ("cover reference number", *candidate.evidence),
+                candidate.command,
+            )
+        out.append(candidate)
+    return out

primarysource/iso/importer.py

@@ -0,0 +1,224 @@
+"""Import the owner's own purchased ISO PDFs into `$PRIMARYSOURCE_HOME/iso/`.
+
+`import_folder` reads every new or changed PDF in the home; `import_file` takes one file and, optionally, the
+designation printed on its cover. Which standard a PDF is comes from its content first (`iso.identify`: the cover's
+`Reference number` line and the running header), then its file name, so a renamed download still imports and a
+file named for the wrong standard is caught. A PDF another collection knows (an ITU-T Recommendation, a NIST
+publication) is refused with that collection's command. Each PDF is read once with `readers.pdf.extract` (cut at
+its bookmarks, or at its numbered headings when it has none, as older ISO editions do) into
+`extracted/<key>.json`, which is what serving reads, so the PDF is never copied and serving never needs `pypdf`.
+`install.json` records the designation, the sha256, size and mtime of the source, the extractor and the
+segmenter. When the PDF's title metadata is useless (`Microsoft Word - C038770e.doc`), the English title on the
+cover is used. After an import the core FTS5 index (`index.sqlite`) is rebuilt atomically. Nothing here fetches,
+and nothing records the buyer's watermark or a distributor's tracking string.
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Iterator
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+from primarysource import index
+from primarysource.errors import LibError
+from primarysource.fsutil import atomic_write, sha256
+from primarysource.iso.designation import Designation, from_filename, parse
+from primarysource.iso.identify import cover_title, useful_title
+from primarysource.model import Document, Section, Unit
+
+INSTALL_RECORD = "install.json"
+INDEX_FILE = "index.sqlite"
+EXTRACTED = "extracted"
+REBUILD_COMMAND = "primarysource iso index"
+IMPORT_COMMAND = "primarysource iso import"
+
+
+@dataclass(frozen=True, slots=True)
+class ImportResult:
+    doc: str
+    designation: str
+    sections: int
+    headings_found: int
+    segmenter: str = ""
+    note: str = ""
+
+
+def read_record(home: Path) -> dict[str, Any]:
+    try:
+        loaded = json.loads((home / INSTALL_RECORD).read_text(encoding="utf-8"))
+    except (OSError, ValueError):
+        return {"documents": {}}
+    if isinstance(loaded, dict) and isinstance(loaded.get("documents"), dict):
+        return loaded
+    return {"documents": {}}
+
+
+def _write_record(home: Path, record: dict[str, Any]) -> None:
+    record["collection"] = "iso"
+    atomic_write(home / INSTALL_RECORD, (json.dumps(record, indent=2, sort_keys=True) + "\n").encode("utf-8"))
+
+
+def _is_pdf(path: Path) -> bool:
+    return path.is_file() and path.suffix.lower() == ".pdf"
+
+
+def designate(path: Path, explicit: str | None = None) -> tuple[Designation, str]:
+    """Which ISO standard `path` is, and a note when the file name disagrees with the cover. An explicit
+    designation wins; then the PDF's own content (a score of 3 or more: the cover, or a running header and the
+    file name); then the file name. A PDF another collection recognises is refused with that collection's
+    command, and one nobody recognises asks for `--designation`."""
+    if explicit:
+        return parse(explicit), ""
+    from primarysource import identify
+    from primarysource.iso.identify import designation_of
+    from primarysource.iso.identify import identify as iso_identify
+
+    probe = identify.probe_pdf(path)
+    named = from_filename(path.name)
+    ours = sorted(iso_identify(probe), key=lambda c: -c.score)
+    if ours and ours[0].score >= 3:
+        found = designation_of(ours[0].designation)
+        note = ""
+        if named is not None and named.doc != found.doc:
+            note = f"the file name says {named.label}, the PDF says {found.label}; using the PDF"
+        return found, note
+    if named is not None:
+        return named, ""
+    others = [c for c in identify.identify(probe, identify.registered_hooks()) if c.collection != "iso"]
+    if others:
+        best = others[0]
+        raise LibError(
+            f"{path.name} is {best.designation}, a {best.collection} document, not an ISO standard", "", best.command
+        )
+    raise LibError(
+        f"{path.name}: no ISO designation found on its cover, in its running headers or in its file name",
+        "",
+        f'{IMPORT_COMMAND} "{path}" --designation "ISO/IEC 9899:2024"  (as printed on the cover)',
+    )
+
+
+def import_file(home: Path, path: Path, designation: str | None = None, *, reindex: bool = True) -> ImportResult:
+    """Identify, extract and record one PDF. Nothing is recorded unless the PDF reads and can be cut into
+    sections (at its bookmarks, or at three or more numbered headings)."""
+    from primarysource.readers import pdf
+
+    if not path.is_file():
+        raise LibError(f"{path} is not a file", "", f"check the path, then {IMPORT_COMMAND}")
+    try:
+        data = path.read_bytes()
+        stat = path.stat()
+    except OSError as exc:
+        raise LibError(f"cannot read {path}: {exc}", "", f"check the file, then {IMPORT_COMMAND}") from exc
+    if len(data) > pdf.MAX_BYTES:
+        raise LibError(f"{path.name} is {len(data)} bytes, over the {pdf.MAX_BYTES} byte bound", "")
+    if not data.startswith(b"%PDF-"):
+        raise LibError(f"{path.name} is not a PDF", "", "use the PDF as delivered by the ISO store")
+    found, note = designate(path, designation)
+    extraction = pdf.extract(data, path, numbered=True)
+    digest = sha256(data)
+    extraction.update({"designation": found.label, "doc": found.doc, "sha256": digest})
+    if not useful_title(str(extraction["title"]), found.label):
+        from primarysource import identify
+
+        extraction["title"] = cover_title(identify.probe_pdf(path))
+    atomic_write(
+        home / EXTRACTED / f"{found.key}.json",
+        (json.dumps(extraction, ensure_ascii=False) + "\n").encode("utf-8"),
+    )
+    record = read_record(home)
+    record["documents"][found.doc] = {
+        "designation": found.label,
+        "key": found.key,
+        "title": extraction["title"],
+        "source": str(path),
+        "sha256": digest,
+        "size": stat.st_size,
+        "mtime_ns": stat.st_mtime_ns,
+        "extractor": extraction["extractor"],
+        "segmenter": extraction["segmenter"],
+        "imported": datetime.now(UTC).isoformat(timespec="seconds"),
+    }
+    _write_record(home, record)
+    if reindex:
+        rebuild_index(home)
+    return ImportResult(
+        found.doc,
+        found.label,
+        len(extraction["sections"]),
+        int(extraction["headings_found"]),
+        str(extraction["segmenter"]),
+        note,
+    )
+
+
+def _unchanged(entry: dict[str, Any] | None, path: Path) -> bool:
+    if not entry or entry.get("source") != str(path):
+        return False
+    try:
+        stat = path.stat()
+    except OSError:
+        return False
+    return entry.get("size") == stat.st_size and entry.get("mtime_ns") == stat.st_mtime_ns
+
+
+def import_folder(home: Path, folder: Path | None = None) -> tuple[list[ImportResult], list[tuple[str, str]]]:
+    """Import every PDF directly in `folder` (default: the home itself) that is new or changed since its last
+    import, identifying each from its content. Returns what was imported and what was skipped, with the reason.
+    The index is rebuilt once at the end when anything was imported."""
+    folder = folder or home
+    if not folder.is_dir():
+        raise LibError(
+            f"{folder} does not exist",
+            "",
+            f"create it (or set PRIMARYSOURCE_ISO_HOME to where your ISO PDFs are), put the PDFs in it, then "
+            f"{IMPORT_COMMAND}",
+        )
+    record = read_record(home)
+    by_source = {entry.get("source"): entry for entry in record["documents"].values()}
+    done: list[ImportResult] = []
+    skipped: list[tuple[str, str]] = []
+    for path in sorted(p for p in folder.iterdir() if _is_pdf(p)):
+        if _unchanged(by_source.get(str(path)), path):
+            skipped.append((path.name, "already imported, unchanged"))
+            continue
+        try:
+            done.append(import_file(home, path, reindex=False))
+        except LibError as exc:
+            skipped.append((path.name, exc.message + (f"; {exc.action}" if exc.action else "")))
+    if done:
+        rebuild_index(home)
+    return done, skipped
+
+
+def documents(home: Path) -> Iterator[Document]:
+    """Every imported standard as a core `Document`, one section per bookmarked clause, sentences as units."""
+    record = read_record(home)
+    for doc, entry in sorted(record["documents"].items()):
+        stored = home / EXTRACTED / f"{entry['key']}.json"
+        try:
+            raw = json.loads(stored.read_text(encoding="utf-8"))
+            sections = tuple(
+                Section(
+                    str(s["id"]),
+                    str(s["title"]),
+                    tuple(line for line in str(s["text"]).split("\n") if line),
+                    int(s["depth"]) + 1,
+                )
+                for s in raw["sections"]
+            )
+        except (OSError, ValueError, KeyError, TypeError) as exc:
+            raise LibError(
+                f"the extraction of {entry.get('designation', doc)} is unreadable: {exc}",
+                "",
+                f"re-import it: {IMPORT_COMMAND}",
+            ) from exc
+        title = f"{entry['designation']} {entry.get('title') or ''}".strip()
+        yield Document(doc, title, Unit.SENTENCE, sections)
+
+
+def rebuild_index(home: Path) -> int:
+    """Rebuild the search index from every stored extraction; returns the number of sections indexed."""
+    return index.rebuild(home / INDEX_FILE, documents(home), rebuild_command=REBUILD_COMMAND)

primarysource/iso/library.py

@@ -0,0 +1,249 @@
+"""Read, outline, search and cite the owner's imported ISO standards. Local files only; nothing here fetches.
+
+A ref is `iso:9899@2024§6.5.2¶3`: the standard's number, `@` and its year (the newest imported year when omitted),
+`§` and a clause as the PDF's bookmarks number it (`6.5.2`, `A` for Annex A, `A.1`) or the slug of an unnumbered
+bookmark (`bibliography`), and `¶n`, the nth sentence of that clause's span. The span is the clause's own text,
+then each subsection as a `§6.5.2.1 Title` line followed by its text. A printed designation works too:
+`ISO/IEC 9899:2024§6.5.2`. A ref with no section is the whole standard.
+
+Search goes through the core FTS5 index (`index.sqlite`, BM25, heading matches weighted 4); read, outline and cite
+read the stored extraction.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from primarysource import budget, cursor, index
+from primarysource.ecma._pdf import PdfDocument, PdfSection
+from primarysource.errors import LibError
+from primarysource.home import collection_home
+from primarysource.iso.designation import parse, split_doc
+from primarysource.iso.importer import (
+    EXTRACTED,
+    IMPORT_COMMAND,
+    INDEX_FILE,
+    REBUILD_COMMAND,
+    read_record,
+)
+from primarysource.model import span_hash
+from primarysource.refs import Ref, RefGrammar
+from primarysource.render import render
+
+PREFIX = re.compile(r"^iso:", re.IGNORECASE)
+_PRINTED = re.compile(r"^(?P<head>ISO[^§¶]*?)\s*(?P<rest>(?:§|¶).*)?$", re.IGNORECASE)
+RANKING = index.Ranking(doc_title=1.0, title=4.0, body=1.0)
+SEARCH_WINDOW = 500
+
+
+def grammar() -> RefGrammar:
+    from primarysource.iso import ISO_DOC, ISO_SECTION
+
+    return RefGrammar(ISO_DOC, ISO_SECTION)
+
+
+@dataclass(frozen=True, slots=True)
+class Loaded:
+    doc: str
+    designation: str
+    title: str
+    document: PdfDocument
+
+
+def default_home() -> Path:
+    return collection_home("iso")
+
+
+def _not_imported(what: str, home: Path) -> LibError:
+    have = ", ".join(sorted(read_record(home)["documents"])) or "nothing"
+    return LibError(
+        f"{what} is not imported (have: {have})",
+        "iso_docs lists what is imported",
+        f"put the PDF in {home}, then run: {IMPORT_COMMAND}",
+    )
+
+
+def parse_ref(text: str) -> Ref:
+    """Parse an iso ref, accepting the `iso:` prefix and a printed designation for the document."""
+    body = PREFIX.sub("", text.strip(), count=1)
+    printed = _PRINTED.match(body)
+    if printed:
+        body = parse(printed["head"]).doc + (printed["rest"] or "")
+    body, doc_para = cursor.split_doc_para(body)
+    ref = grammar().parse(body)
+    if doc_para is not None:
+        ref = Ref(ref.doc, None, doc_para, None, None)
+    if ref.lines is not None or ref.anchor is not None:
+        raise LibError(f"not an iso ref: {text!r}", "iso refs are <number>[@year][§clause][¶n], e.g. 9899@2024§6.5.2")
+    if ref.doc is None:
+        raise LibError(f"not an iso ref: {text!r}", "start with the standard's number, e.g. 9899@2024")
+    return ref
+
+
+def resolve(token: str, home: Path) -> str:
+    """The imported document a token names: exact when it carries `@year`, else the newest imported year of that
+    number (the standard itself before an amendment)."""
+    number, rest = split_doc(token)
+    documents = read_record(home)["documents"]
+    if rest is not None:
+        doc = f"{number}@{rest}"
+        if doc not in documents:
+            raise _not_imported(f"ISO {doc}", home)
+        return doc
+    candidates: list[str] = [str(d) for d in documents if str(d).split("@", 1)[0] == number]
+    if not candidates:
+        raise _not_imported(f"ISO {number}", home)
+    return max(candidates, key=lambda d: ("+" not in d, d.split("@", 1)[1]))
+
+
+def load(token: str, home: Path | None = None) -> Loaded:
+    home = home or default_home()
+    doc = resolve(token, home)
+    entry = read_record(home)["documents"][doc]
+    try:
+        raw: Any = json.loads((home / EXTRACTED / f"{entry['key']}.json").read_text(encoding="utf-8"))
+    except (OSError, ValueError, KeyError) as exc:
+        raise LibError(
+            f"cannot read the import of {entry.get('designation', doc)}: {exc}", "", f"re-import it: {IMPORT_COMMAND}"
+        ) from exc
+    document = PdfDocument.from_json(raw)
+    return Loaded(doc, str(entry["designation"]), document.title or str(entry["designation"]), document)
+
+
+def _title(doc: Loaded) -> str:
+    """The PDF title without the designation it often starts with."""
+    title = doc.title
+    if title.upper().startswith(doc.designation.upper()):
+        title = title[len(doc.designation) :].lstrip(" -—:")
+    return title
+
+
+def _cite_text(doc: Loaded, section: PdfSection | None, para: int | str | None) -> str:
+    where = f", §{section.id} {section.title}" if section else ""
+    if para:
+        where += f" ¶{para}"
+    title = _title(doc)
+    return f"{doc.designation}{', ' + title if title else ''}{where}"
+
+
+def _doc(ref: Ref) -> str:
+    if ref.doc is None:
+        raise LibError("the ref names no document", "start with the standard's number")
+    return ref.doc
+
+
+def _select(ref: Ref, home: Path | None) -> tuple[Loaded, PdfSection | None, tuple[str, ...], str]:
+    doc = load(_doc(ref), home)
+    if ref.section is None:
+        units: list[str] = []
+        for section in doc.document.sections:
+            units.append(f"§{section.id} {section.title}")
+            units.extend(section.sentences)
+        return doc, None, tuple(units), ""
+    position, warning = doc.document.find(ref.section)
+    return doc, doc.document.sections[position], doc.document.span(position), warning
+
+
+def read(ref_text: str, max_tokens: int = 2000, home: Path | None = None) -> str:
+    """The text of a ref, cut at sentence boundaries within `max_tokens`. A `¶n` ref starts at sentence n and runs to
+    the end of its scope; the `next:` footer is the ref that reads the next page."""
+    ref = parse_ref(ref_text)
+    doc, section, units, warning = _select(ref, home)
+    if not units:
+        raise LibError(f"{ref_text}: that clause has no text of its own", "read its subsections, see iso_outline")
+    first = cursor.first_unit(ref.para, len(units), "sentences", ref_text)
+    chosen = budget.page(units, start=first, max_tokens=max_tokens)
+    body = "\n".join(chosen.units)
+    scope = f"iso:{doc.doc}" + (f"§{section.id}" if section else "")
+    footer = {
+        "cite": _cite_text(doc, section, cursor.para_label(ref.para, chosen)),
+        "unit": "sentence",
+        "span": span_hash(body),
+        "next": cursor.read_next(scope, chosen),
+        "warning": warning,
+    }
+    return render(body, footer)
+
+
+def outline(token: str, start: int = 0, limit: int = 200, home: Path | None = None) -> str:
+    """A standard's bookmarked clauses, indented by depth, each with the ref that reads it and its PDF page."""
+    doc = load(token, home)
+    rows = [
+        f"{'  ' * section.depth}{section.id} {section.title}  iso:{doc.doc}§{section.id}  (p. {section.page})"
+        for section in doc.document.sections
+    ]
+    shown, nxt = cursor.window(rows, start, limit)
+    head = f"{doc.designation} {_title(doc)}  [iso:{doc.doc}, {doc.document.pages} pages]"
+    return render("\n".join([head, *shown]), {"next": nxt})
+
+
+def search(query: str, doc: str | None = None, limit: int = 10, start: int = 0, home: Path | None = None) -> str:
+    """Clauses containing every query word (else any, flagged `relaxed`), best first by BM25: one line per hit,
+    `<ref>  <clause title>: <snippet>`."""
+    home = home or default_home()
+    if not read_record(home)["documents"]:
+        raise LibError(
+            "no ISO standards are imported",
+            "iso_docs lists what is imported",
+            f"put your PDFs in {home}, then run: {IMPORT_COMMAND}",
+        )
+    wanted = resolve(doc, home) if doc else None
+    try:
+        conn = index.open_read(home / INDEX_FILE, rebuild_command=REBUILD_COMMAND)
+    except LibError as exc:
+        raise LibError(exc.message, "", REBUILD_COMMAND) from exc
+    try:
+        found = index.search(conn, query, ranking=RANKING, limit=SEARCH_WINDOW)
+    finally:
+        conn.close()
+    hits = [hit for hit in found.hits if wanted is None or hit.doc_id == wanted]
+    if not hits:
+        return render("no matches", {})
+    shown, nxt = cursor.window(hits, start, limit)
+    lines = [f"iso:{hit.doc_id}§{hit.section_id}  {hit.title}: {' '.join(hit.snippet.split())}" for hit in shown]
+    footer = {"next": nxt, "relaxed": "only some query words matched" if found.relaxed else ""}
+    return render("\n".join(lines), footer)
+
+
+def cite(ref_text: str, home: Path | None = None) -> str:
+    """The citation string for a ref, with the span hash of the exact text cited."""
+    ref = parse_ref(ref_text)
+    doc, section, units, warning = _select(ref, home)
+    if ref.para is not None:
+        if ref.para > len(units):
+            raise LibError(f"{ref_text}: the span has {len(units)} sentences", f"use ¶1 to ¶{len(units)}")
+        units = (units[ref.para - 1],)
+    footer = {
+        "cite": _cite_text(doc, section, ref.para),
+        "unit": "sentence",
+        "span": span_hash("\n".join(units)),
+        "warning": warning,
+    }
+    label = f"iso:{doc.doc}" + (f"§{section.id}" if section else "") + (f"¶{ref.para}" if ref.para else "")
+    return render(label, footer)
+
+
+def docs(home: Path | None = None) -> str:
+    """What is imported, with each source file's state: present, changed since import, or missing."""
+    home = home or default_home()
+    record = read_record(home)["documents"]
+    if not record:
+        return render(f"nothing imported; put your purchased ISO PDFs in {home} and run: {IMPORT_COMMAND}", {})
+    lines = []
+    for doc, entry in sorted(record.items()):
+        source = Path(str(entry.get("source", "")))
+        try:
+            stat = source.stat()
+            state = (
+                "source unchanged"
+                if stat.st_size == entry.get("size") and stat.st_mtime_ns == entry.get("mtime_ns")
+                else "source changed since import"
+            )
+        except OSError:
+            state = "source file missing"
+        lines.append(f"iso:{doc}  {entry['designation']}  {entry.get('title', '')}  ({state})")
+    return render("\n".join(lines), {})

primarysource/iso/server.py

@@ -0,0 +1,69 @@
+"""MCP server for the iso collection: `primarysource mcp --collection iso`.
+
+Imported only when served, so `mcp` (the `[mcp]` extra) is needed only then. Tools carry the `iso_` prefix, answer
+in text with the fixed footer, and turn a `LibError` into an MCP tool error with `error:`/`hint:`/`action:` lines.
+Every tool reads local files only; none writes or fetches. Serve it on the buyer's own machine (stdio, or a
+container with the iso home mounted read-only): the PDFs behind it are single-user copies.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from mcp.server.mcpserver import MCPServer
+from mcp.types import ToolAnnotations
+
+from primarysource import __version__
+from primarysource.iso import library
+from primarysource.scaffold import add_tools
+
+READ_ONLY = ToolAnnotations(read_only_hint=True, idempotent_hint=True, destructive_hint=False, open_world_hint=False)
+
+INSTRUCTIONS = (
+    "ISO standards from the user's own purchased PDFs. Loop: iso_search('<words>') -> iso_read('<ref>'). "
+    "Refs: 'iso:9899@2024§6.5.2' (a clause as numbered in the PDF), '...¶3' (from its 3rd sentence), '9899' means "
+    "the newest imported year, 'A' is Annex A; 'ISO/IEC 9899:2024§6.5.2' works too. iso_outline('9899') lists "
+    "clauses as refs; iso_cite('<ref>') gives the citation and the hash of the cited text. iso_docs lists what is "
+    "imported. If a standard is not imported, the error says what the user must do."
+)
+
+
+def iso_read(ref: str, max_tokens: int = 2000) -> str:
+    """Read ISO text by ref, cut at sentence boundaries within max_tokens. A 'next: <ref>' footer line is the ref
+    that reads the next page: pass it back as ref.
+
+    ref: 'iso:9899@2024' (whole standard) | 'iso:9899@2024§6.5.2' (a clause and its subsections) |
+      'iso:9899@2024§6.5.2¶3' (from its 3rd sentence) | 'ISO/IEC 9899:2024§A'. iso_cite gives exactly one sentence."""
+    return library.read(ref, max_tokens=max_tokens)
+
+
+def iso_search(query: str, doc: str | None = None, limit: int = 10, start: int = 0) -> str:
+    """Find clauses containing the query words (all of them, else any, flagged 'relaxed'), best first. One line per
+    hit: '<ref>  <clause title>: <snippet>'. doc restricts to one standard, e.g. '9899' or '9899@2024'. A
+    'next: start=N' footer line means call again with start=N (a 0-based offset)."""
+    return library.search(query, doc=doc, limit=limit, start=start)
+
+
+def iso_outline(doc: str, start: int = 0, limit: int = 200) -> str:
+    """List a standard's clauses, indented by depth, each with the ref that reads it and its PDF page. A
+    'next: start=N' footer line means call again with start=N."""
+    return library.outline(doc, start=start, limit=limit)
+
+
+def iso_cite(ref: str) -> str:
+    """The citation string for a ref, with the sha256 prefix of the exact text cited (cite: and span: lines)."""
+    return library.cite(ref)
+
+
+def iso_docs() -> str:
+    """List the imported ISO standards and whether each source PDF is unchanged, changed or missing."""
+    return library.docs()
+
+
+def _build() -> Any:
+    built = MCPServer("primarysource-iso", version=__version__, instructions=INSTRUCTIONS)
+    add_tools(built, (iso_read, iso_search, iso_outline, iso_cite, iso_docs), annotations=READ_ONLY)
+    return built
+
+
+server = _build()

primarysource/itu/__init__.py

@@ -0,0 +1,41 @@
+"""itulib: ITU-T recommendations as a local, citation-precise library and MCP server.
+
+X.680 and X.690 are the normative source for BER and DER and are not in the RFC series, so `rfclib` cannot
+answer a question about tag-number minimality or DER canonicalisation. This is the same shape of tool pointed
+at the other corpus: clause-precise search, paging, citation, and an errata surface.
+
+The corpus is not in this package and cannot be. ITU's own notice, page iii of X.690 (02/2021), forbids
+reproduction without written permission, so `fetch` downloads it on the user's machine into `ITULIB_HOME`.
+Every entry point raises `CorpusMissing` naming that command rather than returning an empty answer, because a
+missing file and a document with nothing to say must never look the same.
+"""
+
+from __future__ import annotations
+
+__version__ = "0.1.0"
+
+from .corpus import CorpusMissing, fetch, has, home, require
+from .sources import AMENDMENTS, BY_REC, IN_FORCE, SUPERSEDED, Amendment, Edition, current, resolve
+from .text import Clause, clauses, find, pages, tidy
+
+__all__ = [
+    "__version__",
+    "AMENDMENTS",
+    "BY_REC",
+    "IN_FORCE",
+    "SUPERSEDED",
+    "Amendment",
+    "Clause",
+    "CorpusMissing",
+    "Edition",
+    "clauses",
+    "current",
+    "fetch",
+    "find",
+    "has",
+    "home",
+    "pages",
+    "require",
+    "resolve",
+    "tidy",
+]