primarysource 0.1.0__py3-none-any.whl

primarysource/usb/ingest/__init__.py

@@ -0,0 +1 @@
+"""Ingest side of usblib: fetch the publications listed in data/sources.toml and index them."""

primarysource/usb/ingest/acquire.py

@@ -0,0 +1,130 @@
+"""Download the files named in the USB-IF catalogue, unpack ZIP packages, and record what was fetched.
+
+Input: data/usb-catalog.json (from discover_usb). Output, under the data directory:
+  raw/<slug>/<file>        each downloaded file as published
+  raw/<slug>/pdf/<name>    PDFs taken from ZIP packages, with the ZIP kept beside them
+  acquired.json            per file: url, path, size, sha256, and for ZIPs the member list
+
+Only PDFs and ZIPs are kept. Each download goes to a .part file and is renamed only when complete.
+Extraction is safe: members with absolute paths or ".." are refused, and only PDF members are written.
+Requests are throttled to one per second. Pass --dry-run to print the plan and the total size first.
+"""
+
+from __future__ import annotations
+
+from primarysource.usb.paths import PACKAGE_DATA
+
+import argparse
+import hashlib
+import json
+import sys
+import time
+import urllib.error
+import urllib.request
+import zipfile
+from collections.abc import Sequence
+from pathlib import Path, PurePosixPath
+
+USER_AGENT = "usblib-acquire/0.1 (documentation corpus)"
+DELAY = 1.0
+CHUNK = 1 << 20
+
+
+def _size(url: str) -> int | None:
+    req = urllib.request.Request(url, method="HEAD", headers={"User-Agent": USER_AGENT})
+    try:
+        with urllib.request.urlopen(req, timeout=60) as resp:
+            length = resp.headers.get("Content-Length")
+            return int(length) if length else None
+    except urllib.error.URLError:
+        return None
+    finally:
+        time.sleep(DELAY)
+
+
+def _download(url: str, dest: Path) -> tuple[int, str]:
+    dest.parent.mkdir(parents=True, exist_ok=True)
+    part = dest.with_suffix(dest.suffix + ".part")
+    digest = hashlib.sha256()
+    size = 0
+    req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
+    with urllib.request.urlopen(req, timeout=300) as resp, part.open("wb") as out:
+        while chunk := resp.read(CHUNK):
+            out.write(chunk)
+            digest.update(chunk)
+            size += len(chunk)
+    part.replace(dest)
+    time.sleep(DELAY)
+    return size, digest.hexdigest()
+
+
+def _safe_member(name: str) -> bool:
+    p = PurePosixPath(name)
+    return not p.is_absolute() and ".." not in p.parts and name.lower().endswith(".pdf")
+
+
+def _extract_pdfs(archive: Path, out_dir: Path) -> list[str]:
+    written: list[str] = []
+    out_dir.mkdir(parents=True, exist_ok=True)
+    with zipfile.ZipFile(archive) as zf:
+        for info in zf.infolist():
+            if info.is_dir() or not _safe_member(info.filename):
+                continue
+            target = out_dir / PurePosixPath(info.filename).name
+            with zf.open(info) as src, target.open("wb") as dst:
+                dst.write(src.read())
+            written.append(target.name)
+    return written
+
+
+def plan(catalog: dict, only: set[str] | None) -> list[tuple[str, str, str]]:
+    """(slug, url, filename) for every PDF or ZIP the catalogue names."""
+    items = []
+    for entry in catalog["entries"]:
+        if only and entry["slug"] not in only:
+            continue
+        for url in entry["files"]:
+            name = PurePosixPath(url).name
+            items.append((entry["slug"], url, name))
+    return items
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    parser.add_argument("--catalog", type=Path, default=PACKAGE_DATA / "usb-catalog.json")
+    parser.add_argument("--data-dir", type=Path, default=Path("data"))
+    parser.add_argument("--only", nargs="*", default=None, help="catalogue slugs to fetch (default: all)")
+    parser.add_argument("--dry-run", action="store_true")
+    args = parser.parse_args(argv)
+    catalog = json.loads(args.catalog.read_text(encoding="utf-8"))
+    items = plan(catalog, set(args.only) if args.only else None)
+    sizes = {url: _size(url) for _, url, _ in items} if args.dry_run else {}
+    if args.dry_run:
+        total = sum(s or 0 for s in sizes.values())
+        for slug, url, name in items:
+            print(f"{slug:<40} {sizes.get(url) or '?':>12}  {name}")
+        print(f"{len(items)} files, {total / 1e9:.2f} GB known", file=sys.stderr)
+        return 0
+    manifest_path = args.data_dir / "acquired.json"
+    manifest = json.loads(manifest_path.read_text(encoding="utf-8")) if manifest_path.exists() else {"schema": 1, "files": []}
+    done = {f["url"] for f in manifest["files"]}
+    for slug, url, name in items:
+        if url in done:
+            continue
+        dest = args.data_dir / "raw" / slug / name
+        try:
+            size, sha = _download(url, dest)
+        except (urllib.error.URLError, OSError) as e:
+            print(f"FAILED {url}: {e}", file=sys.stderr)
+            continue
+        record = {"slug": slug, "url": url, "path": str(dest.relative_to(args.data_dir)), "size": size, "sha256": sha}
+        if dest.suffix.lower() == ".zip":
+            record["pdfs"] = _extract_pdfs(dest, dest.parent / "pdf")
+        manifest["files"].append(record)
+        manifest_path.write_text(json.dumps(manifest, indent=2), encoding="utf-8")
+        print(f"ok {slug} {name} {size}", file=sys.stderr)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

primarysource/usb/ingest/discover_usb.py

@@ -0,0 +1,88 @@
+"""Build the catalogue of USB-IF document-library entries and the files each one links to.
+
+Walks the paged listing at https://www.usb.org/documents, opens each document page, and records the
+PDF and ZIP links it carries. It writes data/usb-catalog.json. It downloads no specifications.
+Requests are throttled to one per second with a generic User-Agent.
+"""
+
+from __future__ import annotations
+
+from primarysource.usb.paths import PACKAGE_DATA
+
+import argparse
+import json
+import re
+import sys
+import time
+import urllib.error
+import urllib.request
+from collections.abc import Sequence
+from pathlib import Path
+
+BASE = "https://www.usb.org"
+LISTING = BASE + "/documents?page={page}"
+SLUG = re.compile(r"/document-library/([a-z0-9-]+)")
+FILE = re.compile(r'href="(https?://[^"]+?\.(?:pdf|zip))"|href="(/sites/default/files/[^"]+?\.(?:pdf|zip))"', re.I)
+TITLE = re.compile(r"<h1[^>]*>(.*?)</h1>", re.S)
+USER_AGENT = "usblib-discover/0.1 (documentation catalogue)"
+DELAY = 1.0
+
+
+def fetch_text(url: str) -> str | None:
+    req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
+    try:
+        with urllib.request.urlopen(req, timeout=60) as resp:
+            return resp.read().decode("utf-8", errors="replace")
+    except urllib.error.HTTPError as e:
+        print(f"  {url}: HTTP {e.code}", file=sys.stderr)
+        return None
+    finally:
+        time.sleep(DELAY)
+
+
+def listing_slugs(max_pages: int) -> list[str]:
+    seen: list[str] = []
+    for page in range(max_pages):
+        html = fetch_text(LISTING.format(page=page))
+        if html is None:
+            break
+        slugs = [s for s in dict.fromkeys(SLUG.findall(html)) if s not in seen]
+        if not slugs:
+            break
+        seen.extend(slugs)
+        print(f"listing page {page}: {len(slugs)} new, {len(seen)} total", file=sys.stderr)
+    return seen
+
+
+def entry_files(slug: str) -> tuple[str, list[str]]:
+    html = fetch_text(f"{BASE}/document-library/{slug}") or ""
+    title_m = TITLE.search(html)
+    title = re.sub(r"<[^>]+>", "", title_m.group(1)).strip() if title_m else slug
+    urls: list[str] = []
+    for a, b in FILE.findall(html):
+        url = a or (BASE + b)
+        if url not in urls:
+            urls.append(url)
+    return title, urls
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    parser.add_argument("--out", type=Path, default=PACKAGE_DATA / "usb-catalog.json")
+    parser.add_argument("--max-pages", type=int, default=40)
+    args = parser.parse_args(argv)
+    slugs = listing_slugs(args.max_pages)
+    entries = []
+    for slug in slugs:
+        title, files = entry_files(slug)
+        entries.append({"slug": slug, "title": title, "files": files})
+        print(f"{slug}: {len(files)} file(s)", file=sys.stderr)
+    args.out.parent.mkdir(parents=True, exist_ok=True)
+    args.out.write_text(json.dumps({"schema": 1, "source": LISTING.format(page="N"), "entries": entries}, indent=2), encoding="utf-8")
+    with_files = sum(1 for e in entries if e["files"])
+    print(f"wrote {args.out}: {len(entries)} entries, {with_files} with files", file=sys.stderr)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

primarysource/usb/ingest/extract.py

@@ -0,0 +1,48 @@
+"""Text extraction with ``pdftotext -layout``. The section splitter is primarysource.pdfcorpus.split."""
+
+from __future__ import annotations
+
+import shutil
+import subprocess
+from pathlib import Path
+
+
+MISSING_MESSAGE = (
+    "pdftotext was not found on PATH. Install poppler-utils (on MSYS2: pacman -S mingw-w64-x86_64-poppler) "
+    "and make sure pdftotext is on PATH."
+)
+
+
+class ExtractError(RuntimeError):
+    """Raised when text extraction cannot run or fails."""
+
+
+def find_pdftotext() -> str:
+    """Return the path of pdftotext, or raise ExtractError saying how to get it."""
+    found = shutil.which("pdftotext")
+    if found is None:
+        raise ExtractError(MISSING_MESSAGE)
+    return found
+
+
+def extract_text(pdf: Path, *, timeout: float = 300.0) -> str:
+    """Run ``pdftotext -layout`` on ``pdf`` and return the text (pages separated by form feeds)."""
+    exe = find_pdftotext()
+    try:
+        result = subprocess.run(  # noqa: S603
+            [exe, "-layout", "-enc", "UTF-8", str(pdf), "-"],
+            capture_output=True,
+            timeout=timeout,
+            check=False,
+        )
+    except subprocess.TimeoutExpired as error:
+        raise ExtractError(f"pdftotext timed out after {timeout:.0f}s on {pdf}") from error
+    if result.returncode != 0:
+        detail = result.stderr.decode("utf-8", "replace").strip()
+        raise ExtractError(f"pdftotext failed on {pdf} (exit {result.returncode}): {detail}")
+    return result.stdout.decode("utf-8", "replace")
+
+
+# The splitter is shared (primarysource.pdfcorpus.split); it is re-exported here for the importers that have
+# always found it in this module.
+from primarysource.pdfcorpus.split import split_sections as split_sections  # noqa: E402

primarysource/usb/ingest/fetch.py

@@ -0,0 +1,197 @@
+"""``usblib-fetch``: download the publications in data/sources.toml, extract their text and write the index.
+
+This is the only module that touches the network, and only for URLs listed in sources.toml.
+"""
+
+from __future__ import annotations
+
+from primarysource.usb.paths import data_dir as default_data_dir, sources_file
+
+import argparse
+import os
+import sys
+import urllib.error
+import urllib.request
+from collections.abc import Sequence
+from pathlib import Path
+
+from primarysource.usb.ingest.extract import (
+    ExtractError,
+    extract_text,
+    find_pdftotext,
+    split_sections,
+)
+from primarysource.usb.ingest.index import read_index, sha256_bytes, sha256_file, write_index
+from primarysource.usb.ingest.sources import Source, SourcesError, load_sources
+from primarysource.usb.models import Document
+from primarysource.usb.spec import SPEC
+from primarysource.pdfcorpus.fts import build_after_fetch
+
+USER_AGENT = "usblib-fetch/0.1 (corpus fetcher; one request per listed URL)"
+TIMEOUT_SECONDS = 120
+MAX_BYTES = 200 * 1024 * 1024
+
+
+class FetchError(RuntimeError):
+    """Raised when one document cannot be fetched or verified."""
+
+
+def download(url: str, *, timeout: float = TIMEOUT_SECONDS) -> bytes:
+    """GET ``url`` and return the body, which must look like a PDF."""
+    request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})  # noqa: S310
+    try:
+        with urllib.request.urlopen(request, timeout=timeout) as response:  # noqa: S310
+            body: bytes = response.read(MAX_BYTES + 1)
+    except (urllib.error.URLError, TimeoutError, OSError) as error:
+        raise FetchError(f"download failed for {url}: {error}") from error
+    if len(body) > MAX_BYTES:
+        raise FetchError(f"{url}: larger than {MAX_BYTES} bytes, refusing")
+    if not body.startswith(b"%PDF-"):
+        raise FetchError(f"{url}: response is not a PDF")
+    return body
+
+
+def check_digest(doc_id: str, actual: str, expected: str) -> None:
+    """Raise FetchError when a recorded digest exists and differs from ``actual``."""
+    if expected and actual != expected:
+        raise FetchError(
+            f"{doc_id}: sha256 {actual} does not match the recorded {expected}"
+        )
+
+
+def _store(pdf: Path, data: bytes) -> None:
+    temp = pdf.with_name(pdf.name + ".part")
+    temp.write_bytes(data)
+    os.replace(temp, pdf)
+
+
+def obtain_pdf(
+    source: Source, corpus: Path, recorded: str, from_file: Path | None
+) -> str:
+    """Make ``corpus/<doc_id>.pdf`` exist with a matching digest and return that digest.
+
+    ``recorded`` is the digest from sources.toml, or failing that the one recorded by an earlier run.
+    A file already in place whose digest matches (or for which none is recorded) is kept. A mismatching
+    file is replaced by a fresh copy, which is itself verified before it is kept.
+    """
+    pdf = corpus / f"{source.doc_id}.pdf"
+    if from_file is not None:
+        data = from_file.read_bytes()
+        if not data.startswith(b"%PDF-"):
+            raise FetchError(f"{source.doc_id}: {from_file} is not a PDF")
+        digest = sha256_bytes(data)
+        check_digest(source.doc_id, digest, source.sha256)
+        if not (pdf.exists() and sha256_file(pdf) == digest):
+            _store(pdf, data)
+        return digest
+    if pdf.exists():
+        digest = sha256_file(pdf)
+        if not recorded or digest == recorded:
+            return digest
+    data = download(source.url)
+    digest = sha256_bytes(data)
+    check_digest(source.doc_id, digest, recorded)
+    _store(pdf, data)
+    return digest
+
+
+def build_document(source: Source, corpus: Path, digest: str) -> Document:
+    """Extract text for ``source`` into the corpus and return its index entry."""
+    text = extract_text(corpus / f"{source.doc_id}.pdf")
+    text_name = f"{source.doc_id}.txt"
+    (corpus / text_name).write_text(text, encoding="utf-8", newline="")
+    return Document(
+        doc_id=source.doc_id,
+        number=source.number,
+        title=source.title,
+        date=source.date,
+        status=source.status,
+        url=source.url,
+        sha256=digest,
+        text_file=text_name,
+        sections=split_sections(text),
+    )
+
+
+def parse_from_files(values: Sequence[str]) -> dict[str, Path]:
+    """Parse ``DOC_ID=PATH`` arguments."""
+    result: dict[str, Path] = {}
+    for value in values:
+        doc_id, sep, path = value.partition("=")
+        if not sep or not doc_id or not path:
+            raise SourcesError(f"--from-file expects DOC_ID=PATH, got {value!r}")
+        result[doc_id] = Path(path)
+    return result
+
+
+def run(data_dir: Path, from_files: dict[str, Path], only: Sequence[str]) -> int:
+    """Fetch, verify, extract and index. Returns the process exit code."""
+    sources = load_sources(sources_file(data_dir))
+    known = {s.doc_id for s in sources}
+    unknown = sorted((set(from_files) | set(only)) - known)
+    if unknown:
+        raise SourcesError(f"not in sources.toml: {', '.join(unknown)}")
+    find_pdftotext()
+    corpus = data_dir / "corpus"
+    corpus.mkdir(parents=True, exist_ok=True)
+    index_path = corpus / "index.json"
+    previous = {d.doc_id: d for d in read_index(index_path)}
+    wanted = set(only) | set(from_files) if (only or from_files) else known
+    documents: list[Document] = []
+    failures = 0
+    for source in sources:
+        old = previous.get(source.doc_id)
+        if source.doc_id not in wanted:
+            if old is not None and (corpus / old.text_file).exists():
+                documents.append(old)
+            continue
+        recorded = source.sha256 or (old.sha256 if old else "")
+        try:
+            digest = obtain_pdf(source, corpus, recorded, from_files.get(source.doc_id))
+            document = build_document(source, corpus, digest)
+        except (FetchError, ExtractError, OSError) as error:
+            print(f"{source.doc_id}: {error}", file=sys.stderr)
+            failures += 1
+            continue
+        note = "" if source.sha256 else " (digest recorded in index.json)"
+        print(
+            f"{source.doc_id}: {len(document.sections)} sections, sha256 {digest[:12]}{note}"
+        )
+        documents.append(document)
+    write_index(index_path, documents)
+    print(f"wrote {index_path} with {len(documents)} document(s)")
+    if not build_after_fetch(SPEC, corpus):
+        failures += 1
+    return 1 if failures else 0
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="usblib-fetch", description=__doc__)
+    parser.add_argument(
+        "--data-dir",
+        type=Path,
+        default=default_data_dir(),
+        help="directory holding sources.toml and corpus/ (default: $USBLIB_DATA, else the collection's home)",
+    )
+    parser.add_argument(
+        "--from-file",
+        action="append",
+        default=[],
+        metavar="DOC_ID=PATH",
+        help="use a PDF you already have for DOC_ID instead of downloading it (repeatable)",
+    )
+    parser.add_argument(
+        "doc_ids",
+        nargs="*",
+        help="only these documents (default: all, or those named by --from-file)",
+    )
+    args = parser.parse_args(argv)
+    try:
+        return run(args.data_dir, parse_from_files(args.from_file), args.doc_ids)
+    except (SourcesError, ExtractError) as error:
+        print(f"usblib-fetch: {error}", file=sys.stderr)
+        return 2
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

primarysource/usb/ingest/index.py

@@ -0,0 +1,14 @@
+"""The usb corpus index: the shared one (primarysource.pdfcorpus.index)."""
+
+from __future__ import annotations
+
+from primarysource.pdfcorpus.index import (
+    SCHEMA,
+    document_from_dict,
+    read_index,
+    sha256_bytes,
+    sha256_file,
+    write_index,
+)
+
+__all__ = ["SCHEMA", "document_from_dict", "read_index", "sha256_bytes", "sha256_file", "write_index"]

primarysource/usb/ingest/sources.py

@@ -0,0 +1,85 @@
+"""Parser for data/sources.toml, the list of publications ``usblib-fetch`` may download."""
+
+from __future__ import annotations
+
+import re
+import tomllib
+from dataclasses import dataclass
+from pathlib import Path
+
+_DOC_ID = re.compile(r"^[a-z0-9][a-z0-9._-]*$")
+_DATE = re.compile(r"^\d{4}-(0[1-9]|1[0-2])$")
+_SHA256 = re.compile(r"^[0-9a-f]{64}$")
+_REQUIRED = ("doc_id", "url", "number", "title", "date", "status")
+_KNOWN = (*_REQUIRED, "sha256")
+
+
+class SourcesError(ValueError):
+    """Raised when sources.toml is missing, malformed, or inconsistent."""
+
+
+@dataclass(frozen=True, slots=True)
+class Source:
+    """One entry of sources.toml. ``sha256`` is empty until recorded."""
+
+    doc_id: str
+    url: str
+    number: str
+    title: str
+    date: str
+    status: str
+    sha256: str = ""
+
+
+def parse_sources(text: str) -> tuple[Source, ...]:
+    """Parse the text of sources.toml: an array of ``[[source]]`` tables."""
+    try:
+        data = tomllib.loads(text)
+    except tomllib.TOMLDecodeError as error:
+        raise SourcesError(f"sources.toml is not valid TOML: {error}") from error
+    entries = data.get("source")
+    if not isinstance(entries, list) or not entries:
+        raise SourcesError("sources.toml has no [[source]] entries")
+    sources: list[Source] = []
+    seen: set[str] = set()
+    for position, entry in enumerate(entries, start=1):
+        label = f"source #{position}"
+        if not isinstance(entry, dict):
+            raise SourcesError(f"{label}: not a table")
+        missing = [
+            key
+            for key in _REQUIRED
+            if not isinstance(entry.get(key), str) or not entry[key].strip()
+        ]
+        if missing:
+            raise SourcesError(
+                f"{label}: missing or empty field(s): {', '.join(missing)}"
+            )
+        unknown = sorted(set(entry) - set(_KNOWN))
+        if unknown:
+            raise SourcesError(f"{label}: unknown field(s): {', '.join(unknown)}")
+        doc_id = entry["doc_id"]
+        label = f"source {doc_id!r}"
+        if not _DOC_ID.match(doc_id):
+            raise SourcesError(f"{label}: doc_id must be a lowercase file-safe slug")
+        if doc_id in seen:
+            raise SourcesError(f"{label}: duplicate doc_id")
+        seen.add(doc_id)
+        if not entry["url"].startswith("https://"):
+            raise SourcesError(f"{label}: url must be https")
+        if not _DATE.match(entry["date"]):
+            raise SourcesError(f"{label}: date must be YYYY-MM, got {entry['date']!r}")
+        digest = entry.get("sha256", "")
+        if digest and not _SHA256.match(digest):
+            raise SourcesError(f"{label}: sha256 must be 64 lowercase hex digits")
+        sources.append(Source(**{key: entry[key] for key in _KNOWN if key in entry}))
+    return tuple(sources)
+
+
+def load_sources(path: Path) -> tuple[Source, ...]:
+    """Read and parse ``path``."""
+    try:
+        text = path.read_text(encoding="utf-8")
+    except OSError as error:
+        raise SourcesError(f"cannot read {path}: {error}") from error
+    return parse_sources(text)

primarysource/usb/models.py

@@ -0,0 +1,7 @@
+"""The usb collection's document model: the shared one (primarysource.pdfcorpus.models)."""
+
+from __future__ import annotations
+
+from primarysource.pdfcorpus.models import Document, Section
+
+__all__ = ["Document", "Section"]

primarysource/usb/paths.py

@@ -0,0 +1,33 @@
+"""Where the usb collection keeps its files (primarysource PLAN.md §5.8).
+
+The data directory holds `sources.toml` overrides and the fetched `corpus/`: `$USBLIB_DATA` if set (the old
+setting), else `primarysource.home.collection_home("usb")`, which is `$PRIMARYSOURCE_HOME/usb` or the per-user
+`primarysource/usb`. Never the working directory and never beside the installed package. The tracked metadata
+(`sources.toml`, catalogues, digests) ships in the package under `data/`.
+"""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+from primarysource.home import collection_home
+
+PACKAGE_DATA = Path(__file__).resolve().parent / "data"
+
+
+def data_dir() -> Path:
+    env = os.environ.get("USBLIB_DATA")
+    return Path(env) if env else collection_home("usb", legacy_app="usblib")
+
+
+def corpus_dir() -> Path:
+    """`$USBLIB_CORPUS` if set, else `corpus/` in the data directory."""
+    env = os.environ.get("USBLIB_CORPUS")
+    return Path(env) if env else data_dir() / "corpus"
+
+
+def sources_file(data: Path, name: str = "sources.toml") -> Path:
+    """The data directory's own `sources.toml` if it has one, else the one shipped in the package."""
+    local = data / name
+    return local if local.is_file() else PACKAGE_DATA / name

primarysource/usb/server/__init__.py

@@ -0,0 +1,3 @@
+"""Read-only MCP server over the locally fetched USB-IF corpus. Makes no network calls."""
+
+from __future__ import annotations

primarysource/usb/server/corpus.py

@@ -0,0 +1,38 @@
+"""Load the usb corpus: the shared loader (primarysource.pdfcorpus.corpus) with the usb spec."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from primarysource.usb.spec import SPEC
+from primarysource.pdfcorpus import corpus as _shared
+from primarysource.pdfcorpus.corpus import SCHEMA_VERSION, CorpusError
+
+ENV_VAR = SPEC.env_var
+FETCH_COMMAND = SPEC.fetch_command
+__all__ = ["ENV_VAR", "FETCH_COMMAND", "SCHEMA_VERSION", "Corpus", "CorpusError", "default_corpus_dir", "load_corpus"]
+
+
+class Corpus(_shared.Corpus):
+    """A usb corpus: the shared one with this collection's hooks as defaults."""
+
+    def __init__(self, root: Path, documents: tuple[_shared.Document, ...], defaults: dict[str, str] | None = None) -> None:
+        super().__init__(
+            root,
+            documents,
+            defaults or {},
+            SPEC.alias,
+            fetch_command=SPEC.fetch_command,
+            index_command=SPEC.index_command,
+            prefix=f"{SPEC.name}_",
+        )
+
+
+def default_corpus_dir() -> Path:
+    """``$USBLIB_CORPUS`` if set, else ``corpus/`` in the collection's data directory."""
+    return SPEC.corpus_dir()
+
+
+def load_corpus(root: Path | None = None) -> _shared.Corpus:
+    """Read and validate the index; raise ``CorpusError`` whose ``action`` is the fetch command."""
+    return _shared.load_corpus(SPEC, root)

primarysource/usb/server/mcp.py

@@ -0,0 +1,19 @@
+"""MCP server for the local USB-IF specifications: the shared server (primarysource.pdfcorpus.server) with the
+usb spec. `primarysource mcp --collection usb` serves `mcp`."""
+
+from __future__ import annotations
+
+from primarysource.usb.spec import SPEC
+from primarysource.pdfcorpus.server import CorpusServer
+
+SERVER = CorpusServer(SPEC)
+mcp = SERVER.mcp
+
+
+def main() -> None:
+    """Serve over stdio (`primarysource mcp --collection usb` is the usual way)."""
+    mcp.run(transport="stdio")
+
+
+if __name__ == "__main__":
+    main()