primarysource 0.1.0__py3-none-any.whl

primarysource/c2sp/catalogue.json

@@ -0,0 +1,231 @@
+{
+ "repo": "C2SP/C2SP",
+ "commit": "09f3e519fe79bcbdccadcf821182e3b488a14d71",
+ "raw_base": "https://raw.githubusercontent.com/C2SP/C2SP/",
+ "terms": {
+  "licence": "CC-BY-4.0",
+  "placement": "layer",
+  "terms_url": "https://github.com/C2SP/C2SP/blob/09f3e519fe79bcbdccadcf821182e3b488a14d71/.github/README.md",
+  "terms_read": "2026-11-12",
+  "summary": "All C2SP specifications are licensed CC BY 4.0 (the repository's README); attribution required. Code and data in the repository are BSD-1-Clause."
+ },
+ "specs": [
+  {
+   "id": "BLAKE3",
+   "path": "BLAKE3.md",
+   "title": "The BLAKE3 Hashing Framework",
+   "size": 32004,
+   "sha256": "85c336e9900538e572333dd16ddde9f605c000ec8293001dd4c2642193b9c07e"
+  },
+  {
+   "id": "FLOE",
+   "path": "FLOE.md",
+   "title": "Fast Lightweight Online Encryption (FLOE)",
+   "size": 19400,
+   "sha256": "7808f6995f098400009e363674013d71a4d980ad30fc3ca46cb34aae1fafc044"
+  },
+  {
+   "id": "XAES-256-GCM",
+   "path": "XAES-256-GCM.md",
+   "title": "The XAES-256-GCM extended-nonce AEAD",
+   "size": 14100,
+   "sha256": "bcce71811d53d1d19f0298dc52746fd4f92f42bc4aec55abad5d2bd1ab9b2810"
+  },
+  {
+   "id": "age",
+   "path": "age.md",
+   "title": "age",
+   "size": 27736,
+   "sha256": "b0a767b91a184c536e8a04002f7bba7521388fee3989f5891e23cc1aaa00232f"
+  },
+  {
+   "id": "age-plugin",
+   "path": "age-plugin.md",
+   "title": "Plugin system for age",
+   "size": 25210,
+   "sha256": "83623f2d3942fb99c0d59b45c4f9d8bad2b29bf5af178abade558b6fc8341c95"
+  },
+  {
+   "id": "chacha20-poly1305-siv",
+   "path": "chacha20-poly1305-siv.md",
+   "title": "ChaCha20-Poly1305-SIV",
+   "size": 42410,
+   "sha256": "b93e3c9bf3e63669b059947f89bef7f3f55c502a59ef2798b8f084086fb2854b"
+  },
+  {
+   "id": "chacha8rand",
+   "path": "chacha8rand.md",
+   "title": "ChaCha8Rand",
+   "size": 18839,
+   "sha256": "09610ec86f76f1fb0160f354b68801be7fdf9948a04daa2dc88544286b33cd8d"
+  },
+  {
+   "id": "chunked-encryption",
+   "path": "chunked-encryption.md",
+   "title": "Chunked encryption",
+   "size": 18366,
+   "sha256": "4c2efb7e9930d4df06835f64c111e13c475afb6eed314b85fbbaac59a8227201"
+  },
+  {
+   "id": "cocktail-dkg",
+   "path": "cocktail-dkg.md",
+   "title": "COCKTAIL Distributed Key Generation",
+   "size": 148605,
+   "sha256": "e7ded83da0b54d9bf0052368ddcd2ca9ad6e257ed7fd17abf14f7e69f44e717d"
+  },
+  {
+   "id": "det-keygen",
+   "path": "det-keygen.md",
+   "title": "Deterministic Key Generation",
+   "size": 20994,
+   "sha256": "d78969dbe85eb5461c5cbf9136617c85887bed884884ff66a0da8398e411f11a"
+  },
+  {
+   "id": "https-bastion",
+   "path": "https-bastion.md",
+   "title": "HTTPS Bastion",
+   "size": 7053,
+   "sha256": "a3939dd7075f3578fb996f87feafc96e5d6edc71b895c6e965343c0b0565d07c"
+  },
+  {
+   "id": "httpsig-pq",
+   "path": "httpsig-pq.md",
+   "title": "Post-Quantum Algorithms for HTTP Message Signatures",
+   "size": 47902,
+   "sha256": "363ed0952d224a4f4ab1276eb1aa1fe88a046f35e046265eaa4c37dc362027e2"
+  },
+  {
+   "id": "jq255",
+   "path": "jq255.md",
+   "title": "The jq255e and jq255s Groups, Key Exchange and Signatures",
+   "size": 60656,
+   "sha256": "d59fff0aae35128adb3753f72e7f23873d08685951619c9bf0939c4820810021"
+  },
+  {
+   "id": "kopis",
+   "path": "kopis.md",
+   "title": "The Kopis Key Encapsulation Mechanism",
+   "size": 14607,
+   "sha256": "9707bb224408d1ea48f5748c8492472c8968ca2f5318a79ee2594bfcb768c6bd"
+  },
+  {
+   "id": "mtc-tlog",
+   "path": "mtc-tlog.md",
+   "title": "Merkle Tree Certificates With Tiled Transparency Logs",
+   "size": 8019,
+   "sha256": "d4cb827114feb7e5bdf5a5083eddc8805afcf135ee102543319711e607abf978"
+  },
+  {
+   "id": "passkey-record",
+   "path": "passkey-record.md",
+   "title": "Passkey records",
+   "size": 6770,
+   "sha256": "5e33609021db3023bec58c5b9512c7fb9a9f7ffc52853afb9a20356d8f1a7c31"
+  },
+  {
+   "id": "phc-strings",
+   "path": "phc-strings.md",
+   "title": "PHC Strings",
+   "size": 16430,
+   "sha256": "b53e4460a558486388233f8ca5cd30ef1f0543c870b7e0bfeecc282c2fbc2bd9"
+  },
+  {
+   "id": "sequencehash",
+   "path": "sequencehash.md",
+   "title": "SequenceHash and SequenceMAC",
+   "size": 32689,
+   "sha256": "63a3db48455cf3643203ac8665720042ae1f62498e374351b0d228ec4157f101"
+  },
+  {
+   "id": "signed-note",
+   "path": "signed-note.md",
+   "title": "Note",
+   "size": 7841,
+   "sha256": "bd9bf002294385e0cf3c79aac74b193a3831ccc7b32073fb00baa727653816ae"
+  },
+  {
+   "id": "static-ct-api",
+   "path": "static-ct-api.md",
+   "title": "The Static Certificate Transparency API",
+   "size": 14179,
+   "sha256": "e7173ca6fc226ada35d4bffb2cb8bf5a68f6ba7f12a3057fbb696fc6d32b6b5a"
+  },
+  {
+   "id": "tlog-checkpoint",
+   "path": "tlog-checkpoint.md",
+   "title": "Transparency Log Checkpoints",
+   "size": 5999,
+   "sha256": "528317ad4dabb0568a1c20728b74fdab6fbad99749cc2fb089e8fe92c8ddf00e"
+  },
+  {
+   "id": "tlog-cosignature",
+   "path": "tlog-cosignature.md",
+   "title": "Transparency Log Cosignatures",
+   "size": 10715,
+   "sha256": "9cf3f0ef2690319697fcbb4d1e823b593d90a1769912fdcb859ab89a3f924fbe"
+  },
+  {
+   "id": "tlog-list",
+   "path": "tlog-list.md",
+   "title": "Transparency Log Lists",
+   "size": 2295,
+   "sha256": "9c83ee9dbe91b33fdd628f97e35aac7044ff092cfb9cd2e1458721651a0c5934"
+  },
+  {
+   "id": "tlog-mirror",
+   "path": "tlog-mirror.md",
+   "title": "Transparency Log Mirrors",
+   "size": 23273,
+   "sha256": "1aef508bd5fd8c4a341c307c5aa446ce4888a32ba6109c7c6da177a502c049e9"
+  },
+  {
+   "id": "tlog-policy",
+   "path": "tlog-policy.md",
+   "title": "Transparency log trust policy",
+   "size": 9389,
+   "sha256": "2a8745ba91627d976ff2922f18c29a1ecd3567a03107749d5ae164cc5825bbd3"
+  },
+  {
+   "id": "tlog-proof",
+   "path": "tlog-proof.md",
+   "title": "Transparency Log Proofs",
+   "size": 5733,
+   "sha256": "2b46a3a156ddb477fd88091bdb83b984135d9c42cba53cbed867d13d99afd5e8"
+  },
+  {
+   "id": "tlog-tiles",
+   "path": "tlog-tiles.md",
+   "title": "Tiled Transparency Logs",
+   "size": 12447,
+   "sha256": "951b7c96d57b1563a427ecc2c916a44daf4b12b0fa5d0d5188f9c0868855ab1b"
+  },
+  {
+   "id": "tlog-witness",
+   "path": "tlog-witness.md",
+   "title": "Transparency Log Witness Protocol",
+   "size": 14392,
+   "sha256": "c9eaf9f403f0e8b3e61229d19f174022b9d80a3851c8c008bfc0ca9ee1180b89"
+  },
+  {
+   "id": "upki-revocation",
+   "path": "upki-revocation.md",
+   "title": "UPKI Revocation",
+   "size": 37013,
+   "sha256": "e79976dfed9d73cf37ad1a1a7bc2d2e72dadd7ebbc57c4c0664d0181b0f90ab1"
+  },
+  {
+   "id": "vrf-r255",
+   "path": "vrf-r255.md",
+   "title": "ristretto255 VRF",
+   "size": 9450,
+   "sha256": "411c9aed374b4bf342f2afc277e3a3da3ffa22fb7be1f5fe3f4ffb1c4220a4c6"
+  },
+  {
+   "id": "well-known-ssh-hosts",
+   "path": "well-known-ssh-hosts.md",
+   "title": ".well-known/ssh-known-hosts",
+   "size": 2187,
+   "sha256": "9b1573f4d78c56e8cee08928a601cb58ff3a880db3fbffb374fc448958d7a66f"
+  }
+ ]
+}

primarysource/c2sp/catalogue.py

@@ -0,0 +1,95 @@
+"""The C2SP catalogue: which specifications, at which commit, with which sha256. Metadata only, never text.
+
+`catalogue.json` is package data. It pins one commit of https://github.com/C2SP/C2SP and, for each top-level
+specification file at that commit, its size and sha256. The importer refuses any file whose digest differs, so a
+rewritten history or a tampered mirror cannot change what a ref reads.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import dataclass
+from importlib import resources
+from typing import Any
+
+from primarysource.errors import LibError
+
+DOC_ID = re.compile(r"[A-Za-z0-9][A-Za-z0-9-]{0,62}")
+COMMIT = re.compile(r"[0-9a-f]{40}")
+SHA256 = re.compile(r"[0-9a-f]{64}")
+HOSTS = ("raw.githubusercontent.com",)
+
+
+@dataclass(frozen=True, slots=True)
+class Terms:
+    """What the publisher allows, and therefore where the text may be placed."""
+
+    licence: str
+    placement: str
+    terms_url: str
+    terms_read: str
+    summary: str
+
+
+@dataclass(frozen=True, slots=True)
+class Spec:
+    """One specification file at the pinned commit."""
+
+    id: str
+    path: str
+    title: str
+    size: int
+    sha256: str
+
+
+@dataclass(frozen=True, slots=True)
+class Catalogue:
+    repo: str
+    commit: str
+    raw_base: str
+    terms: Terms
+    specs: tuple[Spec, ...]
+
+    def spec(self, doc_id: str) -> Spec:
+        for candidate in self.specs:
+            if candidate.id == doc_id:
+                return candidate
+        known = ", ".join(s.id for s in self.specs)
+        raise LibError(f"no C2SP specification {doc_id!r} in the catalogue", f"known: {known}")
+
+    def url(self, spec: Spec) -> str:
+        return f"{self.raw_base}{self.commit}/{spec.path}"
+
+
+def parse_catalogue(raw: dict[str, Any]) -> Catalogue:
+    """Validate a catalogue document. Raises `ValueError`: a malformed catalogue is a packaging bug."""
+    commit = raw["commit"]
+    if not isinstance(commit, str) or not COMMIT.fullmatch(commit):
+        raise ValueError("catalogue commit must be 40 lowercase hex digits")
+    terms = raw["terms"]
+    specs: list[Spec] = []
+    for entry in raw["specs"]:
+        spec = Spec(entry["id"], entry["path"], entry["title"], int(entry["size"]), entry["sha256"])
+        if not DOC_ID.fullmatch(spec.id) or spec.path != f"{spec.id}.md":
+            raise ValueError(f"catalogue entry {spec.id!r}: id must be a slug and path <id>.md")
+        if not SHA256.fullmatch(spec.sha256):
+            raise ValueError(f"catalogue entry {spec.id!r}: sha256 must be 64 hex digits")
+        specs.append(spec)
+    if len({s.id for s in specs}) != len(specs):
+        raise ValueError("duplicate specification ids in the catalogue")
+    if terms["placement"] not in ("layer", "volume"):
+        raise ValueError("terms.placement must be 'layer' or 'volume'")
+    return Catalogue(
+        raw["repo"],
+        commit,
+        raw["raw_base"],
+        Terms(terms["licence"], terms["placement"], terms["terms_url"], terms["terms_read"], terms["summary"]),
+        tuple(specs),
+    )
+
+
+def catalogue() -> Catalogue:
+    """The packaged catalogue."""
+    text = resources.files("primarysource.c2sp").joinpath("catalogue.json").read_text(encoding="utf-8")
+    return parse_catalogue(json.loads(text))

primarysource/c2sp/cli.py

@@ -0,0 +1,62 @@
+"""`primarysource c2sp ...`: the collection's own commands.
+
+primarysource c2sp import [--only ID ...]
+primarysource c2sp docs
+primarysource c2sp outline <doc>
+primarysource c2sp read <ref> [--max-tokens N]
+primarysource c2sp search <query> [--doc ID] [--limit N] [--start N]
+primarysource c2sp cite <ref>
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from collections.abc import Sequence
+
+from primarysource.c2sp import importer, library
+from primarysource.errors import LibError
+
+
+def _parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(prog="primarysource c2sp", description="C2SP specifications (CC BY 4.0)")
+    commands = parser.add_subparsers(dest="command", required=True)
+    imp = commands.add_parser("import", help="fetch and verify the pinned specifications (one request per second)")
+    imp.add_argument("--only", action="append", default=[], metavar="ID", help="import only this specification")
+    commands.add_parser("docs", help="the catalogue and what is imported")
+    commands.add_parser("outline", help="a specification's headings with refs").add_argument("doc")
+    read = commands.add_parser("read", help="read by ref")
+    read.add_argument("ref")
+    read.add_argument("--max-tokens", type=int, default=2000)
+    search = commands.add_parser("search", help="find blocks containing words")
+    search.add_argument("query")
+    search.add_argument("--doc")
+    search.add_argument("--limit", type=int, default=10)
+    search.add_argument("--start", type=int, default=0)
+    commands.add_parser("cite", help="the citation for a ref").add_argument("ref")
+    return parser
+
+
+def _out(text: str) -> int:
+    sys.stdout.write(text if text.endswith("\n") else text + "\n")
+    return 0
+
+
+def main(argv: Sequence[str]) -> int:
+    args = _parser().parse_args(list(argv))
+    try:
+        if args.command == "import":
+            result = importer.import_specs(library.default_home(), args.only)
+            return _out(f"imported {len(result.files)} specifications at {result.commit[:7]} into {result.directory}")
+        if args.command == "docs":
+            return _out(library.docs())
+        if args.command == "outline":
+            return _out(library.outline(args.doc))
+        if args.command == "read":
+            return _out(library.read(args.ref, max_tokens=args.max_tokens))
+        if args.command == "search":
+            return _out(library.search(args.query, doc=args.doc, limit=args.limit, start=args.start))
+        return _out(library.cite(args.ref))
+    except LibError as exc:
+        print(exc.as_text(), file=sys.stderr)
+        return 1

primarysource/c2sp/importer.py

@@ -0,0 +1,90 @@
+"""Import the pinned C2SP specifications into `$PRIMARYSOURCE_HOME/c2sp/<commit>/`.
+
+Each specification is fetched from `raw.githubusercontent.com/C2SP/C2SP/<commit>/<id>.md`, at most one request per
+second, and written only if its sha256 equals the catalogue's pin. `install.json` is written last, so a reader
+never sees an install that is missing a file. C2SP specifications are licensed CC BY 4.0, so the installed files
+may also be baked into an image layer; this code only ever writes to the home directory.
+
+Run as `primarysource c2sp import [--only ID ...]`.
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Sequence
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+from pathlib import Path
+
+from primarysource.c2sp import markdown
+from primarysource.c2sp._fetch import Fetcher, Transport, sha256, verify
+from primarysource.c2sp.catalogue import DOC_ID, HOSTS, Catalogue, Spec
+from primarysource.c2sp.catalogue import catalogue as packaged_catalogue
+from primarysource.errors import LibError
+from primarysource.home import atomic_write
+
+INSTALL_RECORD = "install.json"
+MAX_FILE = markdown.MAX_BYTES
+
+
+@dataclass(frozen=True, slots=True)
+class ImportResult:
+    commit: str
+    directory: Path
+    files: dict[str, str] = field(default_factory=dict)
+    requests: int = 0
+
+
+def install_dir(home: Path, commit: str) -> Path:
+    return home / commit
+
+
+def import_specs(
+    home: Path,
+    only: Sequence[str] = (),
+    *,
+    cat: Catalogue | None = None,
+    transport: Transport | None = None,
+    fetcher: Fetcher | None = None,
+) -> ImportResult:
+    """Fetch, verify and install the catalogued specs (all, or those named in `only`) under `home`."""
+    cat = cat or packaged_catalogue()
+    chosen = [cat.spec(name) for name in only] if only else list(cat.specs)
+    fetcher = fetcher or Fetcher(HOSTS, transport)
+    target = install_dir(home, cat.commit)
+    record_path = target / INSTALL_RECORD
+    installed: dict[str, str] = {}
+    if record_path.is_file():
+        try:
+            installed = dict(json.loads(record_path.read_text(encoding="utf-8")).get("files", {}))
+        except (ValueError, OSError):
+            installed = {}
+    for spec in chosen:
+        data = fetcher.get(cat.url(spec), min(MAX_FILE, max(spec.size * 2, 1024)))
+        verify(data, spec.sha256, spec.id)
+        markdown.parse(data)  # refuses non-UTF-8, NULs and over-bound text before anything is written
+        atomic_write(target / spec.path, data)
+        installed[spec.id] = spec.sha256
+    record = {
+        "collection": "c2sp",
+        "repo": cat.repo,
+        "commit": cat.commit,
+        "licence": cat.terms.licence,
+        "imported": datetime.now(UTC).isoformat(timespec="seconds"),
+        "files": dict(sorted(installed.items())),
+    }
+    atomic_write(record_path, (json.dumps(record, indent=2) + "\n").encode("utf-8"))
+    return ImportResult(cat.commit, target, dict(sorted(installed.items())), fetcher.requests)
+
+
+def pin_specs(raw_base: str, commit: str, names: Sequence[str], fetcher: Fetcher) -> list[Spec]:
+    """Maintainer step: fetch `names` (file ids) at `commit` and return catalogue entries with size and sha256.
+    The result is what goes in `catalogue.json`; it is a review step, never run by the importer."""
+    specs: list[Spec] = []
+    for name in names:
+        if not DOC_ID.fullmatch(name):
+            raise LibError(f"not a specification id: {name!r}", "")
+        data = fetcher.get(f"{raw_base}{commit}/{name}.md", MAX_FILE)
+        doc = markdown.parse(data)
+        specs.append(Spec(name, f"{name}.md", doc.title or name, len(data), sha256(data)))
+    return specs