primarysource 0.1.0__py3-none-any.whl

primarysource/cavp/importer.py

@@ -0,0 +1,156 @@
+"""Import vector sets named in `vectors.toml`: download, check the pinned sha256, parse, store.
+
+Every entry pins the sha256 of what it downloads (the zip, or the ACVP `prompt.json`), so a first fetch never
+defines the truth. An entry without a pin is refused, and `primarysource cavp pin <file>` prints the digest to
+write. Nothing here unpacks an archive to disk.
+
+    [[vector]]
+    id = "cavp/ecdsa/SigVer.rsp"
+    kind = "cavp"
+    algorithm = "ECDSA"
+    url = "https://csrc.nist.gov/.../186-4ecdsatestvectors.zip"
+    sha256 = "..."                 # of the zip
+    member = "186-4ecdsatestvectors/SigVer.rsp"
+
+    [[vector]]
+    id = "acvp/ECDSA-SigVer-1.0"
+    kind = "acvp"
+    url = ".../prompt.json"            # pin a commit, not a branch
+    sha256 = "..."
+    expected_url = ".../expectedResults.json"
+    expected_sha256 = "..."
+"""
+
+from __future__ import annotations
+
+import tomllib
+import urllib.error
+import urllib.request
+from collections.abc import Callable
+from dataclasses import dataclass
+from pathlib import Path
+from typing import IO, Any
+
+from primarysource.cavp.model import MAX_SOURCE_BYTES, IndexEntry, Store, VectorSet, sha256_bytes
+from primarysource.cavp.parse import check_set_id, parse_acvp, parse_rsp, read_member
+from primarysource.errors import LibError
+
+PACKAGE_DATA = Path(__file__).resolve().parent / "data"
+USER_AGENT = "primarysource-cavp/0.1 (pinned test-vector fetch)"
+TIMEOUT_SECONDS = 120
+_FIELDS = {"id", "kind", "algorithm", "mode", "url", "sha256", "member", "expected_url", "expected_sha256"}
+
+Opener = Callable[[urllib.request.Request, float], IO[bytes]]
+
+
+@dataclass(frozen=True, slots=True)
+class Source:
+    id: str
+    kind: str
+    algorithm: str
+    url: str
+    sha256: str
+    member: str = ""
+    mode: str = ""
+    expected_url: str = ""
+    expected_sha256: str = ""
+
+
+def load_sources(path: Path | None = None) -> tuple[Source, ...]:
+    """The entries of `vectors.toml`: the file given, else the one shipped in the package."""
+    file = path if path is not None else PACKAGE_DATA / "vectors.toml"
+    try:
+        raw: dict[str, Any] = tomllib.loads(file.read_text(encoding="utf-8"))
+    except (OSError, ValueError) as exc:
+        raise LibError(f"cannot read {file}: {exc}", "") from None
+    out: list[Source] = []
+    for row in raw.get("vector", []):
+        unknown = sorted(set(row) - _FIELDS)
+        if unknown:
+            raise LibError(f"{file}: unknown field(s) {', '.join(unknown)} in {row.get('id')!r}", "")
+        try:
+            source = Source(**row)
+        except TypeError as exc:
+            raise LibError(f"{file}: bad entry {row.get('id')!r} ({exc})", "") from None
+        check_set_id(source.id, source.kind)
+        if source.kind == "acvp" and bool(source.expected_url) != bool(source.expected_sha256):
+            raise LibError(f"{source.id}: expected_url and expected_sha256 go together", "")
+        out.append(source)
+    ids = [s.id for s in out]
+    if len(set(ids)) != len(ids):
+        raise LibError(f"{file}: duplicate vector ids", "")
+    return tuple(out)
+
+
+def download(url: str, opener: Opener | None = None) -> bytes:
+    """GET `url`, at most `MAX_SOURCE_BYTES`. https only (a test may pass its own opener)."""
+    if opener is None and not url.startswith("https://"):
+        raise LibError(f"refusing {url!r}: only https downloads", "")
+    request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})  # noqa: S310
+    try:
+        if opener is None:
+            stream = urllib.request.urlopen(request, timeout=TIMEOUT_SECONDS)  # noqa: S310
+        else:
+            stream = opener(request, TIMEOUT_SECONDS)
+        with stream:
+            body: bytes = stream.read(MAX_SOURCE_BYTES + 1)
+    except (urllib.error.URLError, TimeoutError, OSError) as exc:
+        raise LibError(f"download failed for {url}: {exc}", "check the network and the url") from None
+    if len(body) > MAX_SOURCE_BYTES:
+        raise LibError(f"{url}: larger than {MAX_SOURCE_BYTES} bytes, refusing", "")
+    return body
+
+
+def check_pin(what: str, data: bytes, pinned: str) -> str:
+    """The digest of `data`, or a `LibError` when it is not the pinned one or nothing is pinned."""
+    actual = sha256_bytes(data)
+    if not pinned:
+        raise LibError(f"{what}: no sha256 pinned (this file is {actual})", "add sha256 to its vectors.toml entry")
+    if actual != pinned.lower():
+        raise LibError(
+            f"{what}: sha256 {actual} is not the pinned {pinned}", "the upstream file changed: review, then re-pin"
+        )
+    return actual
+
+
+def build(source: Source, artifact: bytes, expected: bytes | None = None) -> VectorSet:
+    """Check pins and parse. `artifact` is the downloaded file (zip or json); `expected` the ACVP answers file."""
+    digest = check_pin(source.id, artifact, source.sha256)
+    if source.kind == "cavp":
+        member = read_member(artifact, source.member) if source.member else artifact
+        return parse_rsp(
+            member,
+            set_id=source.id,
+            algorithm=source.algorithm,
+            source_url=source.url,
+            sha256=digest,
+            member_sha256=sha256_bytes(member),
+            mode=source.mode,
+        )
+    if source.expected_url:
+        if expected is None:
+            raise LibError(f"{source.id}: the expectedResults file is needed", "")
+        check_pin(f"{source.id} expectedResults", expected, source.expected_sha256)
+    return parse_acvp(
+        artifact,
+        expected if source.expected_url else None,
+        set_id=source.id,
+        source_url=source.url,
+        sha256=digest,
+        member_sha256=digest,
+    )
+
+
+def import_files(store: Store, source: Source, artifact: Path, expected: Path | None = None) -> IndexEntry:
+    """Import from files already on disk (the same pins apply)."""
+    size = artifact.stat().st_size
+    if size > MAX_SOURCE_BYTES:
+        raise LibError(f"{artifact}: {size} bytes is over the {MAX_SOURCE_BYTES} byte cap", "")
+    return store.write(build(source, artifact.read_bytes(), expected.read_bytes() if expected else None))
+
+
+def fetch(store: Store, source: Source, opener: Opener | None = None) -> IndexEntry:
+    """Download `source`, verify its pins, parse and store it."""
+    artifact = download(source.url, opener)
+    expected = download(source.expected_url, opener) if source.expected_url else None
+    return store.write(build(source, artifact, expected))

primarysource/cavp/model.py

@@ -0,0 +1,279 @@
+"""Test vectors: the data model, the ref grammar and the on-disk store.
+
+A *set* is one vector file: a CAVP `.rsp` response file, or one ACVP vector set (`prompt.json` joined with
+`expectedResults.json`). A set holds *groups* (CAVP `[...]` header lines, ACVP `testGroups`) and a group holds
+*cases*. Refs, with `nist:` optional on input:
+
+    nist:cavp/ecdsa/SigVer.rsp                      a set
+    nist:cavp/ecdsa/SigVer.rsp#[P-256,SHA-256]      a group
+    nist:cavp/ecdsa/SigVer.rsp#[P-256,SHA-256]/3    a case
+    nist:acvp/ECDSA-SigVer-1.0#1/3                  ACVP: the group is the tgId, the case the tcId
+
+`docs/cavp.md` has the full grammar.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+from dataclasses import asdict, dataclass, field
+from pathlib import Path
+from typing import Any
+
+from primarysource.errors import LibError
+from primarysource.home import atomic_write
+
+SCHEMA_VERSION = 1
+REF_PREFIX = "nist:"
+INDEX_FILE = "index.json"
+SETS_DIR = "sets"
+
+# Bounds. A source over these is refused, never read.
+MAX_SOURCE_BYTES = 64 * 1024 * 1024
+MAX_LINE_BYTES = 1024 * 1024
+MAX_GROUPS = 100_000
+MAX_CASES = 2_000_000
+MAX_FIELDS_PER_CASE = 64
+MAX_HEADER_LINES = 100
+
+SET_ID = re.compile(r"(cavp|acvp)(/[A-Za-z0-9][A-Za-z0-9._+-]*)+")
+KINDS = ("cavp", "acvp")
+
+CREDIT = (
+    "Source: NIST Cryptographic Algorithm Validation Program (CAVP/ACVP). Created by employees of the National "
+    "Institute of Standards and Technology (NIST), not subject to copyright in the United States (17 U.S.C. 105); "
+    "provided AS IS. https://www.nist.gov/open/copyright-fair-use-and-licensing-statements-srd-data-software-"
+    "and-technical-series-publications"
+)
+
+
+@dataclass(frozen=True, slots=True)
+class Case:
+    """`id` is COUNT or tcId when the source has one, else the 1-based position in the group. `fields` are the
+    inputs (and, in CAVP, the expected result as well); `expected` is the joined ACVP expectedResults."""
+
+    id: str
+    fields: dict[str, Any]
+    expected: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass(frozen=True, slots=True)
+class Group:
+    label: str
+    params: dict[str, Any]
+    cases: tuple[Case, ...]
+
+
+@dataclass(frozen=True, slots=True)
+class VectorSet:
+    id: str
+    kind: str
+    algorithm: str
+    mode: str
+    source_url: str
+    sha256: str  # of the downloaded artifact (the zip, or prompt.json)
+    member_sha256: str  # of the file inside the zip; equal to sha256 when the artifact is the file
+    header: tuple[str, ...]
+    groups: tuple[Group, ...]
+    revision: str = ""
+
+    @property
+    def case_count(self) -> int:
+        return sum(len(g.cases) for g in self.groups)
+
+
+@dataclass(frozen=True, slots=True)
+class VectorRef:
+    set_id: str
+    group: str | None = None
+    case: str | None = None
+
+    def __str__(self) -> str:
+        return format_ref(self.set_id, self.group, self.case)
+
+
+def sha256_bytes(data: bytes) -> str:
+    return hashlib.sha256(data).hexdigest()
+
+
+def format_ref(set_id: str, group: str | None = None, case: str | None = None) -> str:
+    out = REF_PREFIX + set_id
+    if group is not None:
+        out += "#" + (f"[{group}]" if set_id.startswith("cavp/") else group)
+        if case is not None:
+            out += "/" + case
+    elif case is not None:
+        raise LibError("a case ref needs its group", "nist:<set>#<group>/<case>")
+    return out
+
+
+def parse_ref(text: str) -> VectorRef:
+    """`nist:<set>[#<group>[/<case>]]`. The case is what follows the last `/`, so a group label may hold one."""
+    shown = text.strip().removeprefix(REF_PREFIX)
+    set_id, sep, rest = shown.partition("#")
+    hint = "e.g. nist:cavp/ecdsa/SigVer.rsp#[P-256,SHA-256]/3 or nist:acvp/ECDSA-SigVer-1.0#1/3"
+    if not SET_ID.fullmatch(set_id):
+        raise LibError(f"not a vector ref: {text!r}", hint)
+    if not sep:
+        return VectorRef(set_id)
+    group, slash, case = rest.rpartition("/")
+    if not slash:
+        group, case = rest, ""
+    if not group:
+        raise LibError(f"not a vector ref: {text!r} (empty group)", hint)
+    if set_id.startswith("cavp/"):
+        if not (group.startswith("[") and group.endswith("]") and len(group) > 2):
+            raise LibError(f"not a vector ref: {text!r} (a CAVP group is written in brackets)", hint)
+        group = group[1:-1]
+    return VectorRef(set_id, group, case or None)
+
+
+# ---- serialisation -----------------------------------------------------------------------------------------
+
+
+def set_to_json(s: VectorSet) -> dict[str, Any]:
+    return {
+        "schema": SCHEMA_VERSION,
+        "id": s.id,
+        "kind": s.kind,
+        "algorithm": s.algorithm,
+        "mode": s.mode,
+        "revision": s.revision,
+        "source_url": s.source_url,
+        "sha256": s.sha256,
+        "member_sha256": s.member_sha256,
+        "header": list(s.header),
+        "groups": [
+            {
+                "label": g.label,
+                "params": g.params,
+                "cases": [{"id": c.id, "fields": c.fields, "expected": c.expected} for c in g.cases],
+            }
+            for g in s.groups
+        ],
+    }
+
+
+def _text(where: str, value: object) -> str:
+    if not isinstance(value, str):
+        raise LibError(f"{where}: expected a string", "", "primarysource cavp fetch")
+    return value
+
+
+def set_from_json(raw: object, where: str) -> VectorSet:
+    if not isinstance(raw, dict) or raw.get("schema") != SCHEMA_VERSION:
+        raise LibError(f"{where}: not a schema {SCHEMA_VERSION} vector set", "", "primarysource cavp fetch")
+    groups: list[Group] = []
+    for g in raw.get("groups", []):
+        cases = tuple(
+            Case(_text("id", c["id"]), dict(c["fields"]), dict(c.get("expected") or {})) for c in g["cases"]
+        )
+        groups.append(Group(_text("label", g["label"]), dict(g.get("params") or {}), cases))
+    return VectorSet(
+        id=_text("id", raw.get("id")),
+        kind=_text("kind", raw.get("kind")),
+        algorithm=_text("algorithm", raw.get("algorithm")),
+        mode=_text("mode", raw.get("mode")),
+        source_url=_text("source_url", raw.get("source_url")),
+        sha256=_text("sha256", raw.get("sha256")),
+        member_sha256=_text("member_sha256", raw.get("member_sha256")),
+        header=tuple(_text("header", h) for h in raw.get("header", [])),
+        groups=tuple(groups),
+        revision=_text("revision", raw.get("revision", "")),
+    )
+
+
+# ---- the store -----------------------------------------------------------------------------------------------
+
+
+@dataclass(frozen=True, slots=True)
+class IndexEntry:
+    id: str
+    kind: str
+    algorithm: str
+    mode: str
+    revision: str
+    source_url: str
+    sha256: str
+    groups: int
+    cases: int
+    file: str
+
+
+class Store:
+    """A directory of imported sets: `index.json` plus `sets/<id with / as __>.json`."""
+
+    def __init__(self, root: Path) -> None:
+        self.root = root
+
+    def entries(self) -> list[IndexEntry]:
+        path = self.root / INDEX_FILE
+        if not path.is_file():
+            return []
+        try:
+            data = json.loads(path.read_text(encoding="utf-8"))
+            if data["schema"] != SCHEMA_VERSION:
+                raise KeyError("schema")
+            return [IndexEntry(**e) for e in data["sets"]]
+        except (ValueError, KeyError, TypeError) as exc:
+            raise LibError(
+                f"{path} is not a schema {SCHEMA_VERSION} index ({exc})", "", "primarysource cavp fetch"
+            ) from None
+
+    def require(self) -> list[IndexEntry]:
+        """The entries, or an error that names the import command when there are none."""
+        found = self.entries()
+        if not found:
+            raise LibError(f"no vector sets imported under {self.root}", "", "primarysource cavp fetch")
+        return found
+
+    def load(self, set_id: str) -> VectorSet:
+        found = self.require()
+        entry = next((e for e in found if e.id == set_id), None)
+        if entry is None:
+            names = ", ".join(e.id for e in found)
+            raise LibError(f"no vector set {set_id!r} (imported: {names})", "cavp_sets lists what is imported")
+        path = (self.root / entry.file).resolve()
+        if not path.is_relative_to(self.root.resolve()):
+            raise LibError(f"{set_id}: {entry.file!r} escapes the store", "", "primarysource cavp fetch")
+        try:
+            return set_from_json(json.loads(path.read_text(encoding="utf-8")), set_id)
+        except FileNotFoundError:
+            raise LibError(f"{set_id}: {entry.file} is missing", "", "primarysource cavp fetch") from None
+        except (ValueError, KeyError, TypeError) as exc:
+            raise LibError(f"{path} is damaged ({exc})", "", "primarysource cavp fetch") from None
+
+    def write(self, s: VectorSet) -> IndexEntry:
+        rel = f"{SETS_DIR}/{s.id.replace('/', '__')}.json"
+        body = json.dumps(set_to_json(s), ensure_ascii=False, separators=(",", ":"))
+        atomic_write(self.root / rel, body.encode("utf-8"))
+        entry = IndexEntry(
+            s.id, s.kind, s.algorithm, s.mode, s.revision, s.source_url, s.sha256, len(s.groups), s.case_count, rel
+        )
+        entries = sorted([e for e in self.entries() if e.id != s.id] + [entry], key=lambda e: e.id)
+        index = {"schema": SCHEMA_VERSION, "sets": [asdict(e) for e in entries]}
+        atomic_write(self.root / INDEX_FILE, (json.dumps(index, indent=2) + "\n").encode("utf-8"))
+        return entry
+
+
+def norm_value(value: object) -> str:
+    """Compare as hex is compared: case-insensitive, without spaces or a `0x` prefix. CAVP writes lower case,
+    ACVP upper case."""
+    s = str(value).strip().lower().replace(" ", "")
+    return s[2:] if s.startswith("0x") else s
+
+
+def find_group(s: VectorSet, label: str) -> Group:
+    for g in s.groups:
+        if g.label == label:
+            return g
+    shown = ", ".join(g.label for g in s.groups[:10]) + (", ..." if len(s.groups) > 10 else "")
+    raise LibError(f"{s.id} has no group {label!r} (groups: {shown})", f"cavp_groups('nist:{s.id}') lists them")
+
+
+def find_case(g: Group, case_id: str) -> Case:
+    for c in g.cases:
+        if c.id == case_id:
+            return c
+    raise LibError(f"group {g.label!r} has no case {case_id!r} (it has {len(g.cases)} cases)", "")