sci-protocol 0.2.0__py3-none-any.whl

pysci/_cli/storage/inputs.py

@@ -0,0 +1,205 @@
+"""Signed local inputs: candidate discovery, verification, sidecars, and ZIP materialization."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, BinaryIO, Literal
+
+from pysci.bundle import (
+    VerificationResult,
+    read_zip_entry,
+    verify_directory,
+    verify_file,
+    verify_zip,
+    write_bundle_zip,
+)
+from pysci.errors import Outcome
+from pysci.manifest import (
+    LICENSE_PATH,
+    MANIFEST_PATH,
+    METADATA_PATH,
+    README_PATH,
+    SIGNATURE_PATH,
+    TIMESTAMP_SIDECAR_PATH,
+    Manifest,
+)
+from pysci.source import FileSource, PathSource
+from pysci.zipformat import parse_zip_layout
+
+from ..errors import CliError, Exit, issue_json
+from .files import copy_file_to
+
+
+@dataclass(frozen=True, slots=True)
+class SignedInput:
+    path: Path
+    kind: Literal["zip", "directory"]
+    result: VerificationResult
+
+    @property
+    def manifest(self) -> Manifest:
+        assert self.result.manifest is not None
+        return self.result.manifest
+
+    @property
+    def digest(self) -> str:
+        return self.manifest.digest
+
+    @property
+    def publication_id(self) -> str:
+        return self.manifest.publication_id
+
+    @property
+    def zip_sha256(self) -> str:
+        assert self.result.zip_sha256 is not None
+        return self.result.zip_sha256
+
+    @property
+    def sidecar(self) -> Path | None:
+        return find_sidecar(self.path, self.kind, self.digest)
+
+
+def find_sidecar(path: Path, kind: str, digest: str) -> Path | None:
+    """Locate a timestamp sidecar: ``manifest.json.sig.tsr`` or ``<digest>.tsr``."""
+    if kind == "directory":
+        candidate = path / TIMESTAMP_SIDECAR_PATH
+        return candidate if candidate.is_file() else None
+    for candidate in (path.parent / f"{digest}.tsr", path.with_suffix(".tsr")):
+        if candidate.is_file():
+            return candidate
+    return None
+
+
+def sidecar_target(path: Path, kind: str, digest: str) -> Path:
+    if kind == "directory":
+        return path / TIMESTAMP_SIDECAR_PATH
+    return path.parent / f"{digest}.tsr"
+
+
+def verify_input(path: Path, expected_id: str | None = None) -> tuple[str, VerificationResult]:
+    if path.is_dir():
+        return "directory", verify_directory(path, expected_id=expected_id, compute_zip_sha256=True)
+    return "zip", verify_zip(path, expected_id=expected_id)
+
+
+def candidates(path: Path) -> list[Path]:
+    """Shared input rules: a ZIP or signed directory, else immediate ``*.zip`` children."""
+    if not path.exists():
+        raise CliError("input_missing", f"{path} does not exist", Exit.USAGE, file=str(path))
+    if path.is_file() or (path / MANIFEST_PATH).exists():
+        return [path]
+    found = sorted(p for p in path.iterdir() if p.suffix == ".zip" and p.is_file())
+    if not found:
+        raise CliError(
+            "no_candidates",
+            f"{path} is neither a signed publication nor a folder containing *.zip bundles",
+            Exit.USAGE,
+            file=str(path),
+        )
+    return found
+
+
+def collect_signed_inputs(
+    paths: list[Path],
+) -> tuple[list[SignedInput], list[dict[str, Any]], list[dict[str, Any]]]:
+    """Verify every candidate (§6.4); return (valid deduplicated, invalid, duplicates)."""
+    valid: list[SignedInput] = []
+    invalid: list[dict[str, Any]] = []
+    duplicates: list[dict[str, Any]] = []
+    seen: dict[str, SignedInput] = {}
+    for root in paths:
+        for candidate in candidates(root):
+            kind, result = verify_input(candidate)
+            if not result.ok or result.manifest is None:
+                invalid.append(
+                    {
+                        "path": str(candidate),
+                        "status": str(result.outcome),
+                        "publication_id": result.publication_id,
+                        "issues": [issue_json(i) for i in result.issues],
+                    }
+                )
+                continue
+            item = SignedInput(candidate, "zip" if kind == "zip" else "directory", result)
+            if item.publication_id in seen:
+                duplicates.append(
+                    {
+                        "path": str(candidate),
+                        "publication_id": item.publication_id,
+                        "same_as": str(seen[item.publication_id].path),
+                    }
+                )
+                continue
+            seen[item.publication_id] = item
+            valid.append(item)
+    return valid, invalid, duplicates
+
+
+def invalid_exit(entries: list[dict[str, Any]]) -> Exit:
+    statuses = {e["status"] for e in entries}
+    if Outcome.UNSUPPORTED in statuses:
+        return Exit.UNSUPPORTED
+    if Outcome.INVALID in statuses:
+        return Exit.INVALID
+    return Exit.INCOMPLETE
+
+
+def zip_manifest_and_signature(source: FileSource) -> tuple[bytes, bytes]:
+    entries = parse_zip_layout(source)
+    return (
+        source.read_at(entries[0].data_offset, entries[0].size),
+        source.read_at(entries[1].data_offset, entries[1].size),
+    )
+
+
+def signature_of(item: SignedInput) -> bytes:
+    if item.kind == "directory":
+        return (item.path / SIGNATURE_PATH).read_bytes()
+    with FileSource(item.path) as source:
+        return zip_manifest_and_signature(source)[1]
+
+
+def read_member(item: SignedInput, path: str) -> bytes:
+    """Return verified bytes of one bundle member (manifest, signature, or listed file)."""
+    if item.kind == "directory":
+        data = (item.path / path).read_bytes()
+        if path == MANIFEST_PATH:
+            if data != item.manifest.raw:
+                raise CliError("input_changed", f"{item.path}: manifest changed", Exit.INCOMPLETE)
+            return data
+        if path != SIGNATURE_PATH:
+            verify_file(item.manifest, path, data)
+        return data
+    with FileSource(item.path) as source:
+        if path in (MANIFEST_PATH, SIGNATURE_PATH):
+            manifest, signature = zip_manifest_and_signature(source)
+            if manifest != item.manifest.raw:
+                raise CliError("input_changed", f"{item.path}: manifest changed", Exit.INCOMPLETE)
+            return manifest if path == MANIFEST_PATH else signature
+        return read_zip_entry(source, path, expected_id=item.publication_id)[1]
+
+
+def reader_paths(manifest: Manifest, display: list[str]) -> list[str]:
+    """Reader files: manifest, signature, metadata, README, any license, and display files."""
+    paths = [MANIFEST_PATH, SIGNATURE_PATH, METADATA_PATH, README_PATH]
+    if LICENSE_PATH in manifest.files:
+        paths.append(LICENSE_PATH)
+    paths.extend(p for p in display if p in manifest.files and p not in paths)
+    return paths
+
+
+def reader_files(item: SignedInput) -> dict[str, bytes]:
+    display = [d.path for d in item.result.metadata.display] if item.result.metadata else []
+    return {path: read_member(item, path) for path in reader_paths(item.manifest, display)}
+
+
+def write_zip(item: SignedInput, handle: BinaryIO) -> None:
+    """Write the canonical ZIP bytes of a verified input to ``handle``."""
+    if item.kind == "zip":
+        copy_file_to(handle, item.path)
+        return
+    manifest_bytes = (item.path / MANIFEST_PATH).read_bytes()
+    signature = (item.path / SIGNATURE_PATH).read_bytes()
+    files = {p: PathSource(item.path / p) for p in item.manifest.files}
+    write_bundle_zip(handle, manifest_bytes, signature, files)

pysci/_cli/storage/keys.py

@@ -0,0 +1,132 @@
+"""Signing key files: locations, creation with owner-only access, and loading."""
+
+from __future__ import annotations
+
+import contextlib
+import os
+import stat
+from pathlib import Path
+
+from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
+
+from pysci.keys import (
+    PublicKey,
+    generate_private_key,
+    load_private_key_pem,
+    load_public_key_pem,
+    private_key_to_pem,
+    public_key_of,
+)
+
+from ..errors import CliError, Exit
+from .files import is_within
+
+
+def public_key_path(private_path: Path) -> Path:
+    """Replace a final ``.pem`` with ``.pub.pem``, or append ``.pub.pem``."""
+    name = private_path.name
+    if name.endswith(".pem"):
+        return private_path.with_name(name[: -len(".pem")] + ".pub.pem")
+    return private_path.with_name(name + ".pub.pem")
+
+
+def resolve_key_path(explicit: str | None, configured: str | None, home: Path) -> Path:
+    if explicit:
+        return Path(explicit).expanduser().absolute()
+    if configured:
+        return Path(configured).expanduser().absolute()
+    return home / "key.pem"
+
+
+def create_key_pair(private_path: Path) -> tuple[PublicKey, Path]:
+    """Create a PKCS#8 private key (mode 0600) and its SubjectPublicKeyInfo public key.
+
+    Raises:
+        CliError: an output already exists (exit 2).
+    """
+    public_path = public_key_path(private_path)
+    for path in (private_path, public_path):
+        if path.exists() or path.is_symlink():
+            raise CliError(
+                "key_exists",
+                f"{path} already exists; keygen never overwrites keys",
+                Exit.USAGE,
+                file=str(path),
+            )
+    private_path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
+    key = generate_private_key()
+    public = public_key_of(key)
+    fd = os.open(private_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
+    try:
+        with os.fdopen(fd, "wb") as handle:
+            os.fchmod(handle.fileno(), 0o600)
+            handle.write(private_key_to_pem(key))
+            handle.flush()
+            os.fsync(handle.fileno())
+        fd = os.open(public_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o644)
+        with os.fdopen(fd, "wb") as handle:
+            handle.write(public.to_pem())
+    except BaseException:
+        with contextlib.suppress(OSError):
+            private_path.unlink()
+        raise
+    return public, public_path
+
+
+def load_signing_key(path: Path) -> Ed25519PrivateKey:
+    try:
+        data = path.read_bytes()
+    except FileNotFoundError:
+        raise CliError(
+            "missing_key",
+            f"no private key at {path}; create one with `sci keygen` or pass --key PATH",
+            Exit.USAGE,
+            file=str(path),
+        ) from None
+    except OSError as exc:
+        raise CliError("key_unreadable", f"cannot read {path}: {exc}", Exit.USAGE) from None
+    try:
+        return load_private_key_pem(data)
+    except (ValueError, TypeError) as exc:
+        raise CliError(
+            "key_invalid",
+            f"{path} is not an unencrypted PKCS#8 Ed25519 private key: {exc}",
+            Exit.USAGE,
+            file=str(path),
+        ) from None
+
+
+def key_permission_warning(path: Path) -> str | None:
+    try:
+        mode = path.stat().st_mode
+    except OSError:
+        return None
+    if os.name == "posix" and mode & (stat.S_IRWXG | stat.S_IRWXO):
+        return f"private key {path} is accessible by other users; run chmod 600 {path}"
+    return None
+
+
+def require_outside(key_path: Path, *directories: Path) -> None:
+    """Private keys must stay outside publication input and output directories."""
+    for directory in directories:
+        if is_within(key_path, directory):
+            raise CliError(
+                "key_inside_publication",
+                f"private key {key_path} is inside {directory}; keep keys outside "
+                "publication input and output directories",
+                Exit.USAGE,
+                file=str(key_path),
+            )
+
+
+def load_public_key_file(path: Path) -> PublicKey:
+    try:
+        return load_public_key_pem(path.read_bytes())
+    except FileNotFoundError:
+        raise CliError("public_key_missing", f"{path} does not exist", Exit.USAGE) from None
+    except (ValueError, TypeError) as exc:
+        raise CliError(
+            "public_key_invalid",
+            f"{path} is not a SubjectPublicKeyInfo PEM Ed25519 public key: {exc}",
+            Exit.USAGE,
+        ) from None

pysci/_cli/storage/local_publish.py

@@ -0,0 +1,88 @@
+"""Local publication: store verified ``<digest>.zip`` bundles and sidecars in folders."""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from typing import Any
+
+from ..clients.tsa import merge_proofs, read_sidecar
+from ..config import Config
+from ..errors import ErrorRecord, Exit
+from .files import InputChanged, StoreOutcome, atomic_write_bytes, store_bytes, store_no_clobber
+from .inputs import SignedInput, write_zip
+
+
+def publish_to_folder(item: SignedInput, folder: Path) -> tuple[dict[str, Any], int]:
+    """Store one verified bundle; return (report, exit code).
+
+    Identical existing bundles are kept; conflicting bytes are rejected and
+    left untouched. Timestamp sidecars are copied as ``<digest>.tsr`` and
+    merged without duplicating proofs.
+    """
+    report: dict[str, Any] = {
+        "publication_id": item.publication_id,
+        "destination": str(folder),
+        "kind": "folder",
+    }
+    try:
+        folder.mkdir(parents=True, exist_ok=True)
+        target = folder / f"{item.digest}.zip"
+        stored = store_no_clobber(target, lambda handle: write_zip(item, handle), item.zip_sha256)
+    except InputChanged as exc:
+        report["status"] = "failed"
+        report["error"] = ErrorRecord("input_changed", str(exc), True).to_json()
+        return report, Exit.INCOMPLETE
+    except OSError as exc:
+        report["status"] = "failed"
+        report["error"] = ErrorRecord("write_failed", f"{folder}: {exc}", True).to_json()
+        return report, Exit.INCOMPLETE
+    report["path"] = str(target)
+    report["bundle"] = str(stored.outcome)
+    if stored.outcome is StoreOutcome.CONFLICT:
+        report["status"] = "failed"
+        report["error"] = ErrorRecord(
+            "destination_conflict",
+            f"{target} exists with different bytes; it was left unchanged",
+            file=str(target),
+            observed=f"sha256 {stored.existing_sha256}",
+            requirement=f"sha256 {item.zip_sha256}",
+        ).to_json()
+        return report, Exit.INVALID
+    report["status"] = "stored"
+    sidecar = item.sidecar
+    if sidecar is not None:
+        report["timestamp_sidecar"] = _copy_sidecar(sidecar, folder / f"{item.digest}.tsr")
+    return report, Exit.OK
+
+
+def _copy_sidecar(source: Path, target: Path) -> dict[str, Any]:
+    if source.exists() and target.exists() and os.path.samefile(source, target):
+        return {"path": str(target), "status": "unchanged"}
+    data = read_sidecar(source) or b""
+    existing = read_sidecar(target)
+    if existing is None:
+        store_bytes(target, data)
+        return {"path": str(target), "status": "written"}
+    if existing == data:
+        return {"path": str(target), "status": "unchanged"}
+    merged, problems = merge_proofs(existing, data)
+    if problems:
+        return {
+            "path": str(target),
+            "status": "preserved",
+            "problems": [p.to_json() for p in problems],
+        }
+    if merged != existing:
+        atomic_write_bytes(target, merged)
+        return {"path": str(target), "status": "merged"}
+    return {"path": str(target), "status": "unchanged"}
+
+
+def remember_folder(config: Config, folder: Path) -> bool:
+    """Add a folder's canonical absolute path to the known folders; True if added."""
+    canonical = str(folder.resolve())
+    if canonical in config.folders:
+        return False
+    config.folders.append(canonical)
+    return True

pysci/accounts.py

@@ -0,0 +1,135 @@
+"""External account methods for key claims (protocol §5.2.2-5.2.4)."""
+
+from __future__ import annotations
+
+import ipaddress
+import re
+from dataclasses import dataclass
+from enum import StrEnum
+from typing import Final
+
+import idna
+
+__all__ = [
+    "Account",
+    "AccountType",
+    "account_problem",
+    "claim_urls",
+    "normalize_account",
+]
+
+_LDH_LABEL: Final = re.compile(r"[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?")
+_GITHUB_LOGIN: Final = re.compile(r"[a-z0-9](?:[a-z0-9-]{0,38})")
+_HF_NAME: Final = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,95}")
+
+
+class AccountType(StrEnum):
+    DOMAIN = "domain"
+    GITHUB = "github"
+    HUGGINGFACE = "huggingface"
+
+
+@dataclass(frozen=True, slots=True)
+class Account:
+    """An account object ``{"type": ..., "id": ...}`` (§5.2.1)."""
+
+    type: str
+    id: str
+
+    def to_json(self) -> dict[str, str]:
+        return {"type": self.type, "id": self.id}
+
+
+def _domain_problem(host: str) -> str | None:
+    if not host or not host.isascii():
+        return "domain must be an ASCII DNS host name (use IDNA2008 A-labels)"
+    if host != host.lower():
+        return "domain must be lowercase"
+    if host.endswith("."):
+        return "domain must not end with a dot"
+    try:
+        ipaddress.ip_address(host.strip("[]"))
+    except ValueError:
+        pass
+    else:
+        return "IP addresses are not permitted"
+    if len(host) > 253:
+        return "domain exceeds 253 characters"
+    labels = host.split(".")
+    for label in labels:
+        if not _LDH_LABEL.fullmatch(label):
+            return f"invalid DNS label {label!r}"
+    if labels[-1].isdigit():
+        return "top-level label must not be numeric"
+    try:
+        if idna.encode(host, uts46=False).decode("ascii") != host:
+            return "domain is not in IDNA2008 A-label form"
+    except idna.IDNAError as exc:
+        return f"invalid IDNA2008 label: {exc}"
+    return None
+
+
+def account_problem(account_type: str, account_id: str) -> str | None:
+    """Why ``(type, id)`` violates its account method, or None if it is valid.
+
+    Unknown account types are reported as unsupported by the caller; this
+    function returns a message for them as well.
+    """
+    if account_type == AccountType.DOMAIN:
+        return _domain_problem(account_id)
+    if account_type == AccountType.GITHUB:
+        if account_id != account_id.lower():
+            return "GitHub login must be lowercase"
+        if not _GITHUB_LOGIN.fullmatch(account_id):
+            return "not a GitHub login"
+        return None
+    if account_type == AccountType.HUGGINGFACE:
+        return None if _HF_NAME.fullmatch(account_id) else "not a Hugging Face user or org name"
+    return f"unsupported account type {account_type!r}"
+
+
+def normalize_account(account_type: str, account_id: str) -> Account:
+    """Normalize user input to the protocol spelling.
+
+    Domains are lowercased, a final dot removed, and Unicode labels converted
+    to IDNA2008 A-labels; GitHub logins are lowercased. Hugging Face names are
+    returned unchanged (their canonical spelling comes from the Hub API).
+
+    Raises:
+        ValueError: the result still violates the account method.
+    """
+    if account_type == AccountType.DOMAIN:
+        host = account_id.strip().rstrip(".")
+        try:
+            host = idna.encode(host, uts46=True).decode("ascii")
+        except idna.IDNAError as exc:
+            raise ValueError(f"invalid domain: {exc}") from None
+        account_id = host.lower()
+    elif account_type == AccountType.GITHUB:
+        account_id = account_id.strip().lower()
+    problem = account_problem(account_type, account_id)
+    if problem is not None:
+        raise ValueError(problem)
+    return Account(account_type, account_id)
+
+
+def claim_urls(account: Account, x: str) -> list[tuple[str, str]]:
+    """Hosting locations ``(claim URL, signature URL)`` for a key's claim.
+
+    GitHub has two permitted repositories; both are returned in preference
+    order. Raises ValueError for unsupported account types.
+    """
+    if account.type == AccountType.DOMAIN:
+        bases = [f"https://{account.id}/.well-known/sci/keys/{x}.json"]
+    elif account.type == AccountType.GITHUB:
+        bases = [
+            f"https://raw.githubusercontent.com/{account.id}/{account.id}/main/sci/keys/{x}.json",
+            f"https://raw.githubusercontent.com/{account.id}/.github/main/sci/keys/{x}.json",
+        ]
+    elif account.type == AccountType.HUGGINGFACE:
+        bases = [
+            f"https://huggingface.co/spaces/{account.id}/README/resolve/main/sci/keys/{x}.json"
+        ]
+    else:
+        raise ValueError(f"unsupported account type {account.type!r}")
+    return [(url, url + ".sig") for url in bases]