sci-protocol 0.2.0__py3-none-any.whl

pysci/_cli/clients/index.py

@@ -0,0 +1,435 @@
+"""Index client: descriptor discovery (§8.1), lookups, submissions, and catalog queries."""
+
+from __future__ import annotations
+
+import json
+import urllib.parse
+from dataclasses import dataclass
+from typing import Any
+
+from pysci.discovery import (
+    IndexDescriptor,
+    is_absolute_url,
+    parse_index_descriptor,
+    parse_publication_lookup,
+    parse_timestamp_lookup,
+)
+from pysci.errors import Outcome, SciError
+from pysci.jsonstrict import JsonValue
+from pysci.submission import SubmissionRequest, parse_submission_response
+from pysci.templates import UrlTemplate, parse_template
+
+from ..errors import CliError, ErrorRecord, Exit
+from .http import Deadline, HttpClient, NetworkError, Response, status_error
+
+DESCRIPTOR_LIMIT = 1 << 20
+CATALOG_SCHEMA = "sciscape_catalog_v1"
+API_SCHEMA = "sciscape-api/0.1"
+CATALOG_FIELDS = "id,title,abstract,public_key,accepted_at"
+
+
+def descriptor_url(origin: str) -> str:
+    return origin.rstrip("/") + "/.well-known/sci/index.json"
+
+
+def unsupported(capability: str, origin: str, detail: str) -> CliError:
+    return CliError(
+        "unsupported_capability",
+        f"index {origin} does not support {capability}: {detail}",
+        Exit.UNSUPPORTED,
+        capability=capability,
+    )
+
+
+@dataclass(frozen=True, slots=True)
+class Index:
+    origin: str
+    descriptor: IndexDescriptor
+
+    def extra_url(self, name: str) -> str | None:
+        value = self.descriptor.extra.get(name)
+        return value if isinstance(value, str) and is_absolute_url(value) else None
+
+    def extra_str(self, name: str) -> str | None:
+        value = self.descriptor.extra.get(name)
+        return value if isinstance(value, str) else None
+
+    def publication_page(self, digest: str) -> str | None:
+        template = self.extra_str("publication_url")
+        if template is None:
+            return None
+        try:
+            parsed: UrlTemplate = parse_template(template)
+            return parsed.expand({"id": digest})
+        except (ValueError, KeyError, SciError):
+            return None
+
+
+def classify_service_error(status: int, code: str, retryable: bool) -> Exit:
+    """Map a submission or Sciscape API error to an exit code."""
+    if code == "unsupported_schema":
+        return Exit.UNSUPPORTED
+    if retryable or status in (408, 409, 425, 429) or status >= 500:
+        return Exit.INCOMPLETE
+    if status in (403, 413) or code in ("policy_refused", "not_baseline"):
+        return Exit.REFUSED
+    return Exit.INVALID
+
+
+class IndexClient:
+    def __init__(self, http: HttpClient, origin: str) -> None:
+        self.http = http
+        self.origin = origin
+        self._index: Index | None = None
+
+    def discover(self, deadline: Deadline) -> Index:
+        if self._index is not None:
+            return self._index
+        url = descriptor_url(self.origin)
+        response = self.http.get(url, deadline=deadline, max_bytes=DESCRIPTOR_LIMIT)
+        if response.status != 200:
+            raise status_error(response, f"index descriptor {url}")
+        try:
+            descriptor = parse_index_descriptor(response.body, file=url)
+        except SciError as exc:
+            error = CliError.from_sci(exc, f"index descriptor {url} is not usable")
+            raise error from None
+        self._index = Index(self.origin, descriptor)
+        return self._index
+
+    # ------------------------------------------------------------------ §8.1.1-8.1.3
+
+    def lookup(self, digest: str, deadline: Deadline) -> list[str]:
+        index = self.discover(deadline)
+        url = index.descriptor.publication_lookup(digest)
+        response = self.http.get(url, deadline=deadline, max_bytes=DESCRIPTOR_LIMIT)
+        try:
+            return parse_publication_lookup(response.status, response.body)
+        except SciError as exc:
+            if exc.outcome is Outcome.INCOMPLETE:
+                raise status_error(response, "publication lookup") from None
+            raise CliError.from_sci(exc, "publication lookup response is invalid") from None
+
+    def timestamps(self, digest: str, deadline: Deadline) -> bytes | None:
+        index = self.discover(deadline)
+        url = index.descriptor.timestamp_lookup(digest)
+        response = self.http.get(url, deadline=deadline, max_bytes=16 << 20)
+        try:
+            lookup = parse_timestamp_lookup(response.status, response.body)
+        except SciError:
+            raise status_error(response, "timestamp lookup") from None
+        return lookup.data
+
+    # ------------------------------------------------------------------ §8.1.4
+
+    def submit(self, request: SubmissionRequest, deadline: Deadline) -> dict[str, Any]:
+        """Submit one publication; return a per-ID result with an exit code.
+
+        Temporary failures and service errors marked retryable are retried
+        with backoff. A lost response stays ``outcome_unknown`` unless a
+        repeated request confirms the result.
+        """
+        index = self.discover(deadline)
+        url = index.descriptor.submit_url
+        if url is None:
+            raise unsupported("publication submission", self.origin, "no submit_url")
+        body = request.to_json_bytes()
+        lost = False
+        try:
+            response = self.http.request(
+                "POST",
+                url,
+                deadline=deadline,
+                headers={"Content-Type": "application/json", "Accept": "application/json"},
+                body=body,
+                paced=True,
+                should_retry=_retryable_error,
+            )
+        except NetworkError as exc:
+            lost = exc.lost_response
+            return {
+                "id": request.id,
+                "status": "outcome_unknown" if lost else "failed",
+                "exit_code": Exit.INCOMPLETE,
+                "error": ErrorRecord(
+                    "outcome_unknown" if lost else "network_error",
+                    (
+                        "the submission response was lost; repeat the same request to "
+                        "confirm the result"
+                    )
+                    if lost
+                    else str(exc),
+                    True,
+                ).to_json(),
+            }
+        except CliError as exc:
+            if exc.code == "deadline_exceeded":
+                return {
+                    "id": request.id,
+                    "status": "failed",
+                    "exit_code": Exit.INCOMPLETE,
+                    "error": exc.records[0].to_json(),
+                }
+            raise
+        try:
+            parsed = parse_submission_response(
+                response.status, response.header("content-type"), response.body, request.id
+            )
+        except SciError as exc:
+            temporary = response.status >= 500 or response.status == 429
+            return {
+                "id": request.id,
+                "status": "failed",
+                "http_status": response.status,
+                "exit_code": Exit.INCOMPLETE if temporary else Exit.INVALID,
+                "error": ErrorRecord(
+                    "invalid_response",
+                    f"submission response violates §8.1.4: {exc.issue.requirement}",
+                    temporary,
+                    observed=exc.issue.observed,
+                ).to_json(),
+            }
+        if parsed.error is not None:
+            exit_code = classify_service_error(
+                response.status, parsed.error.code, parsed.error.retryable
+            )
+            record = ErrorRecord(parsed.error.code, parsed.error.message, parsed.error.retryable)
+            out: dict[str, Any] = {
+                "id": request.id,
+                "status": "refused" if exit_code == Exit.REFUSED else "failed",
+                "http_status": response.status,
+                "exit_code": exit_code,
+                "error": record.to_json(),
+                "response": parsed.body,
+            }
+            if response.header("retry-after"):
+                out["retry_after"] = response.header("retry-after")
+            return out
+        return {
+            "id": request.id,
+            "status": "registered",
+            "http_status": response.status,
+            "exit_code": Exit.OK,
+            "attempts": response.attempts,
+            "response": parsed.body,
+        }
+
+    # ------------------------------------------------------------------ Sciscape extensions
+
+    def _api(self, field: str, deadline: Deadline, capability: str) -> str:
+        index = self.discover(deadline)
+        if index.extra_str("api_schema") != API_SCHEMA:
+            raise unsupported(capability, self.origin, f"api_schema is not {API_SCHEMA}")
+        url = index.extra_url(field)
+        if url is None:
+            raise unsupported(capability, self.origin, f"no {field}")
+        return url
+
+    def submit_multipart(
+        self,
+        field: str,
+        capability: str,
+        parts: list[tuple[str, str | None, bytes, str]],
+        deadline: Deadline,
+    ) -> tuple[Response | None, dict[str, Any]]:
+        from .http import multipart
+
+        url = self._api(field, deadline, capability)
+        body, content_type = multipart(parts)
+        try:
+            response = self.http.request(
+                "POST",
+                url,
+                deadline=deadline,
+                headers={"Content-Type": content_type, "Accept": "application/json"},
+                body=body,
+                paced=True,
+                should_retry=_retryable_error,
+            )
+        except NetworkError as exc:
+            status = "outcome_unknown" if exc.lost_response else "failed"
+            return None, {
+                "status": status,
+                "exit_code": Exit.INCOMPLETE,
+                "error": ErrorRecord(
+                    status if exc.lost_response else "network_error",
+                    "the response was lost; repeat the same request to confirm the result"
+                    if exc.lost_response
+                    else str(exc),
+                    True,
+                ).to_json(),
+            }
+        return response, service_result(response, capability)
+
+    # ------------------------------------------------------------------ catalog
+
+    def catalog(self, deadline: Deadline) -> tuple[str, Index]:
+        index = self.discover(deadline)
+        schema = index.extra_str("catalog_schema")
+        url = index.extra_url("catalog_url")
+        if url is None or schema != CATALOG_SCHEMA:
+            raise unsupported(
+                "catalog search",
+                self.origin,
+                f"requires catalog_url with catalog_schema {CATALOG_SCHEMA} (found {schema!r})",
+            )
+        return url.rstrip("/"), index
+
+    def search(
+        self,
+        deadline: Deadline,
+        *,
+        query: str | None,
+        category: tuple[str, str] | None,
+        domain: str | None,
+        key_x: str | None,
+        after: tuple[str, str] | None,
+        limit: int,
+    ) -> tuple[list[dict[str, Any]], bool, Index]:
+        base, index = self.catalog(deadline)
+        params: list[tuple[str, str]] = [("select", CATALOG_FIELDS)]
+        if query:
+            params.append(("search_text", f"plfts(simple).{query}"))
+        if category is not None:
+            value = json.dumps(
+                [{"scheme": category[0], "code": category[1]}], separators=(",", ":")
+            )
+            params.append(("categories", f"cs.{value}"))
+        if domain is not None:
+            value = json.dumps([{"type": "domain", "id": domain}], separators=(",", ":"))
+            params.append(("admission_accounts", f"cs.{value}"))
+        if key_x is not None:
+            params.append(("public_key->>x", f"eq.{key_x}"))
+        if after is not None:
+            time, last = after
+            params.append(
+                ("or", f"(accepted_at.lt.{time},and(accepted_at.eq.{time},id.lt.{last}))")
+            )
+        params.append(("order", "accepted_at.desc,id.desc"))
+        params.append(("limit", str(limit + 1)))
+        url = f"{base}/publications?" + urllib.parse.urlencode(params, safe="(),.>-:")
+        response = self.http.get(
+            url,
+            deadline=deadline,
+            headers={"Accept": "application/json", "Accept-Profile": CATALOG_SCHEMA},
+            max_bytes=16 << 20,
+        )
+        if response.status != 200:
+            error = status_error(response, "catalog query")
+            detail = _postgrest_message(response)
+            if detail:
+                error.records[0] = ErrorRecord(
+                    error.code, f"{error.records[0].message}: {detail}", error.records[0].retryable
+                )
+            raise error
+        rows = response.json("catalog response")
+        if not isinstance(rows, list):
+            raise CliError(
+                "invalid_response", "catalog response must be a JSON array", Exit.INVALID
+            )
+        results = [_catalog_row(row, index) for row in rows[: limit + 1]]
+        return results[:limit], len(results) > limit, index
+
+
+def _retryable_error(response: Response) -> bool:
+    if response.content_type != "application/json":
+        return False
+    try:
+        record = json.loads(response.body)
+    except ValueError:
+        return False
+    error = record.get("error") if isinstance(record, dict) else None
+    return isinstance(error, dict) and error.get("retryable") is True
+
+
+def _postgrest_message(response: Response) -> str | None:
+    try:
+        record = json.loads(response.body)
+    except ValueError:
+        return None
+    if isinstance(record, dict) and isinstance(record.get("message"), str):
+        return str(record["message"])[:300]
+    return None
+
+
+def service_result(response: Response, capability: str) -> dict[str, Any]:
+    """Interpret a Sciscape custom-write response (success object or error object)."""
+    try:
+        record = response.json_object(f"{capability} response")
+    except CliError as exc:
+        temporary = response.status >= 500 or response.status == 429
+        return {
+            "status": "failed",
+            "http_status": response.status,
+            "exit_code": Exit.INCOMPLETE if temporary else Exit.INVALID,
+            "error": exc.records[0].to_json(),
+        }
+    if 200 <= response.status < 300:
+        return {
+            "status": "ok",
+            "http_status": response.status,
+            "exit_code": Exit.OK,
+            "response": record,
+        }
+    error = record.get("error")
+    if (
+        isinstance(error, dict)
+        and isinstance(error.get("code"), str)
+        and isinstance(error.get("message"), str)
+        and isinstance(error.get("retryable"), bool)
+    ):
+        code, message, retryable = (
+            str(error["code"]),
+            str(error["message"]),
+            bool(error["retryable"]),
+        )
+        exit_code = classify_service_error(response.status, code, retryable)
+        return {
+            "status": "refused" if exit_code == Exit.REFUSED else "failed",
+            "http_status": response.status,
+            "exit_code": exit_code,
+            "error": ErrorRecord(code, message, retryable).to_json(),
+        }
+    temporary = response.status >= 500 or response.status == 429
+    return {
+        "status": "failed",
+        "http_status": response.status,
+        "exit_code": Exit.INCOMPLETE if temporary else Exit.INVALID,
+        "error": ErrorRecord(
+            "invalid_response", f"{capability} returned HTTP {response.status}", temporary
+        ).to_json(),
+    }
+
+
+def _catalog_row(row: JsonValue, index: Index) -> dict[str, Any]:
+    def bad(field: str, value: object) -> CliError:
+        return CliError(
+            "invalid_response",
+            f"catalog row field {field} has an unexpected type",
+            Exit.INVALID,
+            field=f"/{field}",
+            observed=repr(value)[:120],
+        )
+
+    if not isinstance(row, dict):
+        raise bad("", row)
+    digest = row.get("id")
+    if not isinstance(digest, str) or len(digest) != 64:
+        raise bad("id", digest)
+    for name in ("title", "abstract"):
+        if not isinstance(row.get(name), str):
+            raise bad(name, row.get(name))
+    accepted = row.get("accepted_at")
+    if not isinstance(accepted, str):
+        raise bad("accepted_at", accepted)
+    key = row.get("public_key")
+    if not isinstance(key, dict) or not isinstance(key.get("x"), str):
+        raise bad("public_key", key)
+    return {
+        "id": "sha256:" + digest,
+        "title": row["title"],
+        "abstract": row["abstract"],
+        "signer": key,
+        "accepted_at": accepted,
+        "publication_url": index.publication_page(digest),
+    }

pysci/_cli/clients/tsa.py

@@ -0,0 +1,223 @@
+"""RFC 3161 timestamp authority client, trust configuration, and sidecar files."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+
+from cryptography import x509
+
+from pysci.der import frame_values
+from pysci.timestamps import (
+    FREETSA_CRL_URL,
+    FREETSA_URL,
+    TIMESTAMP_CONTENT_TYPE,
+    ProofVerification,
+    RevocationPolicy,
+    TimestampFileVerification,
+    TrustConfig,
+    append_proof,
+    build_timestamp_request,
+    check_response_matches_request,
+    freetsa_ca_certificate,
+    verify_timestamp_file,
+    verify_timestamp_response,
+)
+
+from ..config import TrustSettings
+from ..errors import CliError, ErrorRecord, Exit, issue_record
+from ..storage.files import atomic_write_bytes
+from .http import Deadline, HttpClient, status_error
+
+MAX_RESPONSE = 1 << 20
+
+
+def load_crl(blob: bytes) -> x509.CertificateRevocationList:
+    """Load a PEM or DER CRL through the library's trust loader."""
+    return TrustConfig.from_pem(crls=[blob]).crls[0]
+
+
+MAX_SIDECAR = 16 << 20
+
+__all__ = [
+    "FREETSA_URL",
+    "TsaResult",
+    "append_sidecar",
+    "build_trust",
+    "read_sidecar",
+    "request_proof",
+    "verify_sidecar_bytes",
+]
+
+
+def build_trust(
+    settings: TrustSettings,
+    *,
+    online: bool = False,
+    http: HttpClient | None = None,
+    deadline: Deadline | None = None,
+) -> tuple[TrustConfig, list[str]]:
+    """Trust anchors from configuration; with ``online`` fetch CRLs and require them."""
+    warnings: list[str] = []
+    anchors: list[x509.Certificate] = []
+    if settings.freetsa:
+        anchors.append(freetsa_ca_certificate())
+    for path in settings.anchors:
+        try:
+            anchors.extend(x509.load_pem_x509_certificates(Path(path).expanduser().read_bytes()))
+        except (OSError, ValueError) as exc:
+            raise CliError(
+                "trust_anchor_unreadable",
+                f"timestamp trust anchor {path} could not be loaded: {exc}",
+                Exit.USAGE,
+                file="config.json",
+                field="/timestamp_trust/anchors",
+            ) from None
+    crls = []
+    for path in settings.crls:
+        try:
+            crls.append(load_crl(Path(path).expanduser().read_bytes()))
+        except (OSError, ValueError) as exc:
+            warnings.append(f"CRL {path} could not be loaded: {exc}")
+    policy = RevocationPolicy.CHECK_SUPPLIED_CRLS
+    if online:
+        assert http is not None
+        assert deadline is not None
+        policy = RevocationPolicy.REQUIRE_CURRENT_CRL
+        urls = list(settings.crl_urls)
+        if settings.freetsa:
+            urls.append(FREETSA_CRL_URL)
+        for url in urls:
+            try:
+                response = http.get(url, deadline=deadline, max_bytes=8 << 20)
+                if response.status != 200:
+                    raise status_error(response, f"CRL {url}")
+                crls.append(load_crl(response.body))
+            except (CliError, ValueError) as exc:
+                warnings.append(f"CRL {url} unavailable: {exc}")
+    return TrustConfig(anchors=anchors, crls=crls, revocation_policy=policy), warnings
+
+
+@dataclass(frozen=True, slots=True)
+class TsaResult:
+    response: bytes
+    verification: ProofVerification
+    url: str
+
+
+def request_proof(
+    http: HttpClient,
+    url: str,
+    signature: bytes,
+    trust: TrustConfig,
+    deadline: Deadline,
+) -> TsaResult:
+    """Request a timestamp for SHA-256(signature) and check the reply.
+
+    The response structure, status, imprint, and nonce are required to match
+    the request; trust is reported separately in ``verification``.
+
+    Raises:
+        CliError: (exit 5) the TSA was unavailable or returned an unusable reply.
+    """
+    request = build_timestamp_request(signature)
+    response = http.request(
+        "POST",
+        url,
+        deadline=deadline,
+        headers={"Content-Type": TIMESTAMP_CONTENT_TYPE},
+        body=request.der,
+        max_bytes=MAX_RESPONSE,
+        paced=True,
+    )
+    if response.status != 200:
+        error = status_error(response, f"timestamp authority {url}")
+        error.exit_code = Exit.INCOMPLETE
+        raise error
+    der = response.body
+    framing = frame_values(der, max_value_length=MAX_RESPONSE)
+    if len(framing.values) != 1 or not framing.complete:
+        raise CliError(
+            "timestamp_response_malformed",
+            "timestamp authority reply is not exactly one DER TimeStampResp",
+            Exit.INCOMPLETE,
+            retryable=True,
+            url=url,
+        )
+    issues = check_response_matches_request(der, request)
+    if issues:
+        error = CliError(
+            "timestamp_response_rejected",
+            f"timestamp authority reply does not answer the request: {issues[0].requirement}",
+            Exit.INCOMPLETE,
+            retryable=True,
+            url=url,
+        )
+        error.records.extend(issue_record(i) for i in issues)
+        raise error
+    verification = verify_timestamp_response(der, signature, trust)
+    return TsaResult(der, verification, url)
+
+
+def read_sidecar(path: Path) -> bytes | None:
+    try:
+        size = path.stat().st_size
+    except FileNotFoundError:
+        return None
+    if size > MAX_SIDECAR:
+        raise CliError(
+            "sidecar_too_large",
+            f"{path.name} exceeds the local limit of {MAX_SIDECAR} bytes",
+            Exit.INCOMPLETE,
+            file=str(path),
+        )
+    return path.read_bytes()
+
+
+def append_sidecar(path: Path, response: bytes) -> bool:
+    """Append a DER response to the sidecar atomically; False if it was already present.
+
+    Raises:
+        CliError: (exit 3) the existing sidecar is malformed; it is preserved.
+    """
+    existing = read_sidecar(path) or b""
+    try:
+        combined = append_proof(existing, response)
+    except ValueError as exc:
+        raise CliError(
+            "sidecar_malformed",
+            f"existing timestamp sidecar {path.name} is malformed and was preserved: {exc}",
+            Exit.INVALID,
+            file=str(path),
+        ) from None
+    if combined == existing:
+        return False
+    atomic_write_bytes(path, combined)
+    return True
+
+
+def merge_proofs(existing: bytes, incoming: bytes) -> tuple[bytes, list[ErrorRecord]]:
+    """Union of complete DER values from ``incoming`` appended to ``existing``."""
+    problems: list[ErrorRecord] = []
+    framing = frame_values(incoming, max_value_length=MAX_RESPONSE)
+    combined = existing
+    for offset, length in framing.values:
+        try:
+            combined = append_proof(combined, incoming[offset : offset + length])
+        except ValueError as exc:
+            problems.append(ErrorRecord("sidecar_malformed", str(exc)))
+            return existing, problems
+    if not framing.complete and incoming:
+        problems.append(
+            ErrorRecord(
+                "proof_framing",
+                "retrieved timestamp file has trailing bytes that are not a complete DER value",
+            )
+        )
+    return combined, problems
+
+
+def verify_sidecar_bytes(
+    data: bytes, signature: bytes, trust: TrustConfig
+) -> TimestampFileVerification:
+    return verify_timestamp_file(data, signature, trust)

pysci/_cli/commands/__init__.py

@@ -0,0 +1,44 @@
+"""Command registry, in help order."""
+
+from __future__ import annotations
+
+from ..spec import Command
+from . import (
+    check,
+    claim,
+    folders,
+    index,
+    keygen,
+    license,
+    protocol,
+    publish,
+    pull,
+    read,
+    revoke,
+    search,
+    sign,
+    submit,
+    timestamp,
+    verify,
+)
+
+COMMANDS: tuple[Command, ...] = (
+    search.COMMAND,
+    read.COMMAND,
+    pull.COMMAND,
+    check.COMMAND,
+    license.COMMAND,
+    sign.COMMAND,
+    verify.COMMAND,
+    publish.COMMAND,
+    submit.COMMAND,
+    keygen.COMMAND,
+    claim.COMMAND,
+    timestamp.COMMAND,
+    revoke.COMMAND,
+    index.COMMAND,
+    folders.COMMAND,
+    protocol.COMMAND,
+)
+
+__all__ = ["COMMANDS"]