sci-protocol 0.2.0__py3-none-any.whl

pysci/_cli/commands/submit.py

@@ -0,0 +1,172 @@
+"""``sci submit``: register hosted publications with an indexer (§8.1.4)."""
+
+from __future__ import annotations
+
+import argparse
+from typing import Any
+
+from pysci.accounts import normalize_account
+from pysci.submission import build_submission_request
+
+from ..context import Context
+from ..errors import CliError, ErrorRecord, Exit, combine_exits
+from ..output import Result
+from ..spec import Command, arg
+from ..storage.inputs import collect_signed_inputs, invalid_exit
+from .common import archive_urls, index_client, looks_like_id, parse_id
+
+
+def run(args: argparse.Namespace, ctx: Context) -> Result:
+    archives = archive_urls(args.archive)
+    if not archives:
+        raise CliError(
+            "missing_archive",
+            "submit requires at least one --archive descriptor URL or repository address; "
+            "files must already be hosted",
+        )
+    account = None
+    if args.account:
+        try:
+            account = normalize_account(args.account[0], args.account[1])
+        except ValueError as exc:
+            raise CliError(
+                "invalid_account", f"--account: {exc}", Exit.USAGE, observed=" ".join(args.account)
+            ) from None
+    digests: list[str] = []
+    paths = []
+    for target in args.targets:
+        if looks_like_id(target):
+            digest = parse_id(target)
+            if digest not in digests:
+                digests.append(digest)
+        else:
+            paths.append(ctx.resolve(target))
+    invalid: list[dict[str, Any]] = []
+    if paths:
+        inputs, invalid, _ = collect_signed_inputs(paths)
+        for item in inputs:
+            if item.digest not in digests:
+                digests.append(item.digest)
+    client = index_client(ctx, args.to)
+    result = Result("submit")
+    result.data = {
+        "index": client.origin,
+        "descriptors": archives,
+        "account": account.to_json() if account else None,
+        "invalid_candidates": invalid,
+        "submissions": [],
+    }
+    if not digests:
+        result.exit_code = invalid_exit(invalid) if invalid else Exit.USAGE
+        result.errors.append(ErrorRecord("no_valid_inputs", "no valid publications to submit"))
+        return result
+    client.discover(ctx.deadline(args.timeout))
+    exits: list[int] = []
+    for digest in digests:
+        request = build_submission_request(digest, archives, account)
+        outcome = client.submit(request, ctx.deadline(args.timeout))
+        exits.append(outcome["exit_code"])
+        entry = {k: v for k, v in outcome.items() if k != "exit_code"}
+        entry["id"] = "sha256:" + digest
+        result.data["submissions"].append(entry)
+        _describe(result, entry)
+        if "error" in entry:
+            error = entry["error"]
+            result.errors.append(
+                ErrorRecord(
+                    error["code"],
+                    f"sha256:{digest}: {error['message']}",
+                    error["retryable"],
+                )
+            )
+    if invalid:
+        exits.append(invalid_exit(invalid))
+        for candidate in invalid:
+            result.say(f"not sent (invalid): {candidate['path']} ({candidate['status']})")
+    result.exit_code = combine_exits(exits)
+    result.partial = result.exit_code == Exit.INCOMPLETE and Exit.OK in exits
+    return result
+
+
+def _describe(result: Result, entry: dict[str, Any]) -> None:
+    status = entry["status"]
+    if status == "registered":
+        body = entry.get("response", {})
+        service = body.get("status")
+        line = f"{entry['id']}: registered"
+        if isinstance(service, str):
+            line += f" (service status: {service})"
+        result.say(line)
+        if body.get("publication_url"):
+            result.say(f"    {body['publication_url']}")
+        if body.get("message"):
+            result.say(f"    {body['message']}")
+        if service is None:
+            result.say("    registration confirms lookup only; catalog admission is not implied")
+    else:
+        message = entry.get("error", {}).get("message", "")
+        result.say(f"{entry['id']}: {status} - {message}")
+
+
+COMMAND = Command(
+    name="submit",
+    usage_hint=(
+        "submit ID_OR_PATH ... --archive ARCHIVE [--archive ARCHIVE ...] [--to INDEX] "
+        "[--account TYPE ID] [--json]"
+    ),
+    summary="Register hosted publications with an indexer.",
+    description="""
+Sends one publication submission (SCI protocol §8.1.4) per ID to the index's
+submit_url, naming the archive descriptors that host it. Accepts full IDs
+or local signed inputs (a ZIP, a signed directory, or a folder of *.zip
+bundles; each is verified first and invalid candidates are not sent). Files
+must already be hosted (see `sci publish`); submission uploads nothing.
+
+--archive accepts a descriptor URL or the same GitHub/Hugging Face repository
+address used by publish --to. Repository addresses expand to the descriptor
+created by sci publish; no upload history or provider credentials are needed.
+
+Protocol success confirms location registration only; service fields such as
+Sciscape's status, publication_url, accepted_at, reason, and message are
+reported as returned. Temporary failures are retried with backoff and
+Retry-After; refusals are not retried. A lost response is reported as
+outcome_unknown. Adding or renewing an account claim permits resubmitting an
+unchanged bundle for catalog admission.
+""",
+    args=(
+        arg(
+            "targets",
+            metavar="ID_OR_PATH",
+            nargs="+",
+            help="publication IDs or local signed inputs",
+        ),
+        arg(
+            "--archive",
+            metavar="ARCHIVE",
+            action="append",
+            help="descriptor URL or GitHub/Hugging Face repository address (repeatable)",
+        ),
+        arg("--to", metavar="INDEX", help="index origin (default: configured index)"),
+        arg(
+            "--account",
+            nargs=2,
+            metavar=("TYPE", "ID"),
+            help="unsigned account hint (domain, github, or huggingface)",
+        ),
+    ),
+    outputs="One result per ID (registered, refused, failed, or outcome_unknown) with the "
+    "service response.",
+    files="None.",
+    network="Index descriptor GET, then one paced POST per publication (at most one per second).",
+    exit_codes={
+        3: "a local input failed verification, or the index rejected the content",
+        4: "the index has no submit_url or rejected an unsupported schema",
+        5: "a temporary failure, outcome_unknown, or a mix of successes and failures",
+        6: "the index refused a submission by policy (e.g. not_baseline)",
+    },
+    examples=(
+        "sci submit ./campaign --to https://sciscape.io --archive https://github.com/lab/research",
+    ),
+    run=run,
+    timeout=True,
+)

pysci/_cli/commands/timestamp.py

@@ -0,0 +1,196 @@
+"""``sci timestamp``: obtain or submit timestamp proofs."""
+
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+
+from pysci.bundle import Scope, verify_directory, verify_zip
+from pysci.manifest import TIMESTAMP_SIDECAR_PATH
+from pysci.records import verify_revocation
+
+from ..checks import proof_json
+from ..clients.tsa import read_sidecar, verify_sidecar_bytes
+from ..context import Context
+from ..errors import CliError, ErrorRecord, Exit, exit_for_outcome, issue_record
+from ..output import Result
+from ..spec import Command, arg
+from ..storage.inputs import find_sidecar, sidecar_target
+from .common import index_client, trust
+from .revoke import RECORD, SIDECAR, SIGNATURE, submit_revocation
+from .sign import timestamp_step, tsa_url
+from .verify import signature_bytes
+
+
+def _target(path: Path) -> tuple[str, bytes, Path, str | None, Result | None]:
+    """(kind, signature, sidecar path, publication digest, failure result)."""
+    if path.is_dir() and (path / RECORD).exists():
+        record = (path / RECORD).read_bytes()
+        signature = (path / SIGNATURE).read_bytes() if (path / SIGNATURE).exists() else b""
+        check = verify_revocation(record, signature)
+        if not check.ok:
+            failed = Result("timestamp", exit_code=exit_for_outcome(check.outcome))
+            failed.errors.extend(issue_record(i) for i in check.issues)
+            failed.say("the revocation record failed verification (§6.7); no proof requested")
+            return "revocation", b"", path / SIDECAR, None, failed
+        return "revocation", signature, path / SIDECAR, None, None
+    if path.is_dir():
+        kind = "directory"
+        verification = verify_directory(path, scope=Scope.MANIFEST)
+    else:
+        kind = "zip"
+        verification = verify_zip(path, scope=Scope.MANIFEST)
+    if not verification.ok or verification.manifest is None:
+        failed = Result("timestamp", exit_code=exit_for_outcome(verification.outcome))
+        failed.errors.extend(issue_record(i) for i in verification.issues)
+        failed.say("manifest checks failed (§6.1); no proof requested")
+        return kind, b"", path, None, failed
+    digest = verification.manifest.digest
+    return kind, signature_bytes(path, kind), sidecar_target(path, kind, digest), digest, None
+
+
+def submit(args: argparse.Namespace, ctx: Context, path: Path) -> Result:
+    if path.is_dir() and (path / RECORD).exists():
+        result = submit_revocation(ctx, path, args.to, args.timeout)
+        result.command = "timestamp"
+        return result
+    kind, signature, _, digest, failed = _target(path)
+    if failed is not None:
+        return failed
+    assert digest is not None
+    sidecar = find_sidecar(path, kind, digest)
+    data = read_sidecar(sidecar) if sidecar else None
+    if not data:
+        raise CliError(
+            "no_proofs",
+            f"no timestamp sidecar found for {path}; obtain one with `sci timestamp {path}`",
+            Exit.USAGE,
+        )
+    local = verify_sidecar_bytes(data, signature, trust(ctx))
+    client = index_client(ctx, args.to)
+    _, outcome = client.submit_multipart(
+        "timestamp_submit_url",
+        "timestamp submission",
+        [
+            ("id", None, digest.encode("ascii"), "text/plain"),
+            (TIMESTAMP_SIDECAR_PATH, TIMESTAMP_SIDECAR_PATH, data, "application/octet-stream"),
+        ],
+        ctx.deadline(args.timeout),
+    )
+    if outcome["exit_code"] == Exit.OK:
+        body = outcome["response"]
+        count = body.get("retained_count")
+        if body.get("id") != digest or not isinstance(count, int) or isinstance(count, bool):
+            outcome = {
+                "status": "failed",
+                "exit_code": Exit.INVALID,
+                "error": ErrorRecord(
+                    "invalid_response",
+                    "timestamp submission response needs the submitted id and an integer "
+                    "retained_count",
+                ).to_json(),
+                "response": body,
+            }
+    result = Result("timestamp", exit_code=outcome["exit_code"])
+    result.data = {
+        "mode": "submit",
+        "publication_id": "sha256:" + digest,
+        "index": client.origin,
+        "sidecar": str(sidecar),
+        "local_checks": {
+            "status": str(local.outcome),
+            "proofs": [proof_json(p) for p in local.proofs],
+        },
+        **{k: v for k, v in outcome.items() if k != "exit_code"},
+    }
+    if "error" in outcome:
+        error = outcome["error"]
+        result.errors.append(ErrorRecord(error["code"], error["message"], error["retryable"]))
+        result.say(f"timestamp submission failed: {error['message']}")
+    else:
+        body = outcome["response"]
+        result.say(
+            f"submitted {local.proofs.__len__()} proof(s) for sha256:{digest} to {client.origin}; "
+            f"retained: {body['retained_count']}"
+        )
+    return result
+
+
+def run(args: argparse.Namespace, ctx: Context) -> Result:
+    if args.submit and args.tsa:
+        raise CliError("conflicting_options", "--submit cannot be combined with --tsa")
+    if args.to and not args.submit:
+        raise CliError("invalid_arguments", "--to applies only to --submit")
+    path = ctx.resolve(args.path)
+    if not path.exists():
+        raise CliError("input_missing", f"{path} does not exist", Exit.USAGE, file=str(path))
+    if args.submit:
+        return submit(args, ctx, path)
+    kind, signature, sidecar, digest, failed = _target(path)
+    if failed is not None:
+        return failed
+    url = tsa_url(ctx, args.tsa)
+    result = Result("timestamp")
+    result.data = {
+        "mode": "create",
+        "target": kind,
+        "path": str(path),
+        "publication_id": None if digest is None else "sha256:" + digest,
+    }
+    ok = timestamp_step(ctx, result, sidecar, signature, url, args.timeout)
+    ts = result.data["timestamp"]
+    if not ok:
+        result.exit_code = Exit.INCOMPLETE if ts["error"]["code"] != "sidecar_malformed" else 3
+        result.say(f"timestamp failed: {ts['error']['message']}")
+        return result
+    result.say(f"proof {ts['status']}: {ts['sidecar']}")
+    proof = ts["proof"]
+    result.say(f"genTime: {proof.get('gen_time')}  TSA: {proof.get('tsa') or url}")
+    result.say(
+        f"trust: {proof['result']}"
+        + (f" ({proof['issues'][0]['message']})" if proof["issues"] else "")
+    )
+    return result
+
+
+COMMAND = Command(
+    name="timestamp",
+    usage_hint="timestamp PATH [--tsa URL | --submit [--to URL]] [--json]",
+    summary="Obtain or submit timestamp proofs.",
+    description="""
+Proof creation: PATH is a signed publication directory, a ZIP, or a
+revocation directory. The target signature is verified first (§6.1 or §6.7),
+then an RFC 3161 proof over SHA-256 of the signature bytes is requested from
+the configured TSA (default FreeTSA) or --tsa URL. The reply's structure,
+status, imprint, and nonce are checked; trust checks are reported
+separately. The complete DER reply is appended without duplicates to
+manifest.json.sig.tsr, <digest>.tsr beside a ZIP, or revocation.json.sig.tsr,
+written atomically. A malformed existing sidecar is preserved and the
+command stops.
+
+--submit sends existing publication proofs to the index's timestamp
+submission endpoint (Sciscape `timestamp_submit_url`); the publication must be
+admitted to its catalog. For a revocation directory, --submit sends the proofs
+with the revocation record (as `sci revoke --submit`).
+""",
+    args=(
+        arg("path", metavar="PATH", help="signed directory, ZIP, or revocation directory"),
+        arg(
+            "--tsa", metavar="URL", help="timestamp authority (default: config tsa_url or FreeTSA)"
+        ),
+        arg("--submit", action="store_true", help="submit existing proofs to an index"),
+        arg("--to", metavar="URL", help="index origin for --submit (default: configured index)"),
+    ),
+    outputs="The proof's genTime, TSA, and trust result, or the index's retention result.",
+    files="Creates or appends the timestamp sidecar next to PATH; --submit writes none.",
+    network="One paced request to the TSA, or one paced POST to the index with --submit.",
+    exit_codes={
+        3: "the target fails verification, or the existing sidecar is malformed",
+        4: "the index does not support timestamp submission",
+        5: "the TSA or index request did not complete",
+        6: "the index refused the proofs by policy",
+    },
+    examples=("sci timestamp ./draft", "sci timestamp ./campaign/<digest>.zip --submit"),
+    run=run,
+    timeout=True,
+)

pysci/_cli/commands/verify.py

@@ -0,0 +1,163 @@
+"""``sci verify``: verify a signed directory or ZIP."""
+
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+from typing import Any
+
+from pysci.bundle import VerificationResult
+from pysci.manifest import SIGNATURE_PATH
+from pysci.source import FileSource
+
+from ..checks import (
+    baseline_check,
+    bundle_check,
+    checks,
+    describe_checks,
+    issue_lines,
+    timestamps_check,
+)
+from ..clients.tsa import read_sidecar, verify_sidecar_bytes
+from ..context import Context
+from ..errors import CliError, ErrorRecord, Exit, exit_for_outcome, issue_record
+from ..output import Result
+from ..spec import Command, arg
+from ..storage.inputs import find_sidecar, verify_input, zip_manifest_and_signature
+from ..storage.keys import load_public_key_file
+from .common import trust_with_warnings
+
+
+def signature_bytes(path: Path, kind: str) -> bytes:
+    if kind == "directory":
+        return (path / SIGNATURE_PATH).read_bytes()
+    with FileSource(path) as source:
+        return zip_manifest_and_signature(source)[1]
+
+
+def check_timestamps(
+    ctx: Context,
+    path: Path,
+    kind: str,
+    result: VerificationResult,
+    *,
+    online: bool,
+    timeout: float | None,
+    warnings: list[str],
+) -> dict[str, Any]:
+    if result.manifest is None:
+        return {
+            "status": "not_performed",
+            "reason": "timestamp checks require completed manifest checks (§6.8)",
+        }
+    sidecar = find_sidecar(path, kind, result.manifest.digest)
+    if sidecar is None:
+        return timestamps_check(None, source=None, online=online)
+    data = read_sidecar(sidecar) or b""
+    deadline = ctx.deadline(timeout) if online else None
+    trust_config, trust_warnings = trust_with_warnings(ctx, online=online, deadline=deadline)
+    warnings.extend(trust_warnings)
+    verification = verify_sidecar_bytes(data, signature_bytes(path, kind), trust_config)
+    return timestamps_check(verification, source=str(sidecar), online=online)
+
+
+def run(args: argparse.Namespace, ctx: Context) -> Result:
+    path = ctx.resolve(args.path)
+    if not path.exists():
+        raise CliError("input_missing", f"{path} does not exist", Exit.USAGE, file=str(path))
+    expected = None
+    if args.public_key:
+        expected = load_public_key_file(ctx.resolve(args.public_key))
+    kind, verification = verify_input(path)
+    result = Result("verify")
+    bundle = bundle_check(verification)
+    exit_code = exit_for_outcome(verification.outcome)
+    key_check: dict[str, Any] | None = None
+    if expected is not None:
+        if verification.manifest is None:
+            key_check = {"status": "not_performed"}
+        else:
+            match = verification.manifest.public_key == expected
+            key_check = {"status": "match" if match else "mismatch", "expected": expected.x}
+            if not match:
+                exit_code = Exit.INVALID
+                result.errors.append(
+                    ErrorRecord(
+                        "public_key_mismatch",
+                        "signer does not match --public-key",
+                        file="manifest.json",
+                        field="/public_key/x",
+                        requirement=f"x = {expected.x}",
+                        observed=verification.manifest.public_key.x,
+                    )
+                )
+    timestamps = check_timestamps(
+        ctx,
+        path,
+        kind,
+        verification,
+        online=args.check_timestamp_online,
+        timeout=args.timeout,
+        warnings=result.warnings,
+    )
+    report = checks(bundle=bundle, baseline=baseline_check(verification), timestamps=timestamps)
+    result.data = {
+        "path": str(path),
+        "kind": kind,
+        "publication_id": verification.publication_id,
+        "signer": verification.manifest.public_key.to_jwk() if verification.manifest else None,
+        "public_key_check": key_check,
+        "checks": report,
+    }
+    result.errors.extend(issue_record(i) for i in verification.issues)
+    result.exit_code = exit_code
+    if verification.publication_id:
+        result.say(f"publication: {verification.publication_id}")
+    if verification.manifest is not None:
+        result.say(f"signer x:    {verification.manifest.public_key.x}")
+    if key_check is not None:
+        result.say(f"public key:  {key_check['status']}")
+    result.say("checks:")
+    result.lines.extend(describe_checks(report))
+    if bundle["issues"]:
+        result.lines.extend(issue_lines(bundle["issues"], "    "))
+    result.say("Offline verification does not establish current account associations or the")
+    result.say("absence of key revocation.")
+    return result
+
+
+COMMAND = Command(
+    name="verify",
+    usage_hint="verify PATH [--public-key PATH] [--check-timestamp-online] [--json]",
+    summary="Verify a signed directory or ZIP.",
+    description="""
+Runs the complete bundle checks of SCI protocol §6.4 (ZIP layout, manifest,
+signature, metadata, every file hash, and inventory). Baseline eligibility
+is reported separately; valid non-baseline bundles pass. Timestamp proofs in
+manifest.json.sig.tsr (directories) or <digest>.tsr (beside a ZIP) are
+checked under §6.8 and reported separately; a failed proof never
+invalidates the publication. Associations and revocation are reported as
+not performed.
+""",
+    args=(
+        arg("path", metavar="PATH", help="signed directory or ZIP (any file name)"),
+        arg("--public-key", metavar="PATH", help="require the signer to match this public PEM key"),
+        arg(
+            "--check-timestamp-online",
+            action="store_true",
+            help="fetch current CRLs for timestamp trust checks (otherwise offline)",
+        ),
+    ),
+    outputs="""Publication ID, signer, and the checks object: bundle, baseline, timestamps,
+associations, revocation (each with status; skipped checks are not_performed).""",
+    files="None.",
+    network="None, unless --check-timestamp-online fetches CRLs (FreeTSA and configured URLs).",
+    exit_codes={
+        3: "a protocol, signature, or content check failed, or --public-key does not match",
+        4: "an unsupported schema",
+        5: "a check could not be completed (unreadable input or local limit)",
+    },
+    examples=("sci verify ./campaign/<digest>.zip --public-key ~/.sci/key.pub.pem",),
+    run=run,
+    timeout=True,
+)