sci-protocol 0.2.0__py3-none-any.whl

pysci/_cli/config.py

@@ -0,0 +1,233 @@
+"""CLI state directory and ``config.json`` (index origin, key, folders, TSA, trust)."""
+
+from __future__ import annotations
+
+import ipaddress
+import json
+import os
+import urllib.parse
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from pysci.errors import SciError
+from pysci.jsonstrict import parse_json_object
+
+from .errors import CliError, Exit
+from .storage.files import atomic_write_bytes
+
+CONFIG_FILE = "config.json"
+CATALOG_FILE = "catalog.sqlite"
+DEFAULT_KEY = "key.pem"
+
+
+def state_dir(env: Mapping[str, str]) -> Path:
+    """``SCI_HOME`` if set, else ``~/.sci``."""
+    home = env.get("SCI_HOME")
+    if home:
+        return Path(home).expanduser().absolute()
+    return Path(env.get("HOME") or Path.home()).expanduser() / ".sci"
+
+
+def ensure_state_dir(path: Path) -> Path:
+    path.mkdir(mode=0o700, parents=True, exist_ok=True)
+    return path
+
+
+@dataclass(slots=True)
+class TrustSettings:
+    """Timestamp trust configuration.
+
+    ``freetsa`` enables the bundled FreeTSA root; ``anchors`` and ``crls`` are
+    local PEM/DER files; ``crl_urls`` are fetched only for online checks.
+    """
+
+    freetsa: bool = True
+    anchors: list[str] = field(default_factory=list)
+    crls: list[str] = field(default_factory=list)
+    crl_urls: list[str] = field(default_factory=list)
+
+    def to_json(self) -> dict[str, Any]:
+        return {
+            "freetsa": self.freetsa,
+            "anchors": list(self.anchors),
+            "crls": list(self.crls),
+            "crl_urls": list(self.crl_urls),
+        }
+
+
+@dataclass(slots=True)
+class Config:
+    index: str | None = None
+    key: str | None = None
+    folders: list[str] = field(default_factory=list)
+    tsa_url: str | None = None
+    timestamp_trust: TrustSettings = field(default_factory=TrustSettings)
+    other: dict[str, Any] = field(default_factory=dict)
+
+    def to_json(self) -> dict[str, Any]:
+        out: dict[str, Any] = dict(self.other)
+        out.update(
+            {
+                "index": self.index,
+                "key": self.key,
+                "folders": list(self.folders),
+                "tsa_url": self.tsa_url,
+                "timestamp_trust": self.timestamp_trust.to_json(),
+            }
+        )
+        return out
+
+
+def _config_error(field_name: str, requirement: str, observed: object) -> CliError:
+    return CliError(
+        "config_invalid",
+        f"{CONFIG_FILE} field {field_name}: {requirement}",
+        Exit.USAGE,
+        file=CONFIG_FILE,
+        field="/" + field_name.replace(".", "/"),
+        requirement=requirement,
+        observed=repr(observed)[:120],
+    )
+
+
+def _optional_str(record: Mapping[str, Any], name: str) -> str | None:
+    value = record.get(name)
+    if value is not None and not isinstance(value, str):
+        raise _config_error(name, "must be a string or null", value)
+    return value
+
+
+def _str_list(record: Mapping[str, Any], name: str, label: str | None = None) -> list[str]:
+    value = record.get(name, [])
+    if not isinstance(value, list) or not all(isinstance(item, str) for item in value):
+        raise _config_error(label or name, "must be an array of strings", value)
+    return [str(item) for item in value]
+
+
+def load_config(home: Path) -> Config:
+    path = home / CONFIG_FILE
+    try:
+        data = path.read_bytes()
+    except FileNotFoundError:
+        return Config()
+    except OSError as exc:
+        raise CliError("config_unreadable", f"cannot read {path}: {exc}", Exit.USAGE) from None
+    try:
+        record = parse_json_object(data, file=CONFIG_FILE)
+    except SciError as exc:
+        raise CliError(
+            "config_invalid", f"{path} is not a valid JSON object: {exc}", Exit.USAGE
+        ) from None
+    trust_record = record.get("timestamp_trust", {})
+    if not isinstance(trust_record, dict):
+        raise _config_error("timestamp_trust", "must be an object", trust_record)
+    freetsa = trust_record.get("freetsa", True)
+    if not isinstance(freetsa, bool):
+        raise _config_error("timestamp_trust.freetsa", "must be a Boolean", freetsa)
+    trust = TrustSettings(
+        freetsa=freetsa,
+        anchors=_str_list(trust_record, "anchors", "timestamp_trust.anchors"),
+        crls=_str_list(trust_record, "crls", "timestamp_trust.crls"),
+        crl_urls=_str_list(trust_record, "crl_urls", "timestamp_trust.crl_urls"),
+    )
+    config = Config(
+        index=_optional_str(record, "index"),
+        key=_optional_str(record, "key"),
+        folders=_str_list(record, "folders"),
+        tsa_url=_optional_str(record, "tsa_url"),
+        timestamp_trust=trust,
+        other={
+            k: v
+            for k, v in record.items()
+            if k not in ("index", "key", "folders", "tsa_url", "timestamp_trust")
+        },
+    )
+    if config.index is not None:
+        try:
+            config.index = normalize_origin(config.index)
+        except CliError:
+            raise _config_error("index", "must be an HTTPS origin", config.index) from None
+    if config.tsa_url is not None:
+        try:
+            config.tsa_url = normalize_url(config.tsa_url, what="TSA URL")
+        except CliError:
+            raise _config_error(
+                "tsa_url", "must be an absolute HTTPS URL", config.tsa_url
+            ) from None
+    return config
+
+
+def save_config(home: Path, config: Config) -> None:
+    ensure_state_dir(home)
+    data = json.dumps(config.to_json(), indent=2, ensure_ascii=False).encode("utf-8") + b"\n"
+    atomic_write_bytes(home / CONFIG_FILE, data)
+
+
+# --------------------------------------------------------------------------- URLs
+
+
+def is_loopback(host: str) -> bool:
+    if host == "localhost":
+        return True
+    try:
+        return ipaddress.ip_address(host.strip("[]")).is_loopback
+    except ValueError:
+        return False
+
+
+def _split(value: str, what: str) -> urllib.parse.SplitResult:
+    try:
+        parts = urllib.parse.urlsplit(value)
+        _ = parts.port
+    except ValueError:
+        raise CliError("invalid_url", f"{what} is not a valid URL", observed=value) from None
+    if not parts.scheme or not parts.hostname:
+        raise CliError("invalid_url", f"{what} must be an absolute URL", observed=value)
+    scheme = parts.scheme.lower()
+    if scheme == "http" and not is_loopback(parts.hostname):
+        raise CliError(
+            "insecure_url",
+            f"{what} must use HTTPS (plain HTTP is accepted only for loopback test servers)",
+            observed=value,
+        )
+    if scheme not in ("http", "https"):
+        raise CliError("invalid_url", f"{what} must use HTTPS", observed=value)
+    if parts.username is not None or parts.password is not None:
+        raise CliError("invalid_url", f"{what} must not contain credentials", observed=value)
+    return parts
+
+
+def normalize_origin(value: str, *, what: str = "index origin") -> str:
+    """Validate an HTTPS origin and return ``scheme://host[:port]``."""
+    if "://" not in value:
+        value = "https://" + value
+    parts = _split(value, what)
+    if parts.path not in ("", "/") or parts.query or parts.fragment:
+        raise CliError(
+            "invalid_origin",
+            f"{what} must be an origin without path, query, or fragment",
+            observed=value,
+        )
+    assert parts.hostname is not None
+    host = parts.hostname.lower()
+    if ":" in host:
+        host = f"[{host}]"
+    port = f":{parts.port}" if parts.port is not None else ""
+    return f"{parts.scheme.lower()}://{host}{port}"
+
+
+def normalize_url(value: str, *, what: str = "URL") -> str:
+    """Validate an absolute HTTPS URL (fragment not allowed)."""
+    parts = _split(value, what)
+    if parts.fragment:
+        raise CliError("invalid_url", f"{what} must not contain a fragment", observed=value)
+    return value
+
+
+def same_path(a: str | os.PathLike[str], b: str | os.PathLike[str]) -> bool:
+    try:
+        return os.path.samefile(a, b)
+    except OSError:
+        return Path(a).absolute() == Path(b).absolute()

pysci/_cli/context.py

@@ -0,0 +1,79 @@
+"""Per-invocation context: environment, state directory, configuration, I/O, and HTTP."""
+
+from __future__ import annotations
+
+import subprocess
+import sys
+import time
+from collections.abc import Callable, Mapping, Sequence
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import TextIO
+
+from . import __version__
+from .clients.http import DEFAULT_TIMEOUT, Deadline, HttpClient, Pacer, Transport, UrllibTransport
+from .config import Config, load_config, save_config, state_dir
+
+
+def run_command(argv: Sequence[str], timeout: float) -> str | None:
+    """Run a helper command (e.g. ``gh auth token``); return stripped stdout or None."""
+    try:
+        completed = subprocess.run(
+            list(argv), capture_output=True, text=True, timeout=timeout, check=False
+        )
+    except (OSError, subprocess.SubprocessError):
+        return None
+    if completed.returncode != 0:
+        return None
+    return completed.stdout.strip() or None
+
+
+@dataclass
+class Context:
+    env: Mapping[str, str]
+    stdout: TextIO = field(default_factory=lambda: sys.stdout)
+    stderr: TextIO = field(default_factory=lambda: sys.stderr)
+    stdin: TextIO = field(default_factory=lambda: sys.stdin)
+    transport: Transport | None = None
+    sleep: Callable[[float], None] = time.sleep
+    runner: Callable[[Sequence[str], float], str | None] = run_command
+    cwd: Path = field(default_factory=Path.cwd)
+    _config: Config | None = None
+    _http: HttpClient | None = None
+
+    @property
+    def home(self) -> Path:
+        return state_dir(self.env)
+
+    @property
+    def config(self) -> Config:
+        if self._config is None:
+            self._config = load_config(self.home)
+        return self._config
+
+    def save_config(self) -> None:
+        save_config(self.home, self.config)
+
+    @property
+    def http(self) -> HttpClient:
+        if self._http is None:
+            self._http = HttpClient(
+                self.transport or UrllibTransport(),
+                user_agent=f"pysci/{__version__}",
+                pacer=Pacer(self.home / "pace", sleep=self.sleep),
+                sleep=self.sleep,
+            )
+        return self._http
+
+    def deadline(self, seconds: float | None) -> Deadline:
+        return Deadline(DEFAULT_TIMEOUT if seconds is None else seconds)
+
+    def interactive(self) -> bool:
+        try:
+            return bool(self.stdin.isatty())
+        except (AttributeError, ValueError):
+            return False
+
+    def resolve(self, path: str | Path) -> Path:
+        candidate = Path(path).expanduser()
+        return candidate if candidate.is_absolute() else self.cwd / candidate

pysci/_cli/destinations.py

@@ -0,0 +1,163 @@
+"""Publishing destinations: local folders, ``@local``, GitHub, and Hugging Face datasets."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from pathlib import Path
+
+from .config import normalize_url
+from .errors import CliError, Exit
+
+_NAME = r"[A-Za-z0-9][A-Za-z0-9._-]*"
+_GH_OWNER = r"[A-Za-z0-9](?:[A-Za-z0-9-]{0,38})"
+_SUFFIX = r"(?:\.git)?/?"
+
+_PATTERNS: list[tuple[str, re.Pattern[str]]] = [
+    (
+        "github",
+        re.compile(rf"https://github\.com/(?P<owner>{_GH_OWNER})/(?P<repo>{_NAME}?){_SUFFIX}"),
+    ),
+    ("github", re.compile(rf"git@github\.com:(?P<owner>{_GH_OWNER})/(?P<repo>{_NAME}?){_SUFFIX}")),
+    (
+        "github",
+        re.compile(rf"ssh://git@github\.com/(?P<owner>{_GH_OWNER})/(?P<repo>{_NAME}?){_SUFFIX}"),
+    ),
+    (
+        "huggingface",
+        re.compile(
+            rf"https://huggingface\.co/datasets/(?P<owner>{_NAME})/(?P<repo>{_NAME}?){_SUFFIX}"
+        ),
+    ),
+    (
+        "huggingface",
+        re.compile(
+            rf"git@(?:hf\.co|huggingface\.co):datasets/(?P<owner>{_NAME})/(?P<repo>{_NAME}?)"
+            rf"{_SUFFIX}"
+        ),
+    ),
+    (
+        "huggingface",
+        re.compile(
+            rf"ssh://git@(?:hf\.co|huggingface\.co)/datasets/(?P<owner>{_NAME})/"
+            rf"(?P<repo>{_NAME}?){_SUFFIX}"
+        ),
+    ),
+]
+
+_REMOTE_HINT = re.compile(r"^[A-Za-z][A-Za-z0-9+.-]*://|^[^/\s]+@[^/\s]+:")
+
+
+@dataclass(frozen=True, slots=True)
+class Destination:
+    kind: str
+    """``folder``, ``github``, or ``huggingface``."""
+    key: str
+    display: str
+    path: Path | None = None
+    owner: str | None = None
+    repo: str | None = None
+
+    @property
+    def repository(self) -> str:
+        return f"{self.owner}/{self.repo}"
+
+
+def parse_remote(value: str) -> Destination | None:
+    """Recognize a GitHub or Hugging Face repository address; None if not remote-looking.
+
+    Raises:
+        CliError: the value looks like a remote address but is unsupported.
+    """
+    if not _REMOTE_HINT.search(value):
+        return None
+    if "?" in value or "#" in value:
+        raise CliError(
+            "unsupported_destination",
+            f"{value}: repository addresses must not contain a query or fragment",
+            Exit.USAGE,
+        )
+    for kind, pattern in _PATTERNS:
+        match = pattern.fullmatch(value)
+        if match is None:
+            continue
+        owner, repo = match.group("owner"), match.group("repo")
+        if repo.endswith(".git"):
+            repo = repo[: -len(".git")]
+        if not repo or repo in (".", "..") or repo.endswith("."):
+            break
+        if kind == "github":
+            return Destination(
+                "github",
+                f"github:{owner.lower()}/{repo.lower()}",
+                f"https://github.com/{owner}/{repo}",
+                owner=owner,
+                repo=repo,
+            )
+        return Destination(
+            "huggingface",
+            f"huggingface:{owner.lower()}/{repo.lower()}",
+            f"https://huggingface.co/datasets/{owner}/{repo}",
+            owner=owner,
+            repo=repo,
+        )
+    raise CliError(
+        "unsupported_destination",
+        f"{value} is not a supported destination; use a folder, @local, "
+        "https://github.com/OWNER/REPO, or https://huggingface.co/datasets/OWNER/REPO "
+        "(file and branch URLs are not accepted)",
+        Exit.USAGE,
+    )
+
+
+def resolve_destinations(
+    values: list[str], known_folders: list[str], cwd: Path
+) -> list[Destination]:
+    destinations: list[Destination] = []
+    seen: set[str] = set()
+
+    def add(destination: Destination) -> None:
+        if destination.key not in seen:
+            seen.add(destination.key)
+            destinations.append(destination)
+
+    for value in values:
+        if value == "@local":
+            for folder in known_folders:
+                path = Path(folder)
+                add(Destination("folder", f"folder:{path}", str(path), path=path))
+            continue
+        if value.startswith("@"):
+            raise CliError("unsupported_destination", f"unknown destination {value!r}", Exit.USAGE)
+        remote = parse_remote(value)
+        if remote is not None:
+            add(remote)
+            continue
+        path = Path(value).expanduser()
+        path = path if path.is_absolute() else cwd / path
+        canonical = path.resolve() if path.exists() else path.absolute()
+        add(Destination("folder", f"folder:{canonical}", str(canonical), path=canonical))
+    if not destinations:
+        raise CliError(
+            "no_destinations", "no destinations selected (no known folders?)", Exit.USAGE
+        )
+    return destinations
+
+
+def resolve_archive(value: str) -> str:
+    """Expand supported repository addresses; preserve explicit descriptor URLs.
+
+    Publishing layouts are deterministic, so this needs neither credentials nor
+    local upload history. A repository address does not prove files are hosted.
+    """
+    try:
+        remote = parse_remote(value)
+    except CliError:
+        # Other HTTPS paths may be explicit descriptors (including custom layouts
+        # on a supported host); validate them as URLs without rewriting them.
+        remote = None
+    if remote is not None:
+        if remote.kind == "github":
+            return f"https://raw.githubusercontent.com/{remote.repository}/main/sci/archive.json"
+        return f"https://huggingface.co/datasets/{remote.repository}/resolve/main/sci/archive.json"
+    return normalize_url(value, what="archive descriptor URL or supported repository address")

pysci/_cli/errors.py

@@ -0,0 +1,168 @@
+"""CLI errors, exit codes, and conversion of protocol issues into error records."""
+
+from __future__ import annotations
+
+from collections.abc import Iterable, Sequence
+from dataclasses import dataclass
+from dataclasses import field as dc_field
+from enum import IntEnum
+from typing import Any
+
+from pysci.errors import Issue, Outcome, SciError
+
+
+class Exit(IntEnum):
+    """Process exit codes (cli/DESIGN.md, Results)."""
+
+    OK = 0
+    USAGE = 2
+    """Invalid command or missing configuration."""
+    INVALID = 3
+    """Failed protocol, signature, or content check."""
+    UNSUPPORTED = 4
+    """Unsupported schema or service capability."""
+    INCOMPLETE = 5
+    """Incomplete operation, temporary failure, or partial result."""
+    REFUSED = 6
+    """Baseline refusal before signing or service policy refusal."""
+
+
+EXIT_MEANINGS: dict[int, str] = {
+    Exit.OK: "requested operation completed",
+    Exit.USAGE: "invalid command or missing configuration",
+    Exit.INVALID: "failed protocol, signature, or content check",
+    Exit.UNSUPPORTED: "unsupported schema or service capability",
+    Exit.INCOMPLETE: "incomplete operation, temporary failure, or partial result",
+    Exit.REFUSED: "baseline refusal before signing or service policy refusal",
+}
+
+_OUTCOME_EXIT: dict[Outcome, Exit] = {
+    Outcome.VALID: Exit.OK,
+    Outcome.INVALID: Exit.INVALID,
+    Outcome.UNSUPPORTED: Exit.UNSUPPORTED,
+    Outcome.INCOMPLETE: Exit.INCOMPLETE,
+}
+
+# Severity order used to combine several exit codes into one.
+_SEVERITY = [Exit.OK, Exit.INCOMPLETE, Exit.REFUSED, Exit.UNSUPPORTED, Exit.INVALID, Exit.USAGE]
+
+
+def exit_for_outcome(outcome: Outcome) -> Exit:
+    return _OUTCOME_EXIT[outcome]
+
+
+def combine_exits(codes: Iterable[int]) -> Exit:
+    """The most severe exit code; a mix of successes and failures is partial (5)."""
+    values = [Exit(code) for code in codes]
+    if not values:
+        return Exit.OK
+    failures = [code for code in values if code is not Exit.OK]
+    if not failures:
+        return Exit.OK
+    if len(failures) < len(values):
+        return Exit.INCOMPLETE
+    return max(failures, key=_SEVERITY.index)
+
+
+@dataclass(frozen=True, slots=True)
+class ErrorRecord:
+    """One reported error: code, explanation, retryability, and validation location."""
+
+    code: str
+    message: str
+    retryable: bool = False
+    file: str | None = None
+    field: str | None = None
+    requirement: str | None = None
+    observed: str | None = None
+    section: str | None = None
+    extra: dict[str, Any] = dc_field(default_factory=dict)
+
+    def to_json(self) -> dict[str, Any]:
+        out: dict[str, Any] = {
+            "code": self.code,
+            "message": self.message,
+            "retryable": self.retryable,
+        }
+        for name in ("file", "field", "requirement", "observed", "section"):
+            value = getattr(self, name)
+            if value is not None:
+                out[name] = value
+        out.update(self.extra)
+        return out
+
+    def __str__(self) -> str:
+        where = self.file or ""
+        if self.field:
+            where += ("#" if where else "") + self.field
+        text = f"[{self.code}] "
+        if where:
+            text += where + ": "
+        text += self.message
+        if self.section:
+            text += f" (§{self.section})"
+        if self.observed is not None:
+            text += f"; observed {self.observed}"
+        if self.retryable:
+            text += " (retrying may help)"
+        return text
+
+
+def issue_record(issue: Issue) -> ErrorRecord:
+    return ErrorRecord(
+        code=issue.code,
+        message=issue.requirement,
+        retryable=issue.outcome is Outcome.INCOMPLETE,
+        file=issue.file,
+        field=issue.field,
+        requirement=issue.requirement,
+        observed=issue.observed,
+        section=issue.section,
+    )
+
+
+def issue_json(issue: Issue) -> dict[str, Any]:
+    return issue_record(issue).to_json()
+
+
+class CliError(Exception):
+    """A failure reported to the user with an exit code and error records."""
+
+    def __init__(
+        self,
+        code: str,
+        message: str,
+        exit_code: int = Exit.USAGE,
+        *,
+        retryable: bool = False,
+        records: Sequence[ErrorRecord] = (),
+        result: dict[str, Any] | None = None,
+        **extra: Any,
+    ) -> None:
+        super().__init__(message)
+        self.exit_code = Exit(exit_code)
+        head = ErrorRecord(code, message, retryable, extra=dict(extra.items()))
+        self.records: list[ErrorRecord] = [head, *records]
+        self.result = result or {}
+
+    @property
+    def code(self) -> str:
+        return self.records[0].code
+
+    @classmethod
+    def from_sci(cls, exc: SciError, message: str | None = None) -> CliError:
+        records = [issue_record(issue) for issue in exc.issues]
+        exit_code = exit_for_outcome(exc.outcome)
+        first = records[0]
+        error = cls(
+            first.code,
+            message or first.message,
+            exit_code,
+            retryable=exc.outcome is Outcome.INCOMPLETE,
+        )
+        error.records = records if message is None else [error.records[0], *records]
+        return error
+
+
+def usage(code: str, message: str, **extra: Any) -> CliError:
+    return CliError(code, message, Exit.USAGE, **extra)