reef-vcs 0.4.0__py3-none-any.whl

reef/__init__.py

@@ -0,0 +1,70 @@
+"""reef-vcs: the Python client of the Reef daemon (R2.06, ADR 0054).
+
+Reef gives each agent its own workspace: a copy-on-write copy of a git or jj repository, leased
+to one process and snapshotted after every write. This package talks to a running daemon (it
+never starts one), over its socket or a remote listener::
+
+    import reef
+
+    with reef.connect() as client:
+        repo = client.register_repository(path=".").repository
+        with client.workspace(repo.id, purpose="fix the parser") as ws:
+            (ws.path / "parser.py").write_text("...")
+
+Every RPC of ``reef.api.v1`` is a method on :class:`Client` (and, awaited, on
+``reef.aio.AsyncClient``), generated from the schema; messages and enums are importable from
+``reef.api.v1``. Errors are :class:`ReefError` subclasses by kind, with the daemon's ``hint`` and
+``next``.
+"""
+
+from reef import _env  # noqa: F401  # before grpc is imported
+from reef._build import API_REVISION, COMMIT, VERSION
+from reef.client import Client, connect
+from reef.errors import (
+    AlreadyExistsError,
+    ConfigError,
+    ConflictError,
+    DaemonNotRunningError,
+    ErrorKind,
+    IncompatibleError,
+    InternalError,
+    InvalidArgumentError,
+    IoError,
+    NotFoundError,
+    PermissionDeniedError,
+    ReefError,
+    RemoteConnectError,
+    RemoteMisdirectedError,
+    RemoteTlsError,
+    UnavailableError,
+    UnsupportedError,
+)
+from reef.workspace import WorkspaceHandle
+
+__version__ = VERSION
+
+__all__ = [
+    "API_REVISION",
+    "COMMIT",
+    "AlreadyExistsError",
+    "Client",
+    "ConfigError",
+    "ConflictError",
+    "DaemonNotRunningError",
+    "ErrorKind",
+    "IncompatibleError",
+    "InternalError",
+    "InvalidArgumentError",
+    "IoError",
+    "NotFoundError",
+    "PermissionDeniedError",
+    "ReefError",
+    "RemoteConnectError",
+    "RemoteMisdirectedError",
+    "RemoteTlsError",
+    "UnavailableError",
+    "UnsupportedError",
+    "WorkspaceHandle",
+    "__version__",
+    "connect",
+]

reef/_build.py

@@ -0,0 +1,6 @@
+"""What this build of reef-vcs was generated from. Generated; do not edit."""
+
+VERSION = "0.4.0"
+CARGO_VERSION = "0.4.0"
+API_REVISION = 14
+COMMIT = "aa27f4011a17"

reef/_calls.py

@@ -0,0 +1,174 @@
+"""What the generated methods call: building a request, and the clients' call primitives."""
+
+from __future__ import annotations
+
+import datetime as dt
+from collections.abc import AsyncIterator, Iterable, Iterator, Mapping
+from typing import TYPE_CHECKING, Any, TypeVar
+
+from google.protobuf.descriptor import FieldDescriptor
+from google.protobuf.duration_pb2 import Duration
+from google.protobuf.message import Message
+from google.protobuf.timestamp_pb2 import Timestamp
+
+from reef._rpc import Rpc
+
+if TYPE_CHECKING:
+    from reef._stream import Stream
+    from reef.aio._stream import AsyncStream
+
+Req = TypeVar("Req", bound=Message)
+Resp = TypeVar("Resp", bound=Message)
+
+
+def method_name(rpc: Rpc[Any, Any]) -> str:
+    """The RPC's method name on the clients."""
+    out = []
+    for index, char in enumerate(rpc.name):
+        if char.isupper() and index:
+            out.append("_")
+        out.append(char.lower())
+    return "".join(out)
+
+
+def build(rpc: Rpc[Req, Any], request: Req | None, fields: Mapping[str, Any]) -> Req:
+    """The request: ``request`` as given, or a new one with the keyword fields that were given.
+
+    Raises:
+        TypeError: Both a request and keyword fields, or a request of the wrong type.
+        ValueError: An enum given by a name it does not have.
+    """
+    given = {name: value for name, value in fields.items() if value is not None}
+    if request is not None:
+        if given:
+            raise TypeError(
+                f"{method_name(rpc)}() takes a request or keyword fields, not both "
+                f"(got a request and {', '.join(sorted(given))})"
+            )
+        if not isinstance(request, rpc.request):
+            raise TypeError(
+                f"{method_name(rpc)}() takes a {rpc.request.__name__}, "
+                f"not a {type(request).__name__}"
+            )
+        return request
+    message = rpc.request()
+    descriptor = message.DESCRIPTOR
+    for name, value in given.items():
+        _set(message, descriptor.fields_by_name[name], value)
+    return message
+
+
+def _is_map(field: Any) -> bool:
+    return (
+        field.type == FieldDescriptor.TYPE_MESSAGE
+        and field.message_type is not None
+        and field.message_type.GetOptions().map_entry
+    )
+
+
+def _convert(field: Any, value: Any) -> Any:
+    """One value as the message field takes it."""
+    if field.type == FieldDescriptor.TYPE_ENUM and isinstance(value, str):
+        enum = field.enum_type
+        found = enum.values_by_name.get(value)
+        if found is None:
+            names = ", ".join(enum.values_by_name)
+            raise ValueError(
+                f"{field.name}: {value!r} is not one of {enum.name}'s values ({names})"
+            )
+        return found.number
+    if field.type == FieldDescriptor.TYPE_MESSAGE:
+        full = field.message_type.full_name
+        if full == "google.protobuf.Timestamp" and isinstance(value, dt.datetime):
+            stamp = Timestamp()
+            stamp.FromDatetime(value)
+            return stamp
+        if full == "google.protobuf.Duration" and isinstance(value, dt.timedelta):
+            duration = Duration()
+            duration.FromTimedelta(value)
+            return duration
+    return value
+
+
+def _set(message: Message, field: Any, value: Any) -> None:
+    name = field.name
+    if _is_map(field):
+        target = getattr(message, name)
+        value_field = field.message_type.fields_by_name["value"]
+        for key, item in dict(value).items():
+            converted = _convert(value_field, item)
+            if value_field.type == FieldDescriptor.TYPE_MESSAGE:
+                target[key].CopyFrom(converted)
+            else:
+                target[key] = converted
+        return
+    if field.is_repeated:
+        if isinstance(value, (str, bytes)):
+            raise TypeError(f"{name} takes a list, not a single {type(value).__name__}")
+        getattr(message, name).extend(_convert(field, item) for item in value)
+        return
+    converted = _convert(field, value)
+    if field.type == FieldDescriptor.TYPE_MESSAGE:
+        target = getattr(message, name)
+        target.SetInParent()
+        target.CopyFrom(converted)
+    else:
+        setattr(message, name, converted)
+
+
+class SyncCalls:
+    """The sync client's call primitives, which the generated methods use."""
+
+    def _unary(self, rpc: Rpc[Req, Resp], request: Req) -> Resp:
+        raise NotImplementedError
+
+    def _stream(self, rpc: Rpc[Req, Resp], request: Req) -> Stream[Resp]:
+        raise NotImplementedError
+
+    def _pages(self, rpc: Rpc[Req, Resp], request: Req) -> Iterator[Resp]:
+        """Every page, lazily: the same request with each ``next_page_token``."""
+        page_request = rpc.request()
+        page_request.CopyFrom(request)
+        while True:
+            page = self._unary(rpc, page_request)
+            yield page
+            token: str = page.next_page_token  # type: ignore[attr-defined]
+            if not token:
+                return
+            page_request.page_token = token  # type: ignore[attr-defined]
+
+    def _items(self, rpc: Rpc[Req, Resp], request: Req, field: str) -> Iterator[Any]:
+        """Every page's ``field``, item by item."""
+        for page in self._pages(rpc, request):
+            yield from getattr(page, field)
+
+
+class AsyncCalls:
+    """The asyncio client's call primitives, which the generated methods use."""
+
+    async def _unary(self, rpc: Rpc[Req, Resp], request: Req) -> Resp:
+        raise NotImplementedError
+
+    def _stream(self, rpc: Rpc[Req, Resp], request: Req) -> AsyncStream[Resp]:
+        raise NotImplementedError
+
+    async def _pages(self, rpc: Rpc[Req, Resp], request: Req) -> AsyncIterator[Resp]:
+        """Every page, lazily: the same request with each ``next_page_token``."""
+        page_request = rpc.request()
+        page_request.CopyFrom(request)
+        while True:
+            page = await self._unary(rpc, page_request)
+            yield page
+            token: str = page.next_page_token  # type: ignore[attr-defined]
+            if not token:
+                return
+            page_request.page_token = token  # type: ignore[attr-defined]
+
+    async def _items(self, rpc: Rpc[Req, Resp], request: Req, field: str) -> AsyncIterator[Any]:
+        """Every page's ``field``, item by item."""
+        async for page in self._pages(rpc, request):
+            for item in getattr(page, field):
+                yield item
+
+
+__all__ = ["AsyncCalls", "Iterable", "SyncCalls", "build"]

reef/_channel.py

@@ -0,0 +1,233 @@
+"""How a client reaches the daemon: its socket, or a remote listener (R2.08), as grpc channels.
+
+The socket's channel sets ``:authority`` to ``localhost``: grpc's default, the percent-encoded
+socket path, is refused by the daemon's HTTP/2 server. A remote daemon is checked before grpc
+connects, by a handshake of the SDK's own (``_tls``): against the pin, or against the same roots
+grpc will trust. grpc then trusts only that.
+"""
+
+from __future__ import annotations
+
+import datetime as dt
+import os
+import ssl
+from dataclasses import dataclass, field
+from typing import Any
+
+import grpc
+
+from reef import _tls
+from reef._remote import RemoteTarget
+from reef.errors import RemoteConnectError, RemoteTlsError
+
+MAX_MESSAGE_BYTES = 4 << 20
+"""The largest message a daemon decodes or sends: tonic's default, which the daemon keeps."""
+
+KEEPALIVE_MS = 30_000
+
+ROOTS_ENV = "GRPC_DEFAULT_SSL_ROOTS_FILE_PATH"
+
+
+@dataclass
+class Spec:
+    """Everything needed to open a channel to one daemon."""
+
+    target: str
+    """grpc's target: ``unix:<path>`` or ``<host>:<port>``."""
+    label: str
+    """How errors name it: the socket path, or the remote's origin."""
+    options: list[tuple[str, Any]]
+    credentials: grpc.ChannelCredentials | None = None
+    remote: RemoteTarget | None = None
+    leaf: _tls.Leaf | None = field(default=None, repr=False)
+    """The pinned remote's certificate grpc trusts."""
+    root_certificates: bytes | None = field(default=None, repr=False)
+
+    def open(self) -> grpc.Channel:
+        """A sync channel."""
+        if self.credentials is None:
+            return grpc.insecure_channel(self.target, options=self.options)
+        return grpc.secure_channel(self.target, self.credentials, options=self.options)
+
+    def open_aio(self) -> grpc.aio.Channel:
+        """An asyncio channel, on the running loop."""
+        if self.credentials is None:
+            return grpc.aio.insecure_channel(self.target, options=self.options)
+        return grpc.aio.secure_channel(self.target, self.credentials, options=self.options)
+
+
+def _limits() -> list[tuple[str, Any]]:
+    return [
+        ("grpc.max_send_message_length", MAX_MESSAGE_BYTES),
+        ("grpc.max_receive_message_length", MAX_MESSAGE_BYTES),
+    ]
+
+
+def local(socket: str) -> Spec:
+    """The daemon's socket."""
+    return Spec(
+        target=f"unix:{socket}",
+        label=socket,
+        options=[("grpc.default_authority", "localhost"), *_limits()],
+    )
+
+
+def _grpc_roots() -> bytes:
+    """The roots grpc trusts by default: its environment override, else its bundled file."""
+    path = os.environ.get(ROOTS_ENV) or os.path.join(
+        os.path.dirname(grpc.__file__), "_cython", "_credentials", "roots.pem"
+    )
+    with open(path, "rb") as file:
+        return file.read()
+
+
+def _presented(url_host: str, der: bytes) -> str:
+    del url_host
+    try:
+        return _tls.parse(der).fingerprint
+    except _tls.DerError:
+        return "an unparsable certificate"
+
+
+def remote(
+    target: RemoteTarget,
+    *,
+    connect_timeout: float,
+    root_certificates: bytes | None = None,
+) -> Spec:
+    """A remote daemon, verified before anything is sent.
+
+    Raises:
+        RemoteConnectError: Nothing answered.
+        RemoteTlsError: Its certificate was refused, or the handshake failed.
+    """
+    url = target.url
+    origin = url.origin
+    try:
+        der = _tls.fetch_leaf(url, connect_timeout)
+    except ssl.SSLError as error:
+        raise RemoteTlsError(origin, "handshake", reason=str(error)) from error
+    except OSError as error:
+        raise RemoteConnectError(origin, str(error)) from error
+    options: list[tuple[str, Any]] = [
+        ("grpc.default_authority", url.authority),
+        ("grpc.keepalive_time_ms", KEEPALIVE_MS),
+        ("grpc.keepalive_permit_without_calls", 1),
+        ("grpc.http2.max_pings_without_data", 0),
+        *_limits(),
+    ]
+    call_credentials = grpc.access_token_call_credentials(target.token)
+    if target.fingerprint:
+        try:
+            leaf = _tls.parse(der)
+        except _tls.DerError as error:
+            raise RemoteTlsError(origin, "handshake", reason=f"its certificate: {error}") from error
+        if leaf.fingerprint != target.fingerprint:
+            raise RemoteTlsError(
+                origin,
+                "pin_mismatch",
+                presented=leaf.fingerprint,
+                expected=target.fingerprint,
+            )
+        check_dates(leaf, origin, dt.datetime.now(dt.UTC))
+        # grpc checks the certificate's names, and a call's :authority against them, even with
+        # the key pinned. The pin is what authenticates the daemon, so when the URL's host is not
+        # one of the names (an IP address the certificate does not list, say), the SDK names one
+        # that is, as the daemon's own Host check accepts.
+        name = _tls.target_name(leaf, url.host)
+        if name is not None:
+            options.append(("grpc.ssl_target_name_override", name))
+            if name != url.host:
+                options[0] = ("grpc.default_authority", _authority(name, url.port))
+        credentials = grpc.composite_channel_credentials(
+            grpc.ssl_channel_credentials(root_certificates=leaf.pem), call_credentials
+        )
+        return Spec(
+            target=url.authority,
+            label=origin,
+            options=options,
+            credentials=credentials,
+            remote=target,
+            leaf=leaf,
+        )
+    roots = root_certificates or _grpc_roots()
+    context = ssl.create_default_context(cadata=roots.decode("ascii", "replace"))
+    try:
+        context.check_hostname = True
+        import socket
+
+        with (
+            socket.create_connection((url.host, url.port), timeout=connect_timeout) as raw,
+            context.wrap_socket(raw, server_hostname=url.host),
+        ):
+            pass
+    except ssl.SSLCertVerificationError as error:
+        raise RemoteTlsError(
+            origin,
+            "untrusted",
+            presented=_presented(url.host, der),
+            reason=error.verify_message or str(error),
+        ) from error
+    except ssl.SSLError as error:
+        raise RemoteTlsError(origin, "handshake", reason=str(error)) from error
+    except OSError as error:
+        raise RemoteConnectError(origin, str(error)) from error
+    credentials = grpc.composite_channel_credentials(
+        grpc.ssl_channel_credentials(root_certificates=root_certificates), call_credentials
+    )
+    return Spec(
+        target=url.authority,
+        label=origin,
+        options=options,
+        credentials=credentials,
+        remote=target,
+        root_certificates=root_certificates,
+    )
+
+
+def _authority(host: str, port: int) -> str:
+    return f"[{host}]:{port}" if ":" in host else f"{host}:{port}"
+
+
+def check_dates(leaf: _tls.Leaf, origin: str, now: dt.datetime) -> None:
+    """Raises :class:`RemoteTlsError` (``handshake``) for a pinned certificate outside its dates.
+
+    grpc checks the dates even with the key pinned, which the pin does not need; a lapsed
+    certificate would otherwise fail inside grpc as a connection that never comes up.
+    """
+    if leaf.not_after is not None and now > leaf.not_after:
+        raise RemoteTlsError(
+            origin,
+            "handshake",
+            reason=f"the daemon's certificate expired at {leaf.not_after:%Y-%m-%d %H:%M} UTC",
+            hint=(
+                "The daemon's certificate has lapsed. A generated one is renewed when the "
+                "daemon starts: restart it on its machine. Renew a supplied one "
+                "(`[remote] tls_cert`)."
+            ),
+        )
+    if leaf.not_before is not None and now < leaf.not_before:
+        raise RemoteTlsError(
+            origin,
+            "handshake",
+            reason=f"the daemon's certificate is valid only from "
+            f"{leaf.not_before:%Y-%m-%d %H:%M} UTC; check this machine's clock",
+        )
+
+
+def repinned(spec: Spec, connect_timeout: float) -> Spec | None:
+    """For a pinned remote whose connection failed: a new spec if the daemon now presents a new
+    certificate for the pinned key (a renewal), else None.
+
+    Raises:
+        RemoteTlsError: It now presents another key.
+    """
+    if spec.remote is None or spec.leaf is None or not spec.remote.fingerprint:
+        return None
+    try:
+        der = _tls.fetch_leaf(spec.remote.url, connect_timeout)
+    except (OSError, ssl.SSLError):
+        return None
+    if der == spec.leaf.der:
+        return None
+    return remote(spec.remote, connect_timeout=connect_timeout)

reef/_connect.py

@@ -0,0 +1,162 @@
+"""Settling what ``connect()`` reaches, shared by the sync and asyncio clients."""
+
+from __future__ import annotations
+
+import os
+import re
+from typing import Any
+
+import grpc
+
+from reef import _channel, _paths, _remote
+from reef._build import CARGO_VERSION
+from reef.errors import (
+    DETAIL_TYPE_URL,
+    DaemonNotRunningError,
+    IncompatibleError,
+    ReefError,
+    RemoteConnectError,
+    RemoteMisdirectedError,
+    detail_of,
+    from_status,
+)
+
+V0_STATUS = "/reef.daemon.v0.Daemon/Status"
+DAEMON_STATUS = "/reef.api.v1.DaemonService/GetDaemonStatus"
+UNSERVED_REASONS = ("UNKNOWN_METHOD", "NOT_IMPLEMENTED")
+# The listener's HTTP 421, as grpcio reports it ("Received http2 header with status: 421"), or as
+# tonic does ("... HTTP status code 421").
+MISDIRECTED = re.compile(r"(header with status: |HTTP status code )421$")
+
+__all__ = ["DETAIL_TYPE_URL", "spec_for"]
+
+
+def spec_for(
+    *,
+    socket: str | os.PathLike[str] | None,
+    remote: str | None,
+    token: str | None,
+    fingerprint: str | None,
+    state_dir: str | os.PathLike[str] | None,
+    config: str | os.PathLike[str] | None,
+    connect_timeout: float,
+    root_certificates: bytes | None,
+) -> _channel.Spec:
+    """The socket, or the remote, that ``connect()``'s arguments and the environment name.
+
+    Raises:
+        TypeError: Both ``socket`` and ``remote``.
+        ConfigError: The paths, or the remote's profile, cannot be settled.
+        RemoteConnectError: Nothing answered at the remote.
+        RemoteTlsError: The remote's certificate was refused.
+    """
+    if socket is not None and remote is not None:
+        raise TypeError("connect() takes a socket or a remote, not both")
+    if socket is None and remote is None:
+        from_env = os.environ.get(_remote.REMOTE_ENV, "")
+        remote = from_env.strip() or None
+    if remote is not None:
+        state = _paths.settle_state_dir(state_dir=state_dir)
+        target = _remote.choose(remote, state_dir=state, token=token, fingerprint=fingerprint)
+        return _channel.remote(
+            target, connect_timeout=connect_timeout, root_certificates=root_certificates
+        )
+    settled = _paths.settle(socket=socket, state_dir=state_dir, config=config)
+    return _channel.local(settled.socket)
+
+
+def connect_error(spec: _channel.Spec, failed: bool, timeout: float) -> ReefError:
+    """Why ``connect()`` could not reach the daemon: the connection failed (``failed``), or
+    nothing settled within ``timeout``.
+    """
+    cause = "the connection failed" if failed else f"no connection within {timeout:g} s"
+    if spec.remote is not None:
+        return RemoteConnectError(spec.label, cause)
+    return DaemonNotRunningError(spec.label, cause)
+
+
+def is_unserved(error: grpc.RpcError) -> bool:
+    """Whether an error may say the daemon cannot serve the method: a bare ``UNIMPLEMENTED``,
+    or one whose reason is ``UNKNOWN_METHOD`` or ``NOT_IMPLEMENTED``.
+    """
+    if error.code() != grpc.StatusCode.UNIMPLEMENTED:
+        return False
+    detail = detail_of(error.trailing_metadata())
+    return detail is None or detail.reason in UNSERVED_REASONS
+
+
+def misdirected(spec: _channel.Spec, error: grpc.RpcError) -> ReefError | None:
+    """A remote daemon's HTTP 421 (a host name it does not serve), as its error."""
+    if (
+        spec.remote is not None
+        and error.code() == grpc.StatusCode.UNKNOWN
+        and MISDIRECTED.search(error.details() or "")
+        and detail_of(error.trailing_metadata()) is None
+    ):
+        return RemoteMisdirectedError(spec.remote.url.origin, spec.remote.url.host)
+    return None
+
+
+def needs_repin(spec: _channel.Spec, error: grpc.RpcError) -> bool:
+    """Whether a failure on a pinned channel may be a renewed certificate."""
+    return (
+        spec.leaf is not None
+        and error.code() == grpc.StatusCode.UNAVAILABLE
+        and detail_of(error.trailing_metadata()) is None
+    )
+
+
+def incompatible(
+    spec: _channel.Spec,
+    method: str,
+    error: grpc.RpcError,
+    v1: Any | None,
+    v0: Any | None,
+) -> ReefError:
+    """What a daemon that could not serve ``method`` is, from its answers to the two status
+    calls (``v1``: ``GetDaemonStatusResponse`` or None; ``v0``: 0.1's ``StatusResponse``).
+    """
+    original = from_status(error.code(), error.details() or "", error.trailing_metadata(), method)
+    if v1 is not None:
+        detail = detail_of(error.trailing_metadata())
+        same_version = v1.version == CARGO_VERSION
+        if same_version and detail is not None and detail.reason == "NOT_IMPLEMENTED":
+            return original
+        return IncompatibleError(
+            spec.label,
+            method,
+            daemon_version=v1.version or "unknown",
+            daemon_api="v1",
+            daemon_revision=v1.api_revision,
+            active_workspaces=v1.active_workspace_count,
+            sdk_version=CARGO_VERSION,
+        )
+    if v0 is not None:
+        return IncompatibleError(
+            spec.label,
+            method,
+            daemon_version=v0.version or "unknown",
+            daemon_api="v0",
+            active_workspaces=v0.active_workspaces,
+            sdk_version=CARGO_VERSION,
+        )
+    return original
+
+
+def require(spec: _channel.Spec, revision: int, status: Any) -> None:
+    """Raises :class:`IncompatibleError` when the daemon's API revision is below ``revision``."""
+    if status.api_revision >= revision:
+        return
+    raise IncompatibleError(
+        spec.label,
+        f"API revision {revision}",
+        daemon_version=status.version or "unknown",
+        daemon_api="v1",
+        daemon_revision=status.api_revision,
+        active_workspaces=status.active_workspace_count,
+        sdk_version=CARGO_VERSION,
+        detail=(
+            f"the daemon at {spec.label} is reef {status.version} at API revision "
+            f"{status.api_revision}; this needs revision {revision}"
+        ),
+    )

reef/_env.py

@@ -0,0 +1,12 @@
+"""Process settings applied before grpc is imported.
+
+grpc's fork support logs two lines at INFO from every ``subprocess`` call made while a channel is
+open, which an agent makes all the time, and a child process cannot use the parent's channels
+anyway (create clients after forking). So the SDK turns it off unless the program chose:
+``GRPC_ENABLE_FORK_SUPPORT`` set beforehand wins, and grpc imported before reef keeps its own.
+grpc reads it when it starts, so it stays set, and the program's child processes inherit it.
+"""
+
+import os
+
+os.environ.setdefault("GRPC_ENABLE_FORK_SUPPORT", "false")