clusdr 0.1.0__py3-none-any.whl

clusdr/__init__.py

@@ -0,0 +1,28 @@
+"""Application SDK for the local Clusdr daemon.
+
+Applications talk only to the daemon on the same host (Docker-style).
+The daemon is the cluster member; this package does not dial other nodes.
+
+    from clusdr import local
+
+    cluster = local()
+    members = cluster.members()
+    cluster.publish("deployment", {"sha": "abc"})
+    for event in cluster.watch():
+        ...
+"""
+
+from clusdr._client import Cluster, Event, Member, dial, local
+from clusdr._coord import Lease, Lock
+from clusdr._errors import ClusdrError
+
+__all__ = [
+    "Cluster",
+    "ClusdrError",
+    "Event",
+    "Lease",
+    "Lock",
+    "Member",
+    "dial",
+    "local",
+]

clusdr/_client.py

@@ -0,0 +1,315 @@
+"""gRPC client for the local daemon."""
+
+from __future__ import annotations
+
+import json
+import threading
+import time
+import uuid
+from collections.abc import Iterator
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from typing import Any
+
+import grpc
+
+from clusdr._coord import CoordMixin, Lease, Lock
+from clusdr._errors import ClusdrError
+from clusdr._options import (
+    DEFAULT_READY_TIMEOUT,
+    DEFAULT_REQUEST_TIMEOUT,
+    MAX_PAYLOAD,
+    Options,
+    env_addr,
+    env_insecure,
+)
+from clusdr._retry import retry, transient
+from clusdr._tls import open_channel
+from clusdr.v1alpha1 import (
+    events_pb2,
+    events_pb2_grpc,
+    health_pb2,
+    health_pb2_grpc,
+    leases_pb2_grpc,
+    locks_pb2_grpc,
+    membership_pb2,
+    membership_pb2_grpc,
+    watch_pb2,
+    watch_pb2_grpc,
+)
+
+
+@dataclass(frozen=True)
+class Member:
+    """A cluster node as seen by the local daemon."""
+
+    id: str
+    address: str
+    status: str
+    leader: bool
+    role: str = "voter"
+
+
+@dataclass(frozen=True)
+class Event:
+    """A cluster or custom event from the Watch stream."""
+
+    type: str
+    source: str
+    payload: bytes
+    timestamp: datetime
+    seq: int
+
+
+class Cluster(CoordMixin):
+    """Application view of the local Clusdr daemon."""
+
+    def __init__(self, channel: grpc.Channel, opts: Options) -> None:
+        self._channel = channel
+        self._opts = opts
+        self._closed = threading.Event()
+        self._watch_lock = threading.Lock()
+        self._watch_call: Any = None
+        self._mem = membership_pb2_grpc.MembershipServiceStub(channel)
+        self._watch = watch_pb2_grpc.WatchServiceStub(channel)
+        self._ev = events_pb2_grpc.EventServiceStub(channel)
+        self._health = health_pb2_grpc.HealthServiceStub(channel)
+        self._lock = locks_pb2_grpc.LockServiceStub(channel)
+        self._lease = leases_pb2_grpc.LeaseServiceStub(channel)
+        self._holder = opts.holder or f"sdk-{uuid.uuid4()}"
+        self._coord = threading.Lock()
+        self._held: dict[str, Lock] = {}
+        self._leased: dict[str, Lease] = {}
+
+    def members(self, timeout: float | None = None) -> list[Member]:
+        deadline = self._deadline(timeout)
+
+        def call() -> membership_pb2.ListMembersResponse:
+            return self._mem.ListMembers(
+                membership_pb2.ListMembersRequest(),
+                timeout=self._remaining(deadline),
+            )
+
+        try:
+            resp = retry(call, deadline=deadline)
+        except (grpc.RpcError, TimeoutError) as exc:
+            raise ClusdrError(f"clusdr: members: {exc}") from exc
+        return [
+            Member(
+                id=m.id,
+                address=m.address,
+                status=m.status,
+                leader=m.leader,
+                role=m.role or "voter",
+            )
+            for m in resp.members
+        ]
+
+    def leader(self, timeout: float | None = None) -> Member:
+        deadline = self._deadline(timeout)
+
+        def call() -> membership_pb2.GetLeaderResponse:
+            return self._mem.GetLeader(
+                membership_pb2.GetLeaderRequest(),
+                timeout=self._remaining(deadline),
+            )
+
+        try:
+            resp = retry(call, deadline=deadline)
+        except (grpc.RpcError, TimeoutError) as exc:
+            raise ClusdrError(f"clusdr: leader: {exc}") from exc
+        return Member(
+            id=resp.leader_id,
+            address=resp.address,
+            status="alive",
+            leader=True,
+            role="voter",
+        )
+
+    def publish(
+        self,
+        topic: str,
+        payload: bytes | str | dict[str, Any] | list[Any] | None = None,
+        timeout: float | None = None,
+    ) -> None:
+        body = encode_payload(payload)
+        if len(body) > MAX_PAYLOAD:
+            raise ClusdrError(f"clusdr: publish payload exceeds {MAX_PAYLOAD} bytes")
+        deadline = self._deadline(timeout)
+
+        def call() -> events_pb2.PublishEventResponse:
+            return self._ev.PublishEvent(
+                events_pb2.PublishEventRequest(topic=topic, payload=body),
+                timeout=self._remaining(deadline),
+            )
+
+        try:
+            resp = retry(call, deadline=deadline)
+        except (grpc.RpcError, TimeoutError) as exc:
+            raise ClusdrError(f"clusdr: publish: {exc}") from exc
+        if resp is not None and not resp.accepted:
+            raise ClusdrError(f"clusdr: publish rejected: {resp.message}")
+
+    def watch(self) -> Iterator[Event]:
+        last_seq = 0
+        backoff = 0.05
+        while not self._closed.is_set():
+            try:
+                call = self._watch.Watch(watch_pb2.WatchRequest(last_seq=last_seq))
+            except grpc.RpcError as exc:
+                if self._closed.is_set() or exc.code() == grpc.StatusCode.CANCELLED:
+                    return
+                if not self._sleep(backoff):
+                    return
+                backoff = min(backoff * 2, 2.0)
+                continue
+            with self._watch_lock:
+                self._watch_call = call
+            backoff = 0.05
+            try:
+                for resp in call:
+                    if self._closed.is_set():
+                        return
+                    if resp.seq > last_seq:
+                        last_seq = resp.seq
+                    yield Event(
+                        type=resp.type,
+                        source=resp.source,
+                        payload=bytes(resp.payload),
+                        timestamp=_ts(resp.timestamp_unix_ms),
+                        seq=resp.seq,
+                    )
+            except grpc.RpcError as exc:
+                if self._closed.is_set() or exc.code() == grpc.StatusCode.CANCELLED:
+                    return
+            finally:
+                with self._watch_lock:
+                    if self._watch_call is call:
+                        self._watch_call = None
+            if not self._sleep(backoff):
+                return
+            backoff = min(backoff * 2, 2.0)
+
+    def close(self) -> None:
+        self._closed.set()
+        self._release_grants()
+        with self._watch_lock:
+            if self._watch_call is not None:
+                self._watch_call.cancel()
+                self._watch_call = None
+        self._channel.close()
+
+    def __enter__(self) -> Cluster:
+        return self
+
+    def __exit__(self, *exc: object) -> None:
+        self.close()
+
+    def _deadline(self, timeout: float | None) -> float:
+        if timeout is None:
+            timeout = self._opts.request_timeout
+        return time.monotonic() + timeout
+
+    def _remaining(self, deadline: float) -> float:
+        left = deadline - time.monotonic()
+        if left <= 0:
+            raise TimeoutError("clusdr: request timeout")
+        return left
+
+    def _sleep(self, seconds: float) -> bool:
+        return not self._closed.wait(seconds)
+
+
+def local(
+    *,
+    insecure: bool = False,
+    data_dir: str = "",
+    holder: str = "",
+    request_timeout: float = DEFAULT_REQUEST_TIMEOUT,
+    ready_timeout: float = DEFAULT_READY_TIMEOUT,
+    server_name: str = "",
+) -> Cluster:
+    """Connect to the daemon on this host (CLUSDR_GRPC_ADDR or 127.0.0.1:7947)."""
+    return connect(
+        Options(
+            addr=env_addr(),
+            insecure=insecure,
+            data_dir=data_dir,
+            holder=holder.strip(),
+            request_timeout=request_timeout,
+            ready_timeout=ready_timeout,
+            server_name=server_name,
+        )
+    )
+
+
+def dial(
+    addr: str,
+    *,
+    insecure: bool = False,
+    data_dir: str = "",
+    holder: str = "",
+    request_timeout: float = DEFAULT_REQUEST_TIMEOUT,
+    ready_timeout: float = DEFAULT_READY_TIMEOUT,
+    server_name: str = "",
+) -> Cluster:
+    """Connect to addr. Tests and operators use this; applications use local()."""
+    if not addr.strip():
+        raise ClusdrError("clusdr: empty dial address")
+    return connect(
+        Options(
+            addr=addr.strip(),
+            insecure=insecure,
+            data_dir=data_dir,
+            holder=holder.strip(),
+            request_timeout=request_timeout,
+            ready_timeout=ready_timeout,
+            server_name=server_name,
+        )
+    )
+
+
+def connect(opts: Options) -> Cluster:
+    if not opts.insecure and env_insecure() and not opts.data_dir:
+        opts.insecure = True
+    if not opts.addr:
+        raise ClusdrError("clusdr: empty dial address")
+    channel = open_channel(opts)
+    cluster = Cluster(channel, opts)
+    if opts.ready_timeout > 0:
+        deadline = time.monotonic() + opts.ready_timeout
+        try:
+            retry(
+                lambda: cluster._health.Health(
+                    health_pb2.HealthRequest(),
+                    timeout=min(0.5, max(0.05, deadline - time.monotonic())),
+                ),
+                deadline=deadline,
+                is_transient=_ready_transient,
+            )
+        except (grpc.RpcError, TimeoutError) as exc:
+            cluster.close()
+            raise ClusdrError(f"clusdr: daemon not ready at {opts.addr}: {exc}") from exc
+    return cluster
+
+
+def _ready_transient(exc: BaseException) -> bool:
+    if transient(exc):
+        return True
+    return isinstance(exc, grpc.RpcError) and exc.code() == grpc.StatusCode.DEADLINE_EXCEEDED
+
+
+def encode_payload(payload: bytes | str | dict[str, Any] | list[Any] | None) -> bytes:
+    if payload is None:
+        return b""
+    if isinstance(payload, bytes):
+        return payload
+    if isinstance(payload, str):
+        return payload.encode("utf-8")
+    if isinstance(payload, (dict, list)):
+        return json.dumps(payload, separators=(",", ":")).encode("utf-8")
+    raise TypeError(f"clusdr: payload must be bytes, str, dict, or list; got {type(payload).__name__}")
+
+
+def _ts(unix_ms: int) -> datetime:
+    return datetime.fromtimestamp(unix_ms / 1000.0, tz=timezone.utc)

clusdr/_coord.py

@@ -0,0 +1,345 @@
+"""Lock and lease methods on Cluster. Same surface as the Go SDK."""
+
+from __future__ import annotations
+
+import threading
+import time
+from datetime import datetime, timezone
+from typing import TYPE_CHECKING
+
+import grpc
+
+from clusdr._errors import ClusdrError
+from clusdr._retry import retry
+from clusdr.v1alpha1 import leases_pb2, locks_pb2
+
+if TYPE_CHECKING:
+    from clusdr._client import Cluster
+
+
+class Lock:
+    """A held exclusive lock. token is the fencing token."""
+
+    def __init__(self, name: str, holder: str, token: int, deadline: datetime | None) -> None:
+        self.name = name
+        self.holder = holder
+        self.token = token
+        self._deadline = deadline
+        self._mu = threading.Lock()
+        self._stop = threading.Event()
+        self._cluster: Cluster | None = None
+
+    @property
+    def deadline(self) -> datetime | None:
+        with self._mu:
+            return self._deadline
+
+    def _set_deadline(self, value: datetime | None) -> None:
+        with self._mu:
+            self._deadline = value
+
+
+class Lease:
+    """A held TTL grant. token is the fencing token."""
+
+    def __init__(self, name: str, owner: str, token: int, deadline: datetime | None) -> None:
+        self.name = name
+        self.owner = owner
+        self.token = token
+        self._deadline = deadline
+        self._mu = threading.Lock()
+        self._stop = threading.Event()
+        self._cluster: Cluster | None = None
+
+    @property
+    def deadline(self) -> datetime | None:
+        with self._mu:
+            return self._deadline
+
+    def _set_deadline(self, value: datetime | None) -> None:
+        with self._mu:
+            self._deadline = value
+
+    def stop_renew(self) -> None:
+        """Stop background renewal; the grant then expires at its deadline."""
+        self._stop.set()
+
+
+class CoordMixin:
+    def lock(self: Cluster, name: str, ttl: float | None = None, timeout: float | None = None) -> Lock:
+        existing = self._held_lock(name)
+        if existing is not None:
+            return existing
+        deadline = self._deadline(timeout)
+
+        def call() -> locks_pb2.LockResponse:
+            return self._lock.Lock(
+                locks_pb2.LockRequest(name=name, holder=self._holder, ttl_ms=_ttl_ms(ttl)),
+                timeout=self._remaining(deadline),
+            )
+
+        try:
+            resp = retry(call, deadline=deadline)
+        except (grpc.RpcError, TimeoutError) as exc:
+            raise ClusdrError(f"clusdr: lock {name!r}: {exc}") from exc
+        if resp is None or not resp.acquired:
+            msg = resp.message if resp is not None and resp.message else "not acquired"
+            raise ClusdrError(f"clusdr: lock {name!r}: {msg}")
+        return self._adopt_lock(resp, name, ttl)
+
+    def try_lock(self: Cluster, name: str, ttl: float | None = None, timeout: float | None = None) -> Lock | None:
+        existing = self._held_lock(name)
+        if existing is not None:
+            return existing
+        deadline = self._deadline(timeout)
+
+        def call() -> locks_pb2.LockResponse:
+            return self._lock.TryLock(
+                locks_pb2.LockRequest(name=name, holder=self._holder, ttl_ms=_ttl_ms(ttl)),
+                timeout=self._remaining(deadline),
+            )
+
+        try:
+            resp = retry(call, deadline=deadline)
+        except (grpc.RpcError, TimeoutError) as exc:
+            raise ClusdrError(f"clusdr: trylock {name!r}: {exc}") from exc
+        if resp is None or not resp.acquired:
+            return None
+        return self._adopt_lock(resp, name, ttl)
+
+    def unlock(self: Cluster, name: str, timeout: float | None = None) -> None:
+        lk = self._held_lock(name)
+        if lk is None:
+            raise ClusdrError(f"clusdr: lock {name!r} is not held by this client")
+        self._release_lock(lk, timeout)
+
+    def lease(
+        self: Cluster,
+        name: str,
+        ttl: float | None = None,
+        *,
+        stop: threading.Event | None = None,
+        timeout: float | None = None,
+    ) -> Lease:
+        existing = self._held_lease(name)
+        if existing is not None:
+            return existing
+        deadline = self._deadline(timeout)
+
+        def call() -> leases_pb2.GrantLeaseResponse:
+            return self._lease.Grant(
+                leases_pb2.GrantLeaseRequest(name=name, owner=self._holder, ttl_ms=_ttl_ms(ttl)),
+                timeout=self._remaining(deadline),
+            )
+
+        try:
+            resp = retry(call, deadline=deadline)
+        except (grpc.RpcError, TimeoutError) as exc:
+            raise ClusdrError(f"clusdr: lease {name!r}: {exc}") from exc
+        if resp is None or not resp.granted:
+            msg = resp.message if resp is not None and resp.message else "not granted"
+            owner = resp.owner if resp is not None else ""
+            if owner:
+                raise ClusdrError(f"clusdr: lease {name!r}: {msg} (owner {owner})")
+            raise ClusdrError(f"clusdr: lease {name!r}: {msg}")
+        return self._adopt_lease(resp, name, ttl, stop)
+
+    def renew(self: Cluster, name: str, timeout: float | None = None) -> None:
+        ls = self._held_lease(name)
+        if ls is None:
+            raise ClusdrError(f"clusdr: lease {name!r} is not held by this client")
+        deadline = self._deadline(timeout)
+
+        def call() -> leases_pb2.RenewLeaseResponse:
+            return self._lease.Renew(
+                leases_pb2.RenewLeaseRequest(name=ls.name, owner=ls.owner, fencing_token=ls.token),
+                timeout=self._remaining(deadline),
+            )
+
+        try:
+            resp = retry(call, deadline=deadline)
+        except (grpc.RpcError, TimeoutError) as exc:
+            raise ClusdrError(f"clusdr: renew {name!r}: {exc}") from exc
+        if resp is not None and not resp.renewed:
+            raise ClusdrError(f"clusdr: renew {name!r}: {resp.message}")
+        if resp is not None and resp.deadline_unix_ms:
+            ls._set_deadline(_from_ms(resp.deadline_unix_ms))
+
+    def revoke(self: Cluster, name: str, timeout: float | None = None) -> None:
+        ls = self._held_lease(name)
+        if ls is None:
+            raise ClusdrError(f"clusdr: lease {name!r} is not held by this client")
+        self._drop_lease(ls, timeout)
+
+    def _held_lock(self: Cluster, name: str) -> Lock | None:
+        with self._coord:
+            return self._held.get(name)
+
+    def _held_lease(self: Cluster, name: str) -> Lease | None:
+        with self._coord:
+            return self._leased.get(name)
+
+    def _adopt_lock(self: Cluster, resp: locks_pb2.LockResponse, name: str, ttl: float | None) -> Lock:
+        lk = Lock(name, resp.holder or self._holder, resp.fencing_token, _from_ms(resp.deadline_unix_ms))
+        lk._cluster = self
+        with self._coord:
+            existing = self._held.get(name)
+            if existing is not None and existing.token == lk.token:
+                return existing
+            self._held[name] = lk
+        self._start_lock_renew(lk, ttl)
+        return lk
+
+    def _adopt_lease(
+        self: Cluster,
+        resp: leases_pb2.GrantLeaseResponse,
+        name: str,
+        ttl: float | None,
+        stop: threading.Event | None,
+    ) -> Lease:
+        ls = Lease(name, resp.owner or self._holder, resp.fencing_token, _from_ms(resp.deadline_unix_ms))
+        ls._cluster = self
+        if stop is not None:
+            ls._stop = stop
+        with self._coord:
+            existing = self._leased.get(name)
+            if existing is not None and existing.token == ls.token:
+                return existing
+            self._leased[name] = ls
+        self._start_lease_renew(ls, ttl)
+        return ls
+
+    def _release_lock(self: Cluster, lk: Lock, timeout: float | None) -> None:
+        lk._stop.set()
+        deadline = self._deadline(timeout)
+
+        def call() -> locks_pb2.UnlockResponse:
+            return self._lock.Unlock(
+                locks_pb2.UnlockRequest(name=lk.name, holder=lk.holder, fencing_token=lk.token),
+                timeout=self._remaining(deadline),
+            )
+
+        try:
+            resp = retry(call, deadline=deadline)
+        except grpc.RpcError as exc:
+            if exc.code() == grpc.StatusCode.FAILED_PRECONDITION:
+                self._forget_lock(lk.name, lk.token)
+            raise ClusdrError(f"clusdr: unlock {lk.name!r}: {exc}") from exc
+        except TimeoutError as exc:
+            raise ClusdrError(f"clusdr: unlock {lk.name!r}: {exc}") from exc
+        if resp is not None and not resp.released:
+            raise ClusdrError(f"clusdr: unlock {lk.name!r}: {resp.message}")
+        self._forget_lock(lk.name, lk.token)
+
+    def _drop_lease(self: Cluster, ls: Lease, timeout: float | None) -> None:
+        ls._stop.set()
+        deadline = self._deadline(timeout)
+
+        def call() -> leases_pb2.RevokeLeaseResponse:
+            return self._lease.Revoke(
+                leases_pb2.RevokeLeaseRequest(name=ls.name, owner=ls.owner, fencing_token=ls.token),
+                timeout=self._remaining(deadline),
+            )
+
+        try:
+            resp = retry(call, deadline=deadline)
+        except grpc.RpcError as exc:
+            if exc.code() == grpc.StatusCode.FAILED_PRECONDITION:
+                self._forget_lease(ls.name, ls.token)
+            raise ClusdrError(f"clusdr: revoke {ls.name!r}: {exc}") from exc
+        except TimeoutError as exc:
+            raise ClusdrError(f"clusdr: revoke {ls.name!r}: {exc}") from exc
+        if resp is not None and not resp.revoked:
+            raise ClusdrError(f"clusdr: revoke {ls.name!r}: {resp.message}")
+        self._forget_lease(ls.name, ls.token)
+
+    def _forget_lock(self: Cluster, name: str, token: int) -> None:
+        with self._coord:
+            cur = self._held.get(name)
+            if cur is not None and cur.token == token:
+                del self._held[name]
+
+    def _forget_lease(self: Cluster, name: str, token: int) -> None:
+        with self._coord:
+            cur = self._leased.get(name)
+            if cur is not None and cur.token == token:
+                del self._leased[name]
+
+    def _release_grants(self: Cluster) -> None:
+        with self._coord:
+            locks = list(self._held.values())
+            leases = list(self._leased.values())
+        for lk in locks:
+            try:
+                self._release_lock(lk, None)
+            except ClusdrError:
+                pass
+        for ls in leases:
+            try:
+                self._drop_lease(ls, None)
+            except ClusdrError:
+                pass
+
+    def _start_lock_renew(self: Cluster, lk: Lock, ttl: float | None) -> None:
+        interval = _renew_interval(ttl, lk.deadline)
+        thread = threading.Thread(target=self._renew_lock_loop, args=(lk, interval), daemon=True)
+        thread.start()
+
+    def _start_lease_renew(self: Cluster, ls: Lease, ttl: float | None) -> None:
+        interval = _renew_interval(ttl, ls.deadline)
+        thread = threading.Thread(target=self._renew_lease_loop, args=(ls, interval), daemon=True)
+        thread.start()
+
+    def _renew_lock_loop(self: Cluster, lk: Lock, interval: float) -> None:
+        while not lk._stop.wait(interval):
+            if self._closed.is_set():
+                return
+            try:
+                resp = self._lock.Renew(
+                    locks_pb2.RenewLockRequest(name=lk.name, holder=lk.holder, fencing_token=lk.token),
+                    timeout=self._opts.request_timeout,
+                )
+            except grpc.RpcError as exc:
+                if exc.code() in (grpc.StatusCode.FAILED_PRECONDITION, grpc.StatusCode.CANCELLED):
+                    return
+                continue
+            if resp is not None and resp.deadline_unix_ms:
+                lk._set_deadline(_from_ms(resp.deadline_unix_ms))
+
+    def _renew_lease_loop(self: Cluster, ls: Lease, interval: float) -> None:
+        while not ls._stop.wait(interval):
+            if self._closed.is_set():
+                return
+            try:
+                resp = self._lease.Renew(
+                    leases_pb2.RenewLeaseRequest(name=ls.name, owner=ls.owner, fencing_token=ls.token),
+                    timeout=self._opts.request_timeout,
+                )
+            except grpc.RpcError as exc:
+                if exc.code() in (grpc.StatusCode.FAILED_PRECONDITION, grpc.StatusCode.CANCELLED):
+                    return
+                continue
+            if resp is not None and resp.deadline_unix_ms:
+                ls._set_deadline(_from_ms(resp.deadline_unix_ms))
+
+
+def _ttl_ms(ttl: float | None) -> int:
+    if ttl is None or ttl <= 0:
+        return 0
+    return max(1, int(ttl * 1000))
+
+
+def _from_ms(unix_ms: int) -> datetime | None:
+    if not unix_ms:
+        return None
+    return datetime.fromtimestamp(unix_ms / 1000.0, tz=timezone.utc)
+
+
+def _renew_interval(ttl: float | None, deadline: datetime | None) -> float:
+    d = ttl
+    if d is None or d <= 0:
+        if deadline is not None:
+            d = (deadline - datetime.now(timezone.utc)).total_seconds()
+    if d is None or d <= 0:
+        d = 15.0
+    return max(0.05, d / 3.0)

clusdr/_errors.py

@@ -0,0 +1,5 @@
+"""SDK errors."""
+
+
+class ClusdrError(Exception):
+    """A Clusdr daemon or SDK failure."""