mithril-locks 0.1.0__py3-none-any.whl

mithril/UPGRADING.md

@@ -0,0 +1,86 @@
+# Reliability changes
+
+## Before upgrading an existing deployment
+
+Stop all old-version lock users and drain their leases before starting the new
+version. Redis reader metadata and file lease metadata have changed; old and new
+holders must not run concurrently against the same resources.
+
+Preserve fencing history. Back up existing counters and remove their legacy Redis
+expirations with `PERSIST` during the maintenance window. Names and namespaces
+containing punctuation are now percent-encoded in Redis keys, so migrate those
+counters to the encoded keys without lowering their values. Cluster hash tags use
+the same encoding. Ordinary alphanumeric Redis fencing keys and file fencing
+counter paths are unchanged. While all file-lock users are stopped, remove their
+old `.lock` metadata files so the new format can be created. Keep the adjacent
+`.fence` counters and do not delete live `.guard` files.
+
+If an old counter has already expired, recover a floor from durable history or
+the downstream system's highest accepted token before resuming. Draining holders
+alone does not make resetting a fencing counter safe.
+
+## Redlock requires a fencing authority
+
+`RedlockConfig` now requires `fencing_provider`. Taking the maximum of independent
+node counters cannot order acquisitions across changing quorums. All clients
+locking the same resource must share the same persistent authority and namespace.
+Seed that authority at least as high as every previously issued or downstream-
+accepted token before migrating an existing Redlock deployment.
+
+```python
+from redis import Redis
+
+from mithril import RedisFencingTokenProvider
+from mithril.backends import RedlockBackend, RedlockConfig
+
+authority = Redis(
+    host="fencing-service",
+    socket_timeout=0.1,
+    socket_connect_timeout=0.1,
+)
+backend = RedlockBackend(
+    RedlockConfig(
+        instances=[Redis(host=host) for host in ("redis-a", "redis-b", "redis-c")],
+        fencing_provider=RedisFencingTokenProvider(authority),
+    )
+)
+```
+
+The authority must never roll back, expire, or evict issued counters. A Redis
+authority is only as durable as its deployment: asynchronous failover or restoring
+an old snapshot can violate this requirement. Supply a custom `FencingTokenProvider`
+backed by a suitable durable service when needed. The in-memory provider is for
+isolated tests only. Call `backend.close()` when finished to close its private
+bounded-timeout Redis pools; caller-owned clients are not closed.
+
+## Database backends
+
+An SQLAlchemy `Engine` / `AsyncEngine` is now accepted and recommended. Existing
+`Connection` / `AsyncConnection` arguments still work: Mithril borrows independent
+connections from their engine rather than sharing the caller's transaction.
+Configure bounded pool and driver timeouts appropriate for the lease TTL. Retry
+budgets cannot interrupt synchronous driver calls already in progress.
+
+SQLite requires a file-backed database with an independent-connection pool;
+`StaticPool` and `SingletonThreadPool` are rejected. In-memory databases cannot
+provide persistent fencing or safely shared transactions. Async backends still
+require `await backend.initialize()` before use.
+
+## Lifecycle behavior
+
+Nonblocking acquisition makes one attempt. Concurrent acquisition through the
+same facade is rejected. Expired or released instances remain single-use, expose
+no current token, and context exit raises on ownership loss unless preserving an
+exception from the context body. Renewals refresh the facade's deadline; backend
+latency and Redlock drift are subtracted from validity. A manually shortened TTL
+must still exceed the configured background-renewal interval, including jitter.
+
+File leases now allow takeover after TTL expiry even while the previous process
+is alive, and detect process crashes using OS locks. As with every TTL-based
+backend, downstream writes must enforce fencing tokens. A standalone guard check
+followed by an unrelated write is not an atomic fenced write.
+
+Logging and metrics failures no longer interrupt lease ownership or cleanup.
+Epochs are backend-issued persistent sequence numbers, not caller-supplied counts.
+The backend `epoch` argument remains accepted for compatibility but is ignored by
+the built-in backends.

mithril/__init__.py

@@ -0,0 +1,67 @@
+"""
+Public API for Mithril.
+"""
+
+from __future__ import annotations
+
+from .__version__ import __version__
+from ._fencing import FencingTokenProvider, FileFencingTokenProvider, RedisFencingTokenProvider
+from ._observability import LockHooks, configure_logging, get_logger, set_hooks
+from ._types import LockConfig, LockMode, LockState, LockToken, Result
+from .exceptions import (
+    AlreadyAcquiredError,
+    AutoRenewalStoppedError,
+    BackendError,
+    BackendTimeoutError,
+    BackendUnavailableError,
+    ConfigurationError,
+    FencingTokenError,
+    LockAcquisitionError,
+    LockExpiredError,
+    LockReleaseError,
+    LockRenewalError,
+    LockTimeoutError,
+    MithrilError,
+    NotAcquiredError,
+    RenewalFailedError,
+    StaleTokenError,
+)
+from .lock import Lock
+
+__author__ = "Rohan B. Dalton"
+
+__all__ = [
+    "__version__",
+    "FencingTokenProvider",
+    "FileFencingTokenProvider",
+    "RedisFencingTokenProvider",
+    "LockConfig",
+    "LockMode",
+    "LockState",
+    "LockToken",
+    "Result",
+    "Lock",
+    "LockHooks",
+    "configure_logging",
+    "get_logger",
+    "set_hooks",
+    "MithrilError",
+    "LockAcquisitionError",
+    "LockTimeoutError",
+    "AlreadyAcquiredError",
+    "LockReleaseError",
+    "NotAcquiredError",
+    "LockExpiredError",
+    "LockRenewalError",
+    "RenewalFailedError",
+    "AutoRenewalStoppedError",
+    "FencingTokenError",
+    "StaleTokenError",
+    "BackendError",
+    "BackendUnavailableError",
+    "BackendTimeoutError",
+    "ConfigurationError",
+]
+
+if __name__ == "__main__":
+    pass

mithril/__version__.py

@@ -0,0 +1,3 @@
+"""Single source of truth for the Mithril release version."""
+
+__version__ = "0.1.0"

mithril/_fencing.py

@@ -0,0 +1,164 @@
+"""
+Fencing token providers for Mithril.
+"""
+
+from __future__ import annotations
+
+import threading
+from pathlib import Path
+from typing import Any, Protocol
+from urllib.parse import quote
+
+from ._file_io import file_mutex, read_file, replace_file, validate_component
+from .exceptions import FencingTokenError
+
+__author__ = "Rohan B. Dalton"
+
+
+class FencingTokenProvider(Protocol):
+    """
+    Protocol for fencing token providers.
+    """
+
+    def next_token(self, lock_name: str) -> int: ...
+
+    def current_token(self, lock_name: str) -> int: ...
+
+
+class InMemoryFencingTokenProvider:
+    """
+    Thread-safe in-memory fencing token provider.
+    """
+
+    def __init__(self) -> None:
+        self._lock = threading.Lock()
+        self._tokens: dict[str, int] = {}
+
+    def next_token(self, lock_name: str) -> int:
+        with self._lock:
+            current = self._tokens.get(lock_name, 0)
+            next_value = current + 1
+            self._tokens[lock_name] = next_value
+            return next_value
+
+    def current_token(self, lock_name: str) -> int:
+        with self._lock:
+            return self._tokens.get(lock_name, 0)
+
+
+class FileFencingTokenProvider:
+    """
+    Durable counters protected by a separate OS mutex and atomic replacement.
+    """
+
+    def __init__(self, directory: str | Path, namespace: str = "mithril") -> None:
+        validate_component(namespace)
+        self._directory = Path(directory)
+        self._directory.mkdir(parents=True, exist_ok=True)
+        self._namespace = namespace
+
+    def next_token(self, lock_name: str) -> int:
+        path = self._path_for(lock_name)
+        with file_mutex(path.with_suffix(".fence.guard")):
+            current = self._read_counter(path)
+            replace_file(path, str(value := current + 1))
+            return value
+
+    def current_token(self, lock_name: str) -> int:
+        path = self._path_for(lock_name)
+        with file_mutex(path.with_suffix(".fence.guard")):
+            return self._read_counter(path)
+
+    def _path_for(self, lock_name: str) -> Path:
+        validate_component(lock_name)
+        return self._directory / f"{self._namespace}.{lock_name}.fence"
+
+    @staticmethod
+    def _read_counter(path: Path) -> int:
+        if (data := read_file(path)) is None:
+            return 0
+        if not data.isascii() or not data.isdecimal():
+            raise FencingTokenError(f"Invalid fencing counter: {path}")
+        return int(data)
+
+
+class FencingGuard:
+    """
+    Helper for downstream systems to validate fencing tokens.
+    """
+
+    def validate_and_update(self, lock_name: str, token: int) -> bool:
+        """
+        Validate a fencing token and update the stored max.
+        """
+        raise NotImplementedError("Provide a backend-specific implementation")
+
+
+class RedisFencingTokenProvider:
+    """
+    Explicit fencing authority with persistent counters on one Redis service.
+
+    The service must preserve its counters across failures. Configure bounded
+    client timeouts; availability of this authority is required by Redlock.
+    """
+
+    _SCRIPT = """
+    redis.call('INCR', KEYS[1])
+    redis.call('PERSIST', KEYS[1])
+    return redis.call('GET', KEYS[1])
+    """
+
+    def __init__(self, client: Any, *, namespace: str = "mithril") -> None:
+        self._client = client
+        self._namespace = namespace
+        self._script = client.register_script(self._SCRIPT)
+
+    def _key(self, lock_name: str) -> str:
+        return f"{quote(self._namespace, safe='')}:authority:{quote(lock_name, safe='')}"
+
+    def next_token(self, lock_name: str) -> int:
+        return int(self._script(keys=[self._key(lock_name)]))
+
+    def current_token(self, lock_name: str) -> int:
+        return int(self._client.get(self._key(lock_name)) or 0)
+
+
+class RedisFencingGuard(FencingGuard):
+    """
+    Redis-backed fencing token guard.
+    """
+
+    _SCRIPT = """
+    local current = redis.call('GET', KEYS[1])
+    if not current then
+        redis.call('SET', KEYS[1], ARGV[1])
+        return 1
+    end
+    if tonumber(ARGV[1]) > tonumber(current) then
+        redis.call('SET', KEYS[1], ARGV[1])
+        return 1
+    end
+    return 0
+    """
+
+    def __init__(self, client: Any, *, namespace: str = "mithril") -> None:
+        self._client = client
+        self._namespace = namespace
+        self._script = client.register_script(self._SCRIPT)
+
+    def validate_and_update(self, lock_name: str, token: int) -> bool:
+        """
+        Validate a fencing token and update the stored max.
+        """
+        key = self._key(lock_name)
+        result = self._script(keys=[key], args=[token])
+        return bool(result)
+
+    def _key(self, lock_name: str) -> str:
+        if not self._namespace:
+            return f"fence:{lock_name}:max"
+        return f"{self._namespace}:fence:{lock_name}:max"
+
+
+if __name__ == "__main__":
+    pass

mithril/_file_io.py

@@ -0,0 +1,117 @@
+"""
+Safe file opening, fixed-byte mutexes, and durable metadata replacement.
+"""
+
+from __future__ import annotations
+
+import errno
+import os
+import stat
+import tempfile
+from contextlib import contextmanager
+from pathlib import Path
+
+from .exceptions import BackendError, ConfigurationError
+
+if os.name == "nt":
+    import msvcrt
+else:
+    import fcntl
+
+__author__ = "Rohan B. Dalton"
+
+
+def validate_component(value: str) -> None:
+    if not isinstance(value, str) or not value or "/" in value or "\\" in value:
+        raise ConfigurationError("file names and namespaces must be nonempty path components")
+
+
+def open_file(path: Path, *, create: bool = True):
+    if path.is_symlink():
+        raise ValueError("lock path must not be a symlink")
+    flags = os.O_RDWR | (os.O_CREAT if create else 0) | getattr(os, "O_NOFOLLOW", 0)
+    fd = os.open(path, flags, 0o600)
+    try:
+        info = os.fstat(fd)
+        if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1:
+            raise ValueError("lock path must be a regular file with no hard links")
+        return os.fdopen(fd, "r+", encoding="utf-8")
+    except BaseException:
+        os.close(fd)
+        raise
+
+
+def lock_file(handle, *, blocking: bool) -> bool:
+    if os.name == "nt":
+        handle.seek(0, os.SEEK_END)
+        if handle.tell() == 0:
+            handle.write("\0")
+            handle.flush()
+        handle.seek(0)
+        mode = msvcrt.LK_LOCK if blocking else msvcrt.LK_NBLCK
+        try:
+            msvcrt.locking(handle.fileno(), mode, 1)
+        except OSError as exc:
+            if exc.errno in (errno.EACCES, errno.EAGAIN, errno.EDEADLK):
+                return False
+            raise
+    else:
+        flags = fcntl.LOCK_EX | (0 if blocking else fcntl.LOCK_NB)
+        try:
+            fcntl.flock(handle.fileno(), flags)
+        except OSError as exc:
+            if exc.errno in (errno.EACCES, errno.EAGAIN):
+                return False
+            raise
+    return True
+
+
+def unlock_file(handle) -> None:
+    handle.seek(0)
+    if os.name == "nt":
+        msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1)
+    else:
+        fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
+
+
+@contextmanager
+def file_mutex(path: Path):
+    with open_file(path) as fh:
+        if not lock_file(fh, blocking=True):
+            raise BackendError("Could not acquire file metadata mutex")
+        try:
+            yield
+        finally:
+            unlock_file(fh)
+
+
+def read_file(path: Path) -> str | None:
+    try:
+        with open_file(path, create=False) as fh:
+            return fh.read()
+    except FileNotFoundError:
+        return None
+
+
+def replace_file(path: Path, data: str) -> None:
+    """
+    Replace metadata while a separate, stable mutex file is locked.
+    """
+    if path.is_symlink():
+        raise ValueError("lock path must not be a symlink")
+    fd, temporary = tempfile.mkstemp(prefix=".mithril-", dir=path.parent)
+    try:
+        with os.fdopen(fd, "w", encoding="utf-8") as fh:
+            fh.write(data)
+            fh.flush()
+            os.fsync(fh.fileno())
+        os.replace(temporary, path)
+        if os.name != "nt":
+            directory_fd = os.open(path.parent, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0))
+            try:
+                os.fsync(directory_fd)
+            finally:
+                os.close(directory_fd)
+    finally:
+        if os.path.exists(temporary):
+            os.unlink(temporary)

mithril/_observability.py

@@ -0,0 +1,168 @@
+"""
+Observability helpers for Mithril: structured logging and metrics hooks.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Callable
+
+import structlog
+from opentelemetry.metrics import get_meter
+
+from ._types import LockMode, LockState
+
+__author__ = "Rohan B. Dalton"
+
+
+@dataclass(frozen=True)
+class LockEvent:
+    """
+    Snapshot of a lock lifecycle event.
+    """
+
+    event: str
+    lock_name: str
+    owner_id: str | None
+    mode: LockMode | None
+    fencing_token: int | None
+    epoch: int | None
+    ttl: float | None
+    timeout: float | None
+    elapsed: float | None
+    state: LockState | None
+    backend: str | None
+    namespace: str | None
+
+
+@dataclass(frozen=True)
+class LockHooks:
+    """
+    Optional callbacks for lifecycle events.
+    """
+
+    on_acquire: Callable[[LockEvent], None] | None = None
+    on_release: Callable[[LockEvent], None] | None = None
+    on_renewal: Callable[[LockEvent], None] | None = None
+    on_timeout: Callable[[LockEvent], None] | None = None
+    on_expired: Callable[[LockEvent], None] | None = None
+
+
+_HOOKS = LockHooks()
+_LOGGER = structlog.get_logger("mithril")
+_METER = get_meter("mithril")
+
+_ACQUIRE_COUNTER = _METER.create_counter(
+    "mithril.lock.acquire.total", description="Total lock acquisitions", unit="1"
+)
+_RELEASE_COUNTER = _METER.create_counter(
+    "mithril.lock.release.total", description="Total lock releases", unit="1"
+)
+_RENEW_COUNTER = _METER.create_counter(
+    "mithril.lock.renew.total", description="Total lock renewals", unit="1"
+)
+_TIMEOUT_COUNTER = _METER.create_counter(
+    "mithril.lock.timeout.total", description="Total lock acquisition timeouts", unit="1"
+)
+_EXPIRED_COUNTER = _METER.create_counter(
+    "mithril.lock.expired.total", description="Total lock expirations", unit="1"
+)
+_ACQUIRE_LATENCY = _METER.create_histogram(
+    "mithril.lock.acquire.latency", description="Lock acquisition latency", unit="s"
+)
+
+
+def configure_logging(**kwargs: Any) -> None:
+    """
+    Configure structlog for Mithril consumers.
+
+    kwargs are passed directly to structlog.configure.
+    """
+
+    structlog.configure(**kwargs)
+
+
+def set_hooks(hooks: LockHooks | None) -> None:
+    """
+    Set global lock hooks. Pass None to clear.
+    """
+
+    global _HOOKS
+    _HOOKS = hooks or LockHooks()
+
+
+def get_hooks() -> LockHooks:
+    """
+    Return the global lock hooks.
+    """
+
+    return _HOOKS
+
+
+def get_logger() -> structlog.BoundLogger:
+    """
+    Return the default Mithril logger.
+    """
+
+    return _LOGGER
+
+
+def record_event(event: LockEvent, hooks: LockHooks | None = None) -> None:
+    """
+    Invoke hooks and record OpenTelemetry metrics.
+    """
+
+    try:
+        _record_event(event, hooks)
+    except Exception:
+        log_safely(_LOGGER, "exception", "lock_observability_failed", lock_event=event.event)
+
+
+def log_safely(logger, level: str, event: str, *args, **kwargs) -> None:
+    """
+    Logging failures must never interrupt ownership transitions or cleanup.
+    """
+    try:
+        getattr(logger, level)(event, *args, **kwargs)
+    except Exception:
+        # Retrying through the same broken logging pipeline would fail again.
+        pass
+
+
+def _record_event(event: LockEvent, hooks: LockHooks | None = None) -> None:
+    hooks = hooks or _HOOKS
+    if event.event == "acquire":
+        if hooks.on_acquire is not None:
+            hooks.on_acquire(event)
+        _ACQUIRE_COUNTER.add(1, _attributes(event))
+        if event.elapsed is not None:
+            _ACQUIRE_LATENCY.record(event.elapsed, _attributes(event))
+    elif event.event == "release":
+        if hooks.on_release is not None:
+            hooks.on_release(event)
+        _RELEASE_COUNTER.add(1, _attributes(event))
+    elif event.event == "renew":
+        if hooks.on_renewal is not None:
+            hooks.on_renewal(event)
+        _RENEW_COUNTER.add(1, _attributes(event))
+    elif event.event == "timeout":
+        if hooks.on_timeout is not None:
+            hooks.on_timeout(event)
+        _TIMEOUT_COUNTER.add(1, _attributes(event))
+    elif event.event == "expired":
+        if hooks.on_expired is not None:
+            hooks.on_expired(event)
+        _EXPIRED_COUNTER.add(1, _attributes(event))
+
+
+def _attributes(event: LockEvent) -> dict[str, Any]:
+    return {
+        "lock_name": event.lock_name,
+        "mode": event.mode.value if event.mode else None,
+        "backend": event.backend,
+        "namespace": event.namespace or "",
+    }
+
+
+if __name__ == "__main__":
+    pass