superlocalmemory 3.8.13 → 4.0.0

package/src/superlocalmemory/mesh/discovery.py

@@ -0,0 +1,365 @@
+"""mDNS/DNS-SD advertising for the SLM daemon mesh substrate.
+
+This module provides :class:`MeshAdvertiser`, which registers the running SLM
+daemon as a ``_slm-mesh._tcp.local.`` service so that other SLM instances on
+the same LAN can discover it via :class:`~superlocalmemory.mesh.remote_sync.RemoteSyncClient`.
+
+Advertising is **opt-in and disabled by default** (backward-compatible).
+Existing installations see zero behavior change.  Enable via::
+
+    SLM_MESH_ADVERTISE=1    # also accepts: on, true, yes (case-insensitive)
+
+WHY OPT-IN:  SLM runs on corporate-managed endpoints where unsolicited mDNS
+multicast traffic may violate network policy or trigger security tooling.
+Advertising fires only when an operator explicitly sets ``SLM_MESH_ADVERTISE``.
+
+Service type: ``_slm-mesh._tcp.local.``
+This MUST match the type browsed by :class:`RemoteSyncClient` in
+``superlocalmemory.mesh.remote_sync`` (grep ``_slm-mesh._tcp.local.``).
+
+Instance name: ``slm-{node_id}-{port}._slm-mesh._tcp.local.``
+The port suffix prevents :exc:`~zeroconf.NonUniqueNameException` when two
+SLM daemon instances run on the same host (e.g. primary on 8765, test on 8766).
+
+Properties TXT record: contains ``node_id`` and any caller-supplied metadata.
+**NEVER include secrets, tokens, or passwords** — TXT records are broadcast in
+plaintext across the LAN and are visible to every host on the subnet.
+
+All operations are fail-soft: any error is logged as WARNING and the daemon
+continues.  ``stop()`` is idempotent.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+import socket
+import threading
+from socket import inet_aton
+from typing import Any, Optional
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# Constants
+# ---------------------------------------------------------------------------
+
+_SERVICE_TYPE: str = "_slm-mesh._tcp.local."
+"""DNS-SD service type — must match what RemoteSyncClient browses."""
+
+_ADVERTISE_TRUTHY: frozenset[str] = frozenset({"1", "on", "true", "yes"})
+"""Accepted values for SLM_MESH_ADVERTISE (matched case-insensitively)."""
+
+# ---------------------------------------------------------------------------
+# Optional zeroconf dependency guard
+# Mirrors the exact pattern used in remote_sync.py so both modules behave
+# identically when zeroconf is not installed.
+# ---------------------------------------------------------------------------
+try:
+    from zeroconf import ServiceInfo, Zeroconf
+
+    ZEROCONF_AVAILABLE: bool = True
+except ImportError:
+    ZEROCONF_AVAILABLE = False
+    Zeroconf = None  # type: ignore[assignment,misc]
+    ServiceInfo = None  # type: ignore[assignment]
+
+
+# ---------------------------------------------------------------------------
+# Internal helpers
+# ---------------------------------------------------------------------------
+
+
+def _is_advertising_enabled() -> bool:
+    """Return True only when ``SLM_MESH_ADVERTISE`` is set to a truthy value.
+
+    Case-insensitive: ``'1'``, ``'on'``, ``'true'``, ``'yes'`` are accepted.
+    Everything else (including unset) returns False (default-OFF, BC-safe).
+    """
+    return (
+        os.environ.get("SLM_MESH_ADVERTISE", "").strip().lower()
+        in _ADVERTISE_TRUTHY
+    )
+
+
+def _resolve_advertise_ip(injected_ip: Optional[str] = None) -> str:
+    """Return the best local non-loopback IPv4 address for mDNS advertising.
+
+    Uses a UDP connect trick (no packet is sent — the kernel consults the
+    routing table to decide which source address would be used to reach a
+    well-known public address).  Falls back to ``127.0.0.1`` when:
+
+    * No routable interface is found.
+    * The resolved address is itself a loopback address.
+    * Any :exc:`OSError` is raised (e.g. no network at all).
+
+    The ``127.0.0.1`` fallback allows two-daemon integration tests on the
+    loopback interface to work without a real LAN.
+
+    Args:
+        injected_ip: When provided, skip resolution entirely and return this
+                     value.  Used by tests to avoid real socket calls.
+
+    Returns:
+        A dotted-decimal IPv4 address string.
+    """
+    if injected_ip is not None:
+        return injected_ip
+    # Operator override for multi-homed / VPN hosts where the auto-detected
+    # route would advertise the wrong NIC (audit P2). Explicit wins.
+    env_ip = os.environ.get("SLM_MESH_ADVERTISE_IP", "").strip()
+    if env_ip:
+        return env_ip
+    try:
+        with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sock:
+            sock.connect(("8.8.8.8", 80))
+            ip: str = sock.getsockname()[0]
+        if ip and not ip.startswith("127."):
+            return ip
+    except OSError:
+        pass
+    # No routable non-loopback interface — peers off this host cannot reach a
+    # 127.0.0.1 advertisement. Warn so the operator can set SLM_MESH_ADVERTISE_IP.
+    logger.warning(
+        "MeshAdvertiser: no routable non-loopback IPv4 found; advertising "
+        "127.0.0.1 (off-host peers cannot reach this node). Set "
+        "SLM_MESH_ADVERTISE_IP to advertise a reachable address."
+    )
+    return "127.0.0.1"
+
+
+# ---------------------------------------------------------------------------
+# MeshAdvertiser
+# ---------------------------------------------------------------------------
+
+
+class MeshAdvertiser:
+    """Register this SLM daemon as a discoverable mDNS service on the LAN.
+
+    Advertising is **disabled by default** (opt-in via ``SLM_MESH_ADVERTISE``).
+
+    Lifecycle::
+
+        advertiser = MeshAdvertiser(service_port=8765, node_id="hostname")
+        advertiser.start()   # no-op unless SLM_MESH_ADVERTISE is truthy
+        ...
+        advertiser.stop()    # idempotent; safe to call even if never started
+
+    Thread safety:
+        ``start()`` and ``stop()`` are protected by a :class:`threading.Lock`.
+        The lock prevents double-registration and eliminates the race between
+        a slow ``register_service`` call and a concurrent ``stop()`` call:
+        ``start()`` holds the lock for the entire registration so ``stop()``
+        sees a consistent, fully-initialised state.
+
+    CRIT fixes applied:
+        1. **Instance-name collision** — port is embedded in the instance label
+           (``slm-{node_id}-{port}``), so two daemons on the same host register
+           distinct names and avoid :exc:`~zeroconf.NonUniqueNameException`.
+        2. **Event-loop blocking** — ``register_service`` probes for ~750 ms.
+           Callers on an async startup path MUST wrap ``start()`` in
+           ``asyncio.to_thread(advertiser.start)`` (see lifespan wiring in
+           ``unified_daemon.py``).  ``MeshAdvertiser`` itself remains sync so
+           it is trivially testable without an event loop.
+        3. **stop() / register race** — ``_lock`` is held for the entire
+           duration of ``_start_locked()``, so a concurrent ``stop()`` blocks
+           until registration completes.  ``stop()`` clears state under the
+           lock before releasing it, preventing double-close.
+    """
+
+    def __init__(
+        self,
+        service_port: int,
+        node_id: str,
+        properties: Optional[dict[str, str]] = None,
+        *,
+        _injected_ip: Optional[str] = None,
+    ) -> None:
+        """Create a MeshAdvertiser.
+
+        Args:
+            service_port: TCP port the SLM HTTP daemon is bound to.
+            node_id: Stable unique identifier for this node (hostname is a
+                     good choice).  Must be safe for DNS labels.
+            properties: Optional extra metadata broadcast in the mDNS TXT
+                        record.  Keep tiny.  **NEVER include secrets** — TXT
+                        records are visible in plaintext to every host on the
+                        LAN.
+            _injected_ip: Test hook — override IP resolution.  Not part of the
+                          public API.
+        """
+        self._service_port: int = service_port
+        self._node_id: str = node_id
+        # Defensive copy — immutable style; caller can't mutate our dict later.
+        self._properties: dict[str, str] = dict(properties) if properties else {}
+        self._injected_ip: Optional[str] = _injected_ip
+
+        # Mutable state — all mutations happen under _lock.
+        self._zeroconf: Any = None
+        self._info: Any = None
+        self._advertising: bool = False
+        self._lock: threading.Lock = threading.Lock()
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+
+    def start(self) -> None:
+        """Register the SLM service in mDNS.
+
+        No-op when:
+        * zeroconf is not installed (``ZEROCONF_AVAILABLE is False``).
+        * ``SLM_MESH_ADVERTISE`` is unset or not in the truthy set.
+
+        Any exception from the zeroconf stack is caught, logged as WARNING,
+        and swallowed — the daemon must NEVER fail to start because of
+        optional mDNS advertising.
+
+        Note:
+            ``register_service`` blocks for ~750 ms (mDNS probe phase).
+            Call this from an async context via ``asyncio.to_thread(advertiser.start)``.
+        """
+        if not ZEROCONF_AVAILABLE:
+            logger.debug(
+                "MeshAdvertiser.start: zeroconf not installed; skipping mDNS advertising"
+            )
+            return
+        if not _is_advertising_enabled():
+            logger.debug(
+                "MeshAdvertiser.start: SLM_MESH_ADVERTISE not set (or not truthy); "
+                "mDNS advertising remains disabled (default-OFF, BC-safe)"
+            )
+            return
+
+        with self._lock:
+            if self._advertising:
+                # Idempotent — already registered; nothing to do.
+                return
+            try:
+                self._start_locked()
+            except Exception as exc:
+                logger.warning(
+                    "MeshAdvertiser.start failed (non-fatal, daemon continues): %s", exc
+                )
+                # Clean up any partial state so stop() is still safe.
+                self._zeroconf = None
+                self._info = None
+                self._advertising = False
+
+    def stop(self) -> None:
+        """Unregister the mDNS service and close Zeroconf.
+
+        Idempotent — safe to call multiple times or when ``start()`` was never
+        called.  Any exception is caught and logged as WARNING.
+
+        CRIT note: state is cleared under ``_lock`` *before* calling
+        ``unregister_service`` / ``close``, so a second concurrent ``stop()``
+        sees ``_advertising=False`` and exits immediately without attempting
+        to close an already-closed Zeroconf instance.
+        """
+        with self._lock:
+            if not self._advertising:
+                return
+            # Capture and clear state under the lock atomically.
+            zc: Any = self._zeroconf
+            info: Any = self._info
+            self._advertising = False
+            self._zeroconf = None
+            self._info = None
+
+        # Perform I/O outside the lock so a concurrent stop() call (highly
+        # unlikely given daemon lifecycle, but possible in tests) can't
+        # deadlock waiting for us.
+        # Audit P1: close() MUST run even when unregister_service raises,
+        # otherwise the Zeroconf multicast sockets + threads leak. Best-effort
+        # unregister, then close in a finally.
+        try:
+            if zc is not None and info is not None:
+                try:
+                    zc.unregister_service(info)
+                except Exception as exc:
+                    logger.warning(
+                        "MeshAdvertiser.stop: unregister failed (closing anyway): %s",
+                        exc,
+                    )
+        finally:
+            if zc is not None:
+                try:
+                    zc.close()
+                except Exception as exc:
+                    logger.warning(
+                        "MeshAdvertiser.stop: close failed (non-fatal): %s", exc
+                    )
+
+    @property
+    def is_advertising(self) -> bool:
+        """True when the service is currently registered in mDNS."""
+        return self._advertising
+
+    # ------------------------------------------------------------------
+    # Internal
+    # ------------------------------------------------------------------
+
+    def _start_locked(self) -> None:
+        """Build :class:`~zeroconf.ServiceInfo` and register it.
+
+        Called under ``self._lock``.  Raises on any error; the caller
+        (``start()``) catches and logs.
+        """
+        ip: str = _resolve_advertise_ip(self._injected_ip)
+
+        # Instance label carries port AND pid so the mDNS name is unique:
+        # - port disambiguates two daemons on the SAME host (CRIT fix #1);
+        # - pid disambiguates two machines that share a short hostname on the
+        #   segment (audit P2 — cross-machine NonUniqueNameException).
+        # The TXT node_id stays hostname-only (see _properties below).
+        instance_label: str = f"slm-{self._node_id}-{self._service_port}-{os.getpid()}"
+        service_name: str = f"{instance_label}.{_SERVICE_TYPE}"
+
+        # Build bytes-keyed properties dict (zeroconf API requirement).
+        # SECURITY: only whitelisted string content — never mirror env vars.
+        # TXT records are broadcast in plaintext to the entire LAN subnet.
+        safe_props: dict[str, str] = {
+            "node_id": self._node_id,
+            **self._properties,
+        }
+        byte_props: dict[bytes, bytes] = {
+            k.encode("utf-8"): v.encode("utf-8") for k, v in safe_props.items()
+        }
+
+        info = ServiceInfo(
+            type_=_SERVICE_TYPE,
+            name=service_name,
+            addresses=[inet_aton(ip)],
+            port=self._service_port,
+            properties=byte_props,
+        )
+
+        zc = Zeroconf()
+        # register_service sends mDNS probe + announce packets.
+        # This call blocks for ~750 ms (3 × 250 ms probe interval).
+        # Callers on the async event loop MUST use asyncio.to_thread().
+        try:
+            zc.register_service(info)
+        except Exception:
+            # Audit P1: register_service failed AFTER Zeroconf() opened its
+            # multicast sockets + background threads. Close it before
+            # propagating so those resources are released instead of leaking
+            # for the daemon's lifetime (start() catches and logs).
+            try:
+                zc.close()
+            except Exception:  # pragma: no cover — best-effort cleanup
+                pass
+            raise
+
+        # Only update state after successful registration.
+        self._zeroconf = zc
+        self._info = info
+        self._advertising = True
+
+        logger.info(
+            "MeshAdvertiser: registered '%s' on %s:%d",
+            service_name,
+            ip,
+            self._service_port,
+        )

package/src/superlocalmemory/mesh/lock_protocol.py

@@ -0,0 +1,313 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later - see LICENSE file
+# Part of SuperLocalMemory V4 | https://qualixar.com | https://varunpratap.com
+
+"""SLM Mesh — leaderless cross-node lock coordination (3c-2).
+
+SAFETY MODEL (never overclaim):
+  ``resolve()`` converges the ADVISORY lock VIEW across nodes using a
+  deterministic ``(fencing_token, node_id)`` total order.  Higher wins.
+  When the remote node wins, the local ``mesh_locks`` row is deleted so
+  the local node's token becomes stale.
+
+  The FENCING TOKEN (``broker.validate_lock_fence``) is the single-writer
+  SAFETY guarantee: even during a brief split where both nodes held the
+  lock, any write attempt with the yielded node's old token is REJECTED
+  by the storage layer — because either:
+    (a) the row is gone → "no lock held for this resource", or
+    (b) the winning node has since acquired with a higher token →
+        "fencing token X is stale; current token is Y".
+
+  **The advisory lock alone does NOT guarantee mutual exclusion; only the
+  fence does.**  This is NOT linearizable consensus (which requires a
+  quorum).
+
+  All new behavior is ADDITIVE and only active when ``resolve()`` is called
+  by the sync layer.  Node-local ``lock_action`` behavior is UNCHANGED.
+
+Part of Qualixar | Author: Varun Pratap Bhardwaj
+"""
+
+from __future__ import annotations
+
+import logging
+import sqlite3
+from datetime import datetime, timezone
+from typing import TYPE_CHECKING
+
+from superlocalmemory.mesh.broker import _NEVER_EXPIRES
+from superlocalmemory.mesh.node_identity import get_node_id
+
+if TYPE_CHECKING:
+    from superlocalmemory.mesh.broker import MeshBroker
+
+logger = logging.getLogger("superlocalmemory.mesh.lock_protocol")
+
+
+class LockCoordinator:
+    """Leaderless cross-node advisory lock coordinator.
+
+    Wraps a ``MeshBroker`` instance and adds the cross-machine protocol
+    layer: export local live locks (``local_lock_delta``) and converge
+    the lock view against a remote peer's claims (``resolve``).
+
+    All database operations are fail-soft: errors are logged and an empty
+    / no-op result is returned so callers (including the daemon sync loop)
+    are never interrupted.
+    """
+
+    def __init__(self, broker: "MeshBroker") -> None:
+        self._broker = broker
+        self._db_path: str = broker._db_path
+        self._node_id: str = get_node_id(self._db_path)
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+
+    def local_lock_delta(
+        self,
+        profile_id: str = "default",
+        now_iso: str | None = None,
+    ) -> list[dict]:
+        """Return live local locks for a profile, annotated with this node's id.
+
+        A lock is *live* iff its ``expires_at`` is set, not the legacy
+        ``_NEVER_EXPIRES`` sentinel, and not yet elapsed.
+
+        Args:
+            profile_id: Tenant scope (default ``"default"``).
+            now_iso:    UTC ISO timestamp; injectable for test determinism.
+
+        Returns:
+            List of ``{"file_path","locked_by","locked_at","expires_at",
+            "fencing_token","node_id"}`` dicts.  ``[]`` on DB error (fail-soft).
+        """
+        now: str = now_iso or datetime.now(timezone.utc).isoformat()
+        conn: sqlite3.Connection | None = None
+        try:
+            conn = sqlite3.connect(self._db_path, timeout=5.0)
+            conn.row_factory = sqlite3.Row
+            rows = conn.execute(
+                "SELECT file_path, locked_by, locked_at, expires_at,"
+                " COALESCE(fencing_token, 0) AS fencing_token"
+                " FROM mesh_locks WHERE profile_id=?",
+                (profile_id,),
+            ).fetchall()
+        except sqlite3.Error as exc:
+            logger.error("local_lock_delta: DB error for profile %r: %s", profile_id, exc)
+            return []
+        finally:
+            if conn is not None:
+                conn.close()
+
+        result: list[dict] = []
+        for row in rows:
+            if not self._row_is_live(row["expires_at"], now):
+                continue
+            result.append({
+                "file_path": row["file_path"],
+                "locked_by": row["locked_by"],
+                "locked_at": row["locked_at"],
+                "expires_at": row["expires_at"],
+                "fencing_token": int(row["fencing_token"]),
+                "node_id": self._node_id,
+            })
+        return result
+
+    def resolve(
+        self,
+        profile_id: str,
+        remote_locks: list[dict],
+    ) -> dict:
+        """Converge the local lock view against a remote peer's live claims.
+
+        For each *live* remote lock, compare it to the local ``mesh_locks``
+        row using ``(fencing_token DESC, node_id DESC)``.  Remote wins →
+        delete local row (token goes stale; fence will reject it).  Only
+        ``file_path`` values in ``remote_locks`` are examined (others untouched).
+        Idempotent: a second call after a yield is a no-op.
+
+        Args:
+            profile_id:   Tenant scope.
+            remote_locks: Dicts from the remote peer's ``local_lock_delta``
+                          (must include ``file_path``, ``fencing_token``,
+                          ``node_id``, ``expires_at``).
+
+        Returns:
+            ``{"yielded": [paths], "kept": N}``.
+            ``{"yielded": [], "kept": 0}`` on any error (fail-soft).
+        """
+        try:
+            return self._resolve_inner(profile_id, remote_locks)
+        except Exception as exc:  # broad catch — must never crash daemon
+            logger.error(
+                "resolve: unexpected error for profile %r: %s",
+                profile_id, exc,
+                exc_info=True,
+            )
+            return {"yielded": [], "kept": 0}
+
+    # ------------------------------------------------------------------
+    # Internal helpers
+    # ------------------------------------------------------------------
+
+    def _resolve_inner(
+        self,
+        profile_id: str,
+        remote_locks: list[dict],
+    ) -> dict:
+        """Core resolve logic — separated so the outer method can catch all errors."""
+        now: str = datetime.now(timezone.utc).isoformat()
+
+        # Build a map of file_path → strongest LIVE remote claim.
+        # Audit P2: a non-dict entry must never crash resolve (per-entry skip).
+        # Audit P2: if a caller ever concatenates multiple peers' deltas, keep
+        # the winner under the SAME total order (token, node_id) rather than
+        # letting a later, weaker claim overwrite a stronger one.
+        live_remote: dict[str, dict] = {}
+        for rlock in remote_locks:
+            if not isinstance(rlock, dict):
+                continue
+            fp = rlock.get("file_path", "")
+            if not fp or not self._dict_is_live(rlock, now):
+                continue
+            existing = live_remote.get(fp)
+            if existing is None or self._lock_key(rlock) > self._lock_key(existing):
+                live_remote[fp] = rlock
+
+        if not live_remote:
+            return {"yielded": [], "kept": 0}
+
+        # Fetch local rows for the contested file_paths only — including
+        # expires_at (audit P1: an EXPIRED local row must not win over a live
+        # remote claim just because it carries a higher token).
+        file_paths: tuple[str, ...] = tuple(live_remote.keys())
+        placeholders = ",".join("?" * len(file_paths))
+        conn: sqlite3.Connection = sqlite3.connect(self._db_path, timeout=5.0)
+        try:
+            conn.row_factory = sqlite3.Row
+            try:
+                local_rows = conn.execute(
+                    f"SELECT file_path, expires_at,"
+                    f" COALESCE(fencing_token, 0) AS fencing_token"
+                    f" FROM mesh_locks WHERE profile_id=? AND file_path IN ({placeholders})",
+                    (profile_id, *file_paths),
+                ).fetchall()
+            except sqlite3.Error as exc:
+                logger.error(
+                    "resolve: failed to query local locks for profile %r: %s",
+                    profile_id, exc,
+                )
+                return {"yielded": [], "kept": 0}
+
+            local_by_path: dict[str, dict] = {
+                row["file_path"]: {
+                    "fencing_token": int(row["fencing_token"]),
+                    "expires_at": row["expires_at"],
+                }
+                for row in local_rows
+            }
+
+            yielded: list[str] = []
+            kept: int = 0
+
+            for fp, rlock in live_remote.items():
+                local = local_by_path.get(fp)
+                if local is None:
+                    continue  # No local row — nothing to yield or keep.
+
+                snapshot_token: int = local["fencing_token"]
+                local_live: bool = self._row_is_live(local["expires_at"], now)
+
+                # Yield when: (a) the local row is EXPIRED garbage while a live
+                # remote claim exists (audit P1 — clean it up so its stale token
+                # can't pass the fence), or (b) the live remote strictly wins the
+                # total order.
+                if local_live and not self._remote_wins(rlock, local):
+                    kept += 1
+                    continue
+
+                # Token-CONDITIONAL delete (audit P1/TOCTOU): only delete the
+                # row we compared against. If the local node reacquired to a new
+                # (higher) token between snapshot and now, the predicate misses
+                # and we do NOT wipe the fresher live lock.
+                try:
+                    cur = conn.execute(
+                        "DELETE FROM mesh_locks WHERE profile_id=? AND file_path=?"
+                        " AND COALESCE(fencing_token, 0)=?",
+                        (profile_id, fp, snapshot_token),
+                    )
+                    conn.commit()
+                    if cur.rowcount and cur.rowcount > 0:
+                        logger.info(
+                            "resolve: yielded lock %r to remote node %r "
+                            "(remote (tok=%s,node=%s) vs local (tok=%s), local_live=%s)",
+                            fp, rlock.get("node_id", "?"), self._safe_token(rlock),
+                            rlock.get("node_id", "?"), snapshot_token, local_live,
+                        )
+                        yielded.append(fp)
+                    # rowcount==0 → lost the race (reacquired); leave it, don't count.
+                except sqlite3.Error as exc:
+                    logger.error(
+                        "resolve: failed to yield lock %r for profile %r: %s",
+                        fp, profile_id, exc,
+                    )
+
+            return {"yielded": yielded, "kept": kept}
+        finally:
+            conn.close()
+
+    @staticmethod
+    def _row_is_live(expires_at: str | None, now_iso: str) -> bool:
+        """Return True iff a DB row's expires_at represents a live lock."""
+        if not expires_at:
+            return False
+        if expires_at == _NEVER_EXPIRES:
+            return False
+        return expires_at > now_iso
+
+    @staticmethod
+    def _dict_is_live(lock: dict, now_iso: str) -> bool:
+        """Return True iff a remote lock dict represents a live claim."""
+        exp = lock.get("expires_at", "") or ""
+        if not exp:
+            return False
+        if exp == _NEVER_EXPIRES:
+            return False
+        return exp > now_iso
+
+    @staticmethod
+    def _safe_token(lock: dict) -> int:
+        """Cast fencing_token to int defensively (remote dicts may carry strings)."""
+        try:
+            return int(lock.get("fencing_token", 0))
+        except (TypeError, ValueError):
+            return 0
+
+    def _lock_key(self, lock: dict) -> tuple[int, str]:
+        """Total-order key ``(fencing_token, node_id)`` for comparing claims."""
+        return (self._safe_token(lock), str(lock.get("node_id", "")))
+
+    def _remote_wins(self, remote: dict, local: dict) -> bool:
+        """Total order: ``(fencing_token DESC, node_id DESC)``.
+
+        Compare as ``int`` for the token (critical — JSON may deliver strings,
+        and string comparison ``"10" > "9"`` is ``False``, which would invert
+        the order).  Tie-break via lexicographic ``node_id`` comparison; both
+        sides use consistent lowercase hex from ``get_node_id``, so the order
+        is stable across nodes.
+
+        Returns True iff the remote lock should be considered the effective
+        holder over the local lock.
+        """
+        r_tok: int = self._safe_token(remote)
+        l_tok: int = local.get("fencing_token", 0)
+
+        if r_tok != l_tok:
+            return r_tok > l_tok
+
+        # Token tie — use node_id as deterministic tie-breaker.
+        r_nid: str = str(remote.get("node_id", ""))
+        l_nid: str = self._node_id
+        return r_nid > l_nid