fleet-framework 0.1.0__py3-none-any.whl

fleet/core/sqlite_backend.py

@@ -0,0 +1,1117 @@
+"""Sqlite-backed Backend implementation.
+
+Targets single-host deployments where running Redis is overkill — a CLI tool, a
+laptop demo, a tiny VPS. One file on disk, WAL mode for concurrency, the same
+Backend protocol as RedisBackend and InMemoryBackend.
+
+The implementation uses one sqlite3 connection guarded by a threading.Lock.
+Every public method dispatches the actual SQL to a worker thread via
+asyncio.to_thread so the event loop never blocks on a slow disk.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import secrets
+import sqlite3
+import threading
+import time
+from pathlib import Path
+from typing import Any, Optional
+
+from fleet.core.backend import RawItem
+
+_SCHEMA = """
+CREATE TABLE IF NOT EXISTS kv (
+    ns TEXT NOT NULL,
+    key TEXT NOT NULL,
+    value BLOB NOT NULL,
+    expires_at REAL,
+    PRIMARY KEY(ns, key)
+);
+
+CREATE TABLE IF NOT EXISTS pool_items (
+    name TEXT NOT NULL,
+    item_id TEXT NOT NULL,
+    payload BLOB NOT NULL,
+    tags TEXT NOT NULL,
+    expires_at REAL,
+    PRIMARY KEY(name, item_id)
+);
+
+CREATE TABLE IF NOT EXISTS pool_claims (
+    name TEXT NOT NULL,
+    item_id TEXT NOT NULL,
+    until REAL NOT NULL,
+    PRIMARY KEY(name, item_id)
+);
+
+CREATE TABLE IF NOT EXISTS locks (
+    name TEXT PRIMARY KEY,
+    token TEXT NOT NULL,
+    expires_at REAL NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS counters (
+    name TEXT PRIMARY KEY,
+    value INTEGER NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS queues (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    name TEXT NOT NULL,
+    tier TEXT NOT NULL,
+    body BLOB NOT NULL
+);
+CREATE INDEX IF NOT EXISTS queues_name_tier ON queues(name, tier, id);
+
+CREATE TABLE IF NOT EXISTS queue_processing (
+    name TEXT NOT NULL,
+    task_id TEXT NOT NULL,
+    body BLOB NOT NULL,
+    deadline REAL NOT NULL,
+    PRIMARY KEY(name, task_id)
+);
+
+CREATE TABLE IF NOT EXISTS queue_scheduled (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    name TEXT NOT NULL,
+    available_at REAL NOT NULL,
+    body BLOB NOT NULL
+);
+CREATE INDEX IF NOT EXISTS queue_scheduled_name_at ON queue_scheduled(name, available_at);
+
+CREATE TABLE IF NOT EXISTS queue_dead (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    name TEXT NOT NULL,
+    body BLOB NOT NULL
+);
+CREATE INDEX IF NOT EXISTS queue_dead_name_id ON queue_dead(name, id);
+
+CREATE TABLE IF NOT EXISTS streams (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    name TEXT NOT NULL,
+    score REAL NOT NULL,
+    body BLOB NOT NULL
+);
+CREATE INDEX IF NOT EXISTS streams_name_score ON streams(name, score);
+
+CREATE TABLE IF NOT EXISTS workers (
+    automation_type TEXT NOT NULL,
+    worker_id TEXT NOT NULL,
+    hardware TEXT NOT NULL,
+    config TEXT NOT NULL,
+    config_gen INTEGER NOT NULL DEFAULT 0,
+    state TEXT NOT NULL DEFAULT 'IDLE',
+    last_seen REAL NOT NULL DEFAULT 0,
+    last_error TEXT,
+    stats TEXT NOT NULL DEFAULT '{}',
+    PRIMARY KEY(automation_type, worker_id)
+);
+
+CREATE TABLE IF NOT EXISTS worker_stats_history (
+    automation_type TEXT NOT NULL,
+    worker_id TEXT NOT NULL,
+    ts REAL NOT NULL,
+    frame TEXT NOT NULL
+);
+CREATE INDEX IF NOT EXISTS wsh_at_wid_ts ON worker_stats_history(automation_type, worker_id, ts);
+
+CREATE TABLE IF NOT EXISTS catalog (
+    automation_type TEXT PRIMARY KEY,
+    doc BLOB NOT NULL
+);
+"""
+
+
+def _tier_for_priority(priority: int) -> str:
+    if priority > 0:
+        return "high"
+    if priority < 0:
+        return "low"
+    return "normal"
+
+
+_TIER_ORDER = ("high", "normal", "low")
+
+
+class SqliteBackend:
+    def __init__(self, url_or_path: str = "sqlite://:memory:") -> None:
+        path = _path_from_url(url_or_path)
+        if path != ":memory:":
+            Path(path).parent.mkdir(parents=True, exist_ok=True)
+        self._conn = sqlite3.connect(
+            path,
+            isolation_level=None,  # autocommit; we manage txns by hand
+            check_same_thread=False,
+            timeout=30.0,
+        )
+        self._lock = threading.Lock()
+        self._conn.execute("PRAGMA journal_mode=WAL;")
+        self._conn.execute("PRAGMA synchronous=NORMAL;")
+        self._conn.execute("PRAGMA busy_timeout=5000;")
+        self._conn.executescript(_SCHEMA)
+
+    async def aclose(self) -> None:
+        await asyncio.to_thread(self._close_sync)
+
+    def _close_sync(self) -> None:
+        with self._lock:
+            try:
+                self._conn.close()
+            except Exception:
+                pass
+
+    # ------- KV ----------
+
+    async def kv_set(self, ns: str, key: str, value: bytes, ttl_seconds: Optional[int]) -> None:
+        exp = time.time() + ttl_seconds if (ttl_seconds and ttl_seconds > 0) else None
+        await asyncio.to_thread(self._kv_set_sync, ns, key, value, exp)
+
+    def _kv_set_sync(self, ns: str, key: str, value: bytes, exp: Optional[float]) -> None:
+        with self._lock:
+            self._conn.execute(
+                "INSERT OR REPLACE INTO kv(ns, key, value, expires_at) VALUES(?,?,?,?)",
+                (ns, key, value, exp),
+            )
+
+    async def kv_setnx(
+        self, ns: str, key: str, value: bytes, ttl_seconds: Optional[int]
+    ) -> bool:
+        exp = time.time() + ttl_seconds if (ttl_seconds and ttl_seconds > 0) else None
+        return await asyncio.to_thread(self._kv_setnx_sync, ns, key, value, exp)
+
+    def _kv_setnx_sync(self, ns: str, key: str, value: bytes, exp: Optional[float]) -> bool:
+        now = time.time()
+        with self._lock:
+            self._conn.execute(
+                "DELETE FROM kv WHERE ns=? AND key=? AND expires_at IS NOT NULL AND expires_at < ?",
+                (ns, key, now),
+            )
+            try:
+                self._conn.execute(
+                    "INSERT INTO kv(ns, key, value, expires_at) VALUES(?,?,?,?)",
+                    (ns, key, value, exp),
+                )
+                return True
+            except sqlite3.IntegrityError:
+                return False
+
+    async def kv_get(self, ns: str, key: str) -> Optional[bytes]:
+        return await asyncio.to_thread(self._kv_get_sync, ns, key)
+
+    def _kv_get_sync(self, ns: str, key: str) -> Optional[bytes]:
+        now = time.time()
+        with self._lock:
+            row = self._conn.execute(
+                "SELECT value, expires_at FROM kv WHERE ns=? AND key=?",
+                (ns, key),
+            ).fetchone()
+        if row is None:
+            return None
+        value, exp = row
+        if exp is not None and exp < now:
+            # Lazy expiry: delete and pretend it wasn't there.
+            with self._lock:
+                self._conn.execute("DELETE FROM kv WHERE ns=? AND key=?", (ns, key))
+            return None
+        return value
+
+    async def kv_del(self, ns: str, key: str) -> bool:
+        return await asyncio.to_thread(self._kv_del_sync, ns, key)
+
+    def _kv_del_sync(self, ns: str, key: str) -> bool:
+        with self._lock:
+            cur = self._conn.execute("DELETE FROM kv WHERE ns=? AND key=?", (ns, key))
+        return cur.rowcount > 0
+
+    # ------- Pool ----------
+
+    async def pool_put(
+        self,
+        name: str,
+        item_id: str,
+        payload: bytes,
+        tags: dict[str, Any],
+        ttl_seconds: Optional[int],
+    ) -> None:
+        exp = time.time() + ttl_seconds if (ttl_seconds and ttl_seconds > 0) else None
+        await asyncio.to_thread(self._pool_put_sync, name, item_id, payload, tags, exp)
+
+    def _pool_put_sync(
+        self, name: str, item_id: str, payload: bytes, tags: dict[str, Any], exp: Optional[float]
+    ) -> None:
+        with self._lock:
+            self._conn.execute(
+                "INSERT OR REPLACE INTO pool_items(name, item_id, payload, tags, expires_at) "
+                "VALUES(?,?,?,?,?)",
+                (name, item_id, payload, json.dumps(tags), exp),
+            )
+
+    async def pool_list(self, name: str) -> list[RawItem]:
+        return await asyncio.to_thread(self._pool_list_sync, name)
+
+    def _pool_list_sync(self, name: str) -> list[RawItem]:
+        now = time.time()
+        with self._lock:
+            self._conn.execute(
+                "DELETE FROM pool_items WHERE name=? AND expires_at IS NOT NULL AND expires_at < ?",
+                (name, now),
+            )
+            self._conn.execute(
+                "DELETE FROM pool_claims WHERE name=? AND until < ?",
+                (name, now),
+            )
+            rows = self._conn.execute(
+                "SELECT item_id, payload, tags, expires_at FROM pool_items WHERE name=?",
+                (name,),
+            ).fetchall()
+            claims = dict(
+                self._conn.execute(
+                    "SELECT item_id, until FROM pool_claims WHERE name=?", (name,)
+                ).fetchall()
+            )
+        out: list[RawItem] = []
+        for iid, payload, tags_json, exp in rows:
+            in_use = claims.get(iid)
+            out.append(RawItem(
+                id=iid,
+                payload=payload,
+                tags=json.loads(tags_json),
+                expires_at=exp,
+                in_use_until=in_use if (in_use is not None and in_use > now) else None,
+            ))
+        return out
+
+    async def pool_size(self, name: str) -> int:
+        return await asyncio.to_thread(self._pool_size_sync, name)
+
+    def _pool_size_sync(self, name: str) -> int:
+        now = time.time()
+        with self._lock:
+            row = self._conn.execute(
+                "SELECT COUNT(*) FROM pool_items WHERE name=? AND "
+                "(expires_at IS NULL OR expires_at >= ?)",
+                (name, now),
+            ).fetchone()
+        return int(row[0])
+
+    async def pool_remove(self, name: str, item_id: str) -> bool:
+        return await asyncio.to_thread(self._pool_remove_sync, name, item_id)
+
+    def _pool_remove_sync(self, name: str, item_id: str) -> bool:
+        with self._lock:
+            self._conn.execute(
+                "DELETE FROM pool_claims WHERE name=? AND item_id=?",
+                (name, item_id),
+            )
+            cur = self._conn.execute(
+                "DELETE FROM pool_items WHERE name=? AND item_id=?",
+                (name, item_id),
+            )
+        return cur.rowcount > 0
+
+    async def pool_release(self, name: str, item_id: str) -> bool:
+        return await asyncio.to_thread(self._pool_release_sync, name, item_id)
+
+    def _pool_release_sync(self, name: str, item_id: str) -> bool:
+        with self._lock:
+            cur = self._conn.execute(
+                "DELETE FROM pool_claims WHERE name=? AND item_id=?",
+                (name, item_id),
+            )
+        return cur.rowcount > 0
+
+    async def pool_claim_any(
+        self,
+        name: str,
+        where: dict[str, Any],
+        hold_seconds: int,
+    ) -> Optional[RawItem]:
+        return await asyncio.to_thread(self._pool_claim_any_sync, name, where, hold_seconds)
+
+    def _pool_claim_any_sync(
+        self, name: str, where: dict[str, Any], hold_seconds: int
+    ) -> Optional[RawItem]:
+        now = time.time()
+        until = now + max(1, hold_seconds)
+        with self._lock:
+            # purge expired items + claims first
+            self._conn.execute(
+                "DELETE FROM pool_items WHERE name=? AND expires_at IS NOT NULL AND expires_at < ?",
+                (name, now),
+            )
+            self._conn.execute(
+                "DELETE FROM pool_claims WHERE name=? AND until < ?",
+                (name, now),
+            )
+            # newest items first (largest expires_at; NULL last).
+            rows = self._conn.execute(
+                "SELECT i.item_id, i.payload, i.tags, i.expires_at "
+                "FROM pool_items i "
+                "LEFT JOIN pool_claims c ON c.name=i.name AND c.item_id=i.item_id "
+                "WHERE i.name=? AND c.item_id IS NULL "
+                "ORDER BY (i.expires_at IS NULL), i.expires_at DESC",
+                (name,),
+            ).fetchall()
+            for iid, payload, tags_json, exp in rows:
+                tags = json.loads(tags_json)
+                if not all(tags.get(k) == v for k, v in where.items()):
+                    continue
+                try:
+                    self._conn.execute(
+                        "INSERT INTO pool_claims(name, item_id, until) VALUES(?,?,?)",
+                        (name, iid, until),
+                    )
+                except sqlite3.IntegrityError:
+                    continue
+                return RawItem(
+                    id=iid,
+                    payload=payload,
+                    tags=tags,
+                    expires_at=exp,
+                    in_use_until=until,
+                )
+        return None
+
+    # ------- Locks ----------
+
+    async def lock_acquire(
+        self, name: str, hold_seconds: int, wait_seconds: float
+    ) -> Optional[str]:
+        deadline = time.monotonic() + max(0.0, wait_seconds)
+        backoff = 0.05
+        while True:
+            tok = await asyncio.to_thread(self._lock_try_acquire_sync, name, hold_seconds)
+            if tok is not None:
+                return tok
+            if time.monotonic() >= deadline:
+                return None
+            await asyncio.sleep(min(backoff, max(0.0, deadline - time.monotonic())))
+            backoff = min(backoff * 2, 0.5)
+
+    def _lock_try_acquire_sync(self, name: str, hold_seconds: int) -> Optional[str]:
+        now = time.time()
+        until = now + max(1, hold_seconds)
+        with self._lock:
+            self._conn.execute(
+                "DELETE FROM locks WHERE name=? AND expires_at < ?", (name, now)
+            )
+            token = secrets.token_hex(16)
+            try:
+                self._conn.execute(
+                    "INSERT INTO locks(name, token, expires_at) VALUES(?,?,?)",
+                    (name, token, until),
+                )
+                return token
+            except sqlite3.IntegrityError:
+                return None
+
+    async def lock_release(self, name: str, token: str) -> bool:
+        return await asyncio.to_thread(self._lock_release_sync, name, token)
+
+    def _lock_release_sync(self, name: str, token: str) -> bool:
+        with self._lock:
+            cur = self._conn.execute(
+                "DELETE FROM locks WHERE name=? AND token=?", (name, token)
+            )
+        return cur.rowcount > 0
+
+    # ------- Counters ----------
+
+    async def counter_incr(self, name: str, by: int) -> int:
+        return await asyncio.to_thread(self._counter_incr_sync, name, by)
+
+    def _counter_incr_sync(self, name: str, by: int) -> int:
+        with self._lock:
+            self._conn.execute(
+                "INSERT INTO counters(name, value) VALUES(?,0) "
+                "ON CONFLICT(name) DO NOTHING",
+                (name,),
+            )
+            self._conn.execute(
+                "UPDATE counters SET value = value + ? WHERE name=?",
+                (by, name),
+            )
+            row = self._conn.execute(
+                "SELECT value FROM counters WHERE name=?", (name,)
+            ).fetchone()
+        return int(row[0])
+
+    async def counter_get(self, name: str) -> int:
+        return await asyncio.to_thread(self._counter_get_sync, name)
+
+    def _counter_get_sync(self, name: str) -> int:
+        with self._lock:
+            row = self._conn.execute(
+                "SELECT value FROM counters WHERE name=?", (name,)
+            ).fetchone()
+        return int(row[0]) if row else 0
+
+    async def counter_set(self, name: str, value: int) -> None:
+        await asyncio.to_thread(self._counter_set_sync, name, value)
+
+    def _counter_set_sync(self, name: str, value: int) -> None:
+        with self._lock:
+            self._conn.execute(
+                "INSERT OR REPLACE INTO counters(name, value) VALUES(?,?)",
+                (name, value),
+            )
+
+    async def counter_del(self, name: str) -> None:
+        await asyncio.to_thread(self._counter_del_sync, name)
+
+    def _counter_del_sync(self, name: str) -> None:
+        with self._lock:
+            self._conn.execute("DELETE FROM counters WHERE name=?", (name,))
+
+    # ------- Queues ----------
+
+    async def queue_push(self, name: str, body: bytes, *, priority: int = 0) -> None:
+        await asyncio.to_thread(self._queue_push_sync, name, body, priority)
+
+    def _queue_push_sync(self, name: str, body: bytes, priority: int) -> None:
+        tier = _tier_for_priority(priority)
+        with self._lock:
+            self._conn.execute(
+                "INSERT INTO queues(name, tier, body) VALUES(?,?,?)",
+                (name, tier, body),
+            )
+
+    async def queue_size(self, name: str) -> int:
+        return await asyncio.to_thread(self._queue_size_sync, name)
+
+    def _queue_size_sync(self, name: str) -> int:
+        with self._lock:
+            row = self._conn.execute(
+                "SELECT COUNT(*) FROM queues WHERE name=?", (name,)
+            ).fetchone()
+        return int(row[0])
+
+    async def queue_peek(self, name: str, n: int) -> list[bytes]:
+        if n <= 0:
+            return []
+        return await asyncio.to_thread(self._queue_peek_sync, name, n)
+
+    def _queue_peek_sync(self, name: str, n: int) -> list[bytes]:
+        out: list[bytes] = []
+        with self._lock:
+            for tier in _TIER_ORDER:
+                remaining = n - len(out)
+                if remaining <= 0:
+                    break
+                rows = self._conn.execute(
+                    "SELECT body FROM queues WHERE name=? AND tier=? "
+                    "ORDER BY id ASC LIMIT ?",
+                    (name, tier, remaining),
+                ).fetchall()
+                out.extend(row[0] for row in rows)
+        return out
+
+    async def queue_reserve(
+        self, name: str, timeout_seconds: float, visibility_seconds: int
+    ) -> Optional[bytes]:
+        deadline = time.monotonic() + max(0.0, timeout_seconds)
+        backoff = 0.05
+        while True:
+            body = await asyncio.to_thread(self._queue_reserve_sync, name, visibility_seconds)
+            if body is not None:
+                return body
+            if time.monotonic() >= deadline:
+                return None
+            await asyncio.sleep(min(backoff, max(0.0, deadline - time.monotonic())))
+            backoff = min(backoff * 2, 0.5)
+
+    def _queue_reserve_sync(
+        self, name: str, visibility_seconds: int
+    ) -> Optional[bytes]:
+        now = time.time()
+        with self._lock:
+            # 1. Move due-scheduled into the queue (normal tier).
+            due = self._conn.execute(
+                "SELECT id, body FROM queue_scheduled WHERE name=? AND available_at<=?",
+                (name, now),
+            ).fetchall()
+            if due:
+                ids = [row[0] for row in due]
+                placeholders = ",".join("?" * len(ids))
+                for _id, body in due:
+                    self._conn.execute(
+                        "INSERT INTO queues(name, tier, body) VALUES(?,?,?)",
+                        (name, "normal", body),
+                    )
+                self._conn.execute(
+                    f"DELETE FROM queue_scheduled WHERE id IN ({placeholders})",
+                    ids,
+                )
+            # 2. Pop one body in priority order.
+            body: Optional[bytes] = None
+            for tier in _TIER_ORDER:
+                row = self._conn.execute(
+                    "SELECT id, body FROM queues WHERE name=? AND tier=? "
+                    "ORDER BY id ASC LIMIT 1",
+                    (name, tier),
+                ).fetchone()
+                if row is not None:
+                    self._conn.execute("DELETE FROM queues WHERE id=?", (row[0],))
+                    body = row[1]
+                    break
+            if body is None:
+                return None
+            # 3. Add to processing table.
+            try:
+                envelope = json.loads(body.decode("utf-8"))
+                task_id = str(envelope.get("id", ""))
+            except Exception:
+                task_id = ""
+            if task_id:
+                self._conn.execute(
+                    "INSERT OR REPLACE INTO queue_processing(name, task_id, body, deadline) "
+                    "VALUES(?,?,?,?)",
+                    (name, task_id, body, now + max(1, visibility_seconds)),
+                )
+            return body
+
+    async def queue_ack(self, name: str, task_id: str) -> bool:
+        return await asyncio.to_thread(self._queue_ack_sync, name, task_id)
+
+    def _queue_ack_sync(self, name: str, task_id: str) -> bool:
+        with self._lock:
+            cur = self._conn.execute(
+                "DELETE FROM queue_processing WHERE name=? AND task_id=?",
+                (name, task_id),
+            )
+        return cur.rowcount > 0
+
+    async def queue_nack(
+        self, name: str, task_id: str, body: bytes, *, delay_seconds: float
+    ) -> bool:
+        return await asyncio.to_thread(
+            self._queue_nack_sync, name, task_id, body, delay_seconds,
+        )
+
+    def _queue_nack_sync(
+        self, name: str, task_id: str, body: bytes, delay_seconds: float
+    ) -> bool:
+        with self._lock:
+            cur = self._conn.execute(
+                "DELETE FROM queue_processing WHERE name=? AND task_id=?",
+                (name, task_id),
+            )
+            had = cur.rowcount > 0
+            if delay_seconds > 0:
+                self._conn.execute(
+                    "INSERT INTO queue_scheduled(name, available_at, body) VALUES(?,?,?)",
+                    (name, time.time() + delay_seconds, body),
+                )
+            else:
+                self._conn.execute(
+                    "INSERT INTO queues(name, tier, body) VALUES(?,?,?)",
+                    (name, "normal", body),
+                )
+        return had
+
+    async def queue_sweep_expired(self, name: str, now: Optional[float] = None) -> int:
+        return await asyncio.to_thread(self._queue_sweep_sync, name, now)
+
+    def _queue_sweep_sync(self, name: str, now: Optional[float]) -> int:
+        n = now if now is not None else time.time()
+        with self._lock:
+            rows = self._conn.execute(
+                "SELECT task_id, body FROM queue_processing WHERE name=? AND deadline<=?",
+                (name, n),
+            ).fetchall()
+            for task_id, body in rows:
+                self._conn.execute(
+                    "INSERT INTO queues(name, tier, body) VALUES(?,?,?)",
+                    (name, "normal", body),
+                )
+                self._conn.execute(
+                    "DELETE FROM queue_processing WHERE name=? AND task_id=?",
+                    (name, task_id),
+                )
+        return len(rows)
+
+    async def queue_dead(self, name: str, body: bytes) -> None:
+        await asyncio.to_thread(self._queue_dead_sync, name, body)
+
+    def _queue_dead_sync(self, name: str, body: bytes) -> None:
+        with self._lock:
+            self._conn.execute(
+                "INSERT INTO queue_dead(name, body) VALUES(?,?)", (name, body),
+            )
+
+    async def dlq_peek(self, name: str, n: int) -> list[bytes]:
+        return await asyncio.to_thread(self._dlq_peek_sync, name, n)
+
+    def _dlq_peek_sync(self, name: str, n: int) -> list[bytes]:
+        with self._lock:
+            rows = self._conn.execute(
+                "SELECT body FROM queue_dead WHERE name=? ORDER BY id ASC LIMIT ?",
+                (name, n),
+            ).fetchall()
+        return [r[0] for r in rows]
+
+    async def dlq_pop(self, name: str, n: int) -> list[bytes]:
+        return await asyncio.to_thread(self._dlq_pop_sync, name, n)
+
+    def _dlq_pop_sync(self, name: str, n: int) -> list[bytes]:
+        with self._lock:
+            rows = self._conn.execute(
+                "SELECT id, body FROM queue_dead WHERE name=? ORDER BY id ASC LIMIT ?",
+                (name, n),
+            ).fetchall()
+            ids = [r[0] for r in rows]
+            if ids:
+                placeholders = ",".join("?" * len(ids))
+                self._conn.execute(
+                    f"DELETE FROM queue_dead WHERE id IN ({placeholders})", ids,
+                )
+        return [r[1] for r in rows]
+
+    async def dlq_length(self, name: str) -> int:
+        return await asyncio.to_thread(self._dlq_length_sync, name)
+
+    def _dlq_length_sync(self, name: str) -> int:
+        with self._lock:
+            row = self._conn.execute(
+                "SELECT COUNT(*) FROM queue_dead WHERE name=?", (name,)
+            ).fetchone()
+        return int(row[0])
+
+    async def dlq_clear(self, name: str) -> int:
+        return await asyncio.to_thread(self._dlq_clear_sync, name)
+
+    def _dlq_clear_sync(self, name: str) -> int:
+        with self._lock:
+            cur = self._conn.execute("DELETE FROM queue_dead WHERE name=?", (name,))
+        return cur.rowcount
+
+    # ------- Streams ----------
+
+    async def stream_push(
+        self,
+        name: str,
+        envelope: bytes,
+        *,
+        score: float,
+        max_len: int,
+        ttl_seconds: int,
+    ) -> None:
+        await asyncio.to_thread(
+            self._stream_push_sync, name, envelope, score, max_len, ttl_seconds,
+        )
+
+    def _stream_push_sync(
+        self, name: str, envelope: bytes, score: float, max_len: int, ttl_seconds: int,
+    ) -> None:
+        with self._lock:
+            self._conn.execute(
+                "INSERT INTO streams(name, score, body) VALUES(?,?,?)",
+                (name, score, envelope),
+            )
+            if ttl_seconds > 0:
+                cutoff = score - ttl_seconds
+                self._conn.execute(
+                    "DELETE FROM streams WHERE name=? AND score<?", (name, cutoff),
+                )
+            if max_len > 0:
+                row = self._conn.execute(
+                    "SELECT COUNT(*) FROM streams WHERE name=?", (name,)
+                ).fetchone()
+                count = int(row[0])
+                if count > max_len:
+                    extra = count - max_len
+                    self._conn.execute(
+                        "DELETE FROM streams WHERE id IN ("
+                        "SELECT id FROM streams WHERE name=? ORDER BY score ASC LIMIT ?)",
+                        (name, extra),
+                    )
+
+    async def stream_pop(self, name: str, n: int) -> list[bytes]:
+        return await asyncio.to_thread(self._stream_pop_sync, name, n)
+
+    def _stream_pop_sync(self, name: str, n: int) -> list[bytes]:
+        if n <= 0:
+            return []
+        with self._lock:
+            rows = self._conn.execute(
+                "SELECT id, body FROM streams WHERE name=? ORDER BY score ASC LIMIT ?",
+                (name, n),
+            ).fetchall()
+            ids = [r[0] for r in rows]
+            if ids:
+                placeholders = ",".join("?" * len(ids))
+                self._conn.execute(
+                    f"DELETE FROM streams WHERE id IN ({placeholders})", ids,
+                )
+        return [r[1] for r in rows]
+
+    async def stream_peek(self, name: str, n: int) -> list[bytes]:
+        return await asyncio.to_thread(self._stream_peek_sync, name, n)
+
+    def _stream_peek_sync(self, name: str, n: int) -> list[bytes]:
+        with self._lock:
+            rows = self._conn.execute(
+                "SELECT body FROM streams WHERE name=? ORDER BY score ASC LIMIT ?",
+                (name, n),
+            ).fetchall()
+        return [r[0] for r in rows]
+
+    async def stream_length(self, name: str) -> int:
+        return await asyncio.to_thread(self._stream_length_sync, name)
+
+    def _stream_length_sync(self, name: str) -> int:
+        with self._lock:
+            row = self._conn.execute(
+                "SELECT COUNT(*) FROM streams WHERE name=?", (name,)
+            ).fetchone()
+        return int(row[0])
+
+    async def stream_trim(self, name: str, *, max_len: int, ttl_seconds: int) -> None:
+        await asyncio.to_thread(self._stream_trim_sync, name, max_len, ttl_seconds)
+
+    def _stream_trim_sync(self, name: str, max_len: int, ttl_seconds: int) -> None:
+        with self._lock:
+            if ttl_seconds > 0:
+                cutoff = time.time() - ttl_seconds
+                self._conn.execute(
+                    "DELETE FROM streams WHERE name=? AND score<?", (name, cutoff),
+                )
+            if max_len > 0:
+                row = self._conn.execute(
+                    "SELECT COUNT(*) FROM streams WHERE name=?", (name,)
+                ).fetchone()
+                count = int(row[0])
+                if count > max_len:
+                    extra = count - max_len
+                    self._conn.execute(
+                        "DELETE FROM streams WHERE id IN ("
+                        "SELECT id FROM streams WHERE name=? ORDER BY score ASC LIMIT ?)",
+                        (name, extra),
+                    )
+
+    async def event_publish(self, topic: str, body: bytes) -> None:
+        # No subscribers in a single-process sqlite deploy. Same caveat as
+        # InMemoryBackend: plugins that need pub/sub must use a real broker.
+        return None
+
+    # ------- Workers ----------
+
+    async def worker_register(
+        self, automation_type: str, worker_id: str, hardware: dict[str, Any]
+    ) -> None:
+        await asyncio.to_thread(self._worker_register_sync, automation_type, worker_id, hardware)
+
+    def _worker_register_sync(
+        self, automation_type: str, worker_id: str, hardware: dict[str, Any]
+    ) -> None:
+        hw_json = json.dumps(hardware)
+        with self._lock:
+            row = self._conn.execute(
+                "SELECT 1 FROM workers WHERE automation_type=? AND worker_id=?",
+                (automation_type, worker_id),
+            ).fetchone()
+            if row is None:
+                self._conn.execute(
+                    "INSERT INTO workers(automation_type, worker_id, hardware, config, "
+                    "config_gen, state, last_seen, last_error, stats) "
+                    "VALUES(?,?,?,?,0,'IDLE',0,NULL,'{}')",
+                    (automation_type, worker_id, hw_json, "{}"),
+                )
+            else:
+                self._conn.execute(
+                    "UPDATE workers SET hardware=? WHERE automation_type=? AND worker_id=?",
+                    (hw_json, automation_type, worker_id),
+                )
+
+    async def worker_get_config(
+        self, automation_type: str, worker_id: str
+    ) -> tuple[dict[str, Any], int]:
+        return await asyncio.to_thread(self._worker_get_config_sync, automation_type, worker_id)
+
+    def _worker_get_config_sync(
+        self, automation_type: str, worker_id: str
+    ) -> tuple[dict[str, Any], int]:
+        with self._lock:
+            row = self._conn.execute(
+                "SELECT config, config_gen FROM workers WHERE automation_type=? AND worker_id=?",
+                (automation_type, worker_id),
+            ).fetchone()
+        if row is None:
+            return {}, 0
+        return json.loads(row[0]), int(row[1])
+
+    async def worker_set_config(
+        self, automation_type: str, worker_id: str, config: dict[str, Any]
+    ) -> tuple[dict[str, Any], int]:
+        return await asyncio.to_thread(self._worker_set_config_sync, automation_type, worker_id, config)
+
+    def _worker_set_config_sync(
+        self, automation_type: str, worker_id: str, config: dict[str, Any]
+    ) -> tuple[dict[str, Any], int]:
+        cfg_json = json.dumps(config)
+        with self._lock:
+            row = self._conn.execute(
+                "SELECT config_gen FROM workers WHERE automation_type=? AND worker_id=?",
+                (automation_type, worker_id),
+            ).fetchone()
+            if row is None:
+                self._conn.execute(
+                    "INSERT INTO workers(automation_type, worker_id, hardware, config, "
+                    "config_gen, state, last_seen, last_error, stats) "
+                    "VALUES(?,?,'{}',?,1,'IDLE',0,NULL,'{}')",
+                    (automation_type, worker_id, cfg_json),
+                )
+                new_gen = 1
+            else:
+                new_gen = int(row[0]) + 1
+                self._conn.execute(
+                    "UPDATE workers SET config=?, config_gen=? WHERE automation_type=? AND worker_id=?",
+                    (cfg_json, new_gen, automation_type, worker_id),
+                )
+        return json.loads(cfg_json), new_gen
+
+    async def worker_set_state(
+        self,
+        automation_type: str,
+        worker_id: str,
+        state: str,
+        last_error: Optional[str],
+        last_seen: float,
+    ) -> None:
+        await asyncio.to_thread(
+            self._worker_set_state_sync, automation_type, worker_id, state, last_error, last_seen,
+        )
+
+    def _worker_set_state_sync(
+        self,
+        automation_type: str,
+        worker_id: str,
+        state: str,
+        last_error: Optional[str],
+        last_seen: float,
+    ) -> None:
+        with self._lock:
+            row = self._conn.execute(
+                "SELECT 1 FROM workers WHERE automation_type=? AND worker_id=?",
+                (automation_type, worker_id),
+            ).fetchone()
+            if row is None:
+                self._conn.execute(
+                    "INSERT INTO workers(automation_type, worker_id, hardware, config, "
+                    "config_gen, state, last_seen, last_error, stats) "
+                    "VALUES(?,?,'{}','{}',0,?,?,?,'{}')",
+                    (automation_type, worker_id, state, last_seen, last_error),
+                )
+            else:
+                self._conn.execute(
+                    "UPDATE workers SET state=?, last_seen=?, last_error=? "
+                    "WHERE automation_type=? AND worker_id=?",
+                    (state, last_seen, last_error, automation_type, worker_id),
+                )
+
+    async def worker_set_stats(
+        self, automation_type: str, worker_id: str, stats: dict[str, Any]
+    ) -> None:
+        await asyncio.to_thread(self._worker_set_stats_sync, automation_type, worker_id, stats)
+
+    def _worker_set_stats_sync(
+        self, automation_type: str, worker_id: str, stats: dict[str, Any]
+    ) -> None:
+        stats_json = json.dumps(stats)
+        with self._lock:
+            row = self._conn.execute(
+                "SELECT 1 FROM workers WHERE automation_type=? AND worker_id=?",
+                (automation_type, worker_id),
+            ).fetchone()
+            if row is None:
+                self._conn.execute(
+                    "INSERT INTO workers(automation_type, worker_id, hardware, config, "
+                    "config_gen, state, last_seen, last_error, stats) "
+                    "VALUES(?,?,'{}','{}',0,'IDLE',0,NULL,?)",
+                    (automation_type, worker_id, stats_json),
+                )
+            else:
+                self._conn.execute(
+                    "UPDATE workers SET stats=? WHERE automation_type=? AND worker_id=?",
+                    (stats_json, automation_type, worker_id),
+                )
+
+    async def worker_stats_push(
+        self,
+        automation_type: str,
+        worker_id: str,
+        ts: float,
+        frame: dict[str, Any],
+        *,
+        max_entries: int = 240,
+    ) -> None:
+        await asyncio.to_thread(
+            self._worker_stats_push_sync, automation_type, worker_id, ts, frame, max_entries,
+        )
+
+    def _worker_stats_push_sync(
+        self,
+        automation_type: str,
+        worker_id: str,
+        ts: float,
+        frame: dict[str, Any],
+        max_entries: int,
+    ) -> None:
+        frame_json = json.dumps(frame, default=str)
+        with self._lock:
+            self._conn.execute(
+                "INSERT INTO worker_stats_history(automation_type, worker_id, ts, frame) "
+                "VALUES(?,?,?,?)",
+                (automation_type, worker_id, ts, frame_json),
+            )
+            if max_entries > 0:
+                row = self._conn.execute(
+                    "SELECT COUNT(*) FROM worker_stats_history "
+                    "WHERE automation_type=? AND worker_id=?",
+                    (automation_type, worker_id),
+                ).fetchone()
+                count = int(row[0])
+                if count > max_entries:
+                    extra = count - max_entries
+                    self._conn.execute(
+                        "DELETE FROM worker_stats_history WHERE rowid IN ("
+                        "SELECT rowid FROM worker_stats_history "
+                        "WHERE automation_type=? AND worker_id=? "
+                        "ORDER BY ts ASC LIMIT ?)",
+                        (automation_type, worker_id, extra),
+                    )
+
+    async def worker_stats_history(
+        self,
+        automation_type: str,
+        worker_id: str,
+        *,
+        since: float = 0,
+        limit: int = 240,
+    ) -> list[dict[str, Any]]:
+        return await asyncio.to_thread(
+            self._worker_stats_history_sync, automation_type, worker_id, since, limit,
+        )
+
+    def _worker_stats_history_sync(
+        self,
+        automation_type: str,
+        worker_id: str,
+        since: float,
+        limit: int,
+    ) -> list[dict[str, Any]]:
+        with self._lock:
+            rows = self._conn.execute(
+                "SELECT frame FROM worker_stats_history "
+                "WHERE automation_type=? AND worker_id=? AND ts>=? "
+                "ORDER BY ts ASC LIMIT ?",
+                (automation_type, worker_id, since, limit),
+            ).fetchall()
+        return [json.loads(r[0]) for r in rows]
+
+    async def worker_get(
+        self, automation_type: str, worker_id: str
+    ) -> Optional[dict[str, Any]]:
+        return await asyncio.to_thread(self._worker_get_sync, automation_type, worker_id)
+
+    def _worker_get_sync(
+        self, automation_type: str, worker_id: str
+    ) -> Optional[dict[str, Any]]:
+        with self._lock:
+            row = self._conn.execute(
+                "SELECT hardware, config, config_gen, state, last_seen, last_error, stats "
+                "FROM workers WHERE automation_type=? AND worker_id=?",
+                (automation_type, worker_id),
+            ).fetchone()
+        if row is None:
+            return None
+        return {
+            "hardware": json.loads(row[0]),
+            "config": json.loads(row[1]),
+            "config_gen": int(row[2]),
+            "state": row[3],
+            "last_seen": float(row[4]),
+            "last_error": row[5],
+            "stats": json.loads(row[6]),
+        }
+
+    async def worker_list(
+        self, automation_type: Optional[str] = None
+    ) -> list[tuple[str, str]]:
+        return await asyncio.to_thread(self._worker_list_sync, automation_type)
+
+    def _worker_list_sync(self, automation_type: Optional[str]) -> list[tuple[str, str]]:
+        with self._lock:
+            if automation_type is None:
+                rows = self._conn.execute(
+                    "SELECT automation_type, worker_id FROM workers"
+                ).fetchall()
+            else:
+                rows = self._conn.execute(
+                    "SELECT automation_type, worker_id FROM workers WHERE automation_type=?",
+                    (automation_type,),
+                ).fetchall()
+        return [(r[0], r[1]) for r in rows]
+
+    async def worker_remove(self, automation_type: str, worker_id: str) -> bool:
+        return await asyncio.to_thread(self._worker_remove_sync, automation_type, worker_id)
+
+    def _worker_remove_sync(self, automation_type: str, worker_id: str) -> bool:
+        with self._lock:
+            cur = self._conn.execute(
+                "DELETE FROM workers WHERE automation_type=? AND worker_id=?",
+                (automation_type, worker_id),
+            )
+            self._conn.execute(
+                "DELETE FROM worker_stats_history WHERE automation_type=? AND worker_id=?",
+                (automation_type, worker_id),
+            )
+        return cur.rowcount > 0
+
+    async def catalog_set(self, automation_type: str, doc: bytes) -> None:
+        await asyncio.to_thread(self._catalog_set_sync, automation_type, doc)
+
+    def _catalog_set_sync(self, automation_type: str, doc: bytes) -> None:
+        with self._lock:
+            self._conn.execute(
+                "INSERT OR REPLACE INTO catalog(automation_type, doc) VALUES(?,?)",
+                (automation_type, doc),
+            )
+
+    async def catalog_get(self, automation_type: str) -> Optional[bytes]:
+        return await asyncio.to_thread(self._catalog_get_sync, automation_type)
+
+    def _catalog_get_sync(self, automation_type: str) -> Optional[bytes]:
+        with self._lock:
+            row = self._conn.execute(
+                "SELECT doc FROM catalog WHERE automation_type=?", (automation_type,),
+            ).fetchone()
+        return row[0] if row else None
+
+    async def catalog_all(self) -> dict[str, bytes]:
+        return await asyncio.to_thread(self._catalog_all_sync)
+
+    def _catalog_all_sync(self) -> dict[str, bytes]:
+        with self._lock:
+            rows = self._conn.execute("SELECT automation_type, doc FROM catalog").fetchall()
+        return {r[0]: r[1] for r in rows}
+
+
+def _path_from_url(url: str) -> str:
+    if "://" not in url:
+        return url
+    scheme, rest = url.split("://", 1)
+    if scheme.lower() not in ("sqlite", "file"):
+        raise ValueError(f"SqliteBackend cannot handle scheme: {scheme}")
+    if rest in ("", ":memory:"):
+        return ":memory:"
+    if rest.startswith("/"):
+        # sqlite:///abs/path → /abs/path; sqlite:////abs/path also works
+        return rest
+    return rest
+
+
+__all__ = ["SqliteBackend"]