hivemind-sqlite-database 0.4.0a3__tar.gz → 0.4.3a8__tar.gz

{hivemind_sqlite_database-0.4.0a3 → hivemind_sqlite_database-0.4.3a8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hivemind-sqlite-database
-Version: 0.4.0a3
+Version: 0.4.3a8
 Summary: sqlite database plugin for hivemind-core
 Author-email: jarbasAi <jarbasai@mailfence.com>
 License: Apache-2.0

{hivemind_sqlite_database-0.4.0a3 → hivemind_sqlite_database-0.4.3a8}/README.md

@@ -7,7 +7,7 @@ Implements the [`hivemind-plugin-manager`](https://github.com/JarbasHiveMind/hiv
 encryption is enabled). Client records (API keys, crypto keys, access-control lists) are stored in
 a local `.db` file with WAL journal mode for safe multi-reader, single-writer access.
 
-**This is the default database backend for fresh hivemind-core installations.**
+This is the default database backend for fresh hivemind-core installations.
 
 ## Where it fits
 
@@ -99,13 +99,13 @@ typically `~/.local/share/hivemind-core/clients.db`.
 ## Schema migration
 
 On first open after an upgrade, `SQLiteDB` runs an automatic schema migration.
-Version tracking uses SQLite's built-in `PRAGMA user_version` — no sibling files.
+Version tracking uses SQLite's built-in `PRAGMA user_version`. There are no sibling files.
 
 The v1→v2 migration folds legacy `intent_blacklist` / `skill_blacklist` column
 data into each row's `metadata` JSON field and NULLs the legacy columns.
 `message_blacklist` is purged outright.
 
-The migration is idempotent, crash-safe, and transactional — the row rewrites
+The migration is idempotent and crash-safe. The row rewrites
 and the `user_version` bump happen in one transaction.
 
 To migrate an existing installation to this backend, use hivemind-core's built-in
@@ -117,6 +117,6 @@ hivemind-core migrate-db
 
 ## Docs
 
-- [docs/architecture.md](docs/architecture.md) — internals, WAL mode, schema, migration
-- [docs/configuration.md](docs/configuration.md) — full configuration reference
-- [docs/operations.md](docs/operations.md) — file locations, backup, encryption, authoring a plugin
+- [docs/architecture.md](docs/architecture.md): internals, WAL mode, schema, migration
+- [docs/configuration.md](docs/configuration.md): full configuration reference
+- [docs/operations.md](docs/operations.md): file locations, backup, encryption, authoring a plugin

{hivemind_sqlite_database-0.4.0a3 → hivemind_sqlite_database-0.4.3a8}/hivemind_sqlite_database/__init__.py

@@ -2,7 +2,7 @@ import json
 import os.path
 import sqlite3
 import threading
-from typing import List, Optional, Union, Iterable
+from typing import ClassVar, List, Optional, Union, Iterable
 
 from ovos_utils.log import LOG
 from ovos_utils.xdg_utils import xdg_data_home
@@ -15,7 +15,7 @@ from dataclasses import dataclass
 _VALID_COLUMNS = frozenset({
     "client_id", "api_key", "name", "description", "is_admin",
     "last_seen", "intent_blacklist", "skill_blacklist", "message_blacklist",
-    "allowed_types", "crypto_key", "password",
+    "allowed_types", "password",
     "can_broadcast", "can_escalate", "can_propagate", "metadata",
 })
 
@@ -26,6 +26,17 @@ class SQLiteDB(AbstractDB):
     name: str = "clients"
     subfolder: str = "hivemind-core"
     password: Optional[str] = None
+    # Overrides the computed xdg path entirely, e.g. ":memory:" for tests.
+    # Every thread's connection is opened against this same target, so
+    # nothing ever silently falls back to the real client database.
+    # ":memory:" becomes a named shared in-memory database, one per
+    # SQLiteDB instance, so worker threads see the same tables as the
+    # thread that created them.
+    db_path: Optional[str] = None
+
+    # How long SQLite waits for a file lock held by another connection
+    # before giving up with SQLITE_BUSY, in milliseconds.
+    BUSY_TIMEOUT_MS: ClassVar[int] = 10000
 
     def __post_init__(self):
         """
@@ -38,14 +49,49 @@ class SQLiteDB(AbstractDB):
 
         When *password* is ``None`` (default) the standard ``sqlite3`` module
         is used and the database file is unencrypted.
+
+        Each thread gets its own connection (see :attr:`conn`). A single
+        shared connection is not usable from several threads at once: the
+        transaction state belongs to the connection, so one thread's
+        ``COMMIT`` ends another thread's transaction and resets its
+        in-flight statements. Under a threaded network backend that shows
+        up as ``sqlite3.ProgrammingError: bad parameter or other API
+        misuse`` and as writes that land with corrupted bindings.
         """
-        db_path = os.path.join(xdg_data_home(), self.subfolder, self.name + ".db")
-        LOG.debug(f"sqlite database path: {db_path}")
-        os.makedirs(os.path.dirname(db_path), exist_ok=True)
+        if self.db_path is not None:
+            self._db_path = self.db_path
+            if self._db_path == ":memory:":
+                # A plain ":memory:" database belongs to the one connection
+                # that opened it, so the next thread would open a second,
+                # empty database and find no tables. Give it a name and a
+                # shared cache instead. The name carries the instance id, so
+                # two in-memory databases in one process stay independent.
+                self._db_path = f"file:hivemind-{id(self):x}?mode=memory&cache=shared"
+        else:
+            self._db_path = os.path.join(xdg_data_home(), self.subfolder, self.name + ".db")
+        self._is_uri = self._db_path.startswith("file:")
+        parent = os.path.dirname(self._db_path)
+        if parent and not self._is_uri:
+            os.makedirs(parent, exist_ok=True)
+        LOG.debug(f"sqlite database path: {self._db_path}")
+
+        if self.password is not None and self.password == "":
+            raise ValueError("password must be non-empty when encryption is enabled")
 
+        self._write_lock = threading.Lock()
+        # opening the first connection also applies the WAL pragma
+        self.conn.execute("PRAGMA journal_mode=WAL")
+        if "mode=memory" in self._db_path:
+            # A shared in-memory database lives only while a connection to
+            # it is open. Hold one for the lifetime of this object so the
+            # tables survive a thread closing its own connection.
+            self._keepalive = self._connect()
+        self._initialize_database()
+        self._maybe_migrate()
+
+    def _connect(self):
+        """Open one new connection to the backing file."""
         if self.password is not None:
-            if self.password == "":
-                raise ValueError("password must be non-empty when encryption is enabled")
             try:
                 import sqlcipher3 as _sqlcipher
             except ImportError:
@@ -54,18 +100,63 @@ class SQLiteDB(AbstractDB):
                     "Install the system library (e.g. 'apt install libsqlcipher-dev') "
                     "then: pip install hivemind-sqlite-database[cipher]"
                 )
-            self.conn = _sqlcipher.connect(db_path, check_same_thread=False)
-            self.conn.row_factory = _sqlcipher.Row
+            conn = _sqlcipher.connect(self._db_path, check_same_thread=False,
+                                      uri=self._is_uri)
+            conn.row_factory = _sqlcipher.Row
             escaped_password = self.password.replace("'", "''")
-            self.conn.execute(f"PRAGMA key='{escaped_password}'")
+            conn.execute(f"PRAGMA key='{escaped_password}'")
         else:
-            self.conn = sqlite3.connect(db_path, check_same_thread=False)
-            self.conn.row_factory = sqlite3.Row
-
-        self.conn.execute("PRAGMA journal_mode=WAL")
-        self._write_lock = threading.Lock()
-        self._initialize_database()
-        self._maybe_migrate()
+            conn = sqlite3.connect(self._db_path, check_same_thread=False,
+                                   uri=self._is_uri)
+            conn.row_factory = sqlite3.Row
+        conn.execute(f"PRAGMA busy_timeout={int(self.BUSY_TIMEOUT_MS)}")
+        return conn
+
+    @property
+    def conn(self):
+        """The calling thread's own connection, opened on first use.
+
+        Connections are cheap and WAL lets many readers run beside one
+        writer, so a per-thread connection costs a file handle and buys
+        real thread safety. ``_write_lock`` still serialises writers
+        in-process so they do not fight over the file lock.
+        """
+        local = self._thread_state()
+        conn = getattr(local, "conn", None)
+        if conn is None:
+            conn = local.conn = self._connect()
+        return conn
+
+    @conn.setter
+    def conn(self, value) -> None:
+        """Adopt an already-open connection for the calling thread only.
+
+        Any other thread that later touches ``.conn`` still opens its own
+        connection against ``self._db_path`` — if that is the real client
+        database, that thread silently writes to it. Pass ``db_path``
+        (e.g. ``":memory:"``) to the constructor instead so every thread,
+        present and future, agrees on the same target; this setter exists
+        only for single-threaded call sites that already hold a connection
+        they want reused.
+        """
+        self._thread_state().conn = value
+
+    def _thread_state(self) -> threading.local:
+        local = self.__dict__.get("_local")
+        if local is None:
+            local = self.__dict__["_local"] = threading.local()
+        return local
+
+    def close(self) -> None:
+        """Close the calling thread's connection, if it has one."""
+        local = self._thread_state()
+        conn = getattr(local, "conn", None)
+        if conn is not None:
+            local.conn = None
+            try:
+                conn.close()
+            except sqlite3.Error as e:
+                LOG.error(f"Failed to close SQLite connection: {e}")
 
     def _initialize_database(self):
         """Initialize the database schema."""
@@ -84,7 +175,6 @@ class SQLiteDB(AbstractDB):
                     skill_blacklist TEXT,
                     message_blacklist TEXT,
                     allowed_types TEXT,
-                    crypto_key VARCHAR(16),
                     password TEXT,
                     can_broadcast BOOLEAN DEFAULT TRUE,
                     can_escalate BOOLEAN DEFAULT TRUE,
@@ -92,6 +182,11 @@ class SQLiteDB(AbstractDB):
                     metadata TEXT
                 )
             """)
+            # api_key is looked up on every client connection admission
+            # (get_client_by_api_key), so a full table scan there is hot.
+            self.conn.execute(
+                "CREATE INDEX IF NOT EXISTS idx_clients_api_key ON clients(api_key)"
+            )
             columns = {
                 row["name"]
                 for row in self.conn.execute("PRAGMA table_info(clients)").fetchall()
@@ -191,9 +286,9 @@ class SQLiteDB(AbstractDB):
                     INSERT OR REPLACE INTO clients (
                         client_id, api_key, name, description, is_admin,
                         last_seen, intent_blacklist, skill_blacklist,
-                        message_blacklist, allowed_types, crypto_key, password,
+                        message_blacklist, allowed_types, password,
                         can_broadcast, can_escalate, can_propagate, metadata
-                    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
                 """, (
                     client.client_id, client.api_key, client.name, client.description,
                     client.is_admin, client.last_seen,
@@ -206,7 +301,7 @@ class SQLiteDB(AbstractDB):
                     None,
                     None,
                     json.dumps(client.allowed_types),
-                    client.crypto_key, client.password,
+                    client.password,
                     client.can_broadcast, client.can_escalate, client.can_propagate,
                     metadata_json,
                 ))
@@ -230,10 +325,12 @@ class SQLiteDB(AbstractDB):
             LOG.error(f"Invalid search key: {key!r}")
             return []
         try:
-            with self.conn:
-                cur = self.conn.execute(f"SELECT * FROM clients WHERE {key} = ?", (val,))
-                rows = cur.fetchall()
-                return [self._row_to_client(row) for row in rows]
+            # deliberately NOT wrapped in ``with self.conn`` — that context
+            # manager commits the connection's transaction on exit, which a
+            # read has no business doing.
+            cur = self.conn.execute(f"SELECT * FROM clients WHERE {key} = ?", (val,))
+            rows = cur.fetchall()
+            return [self._row_to_client(row) for row in rows]
         except sqlite3.Error as e:
             LOG.error(f"Failed to search clients in SQLite: {e}")
             return []
@@ -322,7 +419,6 @@ class SQLiteDB(AbstractDB):
             is_admin=bool(row["is_admin"]),
             last_seen=row["last_seen"],
             allowed_types=json.loads(row["allowed_types"] or "[]"),
-            crypto_key=row["crypto_key"],
             password=row["password"],
             can_broadcast=bool(row["can_broadcast"]),
             can_escalate=bool(row["can_escalate"]),

{hivemind_sqlite_database-0.4.0a3 → hivemind_sqlite_database-0.4.3a8}/hivemind_sqlite_database/version.py

@@ -1,8 +1,8 @@
 # START_VERSION_BLOCK
 VERSION_MAJOR = 0
 VERSION_MINOR = 4
-VERSION_BUILD = 0
-VERSION_ALPHA = 3
+VERSION_BUILD = 3
+VERSION_ALPHA = 8
 # END_VERSION_BLOCK
 
 __version__ = f"{VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_BUILD}" + (f"a{VERSION_ALPHA}" if VERSION_ALPHA else "")

{hivemind_sqlite_database-0.4.0a3 → hivemind_sqlite_database-0.4.3a8}/hivemind_sqlite_database.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hivemind-sqlite-database
-Version: 0.4.0a3
+Version: 0.4.3a8
 Summary: sqlite database plugin for hivemind-core
 Author-email: jarbasAi <jarbasai@mailfence.com>
 License: Apache-2.0

{hivemind_sqlite_database-0.4.0a3 → hivemind_sqlite_database-0.4.3a8}/tests/test_sqlitedb.py

@@ -389,7 +389,6 @@ class TestSQLiteDBRoundTrip(unittest.TestCase):
             intent_blacklist=["a:b"],
             skill_blacklist=["c:d"],
             allowed_types=["recognizer_loop:utterance"],
-            crypto_key="1234567890123456",
             password="secret",
             can_broadcast=True,
             can_escalate=False,
@@ -405,7 +404,6 @@ class TestSQLiteDBRoundTrip(unittest.TestCase):
         self.assertEqual(r.description, "a test")
         self.assertFalse(r.is_admin)
         self.assertEqual(r.last_seen, 1234567890.0)
-        self.assertEqual(r.crypto_key, "1234567890123456")
         self.assertEqual(r.password, "secret")
         self.assertFalse(r.can_escalate)
         # After SCHEMA_VERSION=2: legacy skill/intent kwargs auto-migrate
@@ -422,6 +420,56 @@ class TestSQLiteDBRoundTrip(unittest.TestCase):
         self.assertEqual(r.intent_blacklist, ["a:b"])
 
 
+class TestSQLiteDBPathOverride(unittest.TestCase):
+    """A worker thread must never fall back to the real client database
+    just because the test only overrode `.conn` on the main thread."""
+
+    def test_db_path_override_keeps_worker_threads_off_disk(self):
+        import unittest.mock as mock
+        with tempfile.TemporaryDirectory() as tmpdir:
+            with mock.patch(
+                "hivemind_sqlite_database.xdg_data_home", return_value=tmpdir
+            ):
+                db = SQLiteDB(name="clients", subfolder="hivemind-core",
+                              db_path=":memory:")
+
+                # Read a row written on this thread. "SELECT 1" would answer
+                # the same against a private, empty database and so would
+                # never notice a per-thread database.
+                db.add_item(Client(client_id=1, api_key="key",
+                                   name="kitchen"))
+
+                errors = []
+                seen = []
+
+                def worker():
+                    try:
+                        seen.extend(db.search_by_value("name", "kitchen"))
+                    except Exception as e:  # noqa: BLE001
+                        errors.append(e)
+
+                t = threading.Thread(target=worker)
+                t.start()
+                t.join()
+
+            self.assertEqual(errors, [])
+            self.assertEqual([c.client_id for c in seen], [1])
+            real_db_file = os.path.join(tmpdir, "hivemind-core", "clients.db")
+            self.assertFalse(os.path.exists(real_db_file))
+
+    def test_two_in_memory_databases_stay_independent(self):
+        first = SQLiteDB(db_path=":memory:")
+        second = SQLiteDB(db_path=":memory:")
+        first.add_item(Client(client_id=1, api_key="key", name="kitchen"))
+        self.assertEqual(second.search_by_value("name", "kitchen"), [])
+
+    def test_explicit_db_path_creates_missing_directories(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            path = os.path.join(tmpdir, "newdir", "clients.db")
+            SQLiteDB(db_path=path)
+            self.assertTrue(os.path.isfile(path))
+
+
 class TestSQLiteDBCommit(unittest.TestCase):
     def test_commit_returns_true(self):
         db = make_db()
@@ -783,6 +831,26 @@ class TestSQLiteDBGetClientByID(unittest.TestCase):
                 self.assertEqual(db.refresh(1).allowed_types, ["y"])
 
 
+class TestSQLiteDBApiKeyIndex(unittest.TestCase):
+    def test_index_exists_after_init(self):
+        db = make_db()
+        row = db.conn.execute(
+            "SELECT name FROM sqlite_master WHERE type='index' "
+            "AND name='idx_clients_api_key'"
+        ).fetchone()
+        self.assertIsNotNone(row)
+
+    def test_api_key_lookup_uses_index(self):
+        db = make_db()
+        plan = db.conn.execute(
+            "EXPLAIN QUERY PLAN SELECT * FROM clients WHERE api_key = ?",
+            ("k1",),
+        ).fetchall()
+        detail = " ".join(row["detail"] for row in plan)
+        self.assertIn("SEARCH", detail)
+        self.assertNotIn("SCAN", detail)
+
+
 class TestSQLiteDBSchemaV2RoundTrip(unittest.TestCase):
     """v2 schema: allowed_types + skill/intent blacklists (in metadata) survive
     add→search and add→refresh cycles without loss or mutation."""
@@ -863,3 +931,66 @@ class TestSQLiteDBSchemaV2RoundTrip(unittest.TestCase):
 
 if __name__ == "__main__":
     unittest.main()
+
+
+class TestSQLiteDBThreadSafety(unittest.TestCase):
+    """A real file-backed DB hammered from many threads at once.
+
+    Reproduces the failure seen under the threaded (webrockets) network
+    backend: concurrent reads and writes through ONE shared connection
+    raise ``bad parameter or other API misuse`` and write rows with
+    corrupted bindings (``NOT NULL constraint failed: clients.api_key``),
+    because one thread's implicit COMMIT ends another thread's
+    transaction.
+    """
+
+    def test_concurrent_read_write_is_clean(self):
+        import tempfile
+        from unittest.mock import patch
+
+        with tempfile.TemporaryDirectory() as tmp:
+            with patch("hivemind_sqlite_database.xdg_data_home", return_value=tmp):
+                db = SQLiteDB()
+                n = 60
+                for i in range(n):
+                    self.assertTrue(db.add_item(make_client(i, f"key-{i}")))
+
+                errors = []
+                misses = []
+                barrier = threading.Barrier(24)
+
+                def reader(i):
+                    barrier.wait()
+                    try:
+                        for _ in range(40):
+                            got = db.search_by_value("api_key", f"key-{i % n}")
+                            if len(got) != 1:
+                                misses.append((i, len(got)))
+                    except Exception as e:  # noqa: BLE001
+                        errors.append(repr(e))
+
+                def writer(i):
+                    barrier.wait()
+                    try:
+                        for k in range(40):
+                            if not db.add_item(
+                                make_client(i % n, f"key-{i % n}",
+                                            name=f"w{k}")):
+                                errors.append(f"add_item returned False ({i},{k})")
+                    except Exception as e:  # noqa: BLE001
+                        errors.append(repr(e))
+
+                threads = [threading.Thread(target=reader, args=(i,))
+                           for i in range(16)]
+                threads += [threading.Thread(target=writer, args=(i,))
+                            for i in range(8)]
+                for t in threads:
+                    t.start()
+                for t in threads:
+                    t.join(timeout=120)
+
+                self.assertEqual(errors, [])
+                self.assertEqual(misses, [])
+                # every api_key still resolves to exactly one live row
+                for i in range(n):
+                    self.assertEqual(len(db.search_by_value("api_key", f"key-{i}")), 1)