hivemind-sqlite-database 0.4.0a7__tar.gz → 0.4.2a1__tar.gz

{hivemind_sqlite_database-0.4.0a7 → hivemind_sqlite_database-0.4.2a1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hivemind-sqlite-database
-Version: 0.4.0a7
+Version: 0.4.2a1
 Summary: sqlite database plugin for hivemind-core
 Author-email: jarbasAi <jarbasai@mailfence.com>
 License: Apache-2.0

{hivemind_sqlite_database-0.4.0a7 → hivemind_sqlite_database-0.4.2a1}/hivemind_sqlite_database/__init__.py

@@ -2,7 +2,7 @@ import json
 import os.path
 import sqlite3
 import threading
-from typing import List, Optional, Union, Iterable
+from typing import ClassVar, List, Optional, Union, Iterable
 
 from ovos_utils.log import LOG
 from ovos_utils.xdg_utils import xdg_data_home
@@ -27,6 +27,10 @@ class SQLiteDB(AbstractDB):
     subfolder: str = "hivemind-core"
     password: Optional[str] = None
 
+    # How long SQLite waits for a file lock held by another connection
+    # before giving up with SQLITE_BUSY, in milliseconds.
+    BUSY_TIMEOUT_MS: ClassVar[int] = 10000
+
     def __post_init__(self):
         """
         Initialize the SQLiteDB connection.
@@ -38,14 +42,31 @@ class SQLiteDB(AbstractDB):
 
         When *password* is ``None`` (default) the standard ``sqlite3`` module
         is used and the database file is unencrypted.
+
+        Each thread gets its own connection (see :attr:`conn`). A single
+        shared connection is not usable from several threads at once: the
+        transaction state belongs to the connection, so one thread's
+        ``COMMIT`` ends another thread's transaction and resets its
+        in-flight statements. Under a threaded network backend that shows
+        up as ``sqlite3.ProgrammingError: bad parameter or other API
+        misuse`` and as writes that land with corrupted bindings.
         """
-        db_path = os.path.join(xdg_data_home(), self.subfolder, self.name + ".db")
-        LOG.debug(f"sqlite database path: {db_path}")
-        os.makedirs(os.path.dirname(db_path), exist_ok=True)
+        self._db_path = os.path.join(xdg_data_home(), self.subfolder, self.name + ".db")
+        LOG.debug(f"sqlite database path: {self._db_path}")
+        os.makedirs(os.path.dirname(self._db_path), exist_ok=True)
+
+        if self.password is not None and self.password == "":
+            raise ValueError("password must be non-empty when encryption is enabled")
 
+        self._write_lock = threading.Lock()
+        # opening the first connection also applies the WAL pragma
+        self.conn.execute("PRAGMA journal_mode=WAL")
+        self._initialize_database()
+        self._maybe_migrate()
+
+    def _connect(self):
+        """Open one new connection to the backing file."""
         if self.password is not None:
-            if self.password == "":
-                raise ValueError("password must be non-empty when encryption is enabled")
             try:
                 import sqlcipher3 as _sqlcipher
             except ImportError:
@@ -54,18 +75,56 @@ class SQLiteDB(AbstractDB):
                     "Install the system library (e.g. 'apt install libsqlcipher-dev') "
                     "then: pip install hivemind-sqlite-database[cipher]"
                 )
-            self.conn = _sqlcipher.connect(db_path, check_same_thread=False)
-            self.conn.row_factory = _sqlcipher.Row
+            conn = _sqlcipher.connect(self._db_path, check_same_thread=False)
+            conn.row_factory = _sqlcipher.Row
             escaped_password = self.password.replace("'", "''")
-            self.conn.execute(f"PRAGMA key='{escaped_password}'")
+            conn.execute(f"PRAGMA key='{escaped_password}'")
         else:
-            self.conn = sqlite3.connect(db_path, check_same_thread=False)
-            self.conn.row_factory = sqlite3.Row
-
-        self.conn.execute("PRAGMA journal_mode=WAL")
-        self._write_lock = threading.Lock()
-        self._initialize_database()
-        self._maybe_migrate()
+            conn = sqlite3.connect(self._db_path, check_same_thread=False)
+            conn.row_factory = sqlite3.Row
+        conn.execute(f"PRAGMA busy_timeout={int(self.BUSY_TIMEOUT_MS)}")
+        return conn
+
+    @property
+    def conn(self):
+        """The calling thread's own connection, opened on first use.
+
+        Connections are cheap and WAL lets many readers run beside one
+        writer, so a per-thread connection costs a file handle and buys
+        real thread safety. ``_write_lock`` still serialises writers
+        in-process so they do not fight over the file lock.
+        """
+        local = self._thread_state()
+        conn = getattr(local, "conn", None)
+        if conn is None:
+            conn = local.conn = self._connect()
+        return conn
+
+    @conn.setter
+    def conn(self, value) -> None:
+        """Adopt an already-open connection for the calling thread.
+
+        Only the calling thread sees it; other threads still open their
+        own. Tests use this to inject an in-memory database.
+        """
+        self._thread_state().conn = value
+
+    def _thread_state(self) -> threading.local:
+        local = self.__dict__.get("_local")
+        if local is None:
+            local = self.__dict__["_local"] = threading.local()
+        return local
+
+    def close(self) -> None:
+        """Close the calling thread's connection, if it has one."""
+        local = self._thread_state()
+        conn = getattr(local, "conn", None)
+        if conn is not None:
+            local.conn = None
+            try:
+                conn.close()
+            except sqlite3.Error as e:
+                LOG.error(f"Failed to close SQLite connection: {e}")
 
     def _initialize_database(self):
         """Initialize the database schema."""
@@ -92,6 +151,11 @@ class SQLiteDB(AbstractDB):
                     metadata TEXT
                 )
             """)
+            # api_key is looked up on every client connection admission
+            # (get_client_by_api_key), so a full table scan there is hot.
+            self.conn.execute(
+                "CREATE INDEX IF NOT EXISTS idx_clients_api_key ON clients(api_key)"
+            )
             columns = {
                 row["name"]
                 for row in self.conn.execute("PRAGMA table_info(clients)").fetchall()
@@ -230,10 +294,12 @@ class SQLiteDB(AbstractDB):
             LOG.error(f"Invalid search key: {key!r}")
             return []
         try:
-            with self.conn:
-                cur = self.conn.execute(f"SELECT * FROM clients WHERE {key} = ?", (val,))
-                rows = cur.fetchall()
-                return [self._row_to_client(row) for row in rows]
+            # deliberately NOT wrapped in ``with self.conn`` — that context
+            # manager commits the connection's transaction on exit, which a
+            # read has no business doing.
+            cur = self.conn.execute(f"SELECT * FROM clients WHERE {key} = ?", (val,))
+            rows = cur.fetchall()
+            return [self._row_to_client(row) for row in rows]
         except sqlite3.Error as e:
             LOG.error(f"Failed to search clients in SQLite: {e}")
             return []

{hivemind_sqlite_database-0.4.0a7 → hivemind_sqlite_database-0.4.2a1}/hivemind_sqlite_database/version.py

@@ -1,8 +1,8 @@
 # START_VERSION_BLOCK
 VERSION_MAJOR = 0
 VERSION_MINOR = 4
-VERSION_BUILD = 0
-VERSION_ALPHA = 7
+VERSION_BUILD = 2
+VERSION_ALPHA = 1
 # END_VERSION_BLOCK
 
 __version__ = f"{VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_BUILD}" + (f"a{VERSION_ALPHA}" if VERSION_ALPHA else "")

{hivemind_sqlite_database-0.4.0a7 → hivemind_sqlite_database-0.4.2a1}/hivemind_sqlite_database.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hivemind-sqlite-database
-Version: 0.4.0a7
+Version: 0.4.2a1
 Summary: sqlite database plugin for hivemind-core
 Author-email: jarbasAi <jarbasai@mailfence.com>
 License: Apache-2.0

{hivemind_sqlite_database-0.4.0a7 → hivemind_sqlite_database-0.4.2a1}/tests/test_sqlitedb.py

@@ -783,6 +783,26 @@ class TestSQLiteDBGetClientByID(unittest.TestCase):
                 self.assertEqual(db.refresh(1).allowed_types, ["y"])
 
 
+class TestSQLiteDBApiKeyIndex(unittest.TestCase):
+    def test_index_exists_after_init(self):
+        db = make_db()
+        row = db.conn.execute(
+            "SELECT name FROM sqlite_master WHERE type='index' "
+            "AND name='idx_clients_api_key'"
+        ).fetchone()
+        self.assertIsNotNone(row)
+
+    def test_api_key_lookup_uses_index(self):
+        db = make_db()
+        plan = db.conn.execute(
+            "EXPLAIN QUERY PLAN SELECT * FROM clients WHERE api_key = ?",
+            ("k1",),
+        ).fetchall()
+        detail = " ".join(row["detail"] for row in plan)
+        self.assertIn("SEARCH", detail)
+        self.assertNotIn("SCAN", detail)
+
+
 class TestSQLiteDBSchemaV2RoundTrip(unittest.TestCase):
     """v2 schema: allowed_types + skill/intent blacklists (in metadata) survive
     add→search and add→refresh cycles without loss or mutation."""
@@ -863,3 +883,66 @@ class TestSQLiteDBSchemaV2RoundTrip(unittest.TestCase):
 
 if __name__ == "__main__":
     unittest.main()
+
+
+class TestSQLiteDBThreadSafety(unittest.TestCase):
+    """A real file-backed DB hammered from many threads at once.
+
+    Reproduces the failure seen under the threaded (webrockets) network
+    backend: concurrent reads and writes through ONE shared connection
+    raise ``bad parameter or other API misuse`` and write rows with
+    corrupted bindings (``NOT NULL constraint failed: clients.api_key``),
+    because one thread's implicit COMMIT ends another thread's
+    transaction.
+    """
+
+    def test_concurrent_read_write_is_clean(self):
+        import tempfile
+        from unittest.mock import patch
+
+        with tempfile.TemporaryDirectory() as tmp:
+            with patch("hivemind_sqlite_database.xdg_data_home", return_value=tmp):
+                db = SQLiteDB()
+                n = 60
+                for i in range(n):
+                    self.assertTrue(db.add_item(make_client(i, f"key-{i}")))
+
+                errors = []
+                misses = []
+                barrier = threading.Barrier(24)
+
+                def reader(i):
+                    barrier.wait()
+                    try:
+                        for _ in range(40):
+                            got = db.search_by_value("api_key", f"key-{i % n}")
+                            if len(got) != 1:
+                                misses.append((i, len(got)))
+                    except Exception as e:  # noqa: BLE001
+                        errors.append(repr(e))
+
+                def writer(i):
+                    barrier.wait()
+                    try:
+                        for k in range(40):
+                            if not db.add_item(
+                                make_client(i % n, f"key-{i % n}",
+                                            name=f"w{k}")):
+                                errors.append(f"add_item returned False ({i},{k})")
+                    except Exception as e:  # noqa: BLE001
+                        errors.append(repr(e))
+
+                threads = [threading.Thread(target=reader, args=(i,))
+                           for i in range(16)]
+                threads += [threading.Thread(target=writer, args=(i,))
+                            for i in range(8)]
+                for t in threads:
+                    t.start()
+                for t in threads:
+                    t.join(timeout=120)
+
+                self.assertEqual(errors, [])
+                self.assertEqual(misses, [])
+                # every api_key still resolves to exactly one live row
+                for i in range(n):
+                    self.assertEqual(len(db.search_by_value("api_key", f"key-{i}")), 1)