hivemind-sqlite-database 0.4.1a1__tar.gz → 0.4.3a1__tar.gz

{hivemind_sqlite_database-0.4.1a1 → hivemind_sqlite_database-0.4.3a1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hivemind-sqlite-database
-Version: 0.4.1a1
+Version: 0.4.3a1
 Summary: sqlite database plugin for hivemind-core
 Author-email: jarbasAi <jarbasai@mailfence.com>
 License: Apache-2.0

{hivemind_sqlite_database-0.4.1a1 → hivemind_sqlite_database-0.4.3a1}/hivemind_sqlite_database/__init__.py

@@ -2,7 +2,7 @@ import json
 import os.path
 import sqlite3
 import threading
-from typing import List, Optional, Union, Iterable
+from typing import ClassVar, List, Optional, Union, Iterable
 
 from ovos_utils.log import LOG
 from ovos_utils.xdg_utils import xdg_data_home
@@ -26,6 +26,17 @@ class SQLiteDB(AbstractDB):
     name: str = "clients"
     subfolder: str = "hivemind-core"
     password: Optional[str] = None
+    # Overrides the computed xdg path entirely, e.g. ":memory:" for tests.
+    # Every thread's connection is opened against this same target, so
+    # nothing ever silently falls back to the real client database.
+    # ":memory:" becomes a named shared in-memory database, one per
+    # SQLiteDB instance, so worker threads see the same tables as the
+    # thread that created them.
+    db_path: Optional[str] = None
+
+    # How long SQLite waits for a file lock held by another connection
+    # before giving up with SQLITE_BUSY, in milliseconds.
+    BUSY_TIMEOUT_MS: ClassVar[int] = 10000
 
     def __post_init__(self):
         """
@@ -38,14 +49,49 @@ class SQLiteDB(AbstractDB):
 
         When *password* is ``None`` (default) the standard ``sqlite3`` module
         is used and the database file is unencrypted.
+
+        Each thread gets its own connection (see :attr:`conn`). A single
+        shared connection is not usable from several threads at once: the
+        transaction state belongs to the connection, so one thread's
+        ``COMMIT`` ends another thread's transaction and resets its
+        in-flight statements. Under a threaded network backend that shows
+        up as ``sqlite3.ProgrammingError: bad parameter or other API
+        misuse`` and as writes that land with corrupted bindings.
         """
-        db_path = os.path.join(xdg_data_home(), self.subfolder, self.name + ".db")
-        LOG.debug(f"sqlite database path: {db_path}")
-        os.makedirs(os.path.dirname(db_path), exist_ok=True)
+        if self.db_path is not None:
+            self._db_path = self.db_path
+            if self._db_path == ":memory:":
+                # A plain ":memory:" database belongs to the one connection
+                # that opened it, so the next thread would open a second,
+                # empty database and find no tables. Give it a name and a
+                # shared cache instead. The name carries the instance id, so
+                # two in-memory databases in one process stay independent.
+                self._db_path = f"file:hivemind-{id(self):x}?mode=memory&cache=shared"
+        else:
+            self._db_path = os.path.join(xdg_data_home(), self.subfolder, self.name + ".db")
+        self._is_uri = self._db_path.startswith("file:")
+        parent = os.path.dirname(self._db_path)
+        if parent and not self._is_uri:
+            os.makedirs(parent, exist_ok=True)
+        LOG.debug(f"sqlite database path: {self._db_path}")
+
+        if self.password is not None and self.password == "":
+            raise ValueError("password must be non-empty when encryption is enabled")
+
+        self._write_lock = threading.Lock()
+        # opening the first connection also applies the WAL pragma
+        self.conn.execute("PRAGMA journal_mode=WAL")
+        if "mode=memory" in self._db_path:
+            # A shared in-memory database lives only while a connection to
+            # it is open. Hold one for the lifetime of this object so the
+            # tables survive a thread closing its own connection.
+            self._keepalive = self._connect()
+        self._initialize_database()
+        self._maybe_migrate()
 
+    def _connect(self):
+        """Open one new connection to the backing file."""
         if self.password is not None:
-            if self.password == "":
-                raise ValueError("password must be non-empty when encryption is enabled")
             try:
                 import sqlcipher3 as _sqlcipher
             except ImportError:
@@ -54,18 +100,63 @@ class SQLiteDB(AbstractDB):
                     "Install the system library (e.g. 'apt install libsqlcipher-dev') "
                     "then: pip install hivemind-sqlite-database[cipher]"
                 )
-            self.conn = _sqlcipher.connect(db_path, check_same_thread=False)
-            self.conn.row_factory = _sqlcipher.Row
+            conn = _sqlcipher.connect(self._db_path, check_same_thread=False,
+                                      uri=self._is_uri)
+            conn.row_factory = _sqlcipher.Row
             escaped_password = self.password.replace("'", "''")
-            self.conn.execute(f"PRAGMA key='{escaped_password}'")
+            conn.execute(f"PRAGMA key='{escaped_password}'")
         else:
-            self.conn = sqlite3.connect(db_path, check_same_thread=False)
-            self.conn.row_factory = sqlite3.Row
-
-        self.conn.execute("PRAGMA journal_mode=WAL")
-        self._write_lock = threading.Lock()
-        self._initialize_database()
-        self._maybe_migrate()
+            conn = sqlite3.connect(self._db_path, check_same_thread=False,
+                                   uri=self._is_uri)
+            conn.row_factory = sqlite3.Row
+        conn.execute(f"PRAGMA busy_timeout={int(self.BUSY_TIMEOUT_MS)}")
+        return conn
+
+    @property
+    def conn(self):
+        """The calling thread's own connection, opened on first use.
+
+        Connections are cheap and WAL lets many readers run beside one
+        writer, so a per-thread connection costs a file handle and buys
+        real thread safety. ``_write_lock`` still serialises writers
+        in-process so they do not fight over the file lock.
+        """
+        local = self._thread_state()
+        conn = getattr(local, "conn", None)
+        if conn is None:
+            conn = local.conn = self._connect()
+        return conn
+
+    @conn.setter
+    def conn(self, value) -> None:
+        """Adopt an already-open connection for the calling thread only.
+
+        Any other thread that later touches ``.conn`` still opens its own
+        connection against ``self._db_path`` — if that is the real client
+        database, that thread silently writes to it. Pass ``db_path``
+        (e.g. ``":memory:"``) to the constructor instead so every thread,
+        present and future, agrees on the same target; this setter exists
+        only for single-threaded call sites that already hold a connection
+        they want reused.
+        """
+        self._thread_state().conn = value
+
+    def _thread_state(self) -> threading.local:
+        local = self.__dict__.get("_local")
+        if local is None:
+            local = self.__dict__["_local"] = threading.local()
+        return local
+
+    def close(self) -> None:
+        """Close the calling thread's connection, if it has one."""
+        local = self._thread_state()
+        conn = getattr(local, "conn", None)
+        if conn is not None:
+            local.conn = None
+            try:
+                conn.close()
+            except sqlite3.Error as e:
+                LOG.error(f"Failed to close SQLite connection: {e}")
 
     def _initialize_database(self):
         """Initialize the database schema."""
@@ -235,10 +326,12 @@ class SQLiteDB(AbstractDB):
             LOG.error(f"Invalid search key: {key!r}")
             return []
         try:
-            with self.conn:
-                cur = self.conn.execute(f"SELECT * FROM clients WHERE {key} = ?", (val,))
-                rows = cur.fetchall()
-                return [self._row_to_client(row) for row in rows]
+            # deliberately NOT wrapped in ``with self.conn`` — that context
+            # manager commits the connection's transaction on exit, which a
+            # read has no business doing.
+            cur = self.conn.execute(f"SELECT * FROM clients WHERE {key} = ?", (val,))
+            rows = cur.fetchall()
+            return [self._row_to_client(row) for row in rows]
         except sqlite3.Error as e:
             LOG.error(f"Failed to search clients in SQLite: {e}")
             return []

{hivemind_sqlite_database-0.4.1a1 → hivemind_sqlite_database-0.4.3a1}/hivemind_sqlite_database/version.py

@@ -1,7 +1,7 @@
 # START_VERSION_BLOCK
 VERSION_MAJOR = 0
 VERSION_MINOR = 4
-VERSION_BUILD = 1
+VERSION_BUILD = 3
 VERSION_ALPHA = 1
 # END_VERSION_BLOCK
 

{hivemind_sqlite_database-0.4.1a1 → hivemind_sqlite_database-0.4.3a1}/hivemind_sqlite_database.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hivemind-sqlite-database
-Version: 0.4.1a1
+Version: 0.4.3a1
 Summary: sqlite database plugin for hivemind-core
 Author-email: jarbasAi <jarbasai@mailfence.com>
 License: Apache-2.0

{hivemind_sqlite_database-0.4.1a1 → hivemind_sqlite_database-0.4.3a1}/tests/test_sqlitedb.py

@@ -422,6 +422,56 @@ class TestSQLiteDBRoundTrip(unittest.TestCase):
         self.assertEqual(r.intent_blacklist, ["a:b"])
 
 
+class TestSQLiteDBPathOverride(unittest.TestCase):
+    """A worker thread must never fall back to the real client database
+    just because the test only overrode `.conn` on the main thread."""
+
+    def test_db_path_override_keeps_worker_threads_off_disk(self):
+        import unittest.mock as mock
+        with tempfile.TemporaryDirectory() as tmpdir:
+            with mock.patch(
+                "hivemind_sqlite_database.xdg_data_home", return_value=tmpdir
+            ):
+                db = SQLiteDB(name="clients", subfolder="hivemind-core",
+                              db_path=":memory:")
+
+                # Read a row written on this thread. "SELECT 1" would answer
+                # the same against a private, empty database and so would
+                # never notice a per-thread database.
+                db.add_item(Client(client_id=1, api_key="key",
+                                   name="kitchen"))
+
+                errors = []
+                seen = []
+
+                def worker():
+                    try:
+                        seen.extend(db.search_by_value("name", "kitchen"))
+                    except Exception as e:  # noqa: BLE001
+                        errors.append(e)
+
+                t = threading.Thread(target=worker)
+                t.start()
+                t.join()
+
+            self.assertEqual(errors, [])
+            self.assertEqual([c.client_id for c in seen], [1])
+            real_db_file = os.path.join(tmpdir, "hivemind-core", "clients.db")
+            self.assertFalse(os.path.exists(real_db_file))
+
+    def test_two_in_memory_databases_stay_independent(self):
+        first = SQLiteDB(db_path=":memory:")
+        second = SQLiteDB(db_path=":memory:")
+        first.add_item(Client(client_id=1, api_key="key", name="kitchen"))
+        self.assertEqual(second.search_by_value("name", "kitchen"), [])
+
+    def test_explicit_db_path_creates_missing_directories(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            path = os.path.join(tmpdir, "newdir", "clients.db")
+            SQLiteDB(db_path=path)
+            self.assertTrue(os.path.isfile(path))
+
+
 class TestSQLiteDBCommit(unittest.TestCase):
     def test_commit_returns_true(self):
         db = make_db()
@@ -883,3 +933,66 @@ class TestSQLiteDBSchemaV2RoundTrip(unittest.TestCase):
 
 if __name__ == "__main__":
     unittest.main()
+
+
+class TestSQLiteDBThreadSafety(unittest.TestCase):
+    """A real file-backed DB hammered from many threads at once.
+
+    Reproduces the failure seen under the threaded (webrockets) network
+    backend: concurrent reads and writes through ONE shared connection
+    raise ``bad parameter or other API misuse`` and write rows with
+    corrupted bindings (``NOT NULL constraint failed: clients.api_key``),
+    because one thread's implicit COMMIT ends another thread's
+    transaction.
+    """
+
+    def test_concurrent_read_write_is_clean(self):
+        import tempfile
+        from unittest.mock import patch
+
+        with tempfile.TemporaryDirectory() as tmp:
+            with patch("hivemind_sqlite_database.xdg_data_home", return_value=tmp):
+                db = SQLiteDB()
+                n = 60
+                for i in range(n):
+                    self.assertTrue(db.add_item(make_client(i, f"key-{i}")))
+
+                errors = []
+                misses = []
+                barrier = threading.Barrier(24)
+
+                def reader(i):
+                    barrier.wait()
+                    try:
+                        for _ in range(40):
+                            got = db.search_by_value("api_key", f"key-{i % n}")
+                            if len(got) != 1:
+                                misses.append((i, len(got)))
+                    except Exception as e:  # noqa: BLE001
+                        errors.append(repr(e))
+
+                def writer(i):
+                    barrier.wait()
+                    try:
+                        for k in range(40):
+                            if not db.add_item(
+                                make_client(i % n, f"key-{i % n}",
+                                            name=f"w{k}")):
+                                errors.append(f"add_item returned False ({i},{k})")
+                    except Exception as e:  # noqa: BLE001
+                        errors.append(repr(e))
+
+                threads = [threading.Thread(target=reader, args=(i,))
+                           for i in range(16)]
+                threads += [threading.Thread(target=writer, args=(i,))
+                            for i in range(8)]
+                for t in threads:
+                    t.start()
+                for t in threads:
+                    t.join(timeout=120)
+
+                self.assertEqual(errors, [])
+                self.assertEqual(misses, [])
+                # every api_key still resolves to exactly one live row
+                for i in range(n):
+                    self.assertEqual(len(db.search_by_value("api_key", f"key-{i}")), 1)