diskstore 0.2.0__tar.gz → 0.3.1__tar.gz

{diskstore-0.2.0 → diskstore-0.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: diskstore
-Version: 0.2.0
+Version: 0.3.1
 Summary: Simple MutableMapping compatible SQLite based disk storage.
 Author: tds333
 Author-email: tds333 <tds333@mailbox.org>

{diskstore-0.2.0 → diskstore-0.3.1}/pyproject.toml

@@ -1,42 +1,29 @@
 [project]
 name = "diskstore"
-version = "0.2.0"
+version = "0.3.1"
 description = "Simple MutableMapping compatible SQLite based disk storage."
 readme = "README.md"
 requires-python = ">=3.10"
 authors = [{ name = "tds333", email = "tds333@mailbox.org" }]
-dependencies = [
-  "apsw>=3.51.3",
-]
+dependencies = ["apsw>=3.51.3"]
 
 [dependency-groups]
 dev = [
+  "msgspec>=0.20.0",
+  "pydantic>=2.12.5,<=2.99.99b0",
   "pytest",
   "pytest-asyncio>=1.3.0",
   "pytest-cov",
   "pytest-examples>=0.0.18",
-  "pytest-xdist",
-]
-benchmark = [
-  "diskcache",
-  "ipython",
-  "pickleDB<1",
-  "pylibmc",
-  "redis",
-  "sqlitedict",
+  "pytest-xdist"
 ]
 docs = [
-  "mkdocs>=1.6.1",
   "mkdocstrings[python]>=1.0.3",
   "zensical>=0.0.23",
 ]
-ext-dev = [
-  "msgspec>=0.20.0",
-  "pydantic>=2.12.5",
-]
 
 [build-system]
-requires = ["uv_build>=0.9.2,<0.12.0"]
+requires = ["uv_build>=0.9.2,<1"]
 build-backend = "uv_build"
 
 [tool.coverage.run]
@@ -78,3 +65,6 @@ ignore = [
   "RET504",
   "PLR2004",
 ]
+
+[tool.uv]
+exclude-newer = "1 days"

diskstore-0.3.1/src/diskstore/config.py

@@ -0,0 +1,247 @@
+"""DiskStore configuration classes and helpers for config."""
+
+import dataclasses
+import json
+from abc import abstractmethod
+from collections.abc import Sequence
+from typing import Any, Iterable, Protocol
+
+from .const import NO_DEFAULT, TIMEOUT, AnyLite, KeyType
+
+
+def get_sqlite_type(type_) -> str:
+    if type_ in {"BLOB", "TEXT", "INTEGER", "REAL"}:
+        return type_
+    sqlite_type = "BLOB"
+    if type_ is str:
+        sqlite_type = "TEXT"
+    elif type_ is int:
+        sqlite_type = "INTEGER"
+    elif type_ is bool:
+        sqlite_type = "INTEGER"
+    elif type_ is float:
+        sqlite_type = "REAL"
+
+    return sqlite_type
+
+
+def escape_name(name: str) -> str:
+    tablename = '"' + name.replace('"', '""') + '"'
+    return tablename
+
+
+def is_bindable_default(value) -> bool:
+    """Whether *value* can be encoded by apsw as a SQL literal."""
+    return value is None or isinstance(value, (str, bytes, int, float))
+
+
+class ConfigProtocol(Protocol):
+    """Configuration Protocol
+
+    Attributes:
+        tablename: Table name as string
+        key_type: key type as string or basic Python type like str, int, float
+        timeout: Timeout used to wait if someone blocks connections or with writes
+        pragmas: Dictionary with PRAGMAs to set when connections is initialized
+        fields: Iterable used for select, update and create statement with
+            field name, type, default.
+            ``[("value", str, NO_DEFAULT), ("value2", int, 0)]``
+
+    """
+
+    tablename: str
+    key_type: str
+    timeout: float
+    pragmas: dict
+    fields: Iterable
+    auto_migrate: bool
+
+    @abstractmethod
+    def dump_value(self, key: KeyType | None, value: Any) -> Sequence:
+        """Called with the key and value, should return a Sequence
+        with the key as first element, suitable as parameter tuple for
+        the INSERT/SET statement."""
+
+    @abstractmethod
+    def load_data(self, data: tuple) -> Any:
+        """load(db_data): called with the tuple selected from DB,
+        including the key at index 0.  Should be converted to the
+        type normally received as value."""
+
+
+class BaseConfig(ConfigProtocol):
+    """Base default configuratin."""
+
+    def __init__(
+        self,
+        *,
+        tablename=None,
+        key_type=None,
+        timeout=None,
+        pragmas=None,
+        auto_migrate=None,
+    ):
+        self.tablename = "DiskStore" if tablename is None else tablename
+        self.key_type = "BLOB" if key_type is None else get_sqlite_type(key_type)
+        self.timeout = TIMEOUT if timeout is None else float(timeout)
+        self.pragmas = {} if pragmas is None else pragmas
+        self.auto_migrate = True if auto_migrate is None else auto_migrate
+        self.fields = [("value", "BLOB", NO_DEFAULT)]
+
+    def dump_value(self, key: KeyType | None, value: Any) -> Sequence:
+        return (key, value)
+
+    def load_data(self, data: tuple) -> Any:
+        return data[1]
+
+
+class NamedTupleConfig(BaseConfig):
+    def __init__(  # noqa: PLR0913
+        self,
+        value_class,
+        tablename=None,
+        key_type=None,
+        timeout=None,
+        pragmas=None,
+        auto_migrate=None,
+    ):
+        tablename = value_class.__name__ if not tablename else tablename
+        super().__init__(
+            tablename=tablename,
+            key_type=key_type,
+            timeout=timeout,
+            pragmas=pragmas,
+            auto_migrate=auto_migrate,
+        )
+        self.value_class = value_class
+        self.fields = self.get_fields(value_class)
+
+    @staticmethod
+    def get_fields(value_class):
+        fields = []
+        value_columns = tuple(value_class._fields)
+        type_annotations = value_class.__annotations__
+        value_column_defaults: dict[str, AnyLite] = getattr(
+            value_class, "_field_defaults", {}
+        )
+        for name in value_columns:
+            type_cls = type_annotations.get(name, bytes)  # types are optional
+            sqlite_type = get_sqlite_type(type_cls)
+            default = value_column_defaults.get(name, NO_DEFAULT)
+            if not is_bindable_default(default):
+                default = NO_DEFAULT
+            fields.append((name, sqlite_type, default))
+        if "_key" in value_columns:
+            raise ValueError(
+                f"Name _key is not allowed as attribute for {value_class},"
+                " listed as field name in _fields."
+            )
+
+        return tuple(fields)
+
+    def dump_value(self, key: KeyType | None, value: Any) -> Sequence:
+        return (key, *value)
+
+    def load_data(self, data: tuple) -> Any:
+        return self.value_class._make(data[1:])
+
+
+class JsonConfig(BaseConfig):
+    def __init__(
+        self,
+        tablename=None,
+        key_type=None,
+        timeout=None,
+        pragmas=None,
+        auto_migrate=None,
+    ):
+        super().__init__(
+            tablename=tablename,
+            key_type=key_type,
+            timeout=timeout,
+            pragmas=pragmas,
+            auto_migrate=auto_migrate,
+        )
+        self.fields = (("value", "TEXT", NO_DEFAULT),)
+
+    def dump_value(self, key: KeyType | None, value: Any) -> Sequence:
+        return (key, json.dumps(value))
+
+    def load_data(self, data: tuple) -> Any:
+        return json.loads(data[1])
+
+
+class DataclassConfig(BaseConfig):
+    def __init__(  # noqa: PLR0913
+        self,
+        dataclass,
+        tablename=None,
+        key_type=None,
+        timeout=None,
+        pragmas=None,
+        auto_migrate=None,
+    ):
+        tablename = dataclass.__name__ if not tablename else tablename
+        super().__init__(
+            tablename=tablename,
+            key_type=key_type,
+            timeout=timeout,
+            pragmas=pragmas,
+            auto_migrate=auto_migrate,
+        )
+        self.dataclass = dataclass
+        self.fields = self.get_fields(dataclass)
+
+    @staticmethod
+    def get_fields(dataclass):
+        if not dataclasses.is_dataclass(dataclass):
+            raise ValueError("It is not a dataclass.")
+        dc_fields = dataclasses.fields(dataclass)
+        value_columns = tuple(field.name for field in dc_fields)
+        if "_key" in value_columns:
+            raise ValueError("Name _key is not allowed as attribute for dataclass.")
+        type_annotations = getattr(dataclass, "__annotations__", {}) or {}
+        fields = []
+        for f in dc_fields:
+            type_cls = type_annotations.get(f.name, bytes)
+            sqlite_type = get_sqlite_type(type_cls)
+            if f.default is not dataclasses.MISSING and is_bindable_default(f.default):
+                default = f.default
+            else:
+                default = NO_DEFAULT
+            fields.append((f.name, sqlite_type, default))
+        return tuple(fields)
+
+    def dump_value(self, key: KeyType | None, value: Any) -> Sequence:
+        return (key, *dataclasses.astuple(value))
+
+    def load_data(self, data: tuple) -> Any:
+        return self.dataclass(*data[1:])
+
+
+class PydanticConfig(BaseConfig):
+    def __init__(  # noqa: PLR0913
+        self,
+        model,
+        tablename=None,
+        key_type=None,
+        timeout=None,
+        pragmas=None,
+        auto_migrate=None,
+    ):
+        tablename = model.__name__ if not tablename else tablename
+        super().__init__(
+            tablename=tablename,
+            key_type=key_type,
+            timeout=timeout,
+            pragmas=pragmas,
+            auto_migrate=auto_migrate,
+        )
+        self.fields = (("value", "TEXT", NO_DEFAULT),)
+        self.model = model
+
+    def dump_value(self, key: KeyType | None, value: Any) -> Sequence:
+        return (key, value.model_dump_json())
+
+    def load_data(self, data: tuple) -> Any:
+        return self.model.model_validate_json(data[1])

{diskstore-0.2.0 → diskstore-0.3.1}/src/diskstore/const.py

@@ -5,19 +5,31 @@ from typing import (
     Union,
 )
 
-TIMEOUT = 60.0
+TIMEOUT = 10.0
 "default timout in seconds"
 
 MISSING = object()
 
 
+class NO_DEFAULT:  # noqa: N801
+    """Sentinel marker for fields without a default value."""
+
+
+DEFAULT_RO_PRAGMAS = {
+    "cache_size": 2**13,  # 8,192 pages
+    "mmap_size": 2**28,  # 256 MB
+    "temp_store": 2,  # 0=DEFAULT, 1=FILE, 2=MEMORY
+    "synchronous": 1,  # 0=OFF, 1=NORMAL, 2=FULL, 3=EXTRA
+}
+"default read only pragma settings"
+
 DEFAULT_PRAGMAS = {
     "auto_vacuum": 0,  # 1=FULL, 0=None
     "cache_size": 2**13,  # 8,192 pages
     "journal_mode": "wal",
     "mmap_size": 2**28,  # 256 MB
     "synchronous": 1,  # 0=OFF, 1=NORMAL, 2=FULL, 3=EXTRA
-    "temp_store": "memory",
+    "temp_store": 2,  # 0=DEFAULT, 1=FILE, 2=MEMORY
     # "pargma_busy_timeout": 5000,  # milliseconds
 }
 "default pragma settings"

{diskstore-0.2.0 → diskstore-0.3.1}/src/diskstore/diskread.py

@@ -13,16 +13,15 @@ import os
 import os.path
 import threading
 from collections.abc import ItemsView, KeysView, Mapping, ValuesView
-from contextlib import closing, contextmanager
+from contextlib import closing
 from typing import Generator, Optional, Sequence, TypeAlias, Union
 
 import apsw
 
 from .config import BaseConfig, ConfigProtocol, escape_name
-from .const import MISSING, TIMEOUT, KeyType
+from .const import DEFAULT_RO_PRAGMAS, MISSING, TIMEOUT, KeyType
 
 Connection = apsw.Connection
-Cursor = apsw.Cursor
 
 
 BasicType: TypeAlias = Union[bytes, str, int, float]
@@ -76,7 +75,6 @@ class DiskRead(Mapping):
 
         """
         filename = os.path.expanduser(filename)
-        filename = os.path.expandvars(filename)
         self._filename = os.fspath(filename)
         self._config: ConfigProtocol = BaseConfig() if config is None else config
         self._timeout: float = (
@@ -85,13 +83,15 @@ class DiskRead(Mapping):
             else float(self._config.timeout)
         )
         self._load_data = self._config.load_data
+        self._pragmas = DEFAULT_RO_PRAGMAS.copy()
+        self._pragmas.update(self._config.pragmas)
         self._local = threading.local()
 
         # precreated statements based on tablename and value_class
         tablename = escape_name(self._config.tablename)
         fields = ", ".join(f"{escape_name(field)}" for field, *_ in self._config.fields)
         self._statements: dict[str, str] = {
-            "GET": f"SELECT {fields} FROM {tablename} WHERE _key = ? LIMIT 1",
+            "GET": f"SELECT _key, {fields} FROM {tablename} WHERE _key = ? LIMIT 1",
             "CONTAINS": f"SELECT _key FROM {tablename} WHERE _key = ? LIMIT 1",
             "ITER": f"SELECT _key FROM {tablename} ORDER BY rowid ASC",
             "REVERSED": f"SELECT _key FROM {tablename} ORDER BY rowid DESC",
@@ -124,7 +124,7 @@ class DiskRead(Mapping):
 
         if local_pid != pid:
             self.close()
-            self._local.pid: int = pid
+            self._local.pid = pid
 
         con = getattr(self._local, "con", None)
 
@@ -134,20 +134,18 @@ class DiskRead(Mapping):
             )
             con.set_busy_timeout(int(self._timeout * 1000))
 
-        return con
+            # Some SQLite pragmas work on a per-connection basis so
+            # apply them all on fresh connection
+            for key, value in self._pragmas.items():
+                con.pragma(key, value)
 
-    @contextmanager
-    def _cursor(self):
-        cursor: Cursor = self._con.cursor()
-        try:
-            yield cursor
-        finally:
-            cursor.close()
+        return con
 
     def __getitem__(self, key: KeyType):
+        """Get value for *key*, raises KeyError if not found."""
         select = self._statements["GET"]
-        with self._cursor() as cursor:
-            row = next(cursor.execute(select, (key,)), MISSING)
+        with closing(self._con.execute(select, (key,))) as cx:
+            row = next(cx, MISSING)
 
         if row is MISSING:
             raise KeyError(key)
@@ -155,57 +153,83 @@ class DiskRead(Mapping):
         return self._load_data(row)  # ty:ignore[invalid-argument-type]
 
     def keys(self):
+        """Return a set-like view of keys in the mapping."""
+
         return DiskKeysView(self)
 
     def values(self):
+        """Return a set-like view of values in the mapping."""
+
         return DiskValuesView(self)
 
     def items(self):
+        """Return a set-like view of (key, value) pairs in the mapping."""
+
         return DiskItemsView(self)
 
     def query(
         self,
         where: Optional[str] = None,
-        parameters: Optional[Sequence] = None,
+        parameters: Optional[Sequence | dict] = None,
         order: Optional[str] = None,
         limit: int | None = None,
         offset: int | None = None,
     ) -> Generator[tuple, None, None]:
-        where = " WHERE " + where if where else ""
-        parameters = () if parameters is None else parameters
-        order = " ORDER BY " + order if order else ""
-        limit = " LIMIT " + str(limit) if limit is not None else ""
-        offset = " OFFSET " + str(offset) if offset is not None else ""
-        select = self._statements["QUERY"] + where + order + limit + offset
-
-        with self._cursor() as cursor:
-            cursor.execute(select, parameters)
+        """Query rows with optional filtering, ordering, limit and offset.
+
+        Args:
+            where: SQL WHERE clause (without the WHERE keyword).
+            parameters: Parameters for the WHERE clause.
+            order: ORDER BY clause (without the ORDER BY keyword).
+            limit: Maximum number of rows to return.
+            offset: Number of rows to skip.
+
+        Yields:
+            (key, value) tuples.
+
+        Examples:
+            >>> from diskstore import DiskRead
+            >>> ds = DiskRead("/tmp/data.db")
+            >>> list(ds.query(where="_key > ?", parameters=(1,), limit=5))
+            [(2, 'two'), (3, 'three')]
+
+        """
+        where_ = " WHERE " + where if where else ""
+        parameters_ = () if parameters is None else parameters
+        order_ = " ORDER BY " + order if order else ""
+        limit_ = " LIMIT " + str(int(limit)) if limit is not None else ""
+        offset_ = " OFFSET " + str(int(offset)) if offset is not None else ""
+        select = self._statements["QUERY"] + where_ + order_ + limit_ + offset_
+
+        with closing(self._con.execute(select, parameters_)) as cursor:
             for row in cursor:
-                yield row[0], self._load_data(row[1:])
+                yield row[0], self._load_data(row)
 
     def __contains__(self, key: object) -> bool:
-        with self._cursor() as cx:
-            rows = cx.execute(self._statements["CONTAINS"], (key,)).fetchall()
-
-        return bool(rows)
+        """Check if *key* exists in the store."""
+        with closing(self._con.execute(self._statements["CONTAINS"], (key,))) as cx:
+            row = cx.fetchone()
+        return row is not None
 
     def __iter__(self):
-        with self._cursor() as cx:
-            cx.execute(self._statements["ITER"])
+        """Iterate over keys in insertion order."""
+        with closing(self._con.execute(self._statements["ITER"])) as cx:
             for row in cx:
                 yield row[0]
 
     def __reversed__(self):
-        with self._cursor() as cx:
-            cx.execute(self._statements["REVERSED"])
+        """Iterate over keys in reverse insertion order."""
+        with closing(self._con.execute(self._statements["REVERSED"])) as cx:
             for row in cx:
                 yield row[0]
 
-    def open(self):
+    def open(self) -> "DiskRead":
+        """Open (or re-open) the database connection and return self."""
         connection = self._con  # noqa
         return self
 
     def close(self) -> None:
+        """Close the database connection if open."""
         con = getattr(self._local, "con", None)
         if con is None:
             return
@@ -223,6 +247,10 @@ class DiskRead(Mapping):
         self.close()
 
     def __len__(self):
+        """Return the number of items in the store.
+
+        To do this count is used which is not a performant implementation.
+        """
         select = self._statements["COUNT"]
 
         with closing(self._con.execute(select)) as cx:

diskstore-0.3.1/src/diskstore/diskstore.py

@@ -0,0 +1,320 @@
+"""DiskStore (MutableMapping) API.
+
+Examples:
+    >>> from diskstore import DiskStore
+    >>> ds = DiskStore("/tmp/data.db")
+    >>> ds["one"] = 1
+    >>> ds["one"]
+    1
+
+"""
+
+import os
+import os.path
+import threading
+from collections.abc import Mapping, MutableMapping
+from contextlib import closing, contextmanager
+from typing import Any, Iterable
+
+import apsw
+
+from .config import ConfigProtocol, escape_name, get_sqlite_type
+from .const import DEFAULT_PRAGMAS, MISSING, NO_DEFAULT, KeyType
+from .diskread import DiskRead
+
+Connection = apsw.Connection
+Cursor = apsw.Cursor
+SQLError = apsw.SQLError
+BusyError = apsw.BusyError
+
+
+class DiskStore(DiskRead, MutableMapping):
+    def __init__(
+        self, filename: os.PathLike | str, config: ConfigProtocol | None = None
+    ) -> None:
+        """SQLite based MutableMapping disk storage.
+
+        Args:
+            filename: DiskStore DB filename.
+            config: Configuration as specified in ConfigProtocol
+        """
+        super().__init__(filename=filename, config=config)
+        self._txn_id = None
+        tablename = escape_name(self._config.tablename)
+
+        value_columns = self._config.fields
+        fields = ", ".join(f"{escape_name(field)}" for field, *_ in self._config.fields)
+        excluded_fields = ", ".join(
+            f"{field} = excluded.{escape_name(field)}"
+            for field, *_ in self._config.fields
+        )
+        qms = ", ".join("?" for field in value_columns)
+        self._statements.update(
+            {
+                "SET": (
+                    f"INSERT INTO {tablename}(_key, {fields}) VALUES (?, {qms})"
+                    f" ON CONFLICT (_key) DO UPDATE SET {excluded_fields}"
+                ),
+                "ADD": (
+                    f"INSERT INTO {tablename}(_key, {fields}) "
+                    f"VALUES (?, {qms}) ON CONFLICT DO NOTHING RETURNING _key"
+                ),
+                "DELETE": f"DELETE FROM {tablename} WHERE _key = ? RETURNING _key",
+                "POP": (
+                    f"DELETE FROM {tablename} WHERE _key = ? RETURNING _key, {fields}"
+                ),
+                "CLEAR": f"DELETE FROM {tablename};VACUUM;",
+                "POPITEM": (
+                    f"DELETE FROM {tablename}"
+                    f" WHERE rowid = (SELECT MAX(rowid) FROM {tablename})"
+                    f" RETURNING _key, {fields}"
+                ),
+            }
+        )
+        self._pragmas = DEFAULT_PRAGMAS.copy()
+        self._pragmas.update(self._config.pragmas)
+        self._dump_value = self._config.dump_value
+
+    @property
+    def _con(self) -> Connection:
+        # Check process ID to support process forking. If the process
+        # ID changes, close the connection and update the process ID.
+
+        local_pid = getattr(self._local, "pid", None)
+        pid: int = os.getpid()
+
+        if local_pid != pid:
+            self.close()
+            self._local.pid = pid
+
+        con = getattr(self._local, "con", None)
+
+        if con is None:
+            con = Connection(self._filename)
+            con.set_busy_timeout(int(self._timeout * 1000))
+
+            # Some SQLite pragmas work on a per-connection basis so
+            # apply them all on fresh connection
+            for key, value in self._pragmas.items():
+                con.pragma(key, value)
+            self.migrate_table(con, self._config)
+            self._local.con = con
+
+        return con
+
+    @staticmethod
+    def _get_field_create(field_tuple):
+        field_name, field_type, field_default = field_tuple
+        default = " NOT NULL"
+        if field_default is None:
+            default = " DEFAULT NULL"
+        elif field_default is not NO_DEFAULT:
+            default = " NOT NULL DEFAULT " + apsw.format_sql_value(field_default)
+        name = escape_name(field_name)
+        type_ = get_sqlite_type(field_type)
+        field_create = f"{name} {type_}{default}"
+
+        return field_create
+
+    @staticmethod
+    def migrate_table(con: Connection, config: ConfigProtocol) -> None:
+        """Ensure the table exists, optionally migrating to match config.fields.
+
+        If the table does not exist it is created from *config*.  If
+        *config.auto_migrate* is true and the table already exists,
+        columns in *config.fields* missing from the table are added.
+
+        Migration uses ``BEGIN IMMEDIATE`` to serialise concurrent
+        callers.
+        """
+        tablename = escape_name(config.tablename)
+        existing = {
+            row[1] for row in (con.pragma("table_info", config.tablename) or [])
+        }
+
+        if not existing:
+            primary_key_type = get_sqlite_type(config.key_type)
+            fields_create = ", ".join(
+                DiskStore._get_field_create(f) for f in config.fields
+            )
+            create_stmt = (
+                f"CREATE TABLE IF NOT EXISTS {tablename} ("
+                f" _key {primary_key_type} PRIMARY KEY NOT NULL"
+                f", {fields_create})"
+            )
+            con.execute(create_stmt)
+        elif config.auto_migrate:
+            config_names = {f[0] for f in config.fields}
+            if config_names - existing:
+                con.execute("BEGIN IMMEDIATE")
+                existing = {
+                    row[1] for row in con.pragma("table_info", config.tablename)
+                }
+                for field_name, field_type, default_value in config.fields:
+                    if field_name not in existing:
+                        col_def = DiskStore._get_field_create(
+                            (field_name, field_type, default_value)
+                        )
+                        alter_stmt = f"ALTER TABLE {tablename} ADD COLUMN {col_def};"
+                        con.execute(alter_stmt)
+                con.execute("COMMIT")
+
+    @contextmanager
+    def transact(self):
+        """Wrapper for performance sensitive bulk writes.
+
+        Every write operation (``__setitem__``, ``__delitem__``, ``add``,
+        etc.) runs in an implicit transaction when called outside of
+        ``transact()``.  The implicit begin/commit overhead adds up when
+        writing many items in a loop:
+
+        .. code:: python
+
+            # slow: one implicit transaction per write
+            for i in range(1000):
+                store[i] = value
+
+            # fast: single explicit transaction
+            with store.transact():
+                for i in range(1000):
+                    store[i] = value
+
+        Uses ``BEGIN IMMEDIATE`` to avoid deadlocks in concurrent
+        workloads.  Nested calls are idempotent (reuse the same
+        transaction).  Yields an ``apsw.Cursor`` for callers that need
+        direct SQL execution.
+        """
+        cursor: Cursor = self._con.cursor()
+        tid = threading.get_ident()
+        txn_id = self._txn_id
+
+        if tid == txn_id:  # already inside a thread with a transaction
+            begin = False
+        else:
+            cursor.execute("BEGIN IMMEDIATE")
+            begin = True
+            self._txn_id = tid
+
+        try:
+            yield cursor
+        except BaseException:
+            if begin:
+                assert self._txn_id == tid
+                self._txn_id = None
+                cursor.execute("ROLLBACK")
+            cursor.close()
+            raise
+        else:
+            if begin:
+                assert self._txn_id == tid
+                self._txn_id = None
+                cursor.execute("COMMIT")
+            cursor.close()
+
+    def __setitem__(self, key: KeyType, value: Any) -> None:
+        with closing(
+            self._con.execute(self._statements["SET"], self._dump_value(key, value))
+        ):
+            pass
+
+    def add(self, key: KeyType | None, value: Iterable) -> KeyType | None:
+        with closing(
+            self._con.execute(self._statements["ADD"], self._dump_value(key, value))
+        ) as cx:
+            rows = cx.fetchall()
+
+        if not rows:
+            return None
+
+        return rows[0][0]
+
+    def pop(self, key: KeyType, default=MISSING):
+        with closing(self._con.execute(self._statements["POP"], (key,))) as cx:
+            rows = cx.fetchall()
+
+        if not rows:
+            if default is MISSING:
+                raise KeyError(key)
+            return default
+
+        return self._load_data(rows[0])
+
+    def popitem(self):
+        with self.transact() as cursor:
+            cursor.execute(self._statements["POPITEM"])
+            row = next(cursor, None)
+            if not row:
+                raise KeyError()
+        key = row[0]
+        value = self._load_data(row)
+        return key, value
+
+    def __delitem__(self, key: KeyType) -> None:
+        with closing(self._con.execute(self._statements["DELETE"], (key,))) as cx:
+            row = cx.fetchone()
+        if row is None:
+            raise KeyError(key)
+
+    def setdefault(self, key: KeyType, default: Iterable | None = None):
+        with self.transact():
+            try:
+                return self[key]
+            except KeyError:
+                if default is not None:
+                    self.add(key, default)
+                return default
+
+    def check(self, vacuum=False):
+        warns = []
+        sql = self._con.execute
+
+        # Check integrity of database.
+        with closing(sql("PRAGMA integrity_check")) as cx:
+            rows = cx.fetchall()
+
+        if len(rows) != 1 or rows[0][0] != "ok":
+            for (message,) in rows:
+                warns.append(message)
+
+        if vacuum:
+            with closing(sql("VACUUM")):
+                pass
+
+        return warns
+
+    def clear(self) -> None:
+        with closing(self._con.execute(self._statements["CLEAR"])):
+            pass
+
+    def update(self, other=(), /, **kwargs):
+        """Bulk upsert from a mapping or iterable.
+
+        Wrapped in ``transact()`` so all upserts share a single
+        transaction — significantly faster than setting keys
+        individually in a loop.
+        """
+        with self.transact() as cursor:
+            if other:
+                if isinstance(other, Mapping):
+                    cursor.executemany(
+                        self._statements["SET"],
+                        (self._dump_value(key, value) for key, value in other.items()),
+                    )
+                elif hasattr(other, "keys"):
+                    cursor.executemany(
+                        self._statements["SET"],
+                        (self._dump_value(key, other[key]) for key in other.keys()),
+                    )
+                else:
+                    cursor.executemany(
+                        self._statements["SET"],
+                        (self._dump_value(key, value) for key, value in other),
+                    )
+            if kwargs:
+                cursor.executemany(
+                    self._statements["SET"],
+                    (self._dump_value(key, value) for key, value in kwargs.items()),
+                )
+
+    def get_readonly_instance(self):
+        return DiskRead(self._filename, self._config)

diskstore-0.2.0/src/diskstore/config.py

@@ -1,190 +0,0 @@
-"""DiskStore configuration classes and helpers for config."""
-
-import dataclasses
-import json
-from abc import abstractmethod
-from typing import Any, Iterable, Protocol
-
-from .const import TIMEOUT, AnyLite
-
-
-def get_sqlite_type(type_) -> str:
-    if type_ in {"BLOB", "TEXT", "INTEGER", "REAL"}:
-        return type_
-    sqlite_type = "BLOB"
-    if type_ is str:
-        sqlite_type = "TEXT"
-    elif type_ is int:
-        sqlite_type = "INTEGER"
-    elif type_ is float:
-        sqlite_type = "REAL"
-
-    return sqlite_type
-
-
-def escape_name(name: str) -> str:
-    tablename = '"' + name.replace('"', '""') + '"'
-    return tablename
-
-
-class ConfigProtocol(Protocol):
-    """Configuration Protocol
-
-    Attributes:
-        tablename: Table name as string
-        key_type: key type as string or basic Python type like str, int, float
-        timeout: Timeout used to wait if someone blocks connections or with writes
-        pragmas: Dictionary with PRAGMAs to set when connections is initialized
-        fields: Iterable used for select, update and create statement with
-            field name, type and default. `[("value", str), ("value2", int, 0)]`
-
-    """
-
-    tablename: str
-    key_type: str
-    timeout: float
-    pragmas: dict
-    fields: Iterable
-
-    @abstractmethod
-    def dump_value(self, value: Any) -> Iterable:
-        """Called with the value, should return an Iterable
-        which is used to write to the DB."""
-
-    @abstractmethod
-    def load_data(self, data: tuple) -> Any:
-        """load(db_data): called with the tuple selected from DB.
-        Should be converted to the type normally received as value."""
-
-
-class BaseConfig(ConfigProtocol):
-    """Base default configuratin."""
-
-    def __init__(self, *, tablename=None, key_type=None, timeout=None, pragmas=None):
-        self.tablename = "DiskStore" if tablename is None else tablename
-        self.key_type = "BLOB" if key_type is None else get_sqlite_type(key_type)
-        self.timeout = TIMEOUT if timeout is None else float(timeout)
-        self.pragmas = {} if pragmas is None else pragmas
-        self.fields = [("value", "BLOB")]
-
-    def dump_value(self, value: Any) -> Iterable:
-        return (value,)
-
-    def load_data(self, data: tuple) -> Any:
-        return data[0]
-
-
-class NamedTupleConfig(BaseConfig):
-    def __init__(
-        self, value_class, tablename=None, key_type=None, timeout=None, pragmas=None
-    ):
-        tablename = value_class.__name__ if not tablename else tablename
-        super().__init__(
-            tablename=tablename, key_type=key_type, timeout=timeout, pragmas=pragmas
-        )
-        self.value_class = value_class
-        self.fields = self.get_fields(value_class)
-
-    @staticmethod
-    def get_fields(value_class):
-        fields = []
-        value_columns = tuple(value_class._fields)
-        type_annotations = value_class.__annotations__
-        value_column_defaults: dict[str, AnyLite] = getattr(
-            value_class, "_field_defaults", {}
-        )
-        for name in value_columns:
-            type_cls = type_annotations.get(name, bytes)  # types are optional
-            sqlite_type = get_sqlite_type(type_cls)
-            default = value_column_defaults.get(name)
-            fields.append((name, sqlite_type, default))
-        if "_key" in value_columns:
-            raise ValueError(
-                f"Name _key is not allowed as attribute for {value_class},"
-                " listed as field name in _fields."
-            )
-
-        return tuple(fields)
-
-    def dump_value(self, value) -> Iterable:
-        return value
-
-    def load_data(self, data: tuple) -> Any:
-        return self.value_class._make(data)
-
-
-class JsonConfig(BaseConfig):
-    def __init__(self, tablename=None, key_type=None, timeout=None, pragmas=None):
-        super().__init__(
-            tablename=tablename, key_type=key_type, timeout=timeout, pragmas=pragmas
-        )
-        self.fields = (("value", "TEXT"),)
-
-    def dump_value(self, value: Any) -> Iterable:
-        return (json.dumps(value),)
-
-    def load_data(self, data: tuple) -> Any:
-        return json.loads(data[0])
-
-
-class DataclassConfig(BaseConfig):
-    def __init__(
-        self, dataclass, tablename=None, key_type=None, timeout=None, pragmas=None
-    ):
-        tablename = dataclass.__name__ if not tablename else tablename
-        super().__init__(
-            tablename=tablename, key_type=key_type, timeout=timeout, pragmas=pragmas
-        )
-        self.dataclass = dataclass
-        self.fields = self.get_fields(dataclass)
-        self.types = self.get_field_types(dataclass)
-
-    @staticmethod
-    def get_fields(dataclass):
-        if dataclasses.is_dataclass(dataclass):
-            value_columns = tuple(field.name for field in dataclasses.fields(dataclass))
-        else:
-            raise ValueError("It is not a dataclass.")
-        if "_key" in value_columns:
-            raise ValueError("Name _key is not allowed as attribute for dataclass.")
-
-        return value_columns
-
-    @classmethod
-    def get_field_types(cls, value_class):
-        value_columns = cls.get_fields(value_class)
-        if hasattr(value_class, "__annotations__") and value_class.__annotations__:
-            type_annotations = value_class.__annotations__
-            value_types = []
-            for c_name in value_columns:
-                type_cls = type_annotations.get(c_name, bytes)  # types are optional
-                sqlite_type = get_sqlite_type(type_cls)
-                value_types.append(sqlite_type)
-        else:
-            value_types = ["BLOB" for _ in value_columns]
-
-        return tuple(value_types)
-
-    def dump_value(self, value) -> tuple:
-        return dataclasses.astuple(value)
-
-    def load_data(self, data: tuple) -> Any:
-        return self.dataclass(*data)
-
-
-class PydanticConfig(BaseConfig):
-    def __init__(
-        self, model, tablename=None, key_type=None, timeout=None, pragmas=None
-    ):
-        tablename = model.__name__ if not tablename else tablename
-        super().__init__(
-            tablename=tablename, key_type=key_type, timeout=timeout, pragmas=pragmas
-        )
-        self.fields = (("value", "TEXT"),)
-        self.model = model
-
-    def dump_value(self, value):
-        return (value.model_dump_json(),)
-
-    def load_data(self, data):
-        return self.model.model_validate_json(data[0])

diskstore-0.2.0/src/diskstore/diskstore.py

@@ -1,264 +0,0 @@
-"""DiskStore (MutableMapping) API.
-
-Examples:
-    >>> from diskstore import DiskStore
-    >>> ds = DiskStore("/tmp/data.db")
-    >>> ds["one"] = 1
-    >>> ds["one"]
-    1
-
-"""
-
-import os
-import os.path
-import threading
-from collections.abc import Mapping, MutableMapping
-from contextlib import closing, contextmanager
-from typing import Any, Iterable
-
-import apsw
-
-from .config import ConfigProtocol, escape_name, get_sqlite_type
-from .const import DEFAULT_PRAGMAS, MISSING, KeyType
-from .diskread import DiskRead
-
-Connection = apsw.Connection
-Cursor = apsw.Cursor
-SQLError = apsw.SQLError
-BusyError = apsw.BusyError
-
-
-class DiskStore(DiskRead, MutableMapping):
-    def __init__(
-        self, filename: os.PathLike | str, config: ConfigProtocol | None = None
-    ) -> None:
-        """SQLite based MutableMapping disk storage.
-
-        Args:
-            filename: DiskStore DB filename.
-            config: Configuration as specified in ConfigProtocol
-        """
-        super().__init__(filename=filename, config=config)
-        self._txn_id = None
-        tablename = escape_name(self._config.tablename)
-
-        primary_key_type = get_sqlite_type(self._config.key_type)
-        value_columns = self._config.fields
-        fields_create = ", ".join(
-            self._get_field_create(field) for field in self._config.fields
-        )
-        fields = ", ".join(f"{escape_name(field)}" for field, *_ in self._config.fields)
-        excluded_fields = ", ".join(
-            f"{field} = excluded.{escape_name(field)}"
-            for field, *_ in self._config.fields
-        )
-        qms = ", ".join("?" for field in value_columns)
-        self._statements.update(
-            {
-                "CREATE": (
-                    f"CREATE TABLE IF NOT EXISTS {tablename} ("
-                    f" _key {primary_key_type} PRIMARY KEY NOT NULL"
-                    f", {fields_create})"
-                ),
-                "SET": (
-                    f"INSERT INTO {tablename}(_key, {fields}) VALUES (?, {qms})"
-                    f" ON CONFLICT (_key) DO UPDATE SET {excluded_fields}"
-                ),
-                "ADD": (
-                    f"INSERT INTO {tablename}(_key, {fields}) "
-                    f"VALUES (?, {qms}) ON CONFLICT DO NOTHING RETURNING _key"
-                ),
-                "DELETE": f"DELETE FROM {tablename} WHERE _key = ? RETURNING _key",
-                "CLEAR": f"DELETE FROM {tablename};VACUUM;",
-                "POPITEM": (
-                    f"SELECT _key, {fields} FROM {tablename} ORDER BY"
-                    " rowid DESC LIMIT 1"
-                ),
-            }
-        )
-        self._pragmas = DEFAULT_PRAGMAS.copy()
-        self._pragmas.update(self._config.pragmas)
-        self._dump_value = self._config.dump_value
-
-    @property
-    def _con(self) -> Connection:
-        # Check process ID to support process forking. If the process
-        # ID changes, close the connection and update the process ID.
-
-        local_pid = getattr(self._local, "pid", None)
-        pid: int = os.getpid()
-
-        if local_pid != pid:
-            self.close()
-            self._local.pid: int = pid
-
-        con = getattr(self._local, "con", None)
-
-        if con is None:
-            con = Connection(self._filename)
-            con.set_busy_timeout(int(self._timeout * 1000))
-
-            # Some SQLite pragmas work on a per-connection basis so
-            # apply them all on fresh connection
-            for key, value in self._pragmas.items():
-                con.pragma(key, value)
-            con.execute(self._statements["CREATE"])
-            self._local.con = con
-
-        return con
-
-    @staticmethod
-    def _get_field_create(field_tuple):
-        field_name, field_type, *field_default = field_tuple
-        default = ""
-        if field_default and field_default[0] is not None:
-            default = " DEFAULT " + apsw.format_sql_value(field_default[0])
-        name = escape_name(field_name)
-        type_ = get_sqlite_type(field_type)
-        field_create = f"{name} {type_} NOT NULL{default}"
-
-        return field_create
-
-    def _migrate_table(self, new_fields=()):
-        tablename = escape_name(self._config.tablename)
-        existing_fileds = set()
-        table_info_stmt = f"PRAGMA table_info({tablename});"
-        sql = self._con.execute
-        with closing(sql(table_info_stmt)) as cursor:
-            for row in cursor:
-                existing_fileds.add(row[1])
-        for columnname, columntype, default_value in new_fields:
-            if columnname not in existing_fileds:
-                name: str = escape_name(columnname)
-                type_ = get_sqlite_type(columntype)
-                default = apsw.format_sql_value(default_value)
-                alter_stmt = (
-                    f"ALTER TABLE {tablename} ADD COLUMN {name}"
-                    f" {type_} NOT NULL DEFAULT {default};"
-                )
-                with closing(sql(alter_stmt)) as cursor:
-                    pass
-
-    @contextmanager
-    def transact(self):
-        cursor: Cursor = self._con.cursor()
-        tid = threading.get_ident()
-        txn_id = self._txn_id
-
-        if tid == txn_id:  # already inside a thread with a transaction
-            begin = False
-        else:
-            cursor.execute("BEGIN IMMEDIATE")
-            begin = True
-            self._txn_id = tid
-
-        try:
-            yield cursor
-        except BaseException:
-            if begin:
-                assert self._txn_id == tid
-                self._txn_id = None
-                cursor.execute("ROLLBACK")
-            cursor.close()
-            raise
-        else:
-            if begin:
-                assert self._txn_id == tid
-                self._txn_id = None
-                cursor.execute("COMMIT")
-            cursor.close()
-
-    def __setitem__(self, key: KeyType, value: Any) -> None:
-        with self._cursor() as cursor:
-            cursor.execute(self._statements["SET"], (key, *self._dump_value(value)))
-
-    def add(self, key: KeyType | None, value: Iterable) -> KeyType | None:
-        with self._cursor() as cx:
-            rows = cx.execute(
-                self._statements["ADD"], (key, *self._dump_value(value))
-            ).fetchall()
-
-        if not rows:
-            return None
-
-        return rows[0][0]
-
-    def pop(self, key: KeyType, default=MISSING):
-        with self.transact() as cx:
-            value = next(cx.execute(self._statements["GET"], (key,)), None)
-            if value is None:
-                if default is MISSING:
-                    raise KeyError(key)
-                return default
-            cx.execute(self._statements["DELETE"], (key,))
-            return self._load_data(value)
-
-    def popitem(self):
-        with self.transact() as cx:
-            row = next(cx.execute(self._statements["POPITEM"]), None)
-            if not row:
-                raise KeyError()
-            key = row[0]
-            value = self._load_data(row[1:])
-            cx.execute(self._statements["DELETE"], (key,))
-            return key, value
-
-    def __delitem__(self, key: KeyType) -> None:
-        with self._cursor() as cursor:
-            rows = cursor.execute(self._statements["DELETE"], (key,)).fetchall()
-        if not rows:
-            raise KeyError(key)
-
-    def setdefault(self, key: KeyType, default: Iterable | None = None):
-        try:
-            return self[key]
-        except KeyError:
-            if default is not None:
-                self.add(key, default)
-
-        return default
-
-    def check(self, vacuum=False):
-        warns = []
-        sql = self._con.execute
-
-        # Check integrity of database.
-        with closing(sql("PRAGMA integrity_check")) as cx:
-            rows = cx.fetchall()
-
-        if len(rows) != 1 or rows[0][0] != "ok":
-            for (message,) in rows:
-                warns.append(message)
-
-        if vacuum:
-            with closing(sql("VACUUM")):
-                pass
-
-        return warns
-
-    def clear(self) -> None:
-        with closing(self._con.execute(self._statements["CLEAR"])):
-            pass
-
-    def update(self, other=(), /, **kwargs):
-        comps = []
-        if other:
-            if isinstance(other, Mapping):
-                comp = ((key, *self._dump_value(value)) for key, value in other.items())
-                comps.append(comp)
-            elif hasattr(other, "keys"):
-                comp = ((key, *self._dump_value(other[key])) for key in other.keys())
-                comps.append(comp)
-            else:
-                comp = ((key, *self._dump_value(value)) for key, value in other)
-                comps.append(comp)
-        if kwargs:
-            comp = ((key, *self._dump_value(value)) for key, value in kwargs.items())
-            comps.append(comp)
-
-        with self.transact() as cursor:
-            for comp in comps:
-                cursor.executemany(self._statements["SET"], comp)
-
-    def get_readonly_instance(self):
-        return DiskRead(self._filename, self._config)