diskstore 0.1.0__tar.gz → 0.2.0__tar.gz

{diskstore-0.1.0 → diskstore-0.2.0}/PKG-INFO

@@ -1,10 +1,10 @@
 Metadata-Version: 2.3
 Name: diskstore
-Version: 0.1.0
+Version: 0.2.0
 Summary: Simple MutableMapping compatible SQLite based disk storage.
 Author: tds333
 Author-email: tds333 <tds333@mailbox.org>
-Requires-Dist: apsw
+Requires-Dist: apsw>=3.51.3
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 
@@ -15,8 +15,8 @@ in pure Python.
 
 Inspired by DiskCache library it implements a MutableMapping compatible sqlite
 based disk storage. Easy interface and very fast. Keys can be of basic sqlite storage
-classes defined by Python types `int`, `float`, `str`, `bytes`. Value is a `Namedtuple` class
-with same basic Python types.
+classes defined by Python types `int`, `float`, `str`, `bytes`. Values can be of basic sqlite
+type or with custom configuration.
 
 
 ## Features
@@ -51,10 +51,10 @@ Example:
 
 ```python
 
-from diskstore import DiskStore, Value
+from diskstore import DiskStore
 
-ds = DiskStore("/tmp/diskstore.db", value_class=Value)
-ds["key"] = Value("my value")
+ds = DiskStore("/tmp/diskstore.db")
+ds["key"] = "my value"
 print(ds["key"])
 
 ```

{diskstore-0.1.0 → diskstore-0.2.0}/README.md

@@ -5,8 +5,8 @@ in pure Python.
 
 Inspired by DiskCache library it implements a MutableMapping compatible sqlite
 based disk storage. Easy interface and very fast. Keys can be of basic sqlite storage
-classes defined by Python types `int`, `float`, `str`, `bytes`. Value is a `Namedtuple` class
-with same basic Python types.
+classes defined by Python types `int`, `float`, `str`, `bytes`. Values can be of basic sqlite
+type or with custom configuration.
 
 
 ## Features
@@ -41,10 +41,10 @@ Example:
 
 ```python
 
-from diskstore import DiskStore, Value
+from diskstore import DiskStore
 
-ds = DiskStore("/tmp/diskstore.db", value_class=Value)
-ds["key"] = Value("my value")
+ds = DiskStore("/tmp/diskstore.db")
+ds["key"] = "my value"
 print(ds["key"])
 
 ```

diskstore-0.2.0/pyproject.toml

@@ -0,0 +1,80 @@
+[project]
+name = "diskstore"
+version = "0.2.0"
+description = "Simple MutableMapping compatible SQLite based disk storage."
+readme = "README.md"
+requires-python = ">=3.10"
+authors = [{ name = "tds333", email = "tds333@mailbox.org" }]
+dependencies = [
+  "apsw>=3.51.3",
+]
+
+[dependency-groups]
+dev = [
+  "pytest",
+  "pytest-asyncio>=1.3.0",
+  "pytest-cov",
+  "pytest-examples>=0.0.18",
+  "pytest-xdist",
+]
+benchmark = [
+  "diskcache",
+  "ipython",
+  "pickleDB<1",
+  "pylibmc",
+  "redis",
+  "sqlitedict",
+]
+docs = [
+  "mkdocs>=1.6.1",
+  "mkdocstrings[python]>=1.0.3",
+  "zensical>=0.0.23",
+]
+ext-dev = [
+  "msgspec>=0.20.0",
+  "pydantic>=2.12.5",
+]
+
+[build-system]
+requires = ["uv_build>=0.9.2,<0.12.0"]
+build-backend = "uv_build"
+
+[tool.coverage.run]
+parallel = true
+branch = true
+source = ["diskstore"]
+
+[tool.coverage.report]
+show_missing = true
+skip_covered = true
+exclude_lines = ["no cov", "if __name__ == .__main__.:", "if TYPE_CHECKING:"]
+
+[tool.pytest]
+# pythonpath = ["src"]
+testpaths = ["tests"]
+
+[tool.ruff]
+exclude = [".env", ".venv", "venv"]
+line-length = 88
+
+[tool.ruff.lint]
+select = [
+  "E",  # pycodestyle errors
+  "W",  # pycodestyle warnings
+  "F",  # pyflakes
+  "I",  # isort
+  "C",  # flake8-comprehensions
+  "B",  # flake8-bugbear
+  "PT",  # flake8-pytest
+  "N",  # naming
+  # "D",  # docu
+  "PL",  # pylint
+  "RUF",  # ruff
+]
+ignore = [
+  # "E501", # line too long, handled by black
+  "B008",  # do not perform function calls in argument defaults
+  "C901",  # too complex
+  "RET504",
+  "PLR2004",
+]

diskstore-0.2.0/src/diskstore/__init__.py

@@ -0,0 +1,2 @@
+from .diskread import DiskRead  # noqa
+from .diskstore import DiskStore  # noqa

diskstore-0.2.0/src/diskstore/config.py

@@ -0,0 +1,190 @@
+"""DiskStore configuration classes and helpers for config."""
+
+import dataclasses
+import json
+from abc import abstractmethod
+from typing import Any, Iterable, Protocol
+
+from .const import TIMEOUT, AnyLite
+
+
+def get_sqlite_type(type_) -> str:
+    if type_ in {"BLOB", "TEXT", "INTEGER", "REAL"}:
+        return type_
+    sqlite_type = "BLOB"
+    if type_ is str:
+        sqlite_type = "TEXT"
+    elif type_ is int:
+        sqlite_type = "INTEGER"
+    elif type_ is float:
+        sqlite_type = "REAL"
+
+    return sqlite_type
+
+
+def escape_name(name: str) -> str:
+    tablename = '"' + name.replace('"', '""') + '"'
+    return tablename
+
+
+class ConfigProtocol(Protocol):
+    """Configuration Protocol
+
+    Attributes:
+        tablename: Table name as string
+        key_type: key type as string or basic Python type like str, int, float
+        timeout: Timeout used to wait if someone blocks connections or with writes
+        pragmas: Dictionary with PRAGMAs to set when connections is initialized
+        fields: Iterable used for select, update and create statement with
+            field name, type and default. `[("value", str), ("value2", int, 0)]`
+
+    """
+
+    tablename: str
+    key_type: str
+    timeout: float
+    pragmas: dict
+    fields: Iterable
+
+    @abstractmethod
+    def dump_value(self, value: Any) -> Iterable:
+        """Called with the value, should return an Iterable
+        which is used to write to the DB."""
+
+    @abstractmethod
+    def load_data(self, data: tuple) -> Any:
+        """load(db_data): called with the tuple selected from DB.
+        Should be converted to the type normally received as value."""
+
+
+class BaseConfig(ConfigProtocol):
+    """Base default configuratin."""
+
+    def __init__(self, *, tablename=None, key_type=None, timeout=None, pragmas=None):
+        self.tablename = "DiskStore" if tablename is None else tablename
+        self.key_type = "BLOB" if key_type is None else get_sqlite_type(key_type)
+        self.timeout = TIMEOUT if timeout is None else float(timeout)
+        self.pragmas = {} if pragmas is None else pragmas
+        self.fields = [("value", "BLOB")]
+
+    def dump_value(self, value: Any) -> Iterable:
+        return (value,)
+
+    def load_data(self, data: tuple) -> Any:
+        return data[0]
+
+
+class NamedTupleConfig(BaseConfig):
+    def __init__(
+        self, value_class, tablename=None, key_type=None, timeout=None, pragmas=None
+    ):
+        tablename = value_class.__name__ if not tablename else tablename
+        super().__init__(
+            tablename=tablename, key_type=key_type, timeout=timeout, pragmas=pragmas
+        )
+        self.value_class = value_class
+        self.fields = self.get_fields(value_class)
+
+    @staticmethod
+    def get_fields(value_class):
+        fields = []
+        value_columns = tuple(value_class._fields)
+        type_annotations = value_class.__annotations__
+        value_column_defaults: dict[str, AnyLite] = getattr(
+            value_class, "_field_defaults", {}
+        )
+        for name in value_columns:
+            type_cls = type_annotations.get(name, bytes)  # types are optional
+            sqlite_type = get_sqlite_type(type_cls)
+            default = value_column_defaults.get(name)
+            fields.append((name, sqlite_type, default))
+        if "_key" in value_columns:
+            raise ValueError(
+                f"Name _key is not allowed as attribute for {value_class},"
+                " listed as field name in _fields."
+            )
+
+        return tuple(fields)
+
+    def dump_value(self, value) -> Iterable:
+        return value
+
+    def load_data(self, data: tuple) -> Any:
+        return self.value_class._make(data)
+
+
+class JsonConfig(BaseConfig):
+    def __init__(self, tablename=None, key_type=None, timeout=None, pragmas=None):
+        super().__init__(
+            tablename=tablename, key_type=key_type, timeout=timeout, pragmas=pragmas
+        )
+        self.fields = (("value", "TEXT"),)
+
+    def dump_value(self, value: Any) -> Iterable:
+        return (json.dumps(value),)
+
+    def load_data(self, data: tuple) -> Any:
+        return json.loads(data[0])
+
+
+class DataclassConfig(BaseConfig):
+    def __init__(
+        self, dataclass, tablename=None, key_type=None, timeout=None, pragmas=None
+    ):
+        tablename = dataclass.__name__ if not tablename else tablename
+        super().__init__(
+            tablename=tablename, key_type=key_type, timeout=timeout, pragmas=pragmas
+        )
+        self.dataclass = dataclass
+        self.fields = self.get_fields(dataclass)
+        self.types = self.get_field_types(dataclass)
+
+    @staticmethod
+    def get_fields(dataclass):
+        if dataclasses.is_dataclass(dataclass):
+            value_columns = tuple(field.name for field in dataclasses.fields(dataclass))
+        else:
+            raise ValueError("It is not a dataclass.")
+        if "_key" in value_columns:
+            raise ValueError("Name _key is not allowed as attribute for dataclass.")
+
+        return value_columns
+
+    @classmethod
+    def get_field_types(cls, value_class):
+        value_columns = cls.get_fields(value_class)
+        if hasattr(value_class, "__annotations__") and value_class.__annotations__:
+            type_annotations = value_class.__annotations__
+            value_types = []
+            for c_name in value_columns:
+                type_cls = type_annotations.get(c_name, bytes)  # types are optional
+                sqlite_type = get_sqlite_type(type_cls)
+                value_types.append(sqlite_type)
+        else:
+            value_types = ["BLOB" for _ in value_columns]
+
+        return tuple(value_types)
+
+    def dump_value(self, value) -> tuple:
+        return dataclasses.astuple(value)
+
+    def load_data(self, data: tuple) -> Any:
+        return self.dataclass(*data)
+
+
+class PydanticConfig(BaseConfig):
+    def __init__(
+        self, model, tablename=None, key_type=None, timeout=None, pragmas=None
+    ):
+        tablename = model.__name__ if not tablename else tablename
+        super().__init__(
+            tablename=tablename, key_type=key_type, timeout=timeout, pragmas=pragmas
+        )
+        self.fields = (("value", "TEXT"),)
+        self.model = model
+
+    def dump_value(self, value):
+        return (value.model_dump_json(),)
+
+    def load_data(self, data):
+        return self.model.model_validate_json(data[0])

{diskstore-0.1.0 → diskstore-0.2.0}/src/diskstore/const.py

@@ -1,11 +1,12 @@
-"""Core disktore API."""
+"""Default constants used in diskstore."""
 
 from typing import (
     TypeAlias,
     Union,
 )
 
-TIMEOUT = 60.0  # in seconds
+TIMEOUT = 60.0
+"default timout in seconds"
 
 MISSING = object()
 
@@ -19,6 +20,7 @@ DEFAULT_PRAGMAS = {
     "temp_store": "memory",
     # "pargma_busy_timeout": 5000,  # milliseconds
 }
+"default pragma settings"
 
 
 AnyLite: TypeAlias = Union[bytes, str, int, float]

{diskstore-0.1.0 → diskstore-0.2.0}/src/diskstore/diskread.py

@@ -1,14 +1,24 @@
-"""DiskRead (Mapping) API."""
+"""DiskRead (Mapping) API.
+
+Examples:
+    >>> from diskstore import DiskRead
+    # DB must exist!
+    >>> ds = DiskRead("/tmp/data.db")
+    >>> ds["one"]
+    1
+
+"""
 
 import os
 import os.path
 import threading
 from collections.abc import ItemsView, KeysView, Mapping, ValuesView
 from contextlib import closing, contextmanager
-from typing import Generator, NamedTuple, Optional, Sequence, TypeAlias, Union
+from typing import Generator, Optional, Sequence, TypeAlias, Union
 
 import apsw
 
+from .config import BaseConfig, ConfigProtocol, escape_name
 from .const import MISSING, TIMEOUT, KeyType
 
 Connection = apsw.Connection
@@ -18,31 +28,6 @@ Cursor = apsw.Cursor
 BasicType: TypeAlias = Union[bytes, str, int, float]
 
 
-class Value(NamedTuple):
-    value: BasicType
-
-    def __float__(self):
-        data = self[0]
-        if isinstance(data, float):
-            return data
-        raise ValueError("Internal value is not an instance of float.")
-
-    def __int__(self):
-        data = self[0]
-        if isinstance(data, int):
-            return data
-        raise ValueError("Internal value is not an instance of int.")
-
-    def __bytes__(self):
-        data = self[0]
-        if isinstance(data, bytes):
-            return data
-        raise ValueError("Internal value is not an instance of bytes.")
-
-    def __str__(self):
-        return str(self[0])
-
-
 class DiskKeysView(KeysView):
     __slots__ = ()
 
@@ -76,44 +61,35 @@ class DiskItemsView(ItemsView):
 
 
 class DiskRead(Mapping):
-    """SQLite read only disk storage. Does not connect on init!"""
-
-    def __init__(  # noqa: PLR0915
+    def __init__(
         self,
         filename: os.PathLike | str,
-        value_class=None,
-        tablename: str | None = None,
-        timeout=None,
+        config: ConfigProtocol | None = None,
     ) -> None:
-        """Initialize cache instance.
-
-        :param str filename: filename for DB to use.
-        :param str tablename: Optional table name to use.
-        :param type value_class: Class type (inherited from NamedTuple)
-                                 used to get back data.
-        :param float timeout: SQLite connection timeout
-                              (also used if DB is busy to retry)
+        """SQLite read only disk storage.
+
+        Database is opened read only on demand.
+
+        Args:
+           filename: filename for DB to use.
+           config: Configuration
+
         """
         filename = os.path.expanduser(filename)
         filename = os.path.expandvars(filename)
-        if value_class is None:
-            self._value_class = Value
-        else:
-            self._value_class = value_class
-        self._tablename = (
-            self._value_class.__name__
-            if tablename is None or tablename == "_Settings"
-            else tablename
-        )
-
         self._filename = os.fspath(filename)
-        self._timeout: float = TIMEOUT if timeout is None else float(timeout)
+        self._config: ConfigProtocol = BaseConfig() if config is None else config
+        self._timeout: float = (
+            TIMEOUT
+            if (self._config.timeout is None or self._config.timeout < 0.0)
+            else float(self._config.timeout)
+        )
+        self._load_data = self._config.load_data
         self._local = threading.local()
 
         # precreated statements based on tablename and value_class
-        tablename = self._escape_table_name(self._tablename)
-        value_columns = self.get_fields(self._value_class)
-        fields = ", ".join(f"{field}" for field in value_columns)
+        tablename = escape_name(self._config.tablename)
+        fields = ", ".join(f"{escape_name(field)}" for field, *_ in self._config.fields)
         self._statements: dict[str, str] = {
             "GET": f"SELECT {fields} FROM {tablename} WHERE _key = ? LIMIT 1",
             "CONTAINS": f"SELECT _key FROM {tablename} WHERE _key = ? LIMIT 1",
@@ -123,22 +99,6 @@ class DiskRead(Mapping):
             "QUERY": f"SELECT _key, {fields} FROM {tablename}",
         }
 
-    @staticmethod
-    def get_fields(value_class):
-        value_columns = tuple(value_class._fields)
-        if "_key" in value_columns:
-            raise ValueError(
-                f"Name _key is not allowed as attribute for {value_class},"
-                " listed as field name in _fields."
-            )
-
-        return value_columns
-
-    @staticmethod
-    def _escape_table_name(name: str) -> str:
-        tablename = '"' + name.replace('"', '""') + '"'
-        return tablename
-
     @property
     def filename(self) -> str:
         """DiskStore filename for DB."""
@@ -151,7 +111,8 @@ class DiskRead(Mapping):
 
     @property
     def tablename(self) -> str:
-        return self._tablename
+        """Tablename used to get data from."""
+        return self._config.tablename
 
     @property
     def _con(self) -> Connection:
@@ -191,7 +152,7 @@ class DiskRead(Mapping):
         if row is MISSING:
             raise KeyError(key)
 
-        return self._value_class(*row)  # ty:ignore[not-iterable]
+        return self._load_data(row)  # ty:ignore[invalid-argument-type]
 
     def keys(self):
         return DiskKeysView(self)
@@ -207,16 +168,20 @@ class DiskRead(Mapping):
         where: Optional[str] = None,
         parameters: Optional[Sequence] = None,
         order: Optional[str] = None,
+        limit: int | None = None,
+        offset: int | None = None,
     ) -> Generator[tuple, None, None]:
         where = " WHERE " + where if where else ""
         parameters = () if parameters is None else parameters
         order = " ORDER BY " + order if order else ""
-        select = self._statements["QUERY"] + where + order
+        limit = " LIMIT " + str(limit) if limit is not None else ""
+        offset = " OFFSET " + str(offset) if offset is not None else ""
+        select = self._statements["QUERY"] + where + order + limit + offset
 
         with self._cursor() as cursor:
             cursor.execute(select, parameters)
             for row in cursor:
-                yield row[0], self._value_class(*row[1:])
+                yield row[0], self._load_data(row[1:])
 
     def __contains__(self, key: object) -> bool:
         with self._cursor() as cx:
@@ -236,6 +201,10 @@ class DiskRead(Mapping):
             for row in cx:
                 yield row[0]
 
+    def open(self):
+        connection = self._con  # noqa
+        return self
+
     def close(self) -> None:
         con = getattr(self._local, "con", None)
         if con is None:
@@ -263,9 +232,7 @@ class DiskRead(Mapping):
     def __getstate__(self):
         return {
             "filename": self.filename,
-            "timeout": self.timeout,
-            "tablename": self._tablename,
-            "value_class": self._value_class,
+            "config": self._config,
         }
 
     def __setstate__(self, state) -> None:

{diskstore-0.1.0 → diskstore-0.2.0}/src/diskstore/diskstore.py

@@ -1,16 +1,25 @@
-"""DiskStore (MutableMapping) API."""
+"""DiskStore (MutableMapping) API.
+
+Examples:
+    >>> from diskstore import DiskStore
+    >>> ds = DiskStore("/tmp/data.db")
+    >>> ds["one"] = 1
+    >>> ds["one"]
+    1
+
+"""
 
 import os
 import os.path
 import threading
 from collections.abc import Mapping, MutableMapping
 from contextlib import closing, contextmanager
-from time import sleep, time
-from typing import Iterable
+from typing import Any, Iterable
 
 import apsw
 
-from .const import DEFAULT_PRAGMAS, MISSING, TIMEOUT, KeyType
+from .config import ConfigProtocol, escape_name, get_sqlite_type
+from .const import DEFAULT_PRAGMAS, MISSING, KeyType
 from .diskread import DiskRead
 
 Connection = apsw.Connection
@@ -20,55 +29,28 @@ BusyError = apsw.BusyError
 
 
 class DiskStore(DiskRead, MutableMapping):
-    """SQLite disk storage."""
-
-    def __init__(  # noqa: PLR0915
-        self,
-        filename: os.PathLike | str,
-        value_class=None,
-        tablename: str | None = None,
-        key_type=None,
-        timeout: float | None = None,
-        alter_table: bool = True,
-        **pragmas,
+    def __init__(
+        self, filename: os.PathLike | str, config: ConfigProtocol | None = None
     ) -> None:
-        """Initialize Diskstore instance.
-
-        :param str filename: DiskStore DB filename.
-                              If not set a tmp directory and default DB name is created.
-        :param str tablename: Optional table name to use.
-                              If not set the name from value_class is used.
-        :param type value_class: Class type (inherited from NamedTuple)
-                                 used to get back data. Default is Value class.
-        :param float timeout: SQLite connection timeout
-                              (also used if DB is busy to retry)
-        :param pragmas: Any of DEFAULT_PRAGMAS or additional PRAGMA's to apply
+        """SQLite based MutableMapping disk storage.
+
+        Args:
+            filename: DiskStore DB filename.
+            config: Configuration as specified in ConfigProtocol
         """
-        super().__init__(
-            filename=filename,
-            value_class=value_class,
-            tablename=tablename,
-            timeout=timeout,
-        )
+        super().__init__(filename=filename, config=config)
         self._txn_id = None
-        tablename = self._escape_table_name(self._tablename)
-
-        primary_key_type = "BLOB"
-        if key_type is str:
-            primary_key_type = "TEXT"
-        elif key_type is int:
-            primary_key_type = "INTEGER"
-        elif key_type is float:
-            primary_key_type = "REAL"
-        value_columns = self.get_fields(self._value_class)
-        column_types = self.get_field_types(self._value_class)
+        tablename = escape_name(self._config.tablename)
+
+        primary_key_type = get_sqlite_type(self._config.key_type)
+        value_columns = self._config.fields
         fields_create = ", ".join(
-            f"{field} {ctype} NOT NULL"
-            for field, ctype in zip(value_columns, column_types, strict=True)
+            self._get_field_create(field) for field in self._config.fields
         )
-        fields = ", ".join(f"{field}" for field in value_columns)
+        fields = ", ".join(f"{escape_name(field)}" for field, *_ in self._config.fields)
         excluded_fields = ", ".join(
-            f"{field} = excluded.{field}" for field in value_columns
+            f"{field} = excluded.{escape_name(field)}"
+            for field, *_ in self._config.fields
         )
         qms = ", ".join("?" for field in value_columns)
         self._statements.update(
@@ -95,43 +77,8 @@ class DiskStore(DiskRead, MutableMapping):
             }
         )
         self._pragmas = DEFAULT_PRAGMAS.copy()
-        self._pragmas.update(pragmas)
-        sql = self._con.execute
-        # Setup table.
-        with closing(sql(self._statements["CREATE"])):
-            pass
-        # try alter table if needed
-        if alter_table:
-            self._alter_table(sql)
-
-    @staticmethod
-    def get_field_types(value_class):
-        value_columns = tuple(value_class._fields)
-        if hasattr(value_class, "__annotations__") and value_class.__annotations__:
-            type_annotations = value_class.__annotations__
-            value_types = []
-            for c_name in value_columns:
-                type_cls = type_annotations.get(c_name, bytes)  # types are optional
-                if type_cls is str:
-                    value_types.append("TEXT")
-                elif type_cls is int:
-                    value_types.append("INTEGER")
-                elif type_cls is float:
-                    value_types.append("REAL")
-                else:
-                    value_types.append("BLOB")
-        else:
-            value_types = ["BLOB" for _ in value_columns]
-
-        return value_types
-
-    def _validate_value(self, value):
-        if isinstance(value, self._value_class):
-            return value
-        value_type = type(value)
-        raise ValueError(
-            f"Value must be instance of {self._value_class} but is {value_type}."
-        )
+        self._pragmas.update(self._config.pragmas)
+        self._dump_value = self._config.dump_value
 
     @property
     def _con(self) -> Connection:
@@ -148,57 +95,62 @@ class DiskStore(DiskRead, MutableMapping):
         con = getattr(self._local, "con", None)
 
         if con is None:
-            con = self._local.con = Connection(self._filename)
+            con = Connection(self._filename)
             con.set_busy_timeout(int(self._timeout * 1000))
 
             # Some SQLite pragmas work on a per-connection basis so
             # apply them all on fresh connection
             for key, value in self._pragmas.items():
                 con.pragma(key, value)
+            con.execute(self._statements["CREATE"])
+            self._local.con = con
 
         return con
 
-    def _alter_table(self, sql):
-        value_columns = self.get_fields(self._value_class)
-        column_types = self.get_field_types(self._value_class)
-        tablename = self._escape_table_name(self._tablename)
-        value_column_defaults = getattr(self._value_class, "_field_defaults", {})
+    @staticmethod
+    def _get_field_create(field_tuple):
+        field_name, field_type, *field_default = field_tuple
+        default = ""
+        if field_default and field_default[0] is not None:
+            default = " DEFAULT " + apsw.format_sql_value(field_default[0])
+        name = escape_name(field_name)
+        type_ = get_sqlite_type(field_type)
+        field_create = f"{name} {type_} NOT NULL{default}"
+
+        return field_create
+
+    def _migrate_table(self, new_fields=()):
+        tablename = escape_name(self._config.tablename)
         existing_fileds = set()
         table_info_stmt = f"PRAGMA table_info({tablename});"
+        sql = self._con.execute
         with closing(sql(table_info_stmt)) as cursor:
             for row in cursor:
                 existing_fileds.add(row[1])
-        for columnname, columntype in zip(value_columns, column_types, strict=True):
+        for columnname, columntype, default_value in new_fields:
             if columnname not in existing_fileds:
-                default = apsw.format_sql_value(value_column_defaults[columnname])
+                name: str = escape_name(columnname)
+                type_ = get_sqlite_type(columntype)
+                default = apsw.format_sql_value(default_value)
                 alter_stmt = (
-                    f"ALTER TABLE {tablename} ADD COLUMN {columnname}"
-                    f" {columntype} NOT NULL DEFAULT {default};"
+                    f"ALTER TABLE {tablename} ADD COLUMN {name}"
+                    f" {type_} NOT NULL DEFAULT {default};"
                 )
                 with closing(sql(alter_stmt)) as cursor:
                     pass
 
     @contextmanager
-    def transact(self, retry=False):
+    def transact(self):
         cursor: Cursor = self._con.cursor()
         tid = threading.get_ident()
         txn_id = self._txn_id
-        retry_until = time() + TIMEOUT
 
         if tid == txn_id:  # already inside a thread with a transaction
             begin = False
         else:
-            while True:
-                try:
-                    cursor.execute("BEGIN IMMEDIATE")
-                    begin = True
-                    self._txn_id = tid
-                    break
-                except BusyError:
-                    if retry and time() < retry_until:
-                        sleep(0.001)
-                        continue
-                    raise
+            cursor.execute("BEGIN IMMEDIATE")
+            begin = True
+            self._txn_id = tid
 
         try:
             yield cursor
@@ -216,38 +168,38 @@ class DiskStore(DiskRead, MutableMapping):
                 cursor.execute("COMMIT")
             cursor.close()
 
-    def __setitem__(self, key: KeyType, value: Iterable) -> None:
-        value = self._validate_value(value)
+    def __setitem__(self, key: KeyType, value: Any) -> None:
         with self._cursor() as cursor:
-            cursor.execute(self._statements["SET"], (key, *value))
+            cursor.execute(self._statements["SET"], (key, *self._dump_value(value)))
 
     def add(self, key: KeyType | None, value: Iterable) -> KeyType | None:
-        value = self._validate_value(value)
         with self._cursor() as cx:
-            rows = cx.execute(self._statements["ADD"], (key, *value)).fetchall()
+            rows = cx.execute(
+                self._statements["ADD"], (key, *self._dump_value(value))
+            ).fetchall()
 
         if not rows:
             return None
 
         return rows[0][0]
 
-    def pop(self, key: KeyType, default=MISSING):  # noqa: E501
-        with self.transact(retry=True) as cx:
+    def pop(self, key: KeyType, default=MISSING):
+        with self.transact() as cx:
             value = next(cx.execute(self._statements["GET"], (key,)), None)
             if value is None:
                 if default is MISSING:
                     raise KeyError(key)
                 return default
             cx.execute(self._statements["DELETE"], (key,))
-            return self._value_class(*value)
+            return self._load_data(value)
 
     def popitem(self):
-        with self.transact(retry=True) as cx:
+        with self.transact() as cx:
             row = next(cx.execute(self._statements["POPITEM"]), None)
             if not row:
                 raise KeyError()
             key = row[0]
-            value = self._value_class(*row[1:])
+            value = self._load_data(row[1:])
             cx.execute(self._statements["DELETE"], (key,))
             return key, value
 
@@ -262,7 +214,6 @@ class DiskStore(DiskRead, MutableMapping):
             return self[key]
         except KeyError:
             if default is not None:
-                default = self._validate_value(default)
                 self.add(key, default)
 
         return default
@@ -289,27 +240,25 @@ class DiskStore(DiskRead, MutableMapping):
         with closing(self._con.execute(self._statements["CLEAR"])):
             pass
 
-    def update(self, other=(), /, **kwargs):  # type: ignore
+    def update(self, other=(), /, **kwargs):
         comps = []
         if other:
             if isinstance(other, Mapping):
-                comp = ((key, *value) for key, value in other.items())
+                comp = ((key, *self._dump_value(value)) for key, value in other.items())
                 comps.append(comp)
             elif hasattr(other, "keys"):
-                comp = ((key, *other[key]) for key in other.keys())
+                comp = ((key, *self._dump_value(other[key])) for key in other.keys())
                 comps.append(comp)
             else:
-                comp = ((key, *value) for key, value in other)
+                comp = ((key, *self._dump_value(value)) for key, value in other)
                 comps.append(comp)
         if kwargs:
-            comp = ((key, *value) for key, value in kwargs.items())
+            comp = ((key, *self._dump_value(value)) for key, value in kwargs.items())
             comps.append(comp)
 
-        with self.transact(retry=True) as cursor:
+        with self.transact() as cursor:
             for comp in comps:
                 cursor.executemany(self._statements["SET"], comp)
 
     def get_readonly_instance(self):
-        return DiskRead(
-            self._filename, self._value_class, self._tablename, self._timeout
-        )
+        return DiskRead(self._filename, self._config)

diskstore-0.1.0/pyproject.toml

@@ -1,67 +0,0 @@
-[project]
-name = "diskstore"
-version = "0.1.0"
-description = "Simple MutableMapping compatible SQLite based disk storage."
-readme = "README.md"
-authors = [{ name = "tds333", email = "tds333@mailbox.org" }]
-requires-python = ">=3.10"
-dependencies = ["apsw"]
-
-
-[build-system]
-requires = ["uv_build>=0.9.2,<0.11.0"]
-build-backend = "uv_build"
-
-
-[tool.coverage.run]
-parallel = true
-branch = true
-source = ["diskstore"]
-
-
-[tool.coverage.report]
-show_missing = true
-skip_covered = true
-exclude_lines = ["no cov", "if __name__ == .__main__.:", "if TYPE_CHECKING:"]
-
-
-[tool.pytest.ini_options]
-#pythonpath = ["src"]
-testpaths = ["tests"]
-
-[tool.ruff]
-exclude = [".env", ".venv", "venv"]
-line-length = 88
-[tool.ruff.lint]
-select = [
-    "E",  # pycodestyle errors
-    "W",  # pycodestyle warnings
-    "F",  # pyflakes
-    "I",  # isort
-    "C",  # flake8-comprehensions
-    "B",  # flake8-bugbear
-    "PT", # flake8-pytest
-    "N",  # naming
-    #"D",  # docu
-    "PL", # pylint
-    "R",  # ruff
-
-]
-ignore = [
-    #"E501", # line too long, handled by black
-    "B008",   # do not perform function calls in argument defaults
-    "C901",   # too complex
-    "RET504",
-]
-
-[dependency-groups]
-dev = ["pytest", "pytest-cov", "pytest-env", "pytest-xdist"]
-
-benchmark = [
-    "ipython",
-    "pickleDB<1",
-    "pylibmc",
-    "sqlitedict",
-    "redis",
-    "diskcache",
-]

diskstore-0.1.0/src/diskstore/__init__.py

@@ -1,4 +0,0 @@
-from .const import DEFAULT_PRAGMAS, TIMEOUT, AnyLite, KeyType  # noqa
-from .diskread import DiskRead, Value  # noqa
-from .diskstore import DiskStore  # noqa
-from .diskcache import DiskCache  # noqa

diskstore-0.1.0/src/diskstore/diskcache.py

@@ -1,136 +0,0 @@
-"""Core disktore API."""
-
-import os
-import os.path
-import pickle
-from time import time
-from typing import (
-    NamedTuple,
-)
-
-from .const import MISSING, TIMEOUT, KeyType
-from .diskstore import DiskStore
-
-
-class CacheValue(NamedTuple):
-    value: bytes
-    expires_at: float
-
-    def load(self, expire_timestamp, default=MISSING):
-        expires_at = self[1]
-        if expires_at == 0.0 or expires_at > time():
-            return pickle.loads(self[0])
-        return default
-
-    @classmethod
-    def create(cls, value, expires=None):
-        expires_at = 0.0
-        if expires:
-            expires_at = time() + expires
-        return cls(value=pickle.dumps(value), expires_at=expires_at)
-
-
-class DiskCache:
-    def __init__(
-        self,
-        filename: os.PathLike | str,
-        default_expiration=60.0,
-    ) -> None:
-        """Initialize DiskCache instance.
-
-        :param str filename: DiskStore DB filename.
-                              If not set a tmp directory and default DB name is created.
-        :param float default_expiration: Default expiration value in seconds (float)
-        """
-        self._diskstore = DiskStore(
-            filename=filename,
-            value_class=CacheValue,
-            tablename="Cache",
-            timeout=TIMEOUT,
-            pragma_auto_vacuum=1,
-            pragma_synchronous=0,
-            alter_table=False,
-        )
-        self._default_expiration = default_expiration
-
-    def delete(self, key: KeyType) -> bool:
-        bin_key = pickle.dumps(key)
-        try:
-            del self._diskstore[bin_key]
-            return True
-        except KeyError:
-            return False
-
-    def set(self, key, value, expire=None):
-        bin_key = pickle.dumps(key)
-        if expire is None:
-            expire = self._default_expiration
-        v = CacheValue.create(value, expire)
-        self._diskstore[bin_key] = v
-
-    def get(
-        self,
-        key: KeyType,
-        default=None,
-    ):
-        bin_key = pickle.dumps(key)
-        try:
-            result = self._diskstore[bin_key]
-        except KeyError:
-            return default
-
-        return result.load(expire_timestamp=time(), default=default)
-
-    def add(self, key, value, expire=None):
-        bin_key = pickle.dumps(key)
-        v = CacheValue.create(value, expire)
-        ret_key = self._diskstore.add(bin_key, v)
-        if ret_key is None:
-            return False
-
-        return True
-
-    def delete_expired(self):
-        delete_expired = "DELETE FROM Cache WHERE expires_at != 0.0 AND expires_at < ?"
-        with self._diskstore.transact() as cx:
-            cx.execute(delete_expired, (time(),))
-
-    def touch(self, key, expire=None):
-        bin_key = pickle.dumps(key)
-        if expire is None:
-            expire = self._default_expiration
-        expires_at = time() + expire if expire else 0.0
-        stmt = "UPDATE Cache SET expires_at = ? WHERE _key = ?"
-        with self._diskstore.transact() as cx:
-            cx.execute(stmt, (expires_at, bin_key))
-
-    @property
-    def filename(self):
-        return self._diskstore.filename
-
-    def check(self, *args, **kwargs):
-        return self._diskstore.check(*args, **kwargs)
-
-    def clear(self):
-        self._diskstore.clear()
-
-    def close(self):
-        self._diskstore.close()
-
-    def __enter__(self):
-        self._diskstore.__enter__()
-        return self
-
-    def __exit__(self, *args, **kwargs) -> None:
-        self._diskstore.__exit__(*args, **kwargs)
-
-    def __len__(self):
-        return len(self._diskstore)
-
-    def __iter__(self):
-        for key in self._diskstore:
-            yield pickle.loads(key)
-
-    def __reversed__(self):
-        for key in reversed(self._diskstore):
-            yield pickle.loads(key)