diskstore 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- diskstore/__init__.py +4 -0
- diskstore/const.py +25 -0
- diskstore/diskcache.py +136 -0
- diskstore/diskread.py +272 -0
- diskstore/diskstore.py +315 -0
- diskstore-0.1.0.dist-info/METADATA +100 -0
- diskstore-0.1.0.dist-info/RECORD +8 -0
- diskstore-0.1.0.dist-info/WHEEL +4 -0
diskstore/__init__.py
ADDED
diskstore/const.py
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
"""Core disktore API."""
|
|
2
|
+
|
|
3
|
+
from typing import (
|
|
4
|
+
TypeAlias,
|
|
5
|
+
Union,
|
|
6
|
+
)
|
|
7
|
+
|
|
8
|
+
TIMEOUT = 60.0 # in seconds
|
|
9
|
+
|
|
10
|
+
MISSING = object()
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
DEFAULT_PRAGMAS = {
|
|
14
|
+
"auto_vacuum": 0, # 1=FULL, 0=None
|
|
15
|
+
"cache_size": 2**13, # 8,192 pages
|
|
16
|
+
"journal_mode": "wal",
|
|
17
|
+
"mmap_size": 2**28, # 256 MB
|
|
18
|
+
"synchronous": 1, # 0=OFF, 1=NORMAL, 2=FULL, 3=EXTRA
|
|
19
|
+
"temp_store": "memory",
|
|
20
|
+
# "pargma_busy_timeout": 5000, # milliseconds
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
AnyLite: TypeAlias = Union[bytes, str, int, float]
|
|
25
|
+
KeyType = AnyLite
|
diskstore/diskcache.py
ADDED
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
"""Core disktore API."""
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
import os.path
|
|
5
|
+
import pickle
|
|
6
|
+
from time import time
|
|
7
|
+
from typing import (
|
|
8
|
+
NamedTuple,
|
|
9
|
+
)
|
|
10
|
+
|
|
11
|
+
from .const import MISSING, TIMEOUT, KeyType
|
|
12
|
+
from .diskstore import DiskStore
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class CacheValue(NamedTuple):
|
|
16
|
+
value: bytes
|
|
17
|
+
expires_at: float
|
|
18
|
+
|
|
19
|
+
def load(self, expire_timestamp, default=MISSING):
|
|
20
|
+
expires_at = self[1]
|
|
21
|
+
if expires_at == 0.0 or expires_at > time():
|
|
22
|
+
return pickle.loads(self[0])
|
|
23
|
+
return default
|
|
24
|
+
|
|
25
|
+
@classmethod
|
|
26
|
+
def create(cls, value, expires=None):
|
|
27
|
+
expires_at = 0.0
|
|
28
|
+
if expires:
|
|
29
|
+
expires_at = time() + expires
|
|
30
|
+
return cls(value=pickle.dumps(value), expires_at=expires_at)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class DiskCache:
|
|
34
|
+
def __init__(
|
|
35
|
+
self,
|
|
36
|
+
filename: os.PathLike | str,
|
|
37
|
+
default_expiration=60.0,
|
|
38
|
+
) -> None:
|
|
39
|
+
"""Initialize DiskCache instance.
|
|
40
|
+
|
|
41
|
+
:param str filename: DiskStore DB filename.
|
|
42
|
+
If not set a tmp directory and default DB name is created.
|
|
43
|
+
:param float default_expiration: Default expiration value in seconds (float)
|
|
44
|
+
"""
|
|
45
|
+
self._diskstore = DiskStore(
|
|
46
|
+
filename=filename,
|
|
47
|
+
value_class=CacheValue,
|
|
48
|
+
tablename="Cache",
|
|
49
|
+
timeout=TIMEOUT,
|
|
50
|
+
pragma_auto_vacuum=1,
|
|
51
|
+
pragma_synchronous=0,
|
|
52
|
+
alter_table=False,
|
|
53
|
+
)
|
|
54
|
+
self._default_expiration = default_expiration
|
|
55
|
+
|
|
56
|
+
def delete(self, key: KeyType) -> bool:
|
|
57
|
+
bin_key = pickle.dumps(key)
|
|
58
|
+
try:
|
|
59
|
+
del self._diskstore[bin_key]
|
|
60
|
+
return True
|
|
61
|
+
except KeyError:
|
|
62
|
+
return False
|
|
63
|
+
|
|
64
|
+
def set(self, key, value, expire=None):
|
|
65
|
+
bin_key = pickle.dumps(key)
|
|
66
|
+
if expire is None:
|
|
67
|
+
expire = self._default_expiration
|
|
68
|
+
v = CacheValue.create(value, expire)
|
|
69
|
+
self._diskstore[bin_key] = v
|
|
70
|
+
|
|
71
|
+
def get(
|
|
72
|
+
self,
|
|
73
|
+
key: KeyType,
|
|
74
|
+
default=None,
|
|
75
|
+
):
|
|
76
|
+
bin_key = pickle.dumps(key)
|
|
77
|
+
try:
|
|
78
|
+
result = self._diskstore[bin_key]
|
|
79
|
+
except KeyError:
|
|
80
|
+
return default
|
|
81
|
+
|
|
82
|
+
return result.load(expire_timestamp=time(), default=default)
|
|
83
|
+
|
|
84
|
+
def add(self, key, value, expire=None):
|
|
85
|
+
bin_key = pickle.dumps(key)
|
|
86
|
+
v = CacheValue.create(value, expire)
|
|
87
|
+
ret_key = self._diskstore.add(bin_key, v)
|
|
88
|
+
if ret_key is None:
|
|
89
|
+
return False
|
|
90
|
+
|
|
91
|
+
return True
|
|
92
|
+
|
|
93
|
+
def delete_expired(self):
|
|
94
|
+
delete_expired = "DELETE FROM Cache WHERE expires_at != 0.0 AND expires_at < ?"
|
|
95
|
+
with self._diskstore.transact() as cx:
|
|
96
|
+
cx.execute(delete_expired, (time(),))
|
|
97
|
+
|
|
98
|
+
def touch(self, key, expire=None):
|
|
99
|
+
bin_key = pickle.dumps(key)
|
|
100
|
+
if expire is None:
|
|
101
|
+
expire = self._default_expiration
|
|
102
|
+
expires_at = time() + expire if expire else 0.0
|
|
103
|
+
stmt = "UPDATE Cache SET expires_at = ? WHERE _key = ?"
|
|
104
|
+
with self._diskstore.transact() as cx:
|
|
105
|
+
cx.execute(stmt, (expires_at, bin_key))
|
|
106
|
+
|
|
107
|
+
@property
|
|
108
|
+
def filename(self):
|
|
109
|
+
return self._diskstore.filename
|
|
110
|
+
|
|
111
|
+
def check(self, *args, **kwargs):
|
|
112
|
+
return self._diskstore.check(*args, **kwargs)
|
|
113
|
+
|
|
114
|
+
def clear(self):
|
|
115
|
+
self._diskstore.clear()
|
|
116
|
+
|
|
117
|
+
def close(self):
|
|
118
|
+
self._diskstore.close()
|
|
119
|
+
|
|
120
|
+
def __enter__(self):
|
|
121
|
+
self._diskstore.__enter__()
|
|
122
|
+
return self
|
|
123
|
+
|
|
124
|
+
def __exit__(self, *args, **kwargs) -> None:
|
|
125
|
+
self._diskstore.__exit__(*args, **kwargs)
|
|
126
|
+
|
|
127
|
+
def __len__(self):
|
|
128
|
+
return len(self._diskstore)
|
|
129
|
+
|
|
130
|
+
def __iter__(self):
|
|
131
|
+
for key in self._diskstore:
|
|
132
|
+
yield pickle.loads(key)
|
|
133
|
+
|
|
134
|
+
def __reversed__(self):
|
|
135
|
+
for key in reversed(self._diskstore):
|
|
136
|
+
yield pickle.loads(key)
|
diskstore/diskread.py
ADDED
|
@@ -0,0 +1,272 @@
|
|
|
1
|
+
"""DiskRead (Mapping) API."""
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
import os.path
|
|
5
|
+
import threading
|
|
6
|
+
from collections.abc import ItemsView, KeysView, Mapping, ValuesView
|
|
7
|
+
from contextlib import closing, contextmanager
|
|
8
|
+
from typing import Generator, NamedTuple, Optional, Sequence, TypeAlias, Union
|
|
9
|
+
|
|
10
|
+
import apsw
|
|
11
|
+
|
|
12
|
+
from .const import MISSING, TIMEOUT, KeyType
|
|
13
|
+
|
|
14
|
+
Connection = apsw.Connection
|
|
15
|
+
Cursor = apsw.Cursor
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
BasicType: TypeAlias = Union[bytes, str, int, float]
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class Value(NamedTuple):
|
|
22
|
+
value: BasicType
|
|
23
|
+
|
|
24
|
+
def __float__(self):
|
|
25
|
+
data = self[0]
|
|
26
|
+
if isinstance(data, float):
|
|
27
|
+
return data
|
|
28
|
+
raise ValueError("Internal value is not an instance of float.")
|
|
29
|
+
|
|
30
|
+
def __int__(self):
|
|
31
|
+
data = self[0]
|
|
32
|
+
if isinstance(data, int):
|
|
33
|
+
return data
|
|
34
|
+
raise ValueError("Internal value is not an instance of int.")
|
|
35
|
+
|
|
36
|
+
def __bytes__(self):
|
|
37
|
+
data = self[0]
|
|
38
|
+
if isinstance(data, bytes):
|
|
39
|
+
return data
|
|
40
|
+
raise ValueError("Internal value is not an instance of bytes.")
|
|
41
|
+
|
|
42
|
+
def __str__(self):
|
|
43
|
+
return str(self[0])
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
class DiskKeysView(KeysView):
|
|
47
|
+
__slots__ = ()
|
|
48
|
+
|
|
49
|
+
def __iter__(self):
|
|
50
|
+
return iter(self._mapping) # ty:ignore[unresolved-attribute]
|
|
51
|
+
|
|
52
|
+
def __reversed__(self):
|
|
53
|
+
return reversed(self._mapping) # ty:ignore[unresolved-attribute]
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
class DiskValuesView(ValuesView):
|
|
57
|
+
__slots__ = ()
|
|
58
|
+
|
|
59
|
+
def __iter__(self):
|
|
60
|
+
for _, value in self._mapping.query(order="rowid ASC"): # ty:ignore[unresolved-attribute]
|
|
61
|
+
yield value
|
|
62
|
+
|
|
63
|
+
def __reversed__(self):
|
|
64
|
+
for _, value in self._mapping.query(order="rowid DESC"): # ty:ignore[unresolved-attribute]
|
|
65
|
+
yield value
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
class DiskItemsView(ItemsView):
|
|
69
|
+
__slots__ = ()
|
|
70
|
+
|
|
71
|
+
def __iter__(self):
|
|
72
|
+
return self._mapping.query(order="rowid ASC") # ty:ignore[unresolved-attribute]
|
|
73
|
+
|
|
74
|
+
def __reversed__(self):
|
|
75
|
+
return self._mapping.query(order="rowid DESC") # ty:ignore[unresolved-attribute]
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
class DiskRead(Mapping):
|
|
79
|
+
"""SQLite read only disk storage. Does not connect on init!"""
|
|
80
|
+
|
|
81
|
+
def __init__( # noqa: PLR0915
|
|
82
|
+
self,
|
|
83
|
+
filename: os.PathLike | str,
|
|
84
|
+
value_class=None,
|
|
85
|
+
tablename: str | None = None,
|
|
86
|
+
timeout=None,
|
|
87
|
+
) -> None:
|
|
88
|
+
"""Initialize cache instance.
|
|
89
|
+
|
|
90
|
+
:param str filename: filename for DB to use.
|
|
91
|
+
:param str tablename: Optional table name to use.
|
|
92
|
+
:param type value_class: Class type (inherited from NamedTuple)
|
|
93
|
+
used to get back data.
|
|
94
|
+
:param float timeout: SQLite connection timeout
|
|
95
|
+
(also used if DB is busy to retry)
|
|
96
|
+
"""
|
|
97
|
+
filename = os.path.expanduser(filename)
|
|
98
|
+
filename = os.path.expandvars(filename)
|
|
99
|
+
if value_class is None:
|
|
100
|
+
self._value_class = Value
|
|
101
|
+
else:
|
|
102
|
+
self._value_class = value_class
|
|
103
|
+
self._tablename = (
|
|
104
|
+
self._value_class.__name__
|
|
105
|
+
if tablename is None or tablename == "_Settings"
|
|
106
|
+
else tablename
|
|
107
|
+
)
|
|
108
|
+
|
|
109
|
+
self._filename = os.fspath(filename)
|
|
110
|
+
self._timeout: float = TIMEOUT if timeout is None else float(timeout)
|
|
111
|
+
self._local = threading.local()
|
|
112
|
+
|
|
113
|
+
# precreated statements based on tablename and value_class
|
|
114
|
+
tablename = self._escape_table_name(self._tablename)
|
|
115
|
+
value_columns = self.get_fields(self._value_class)
|
|
116
|
+
fields = ", ".join(f"{field}" for field in value_columns)
|
|
117
|
+
self._statements: dict[str, str] = {
|
|
118
|
+
"GET": f"SELECT {fields} FROM {tablename} WHERE _key = ? LIMIT 1",
|
|
119
|
+
"CONTAINS": f"SELECT _key FROM {tablename} WHERE _key = ? LIMIT 1",
|
|
120
|
+
"ITER": f"SELECT _key FROM {tablename} ORDER BY rowid ASC",
|
|
121
|
+
"REVERSED": f"SELECT _key FROM {tablename} ORDER BY rowid DESC",
|
|
122
|
+
"COUNT": f"SELECT COUNT (_key) FROM {tablename}",
|
|
123
|
+
"QUERY": f"SELECT _key, {fields} FROM {tablename}",
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
@staticmethod
|
|
127
|
+
def get_fields(value_class):
|
|
128
|
+
value_columns = tuple(value_class._fields)
|
|
129
|
+
if "_key" in value_columns:
|
|
130
|
+
raise ValueError(
|
|
131
|
+
f"Name _key is not allowed as attribute for {value_class},"
|
|
132
|
+
" listed as field name in _fields."
|
|
133
|
+
)
|
|
134
|
+
|
|
135
|
+
return value_columns
|
|
136
|
+
|
|
137
|
+
@staticmethod
|
|
138
|
+
def _escape_table_name(name: str) -> str:
|
|
139
|
+
tablename = '"' + name.replace('"', '""') + '"'
|
|
140
|
+
return tablename
|
|
141
|
+
|
|
142
|
+
@property
|
|
143
|
+
def filename(self) -> str:
|
|
144
|
+
"""DiskStore filename for DB."""
|
|
145
|
+
return self._filename
|
|
146
|
+
|
|
147
|
+
@property
|
|
148
|
+
def timeout(self) -> float:
|
|
149
|
+
"""SQLite connection timeout value in seconds."""
|
|
150
|
+
return self._timeout
|
|
151
|
+
|
|
152
|
+
@property
|
|
153
|
+
def tablename(self) -> str:
|
|
154
|
+
return self._tablename
|
|
155
|
+
|
|
156
|
+
@property
|
|
157
|
+
def _con(self) -> Connection:
|
|
158
|
+
# Check process ID to support process forking. If the process
|
|
159
|
+
# ID changes, close the connection and update the process ID.
|
|
160
|
+
|
|
161
|
+
local_pid = getattr(self._local, "pid", None)
|
|
162
|
+
pid: int = os.getpid()
|
|
163
|
+
|
|
164
|
+
if local_pid != pid:
|
|
165
|
+
self.close()
|
|
166
|
+
self._local.pid: int = pid
|
|
167
|
+
|
|
168
|
+
con = getattr(self._local, "con", None)
|
|
169
|
+
|
|
170
|
+
if con is None:
|
|
171
|
+
con = self._local.con = Connection(
|
|
172
|
+
self._filename, flags=apsw.SQLITE_OPEN_READONLY
|
|
173
|
+
)
|
|
174
|
+
con.set_busy_timeout(int(self._timeout * 1000))
|
|
175
|
+
|
|
176
|
+
return con
|
|
177
|
+
|
|
178
|
+
@contextmanager
|
|
179
|
+
def _cursor(self):
|
|
180
|
+
cursor: Cursor = self._con.cursor()
|
|
181
|
+
try:
|
|
182
|
+
yield cursor
|
|
183
|
+
finally:
|
|
184
|
+
cursor.close()
|
|
185
|
+
|
|
186
|
+
def __getitem__(self, key: KeyType):
|
|
187
|
+
select = self._statements["GET"]
|
|
188
|
+
with self._cursor() as cursor:
|
|
189
|
+
row = next(cursor.execute(select, (key,)), MISSING)
|
|
190
|
+
|
|
191
|
+
if row is MISSING:
|
|
192
|
+
raise KeyError(key)
|
|
193
|
+
|
|
194
|
+
return self._value_class(*row) # ty:ignore[not-iterable]
|
|
195
|
+
|
|
196
|
+
def keys(self):
|
|
197
|
+
return DiskKeysView(self)
|
|
198
|
+
|
|
199
|
+
def values(self):
|
|
200
|
+
return DiskValuesView(self)
|
|
201
|
+
|
|
202
|
+
def items(self):
|
|
203
|
+
return DiskItemsView(self)
|
|
204
|
+
|
|
205
|
+
def query(
|
|
206
|
+
self,
|
|
207
|
+
where: Optional[str] = None,
|
|
208
|
+
parameters: Optional[Sequence] = None,
|
|
209
|
+
order: Optional[str] = None,
|
|
210
|
+
) -> Generator[tuple, None, None]:
|
|
211
|
+
where = " WHERE " + where if where else ""
|
|
212
|
+
parameters = () if parameters is None else parameters
|
|
213
|
+
order = " ORDER BY " + order if order else ""
|
|
214
|
+
select = self._statements["QUERY"] + where + order
|
|
215
|
+
|
|
216
|
+
with self._cursor() as cursor:
|
|
217
|
+
cursor.execute(select, parameters)
|
|
218
|
+
for row in cursor:
|
|
219
|
+
yield row[0], self._value_class(*row[1:])
|
|
220
|
+
|
|
221
|
+
def __contains__(self, key: object) -> bool:
|
|
222
|
+
with self._cursor() as cx:
|
|
223
|
+
rows = cx.execute(self._statements["CONTAINS"], (key,)).fetchall()
|
|
224
|
+
|
|
225
|
+
return bool(rows)
|
|
226
|
+
|
|
227
|
+
def __iter__(self):
|
|
228
|
+
with self._cursor() as cx:
|
|
229
|
+
cx.execute(self._statements["ITER"])
|
|
230
|
+
for row in cx:
|
|
231
|
+
yield row[0]
|
|
232
|
+
|
|
233
|
+
def __reversed__(self):
|
|
234
|
+
with self._cursor() as cx:
|
|
235
|
+
cx.execute(self._statements["REVERSED"])
|
|
236
|
+
for row in cx:
|
|
237
|
+
yield row[0]
|
|
238
|
+
|
|
239
|
+
def close(self) -> None:
|
|
240
|
+
con = getattr(self._local, "con", None)
|
|
241
|
+
if con is None:
|
|
242
|
+
return
|
|
243
|
+
con.close()
|
|
244
|
+
try:
|
|
245
|
+
delattr(self._local, "con")
|
|
246
|
+
except AttributeError:
|
|
247
|
+
pass
|
|
248
|
+
|
|
249
|
+
def __enter__(self):
|
|
250
|
+
connection = self._con # noqa
|
|
251
|
+
return self
|
|
252
|
+
|
|
253
|
+
def __exit__(self, *args, **kwargs) -> None:
|
|
254
|
+
self.close()
|
|
255
|
+
|
|
256
|
+
def __len__(self):
|
|
257
|
+
select = self._statements["COUNT"]
|
|
258
|
+
|
|
259
|
+
with closing(self._con.execute(select)) as cx:
|
|
260
|
+
rows = next(cx, (0,))
|
|
261
|
+
return rows[0]
|
|
262
|
+
|
|
263
|
+
def __getstate__(self):
|
|
264
|
+
return {
|
|
265
|
+
"filename": self.filename,
|
|
266
|
+
"timeout": self.timeout,
|
|
267
|
+
"tablename": self._tablename,
|
|
268
|
+
"value_class": self._value_class,
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
def __setstate__(self, state) -> None:
|
|
272
|
+
self.__init__(**state)
|
diskstore/diskstore.py
ADDED
|
@@ -0,0 +1,315 @@
|
|
|
1
|
+
"""DiskStore (MutableMapping) API."""
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
import os.path
|
|
5
|
+
import threading
|
|
6
|
+
from collections.abc import Mapping, MutableMapping
|
|
7
|
+
from contextlib import closing, contextmanager
|
|
8
|
+
from time import sleep, time
|
|
9
|
+
from typing import Iterable
|
|
10
|
+
|
|
11
|
+
import apsw
|
|
12
|
+
|
|
13
|
+
from .const import DEFAULT_PRAGMAS, MISSING, TIMEOUT, KeyType
|
|
14
|
+
from .diskread import DiskRead
|
|
15
|
+
|
|
16
|
+
Connection = apsw.Connection
|
|
17
|
+
Cursor = apsw.Cursor
|
|
18
|
+
SQLError = apsw.SQLError
|
|
19
|
+
BusyError = apsw.BusyError
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class DiskStore(DiskRead, MutableMapping):
|
|
23
|
+
"""SQLite disk storage."""
|
|
24
|
+
|
|
25
|
+
def __init__( # noqa: PLR0915
|
|
26
|
+
self,
|
|
27
|
+
filename: os.PathLike | str,
|
|
28
|
+
value_class=None,
|
|
29
|
+
tablename: str | None = None,
|
|
30
|
+
key_type=None,
|
|
31
|
+
timeout: float | None = None,
|
|
32
|
+
alter_table: bool = True,
|
|
33
|
+
**pragmas,
|
|
34
|
+
) -> None:
|
|
35
|
+
"""Initialize Diskstore instance.
|
|
36
|
+
|
|
37
|
+
:param str filename: DiskStore DB filename.
|
|
38
|
+
If not set a tmp directory and default DB name is created.
|
|
39
|
+
:param str tablename: Optional table name to use.
|
|
40
|
+
If not set the name from value_class is used.
|
|
41
|
+
:param type value_class: Class type (inherited from NamedTuple)
|
|
42
|
+
used to get back data. Default is Value class.
|
|
43
|
+
:param float timeout: SQLite connection timeout
|
|
44
|
+
(also used if DB is busy to retry)
|
|
45
|
+
:param pragmas: Any of DEFAULT_PRAGMAS or additional PRAGMA's to apply
|
|
46
|
+
"""
|
|
47
|
+
super().__init__(
|
|
48
|
+
filename=filename,
|
|
49
|
+
value_class=value_class,
|
|
50
|
+
tablename=tablename,
|
|
51
|
+
timeout=timeout,
|
|
52
|
+
)
|
|
53
|
+
self._txn_id = None
|
|
54
|
+
tablename = self._escape_table_name(self._tablename)
|
|
55
|
+
|
|
56
|
+
primary_key_type = "BLOB"
|
|
57
|
+
if key_type is str:
|
|
58
|
+
primary_key_type = "TEXT"
|
|
59
|
+
elif key_type is int:
|
|
60
|
+
primary_key_type = "INTEGER"
|
|
61
|
+
elif key_type is float:
|
|
62
|
+
primary_key_type = "REAL"
|
|
63
|
+
value_columns = self.get_fields(self._value_class)
|
|
64
|
+
column_types = self.get_field_types(self._value_class)
|
|
65
|
+
fields_create = ", ".join(
|
|
66
|
+
f"{field} {ctype} NOT NULL"
|
|
67
|
+
for field, ctype in zip(value_columns, column_types, strict=True)
|
|
68
|
+
)
|
|
69
|
+
fields = ", ".join(f"{field}" for field in value_columns)
|
|
70
|
+
excluded_fields = ", ".join(
|
|
71
|
+
f"{field} = excluded.{field}" for field in value_columns
|
|
72
|
+
)
|
|
73
|
+
qms = ", ".join("?" for field in value_columns)
|
|
74
|
+
self._statements.update(
|
|
75
|
+
{
|
|
76
|
+
"CREATE": (
|
|
77
|
+
f"CREATE TABLE IF NOT EXISTS {tablename} ("
|
|
78
|
+
f" _key {primary_key_type} PRIMARY KEY NOT NULL"
|
|
79
|
+
f", {fields_create})"
|
|
80
|
+
),
|
|
81
|
+
"SET": (
|
|
82
|
+
f"INSERT INTO {tablename}(_key, {fields}) VALUES (?, {qms})"
|
|
83
|
+
f" ON CONFLICT (_key) DO UPDATE SET {excluded_fields}"
|
|
84
|
+
),
|
|
85
|
+
"ADD": (
|
|
86
|
+
f"INSERT INTO {tablename}(_key, {fields}) "
|
|
87
|
+
f"VALUES (?, {qms}) ON CONFLICT DO NOTHING RETURNING _key"
|
|
88
|
+
),
|
|
89
|
+
"DELETE": f"DELETE FROM {tablename} WHERE _key = ? RETURNING _key",
|
|
90
|
+
"CLEAR": f"DELETE FROM {tablename};VACUUM;",
|
|
91
|
+
"POPITEM": (
|
|
92
|
+
f"SELECT _key, {fields} FROM {tablename} ORDER BY"
|
|
93
|
+
" rowid DESC LIMIT 1"
|
|
94
|
+
),
|
|
95
|
+
}
|
|
96
|
+
)
|
|
97
|
+
self._pragmas = DEFAULT_PRAGMAS.copy()
|
|
98
|
+
self._pragmas.update(pragmas)
|
|
99
|
+
sql = self._con.execute
|
|
100
|
+
# Setup table.
|
|
101
|
+
with closing(sql(self._statements["CREATE"])):
|
|
102
|
+
pass
|
|
103
|
+
# try alter table if needed
|
|
104
|
+
if alter_table:
|
|
105
|
+
self._alter_table(sql)
|
|
106
|
+
|
|
107
|
+
@staticmethod
|
|
108
|
+
def get_field_types(value_class):
|
|
109
|
+
value_columns = tuple(value_class._fields)
|
|
110
|
+
if hasattr(value_class, "__annotations__") and value_class.__annotations__:
|
|
111
|
+
type_annotations = value_class.__annotations__
|
|
112
|
+
value_types = []
|
|
113
|
+
for c_name in value_columns:
|
|
114
|
+
type_cls = type_annotations.get(c_name, bytes) # types are optional
|
|
115
|
+
if type_cls is str:
|
|
116
|
+
value_types.append("TEXT")
|
|
117
|
+
elif type_cls is int:
|
|
118
|
+
value_types.append("INTEGER")
|
|
119
|
+
elif type_cls is float:
|
|
120
|
+
value_types.append("REAL")
|
|
121
|
+
else:
|
|
122
|
+
value_types.append("BLOB")
|
|
123
|
+
else:
|
|
124
|
+
value_types = ["BLOB" for _ in value_columns]
|
|
125
|
+
|
|
126
|
+
return value_types
|
|
127
|
+
|
|
128
|
+
def _validate_value(self, value):
|
|
129
|
+
if isinstance(value, self._value_class):
|
|
130
|
+
return value
|
|
131
|
+
value_type = type(value)
|
|
132
|
+
raise ValueError(
|
|
133
|
+
f"Value must be instance of {self._value_class} but is {value_type}."
|
|
134
|
+
)
|
|
135
|
+
|
|
136
|
+
@property
|
|
137
|
+
def _con(self) -> Connection:
|
|
138
|
+
# Check process ID to support process forking. If the process
|
|
139
|
+
# ID changes, close the connection and update the process ID.
|
|
140
|
+
|
|
141
|
+
local_pid = getattr(self._local, "pid", None)
|
|
142
|
+
pid: int = os.getpid()
|
|
143
|
+
|
|
144
|
+
if local_pid != pid:
|
|
145
|
+
self.close()
|
|
146
|
+
self._local.pid: int = pid
|
|
147
|
+
|
|
148
|
+
con = getattr(self._local, "con", None)
|
|
149
|
+
|
|
150
|
+
if con is None:
|
|
151
|
+
con = self._local.con = Connection(self._filename)
|
|
152
|
+
con.set_busy_timeout(int(self._timeout * 1000))
|
|
153
|
+
|
|
154
|
+
# Some SQLite pragmas work on a per-connection basis so
|
|
155
|
+
# apply them all on fresh connection
|
|
156
|
+
for key, value in self._pragmas.items():
|
|
157
|
+
con.pragma(key, value)
|
|
158
|
+
|
|
159
|
+
return con
|
|
160
|
+
|
|
161
|
+
def _alter_table(self, sql):
|
|
162
|
+
value_columns = self.get_fields(self._value_class)
|
|
163
|
+
column_types = self.get_field_types(self._value_class)
|
|
164
|
+
tablename = self._escape_table_name(self._tablename)
|
|
165
|
+
value_column_defaults = getattr(self._value_class, "_field_defaults", {})
|
|
166
|
+
existing_fileds = set()
|
|
167
|
+
table_info_stmt = f"PRAGMA table_info({tablename});"
|
|
168
|
+
with closing(sql(table_info_stmt)) as cursor:
|
|
169
|
+
for row in cursor:
|
|
170
|
+
existing_fileds.add(row[1])
|
|
171
|
+
for columnname, columntype in zip(value_columns, column_types, strict=True):
|
|
172
|
+
if columnname not in existing_fileds:
|
|
173
|
+
default = apsw.format_sql_value(value_column_defaults[columnname])
|
|
174
|
+
alter_stmt = (
|
|
175
|
+
f"ALTER TABLE {tablename} ADD COLUMN {columnname}"
|
|
176
|
+
f" {columntype} NOT NULL DEFAULT {default};"
|
|
177
|
+
)
|
|
178
|
+
with closing(sql(alter_stmt)) as cursor:
|
|
179
|
+
pass
|
|
180
|
+
|
|
181
|
+
@contextmanager
|
|
182
|
+
def transact(self, retry=False):
|
|
183
|
+
cursor: Cursor = self._con.cursor()
|
|
184
|
+
tid = threading.get_ident()
|
|
185
|
+
txn_id = self._txn_id
|
|
186
|
+
retry_until = time() + TIMEOUT
|
|
187
|
+
|
|
188
|
+
if tid == txn_id: # already inside a thread with a transaction
|
|
189
|
+
begin = False
|
|
190
|
+
else:
|
|
191
|
+
while True:
|
|
192
|
+
try:
|
|
193
|
+
cursor.execute("BEGIN IMMEDIATE")
|
|
194
|
+
begin = True
|
|
195
|
+
self._txn_id = tid
|
|
196
|
+
break
|
|
197
|
+
except BusyError:
|
|
198
|
+
if retry and time() < retry_until:
|
|
199
|
+
sleep(0.001)
|
|
200
|
+
continue
|
|
201
|
+
raise
|
|
202
|
+
|
|
203
|
+
try:
|
|
204
|
+
yield cursor
|
|
205
|
+
except BaseException:
|
|
206
|
+
if begin:
|
|
207
|
+
assert self._txn_id == tid
|
|
208
|
+
self._txn_id = None
|
|
209
|
+
cursor.execute("ROLLBACK")
|
|
210
|
+
cursor.close()
|
|
211
|
+
raise
|
|
212
|
+
else:
|
|
213
|
+
if begin:
|
|
214
|
+
assert self._txn_id == tid
|
|
215
|
+
self._txn_id = None
|
|
216
|
+
cursor.execute("COMMIT")
|
|
217
|
+
cursor.close()
|
|
218
|
+
|
|
219
|
+
def __setitem__(self, key: KeyType, value: Iterable) -> None:
|
|
220
|
+
value = self._validate_value(value)
|
|
221
|
+
with self._cursor() as cursor:
|
|
222
|
+
cursor.execute(self._statements["SET"], (key, *value))
|
|
223
|
+
|
|
224
|
+
def add(self, key: KeyType | None, value: Iterable) -> KeyType | None:
|
|
225
|
+
value = self._validate_value(value)
|
|
226
|
+
with self._cursor() as cx:
|
|
227
|
+
rows = cx.execute(self._statements["ADD"], (key, *value)).fetchall()
|
|
228
|
+
|
|
229
|
+
if not rows:
|
|
230
|
+
return None
|
|
231
|
+
|
|
232
|
+
return rows[0][0]
|
|
233
|
+
|
|
234
|
+
def pop(self, key: KeyType, default=MISSING): # noqa: E501
|
|
235
|
+
with self.transact(retry=True) as cx:
|
|
236
|
+
value = next(cx.execute(self._statements["GET"], (key,)), None)
|
|
237
|
+
if value is None:
|
|
238
|
+
if default is MISSING:
|
|
239
|
+
raise KeyError(key)
|
|
240
|
+
return default
|
|
241
|
+
cx.execute(self._statements["DELETE"], (key,))
|
|
242
|
+
return self._value_class(*value)
|
|
243
|
+
|
|
244
|
+
def popitem(self):
|
|
245
|
+
with self.transact(retry=True) as cx:
|
|
246
|
+
row = next(cx.execute(self._statements["POPITEM"]), None)
|
|
247
|
+
if not row:
|
|
248
|
+
raise KeyError()
|
|
249
|
+
key = row[0]
|
|
250
|
+
value = self._value_class(*row[1:])
|
|
251
|
+
cx.execute(self._statements["DELETE"], (key,))
|
|
252
|
+
return key, value
|
|
253
|
+
|
|
254
|
+
def __delitem__(self, key: KeyType) -> None:
|
|
255
|
+
with self._cursor() as cursor:
|
|
256
|
+
rows = cursor.execute(self._statements["DELETE"], (key,)).fetchall()
|
|
257
|
+
if not rows:
|
|
258
|
+
raise KeyError(key)
|
|
259
|
+
|
|
260
|
+
def setdefault(self, key: KeyType, default: Iterable | None = None):
|
|
261
|
+
try:
|
|
262
|
+
return self[key]
|
|
263
|
+
except KeyError:
|
|
264
|
+
if default is not None:
|
|
265
|
+
default = self._validate_value(default)
|
|
266
|
+
self.add(key, default)
|
|
267
|
+
|
|
268
|
+
return default
|
|
269
|
+
|
|
270
|
+
def check(self, vacuum=False):
|
|
271
|
+
warns = []
|
|
272
|
+
sql = self._con.execute
|
|
273
|
+
|
|
274
|
+
# Check integrity of database.
|
|
275
|
+
with closing(sql("PRAGMA integrity_check")) as cx:
|
|
276
|
+
rows = cx.fetchall()
|
|
277
|
+
|
|
278
|
+
if len(rows) != 1 or rows[0][0] != "ok":
|
|
279
|
+
for (message,) in rows:
|
|
280
|
+
warns.append(message)
|
|
281
|
+
|
|
282
|
+
if vacuum:
|
|
283
|
+
with closing(sql("VACUUM")):
|
|
284
|
+
pass
|
|
285
|
+
|
|
286
|
+
return warns
|
|
287
|
+
|
|
288
|
+
def clear(self) -> None:
|
|
289
|
+
with closing(self._con.execute(self._statements["CLEAR"])):
|
|
290
|
+
pass
|
|
291
|
+
|
|
292
|
+
def update(self, other=(), /, **kwargs): # type: ignore
|
|
293
|
+
comps = []
|
|
294
|
+
if other:
|
|
295
|
+
if isinstance(other, Mapping):
|
|
296
|
+
comp = ((key, *value) for key, value in other.items())
|
|
297
|
+
comps.append(comp)
|
|
298
|
+
elif hasattr(other, "keys"):
|
|
299
|
+
comp = ((key, *other[key]) for key in other.keys())
|
|
300
|
+
comps.append(comp)
|
|
301
|
+
else:
|
|
302
|
+
comp = ((key, *value) for key, value in other)
|
|
303
|
+
comps.append(comp)
|
|
304
|
+
if kwargs:
|
|
305
|
+
comp = ((key, *value) for key, value in kwargs.items())
|
|
306
|
+
comps.append(comp)
|
|
307
|
+
|
|
308
|
+
with self.transact(retry=True) as cursor:
|
|
309
|
+
for comp in comps:
|
|
310
|
+
cursor.executemany(self._statements["SET"], comp)
|
|
311
|
+
|
|
312
|
+
def get_readonly_instance(self):
|
|
313
|
+
return DiskRead(
|
|
314
|
+
self._filename, self._value_class, self._tablename, self._timeout
|
|
315
|
+
)
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
Metadata-Version: 2.3
|
|
2
|
+
Name: diskstore
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Simple MutableMapping compatible SQLite based disk storage.
|
|
5
|
+
Author: tds333
|
|
6
|
+
Author-email: tds333 <tds333@mailbox.org>
|
|
7
|
+
Requires-Dist: apsw
|
|
8
|
+
Requires-Python: >=3.10
|
|
9
|
+
Description-Content-Type: text/markdown
|
|
10
|
+
|
|
11
|
+
# DiskStore Sqlite based disk store
|
|
12
|
+
|
|
13
|
+
`DiskStore` is an Apache2/MIT licensed disk storage library, written
|
|
14
|
+
in pure Python.
|
|
15
|
+
|
|
16
|
+
Inspired by DiskCache library it implements a MutableMapping compatible sqlite
|
|
17
|
+
based disk storage. Easy interface and very fast. Keys can be of basic sqlite storage
|
|
18
|
+
classes defined by Python types `int`, `float`, `str`, `bytes`. Value is a `Namedtuple` class
|
|
19
|
+
with same basic Python types.
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
## Features
|
|
23
|
+
|
|
24
|
+
- Pure-Python
|
|
25
|
+
- nearly 100% test coverage
|
|
26
|
+
- Performance matters
|
|
27
|
+
- Thread-safe and process-safe
|
|
28
|
+
- Developed on Python 3.10
|
|
29
|
+
- Tested on CPython 3.10, 3.11, 3.12, 3.13, 3.14, 3.15
|
|
30
|
+
- Tested using GitHub Actions
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
## Quickstart
|
|
34
|
+
|
|
35
|
+
Installing `DiskStore`:
|
|
36
|
+
|
|
37
|
+
$ uv pip install diskstore
|
|
38
|
+
|
|
39
|
+
or
|
|
40
|
+
|
|
41
|
+
$ uv add diskstore
|
|
42
|
+
|
|
43
|
+
There are three basic storage classes available.
|
|
44
|
+
`DiskStore`, `DiskRead` and `DiskCache`.
|
|
45
|
+
|
|
46
|
+
## User Guide
|
|
47
|
+
|
|
48
|
+
Work in progress, look at unit tests for examples.
|
|
49
|
+
|
|
50
|
+
Example:
|
|
51
|
+
|
|
52
|
+
```python
|
|
53
|
+
|
|
54
|
+
from diskstore import DiskStore, Value
|
|
55
|
+
|
|
56
|
+
ds = DiskStore("/tmp/diskstore.db", value_class=Value)
|
|
57
|
+
ds["key"] = Value("my value")
|
|
58
|
+
print(ds["key"])
|
|
59
|
+
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
Everything is mostly stable and test coverage is nearly 100%. Documentation is missing.
|
|
63
|
+
|
|
64
|
+
### Timings
|
|
65
|
+
|
|
66
|
+
These are rough measurements. Compared to Diskcache library.
|
|
67
|
+
|
|
68
|
+
Diskstore library:
|
|
69
|
+
|
|
70
|
+
set: 28.8 µs
|
|
71
|
+
get: 3.23 µs
|
|
72
|
+
set/delete: 84 µs
|
|
73
|
+
|
|
74
|
+
Diskcache library:
|
|
75
|
+
|
|
76
|
+
set: 196 µs
|
|
77
|
+
get: 15.5 µs
|
|
78
|
+
set/delete: 535 µs
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
Why is the Diskstore library faster than others. Every overhead is eliminitated and it uses the
|
|
82
|
+
faster sqlite library apsw with up to date sqlite library version.
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
License
|
|
86
|
+
-------
|
|
87
|
+
|
|
88
|
+
Copyright 2025-2026 Wolfgang Langner
|
|
89
|
+
|
|
90
|
+
Licensed under the Apache License, Version 2.0 (the "License") or MIT; you may not use
|
|
91
|
+
this file except in compliance with the License. You may obtain a copy of the
|
|
92
|
+
License at
|
|
93
|
+
|
|
94
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
95
|
+
https://mit-license.org/
|
|
96
|
+
|
|
97
|
+
Unless required by applicable law or agreed to in writing, software distributed
|
|
98
|
+
under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
|
|
99
|
+
CONDITIONS OF ANY KIND, either express or implied. See the License for the
|
|
100
|
+
specific language governing permissions and limitations under the License.
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
diskstore/__init__.py,sha256=7ZZHY-ok2S01xGdZ9jebyaFtXGz3ddMAbvw1LZnQYt0,198
|
|
2
|
+
diskstore/const.py,sha256=VPBso0bMQyvrwUmAriseWbdrfBRndLErk5340jUh0z4,498
|
|
3
|
+
diskstore/diskcache.py,sha256=jadCJ1WRJgvpnUpC409nQVLWLRtCknbi8HERFfGwLu4,3743
|
|
4
|
+
diskstore/diskread.py,sha256=r8FA_Nn0iDZQ5ssXWVVnA-R3XyBrRuVvI6NHycZFz9E,8063
|
|
5
|
+
diskstore/diskstore.py,sha256=_p3gZXhbaq6EY4TOas8zKB76F1ZO03TEAv2pauut37U,11162
|
|
6
|
+
diskstore-0.1.0.dist-info/WHEEL,sha256=jROcLULcdzropX2J55opKw4UHhPFREZax2XzS-Mvpxs,80
|
|
7
|
+
diskstore-0.1.0.dist-info/METADATA,sha256=vw7LuKhkEFK4YbmpAT0S6wrMm4cuz8-no3IkORlS1ks,2406
|
|
8
|
+
diskstore-0.1.0.dist-info/RECORD,,
|