sql3-lite-saver 0.99.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sql3_lite_saver/__init__.py +6 -0
- sql3_lite_saver/pool.py +403 -0
- sql3_lite_saver-0.99.0.dist-info/METADATA +225 -0
- sql3_lite_saver-0.99.0.dist-info/RECORD +7 -0
- sql3_lite_saver-0.99.0.dist-info/WHEEL +5 -0
- sql3_lite_saver-0.99.0.dist-info/licenses/LICENSE +22 -0
- sql3_lite_saver-0.99.0.dist-info/top_level.txt +1 -0
sql3_lite_saver/pool.py
ADDED
|
@@ -0,0 +1,403 @@
|
|
|
1
|
+
import time
|
|
2
|
+
import queue
|
|
3
|
+
import atexit
|
|
4
|
+
import typing as tp
|
|
5
|
+
import logging
|
|
6
|
+
import sqlite3
|
|
7
|
+
import threading
|
|
8
|
+
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
logger = logging.getLogger(__name__)
|
|
12
|
+
|
|
13
|
+
try:
|
|
14
|
+
from tenacity import retry, retry_if_exception, stop_after_attempt, wait_chain, wait_fixed
|
|
15
|
+
HAVE_TENACITY = True
|
|
16
|
+
except ImportError:
|
|
17
|
+
retry = retry_if_exception = stop_after_attempt = wait_chain = wait_fixed = None
|
|
18
|
+
HAVE_TENACITY = False
|
|
19
|
+
|
|
20
|
+
'''
|
|
21
|
+
@tp.runtime_checkable
|
|
22
|
+
class ConnectionLike(tp.Protocol):
|
|
23
|
+
def execute(self, *a: tp.Any, **kw: tp.Any): ...
|
|
24
|
+
def executemany(self, *a: tp.Any, **kw: tp.Any): ...
|
|
25
|
+
def executescript(self, *a: tp.Any, **kw: tp.Any): ...
|
|
26
|
+
def close(self) -> None: ...
|
|
27
|
+
'''
|
|
28
|
+
|
|
29
|
+
class _ConnectionProxy:
|
|
30
|
+
"""Lightweight proxy that adds retry to SQLite connection methods with proper type support."""
|
|
31
|
+
def __init__(self, conn: sqlite3.Connection, wrapper: tp.Callable):
|
|
32
|
+
self._conn = conn
|
|
33
|
+
self._wrap = wrapper
|
|
34
|
+
|
|
35
|
+
# Explicit methods with retry and proper type hints
|
|
36
|
+
def execute(self, sql: str, parameters: tp.Sequence[tp.Any] = ()) -> sqlite3.Cursor:
|
|
37
|
+
"""Execute a single SQL statement with retry logic."""
|
|
38
|
+
return self._wrap(self._conn.execute)(sql, parameters)
|
|
39
|
+
|
|
40
|
+
def executemany(self, sql: str, seq_of_parameters: tp.Iterable[tp.Sequence[tp.Any]]) -> sqlite3.Cursor:
|
|
41
|
+
"""Execute SQL statement multiple times with retry logic."""
|
|
42
|
+
return self._wrap(self._conn.executemany)(sql, seq_of_parameters)
|
|
43
|
+
|
|
44
|
+
def executescript(self, sql_script: str) -> sqlite3.Cursor:
|
|
45
|
+
"""Execute multiple SQL statements with retry logic."""
|
|
46
|
+
return self._wrap(self._conn.executescript)(sql_script)
|
|
47
|
+
|
|
48
|
+
# Pass through other commonly used connection methods without retry
|
|
49
|
+
def commit(self) -> None:
|
|
50
|
+
"""Commit the current transaction."""
|
|
51
|
+
self._conn.commit()
|
|
52
|
+
|
|
53
|
+
def rollback(self) -> None:
|
|
54
|
+
"""Roll back the current transaction."""
|
|
55
|
+
self._conn.rollback()
|
|
56
|
+
|
|
57
|
+
def close(self) -> None:
|
|
58
|
+
"""Close the connection."""
|
|
59
|
+
self._conn.close()
|
|
60
|
+
|
|
61
|
+
@property
|
|
62
|
+
def row_factory(self) -> tp.Optional[tp.Callable]:
|
|
63
|
+
"""Get/set the row factory for this connection."""
|
|
64
|
+
return self._conn.row_factory
|
|
65
|
+
|
|
66
|
+
@row_factory.setter
|
|
67
|
+
def row_factory(self, factory: tp.Optional[tp.Callable]) -> None:
|
|
68
|
+
self._conn.row_factory = factory
|
|
69
|
+
|
|
70
|
+
@property
|
|
71
|
+
def total_changes(self) -> int:
|
|
72
|
+
"""Get the total number of changes made to the database."""
|
|
73
|
+
return self._conn.total_changes
|
|
74
|
+
|
|
75
|
+
# Fallback for any other attributes/methods
|
|
76
|
+
def __getattr__(self, name: str) -> tp.Any:
|
|
77
|
+
return getattr(self._conn, name)
|
|
78
|
+
|
|
79
|
+
# Allow 'with conn:' usage
|
|
80
|
+
def __enter__(self) -> sqlite3.Connection:
|
|
81
|
+
return tp.cast(sqlite3.Connection, self._conn)
|
|
82
|
+
|
|
83
|
+
def __exit__(self, exc_type: tp.Optional[tp.Type[BaseException]],
|
|
84
|
+
exc_val: tp.Optional[BaseException],
|
|
85
|
+
exc_tb: tp.Optional[tp.Any]) -> tp.Optional[bool]:
|
|
86
|
+
return self._conn.__exit__(exc_type, exc_val, exc_tb)
|
|
87
|
+
|
|
88
|
+
class ConnectionPool:
|
|
89
|
+
def __init__(
|
|
90
|
+
self,
|
|
91
|
+
db_path: Path,
|
|
92
|
+
*,
|
|
93
|
+
max_size: int = 5,
|
|
94
|
+
acquire_timeout: tp.Optional[float] = None,
|
|
95
|
+
enable_retry: bool = True,
|
|
96
|
+
base_delay: float = 1.0,
|
|
97
|
+
max_backoff_total: float = 60.0,
|
|
98
|
+
retry_attempts: tp.Optional[int] = None,
|
|
99
|
+
retry_jitter: float = 0.1,
|
|
100
|
+
auto_cleanup: bool = True,
|
|
101
|
+
read_only: bool = False,
|
|
102
|
+
warmup_callback: tp.Optional[tp.Callable[[sqlite3.Connection], None]] = None,
|
|
103
|
+
):
|
|
104
|
+
self.db_path = db_path
|
|
105
|
+
self.max_size = max_size
|
|
106
|
+
self.acquire_timeout = acquire_timeout
|
|
107
|
+
self.enable_retry = enable_retry
|
|
108
|
+
self.base_delay = base_delay
|
|
109
|
+
self.max_backoff_total = max_backoff_total
|
|
110
|
+
self.retry_attempts = retry_attempts
|
|
111
|
+
self.retry_jitter = retry_jitter
|
|
112
|
+
self.read_only = read_only
|
|
113
|
+
self.warmup_callback = warmup_callback
|
|
114
|
+
|
|
115
|
+
self._pool: "queue.Queue[sqlite3.Connection]" = queue.Queue(max_size)
|
|
116
|
+
self._all_conns: tp.List[sqlite3.Connection] = []
|
|
117
|
+
self._lock = threading.RLock()
|
|
118
|
+
self._wait_count = 0
|
|
119
|
+
self._is_closed = False
|
|
120
|
+
|
|
121
|
+
for _ in range(max_size):
|
|
122
|
+
conn = self._create_connection()
|
|
123
|
+
self._pool.put(conn)
|
|
124
|
+
self._all_conns.append(conn)
|
|
125
|
+
|
|
126
|
+
if auto_cleanup:
|
|
127
|
+
atexit.register(self._atexit_cleanup)
|
|
128
|
+
logger.debug("Registered atexit cleanup for SQLiteConnectionPool")
|
|
129
|
+
|
|
130
|
+
logger.info(f"SQLiteConnectionPool created with max_size={max_size}")
|
|
131
|
+
|
|
132
|
+
# ------------------------------------------------------------------
|
|
133
|
+
# Connection creation / validation
|
|
134
|
+
# ------------------------------------------------------------------
|
|
135
|
+
def _create_connection(self) -> sqlite3.Connection:
|
|
136
|
+
uri = f"file:{self.db_path}?mode=ro" if self.read_only else str(self.db_path)
|
|
137
|
+
conn = sqlite3.connect(
|
|
138
|
+
uri,
|
|
139
|
+
timeout=10.0,
|
|
140
|
+
isolation_level=None, # autocommit
|
|
141
|
+
check_same_thread=False, # allow cross-thread usage
|
|
142
|
+
uri=self.read_only,
|
|
143
|
+
)
|
|
144
|
+
conn.row_factory = sqlite3.Row
|
|
145
|
+
conn.execute("PRAGMA foreign_keys = ON")
|
|
146
|
+
conn.execute("PRAGMA busy_timeout = 10000")
|
|
147
|
+
conn.execute("PRAGMA journal_mode = WAL")
|
|
148
|
+
conn.execute("PRAGMA synchronous = NORMAL")
|
|
149
|
+
if self.warmup_callback:
|
|
150
|
+
self.warmup_callback(conn)
|
|
151
|
+
return self._wrap_connection(conn)
|
|
152
|
+
|
|
153
|
+
def _validate_connection(self, conn: sqlite3.Connection) -> sqlite3.Connection:
|
|
154
|
+
try:
|
|
155
|
+
# Get the underlying connection if this is a proxy
|
|
156
|
+
underlying_conn = getattr(conn, '_conn', conn)
|
|
157
|
+
underlying_conn.execute("SELECT 1")
|
|
158
|
+
return conn
|
|
159
|
+
except sqlite3.Error:
|
|
160
|
+
logger.warning("Recreating stale SQLite connection")
|
|
161
|
+
# Create new connection without wrapping first
|
|
162
|
+
uri = f"file:{self.db_path}?mode=ro" if self.read_only else str(self.db_path)
|
|
163
|
+
new_conn = sqlite3.connect(
|
|
164
|
+
uri,
|
|
165
|
+
timeout=10.0,
|
|
166
|
+
isolation_level=None,
|
|
167
|
+
check_same_thread=False,
|
|
168
|
+
uri=self.read_only,
|
|
169
|
+
)
|
|
170
|
+
new_conn.row_factory = sqlite3.Row
|
|
171
|
+
new_conn.execute("PRAGMA foreign_keys = ON")
|
|
172
|
+
new_conn.execute("PRAGMA busy_timeout = 10000")
|
|
173
|
+
new_conn.execute("PRAGMA journal_mode = WAL")
|
|
174
|
+
new_conn.execute("PRAGMA synchronous = NORMAL")
|
|
175
|
+
if self.warmup_callback:
|
|
176
|
+
self.warmup_callback(new_conn)
|
|
177
|
+
|
|
178
|
+
# Now wrap it
|
|
179
|
+
wrapped_conn = self._wrap_connection(new_conn)
|
|
180
|
+
|
|
181
|
+
with self._lock:
|
|
182
|
+
try:
|
|
183
|
+
self._all_conns.remove(conn)
|
|
184
|
+
except ValueError:
|
|
185
|
+
pass
|
|
186
|
+
self._all_conns.append(wrapped_conn)
|
|
187
|
+
return wrapped_conn
|
|
188
|
+
|
|
189
|
+
# ------------------------------------------------------------------
|
|
190
|
+
# Retry wrapping
|
|
191
|
+
# ------------------------------------------------------------------
|
|
192
|
+
def _compute_backoff_delays(self) -> tp.List[float]:
|
|
193
|
+
delays, total, d = [], 0.0, self.base_delay
|
|
194
|
+
while total + d <= self.max_backoff_total:
|
|
195
|
+
delays.append(d)
|
|
196
|
+
total += d
|
|
197
|
+
d *= 2
|
|
198
|
+
if total < self.max_backoff_total:
|
|
199
|
+
delays.append(round(self.max_backoff_total - total, 1))
|
|
200
|
+
return delays
|
|
201
|
+
|
|
202
|
+
def _build_tenacity_retry(self):
|
|
203
|
+
if not HAVE_TENACITY:
|
|
204
|
+
return None
|
|
205
|
+
|
|
206
|
+
if tp.TYPE_CHECKING:
|
|
207
|
+
assert retry is not None
|
|
208
|
+
assert retry_if_exception is not None
|
|
209
|
+
assert stop_after_attempt is not None
|
|
210
|
+
assert wait_chain is not None
|
|
211
|
+
assert wait_fixed is not None
|
|
212
|
+
|
|
213
|
+
base_delay = self.base_delay
|
|
214
|
+
max_delay = self.max_backoff_total
|
|
215
|
+
jitter = self.retry_jitter
|
|
216
|
+
|
|
217
|
+
delays = []
|
|
218
|
+
total = 0.0
|
|
219
|
+
d = base_delay
|
|
220
|
+
while total + d <= max_delay:
|
|
221
|
+
delays.append(d)
|
|
222
|
+
total += d
|
|
223
|
+
d *= 2
|
|
224
|
+
if total < max_delay:
|
|
225
|
+
delays.append(round(max_delay - total, 1))
|
|
226
|
+
|
|
227
|
+
waits = [wait_fixed(d).__add__(wait_fixed(jitter)) for d in delays]
|
|
228
|
+
|
|
229
|
+
return retry(
|
|
230
|
+
retry=retry_if_exception(lambda e: "locked" in str(e).lower()),
|
|
231
|
+
wait=wait_chain(*waits),
|
|
232
|
+
stop=stop_after_attempt(len(waits)),
|
|
233
|
+
reraise=True,
|
|
234
|
+
)
|
|
235
|
+
|
|
236
|
+
def _wrap_with_retry(self, func):
|
|
237
|
+
if not self.enable_retry:
|
|
238
|
+
return func
|
|
239
|
+
|
|
240
|
+
if HAVE_TENACITY:
|
|
241
|
+
retry_obj = self._build_tenacity_retry()
|
|
242
|
+
if retry_obj:
|
|
243
|
+
return retry_obj(func)
|
|
244
|
+
|
|
245
|
+
delays = self._compute_backoff_delays()
|
|
246
|
+
def wrapper(*args, **kwargs):
|
|
247
|
+
for i, delay in enumerate(delays, 1):
|
|
248
|
+
try:
|
|
249
|
+
return func(*args, **kwargs)
|
|
250
|
+
except sqlite3.OperationalError as e:
|
|
251
|
+
if "locked" not in str(e).lower() or i == len(delays):
|
|
252
|
+
raise
|
|
253
|
+
logger.debug(f"DB locked; retry {i}/{len(delays)} in {delay:.1f}s")
|
|
254
|
+
time.sleep(delay)
|
|
255
|
+
return wrapper
|
|
256
|
+
|
|
257
|
+
def _wrap_connection(self, conn: sqlite3.Connection) -> sqlite3.Connection:
|
|
258
|
+
"""Return a proxy connection with retry-wrapped methods."""
|
|
259
|
+
return tp.cast(sqlite3.Connection, _ConnectionProxy(conn, self._wrap_with_retry))
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
# ------------------------------------------------------------------
|
|
263
|
+
# Acquire / Try-Acquire context managers
|
|
264
|
+
# ------------------------------------------------------------------
|
|
265
|
+
class _ConnContext:
|
|
266
|
+
def __init__(self, pool: "ConnectionPool", conn: sqlite3.Connection):
|
|
267
|
+
self.pool = pool
|
|
268
|
+
self.conn = conn
|
|
269
|
+
|
|
270
|
+
def __enter__(self) -> sqlite3.Connection:
|
|
271
|
+
return self.conn
|
|
272
|
+
|
|
273
|
+
def __exit__(self, exc_type, exc_val, exc_tb):
|
|
274
|
+
self.pool._release(self.conn)
|
|
275
|
+
|
|
276
|
+
def acquire(self) -> "_ConnContext":
|
|
277
|
+
start = time.perf_counter()
|
|
278
|
+
with self._lock:
|
|
279
|
+
self._wait_count += 1
|
|
280
|
+
try:
|
|
281
|
+
conn = self._pool.get(timeout=self.acquire_timeout)
|
|
282
|
+
conn = self._validate_connection(conn)
|
|
283
|
+
return self._ConnContext(self, conn)
|
|
284
|
+
except queue.Empty:
|
|
285
|
+
raise TimeoutError("No available database connections in pool")
|
|
286
|
+
finally:
|
|
287
|
+
with self._lock:
|
|
288
|
+
self._wait_count -= 1
|
|
289
|
+
elapsed = time.perf_counter() - start
|
|
290
|
+
if elapsed > 0.01:
|
|
291
|
+
logger.debug(f"Waited {elapsed:.2f}s to acquire connection")
|
|
292
|
+
|
|
293
|
+
def try_acquire(self) -> tp.Optional["_ConnContext"]:
|
|
294
|
+
try:
|
|
295
|
+
conn = self._pool.get_nowait()
|
|
296
|
+
conn = self._validate_connection(conn)
|
|
297
|
+
logger.debug(f"Non-blocking acquire succeeded (in_use={self.in_use})")
|
|
298
|
+
return self._ConnContext(self, conn)
|
|
299
|
+
except queue.Empty:
|
|
300
|
+
logger.debug("Non-blocking acquire failed (pool full)")
|
|
301
|
+
return None
|
|
302
|
+
|
|
303
|
+
def _release(self, conn: sqlite3.Connection):
|
|
304
|
+
self._pool.put(conn)
|
|
305
|
+
logger.debug(f"Released connection (in_use={self.in_use})")
|
|
306
|
+
|
|
307
|
+
# ------------------------------------------------------------------
|
|
308
|
+
# Metrics
|
|
309
|
+
# ------------------------------------------------------------------
|
|
310
|
+
@property
|
|
311
|
+
def in_use(self) -> int:
|
|
312
|
+
if self._is_closed:
|
|
313
|
+
return 0
|
|
314
|
+
return self.max_size - self._pool.qsize()
|
|
315
|
+
|
|
316
|
+
@property
|
|
317
|
+
def available(self) -> int:
|
|
318
|
+
if self._is_closed:
|
|
319
|
+
return 0
|
|
320
|
+
return self._pool.qsize()
|
|
321
|
+
|
|
322
|
+
@property
|
|
323
|
+
def wait_count(self) -> int:
|
|
324
|
+
return self._wait_count
|
|
325
|
+
|
|
326
|
+
# ------------------------------------------------------------------
|
|
327
|
+
# WAL Checkpoint Management
|
|
328
|
+
# ------------------------------------------------------------------
|
|
329
|
+
def checkpoint(
|
|
330
|
+
self,
|
|
331
|
+
mode: tp.Literal["PASSIVE", "FULL", "RESTART", "TRUNCATE"] = "PASSIVE"
|
|
332
|
+
) -> tp.Dict[str, int]:
|
|
333
|
+
"""
|
|
334
|
+
Perform a WAL checkpoint to transfer WAL data to the main database file.
|
|
335
|
+
|
|
336
|
+
Without regular checkpoints, the WAL file grows indefinitely, causing:
|
|
337
|
+
- Wasted disk space (WAL can become massive)
|
|
338
|
+
- Degraded read performance (must scan through entire WAL)
|
|
339
|
+
- Potential file system issues with very large files
|
|
340
|
+
|
|
341
|
+
Modes (from least to most aggressive):
|
|
342
|
+
- PASSIVE (default): Checkpoint without blocking readers/writers
|
|
343
|
+
- FULL: Checkpoint all frames, may block briefly
|
|
344
|
+
- RESTART: Like FULL, then resets WAL for reuse
|
|
345
|
+
- TRUNCATE: Like RESTART, then truncates WAL file to zero bytes
|
|
346
|
+
|
|
347
|
+
Returns dict with:
|
|
348
|
+
- busy: Number of pages not checkpointed due to locks (0 = full success)
|
|
349
|
+
- log: Total pages in WAL after checkpoint
|
|
350
|
+
- checkpointed: Pages successfully transferred
|
|
351
|
+
|
|
352
|
+
See: https://www.sqlite.org/pragma.html#pragma_wal_checkpoint
|
|
353
|
+
"""
|
|
354
|
+
with self.acquire() as conn:
|
|
355
|
+
# Get the underlying connection if wrapped
|
|
356
|
+
underlying_conn = getattr(conn, '_conn', conn)
|
|
357
|
+
result = underlying_conn.execute(f"PRAGMA wal_checkpoint({mode})").fetchone()
|
|
358
|
+
|
|
359
|
+
if result:
|
|
360
|
+
return {
|
|
361
|
+
"busy": result[0], # Pages not checkpointed due to locks
|
|
362
|
+
"log": result[1], # Total pages in WAL
|
|
363
|
+
"checkpointed": result[2] # Pages successfully checkpointed
|
|
364
|
+
}
|
|
365
|
+
return {"busy": 0, "log": 0, "checkpointed": 0}
|
|
366
|
+
|
|
367
|
+
# ------------------------------------------------------------------
|
|
368
|
+
# Cleanup
|
|
369
|
+
# ------------------------------------------------------------------
|
|
370
|
+
def close_all(self) -> None:
|
|
371
|
+
"""Close and clear all pooled SQLite connections."""
|
|
372
|
+
logger.info("Closing all SQLite connections")
|
|
373
|
+
with self._lock:
|
|
374
|
+
# Close every tracked connection (both pooled and in-use)
|
|
375
|
+
for conn in list(self._all_conns):
|
|
376
|
+
try:
|
|
377
|
+
# Get the underlying connection if this is a proxy
|
|
378
|
+
underlying_conn = getattr(conn, '_conn', conn)
|
|
379
|
+
underlying_conn.close()
|
|
380
|
+
except Exception as e:
|
|
381
|
+
logger.warning(f"Error closing connection: {e}")
|
|
382
|
+
|
|
383
|
+
# Clear the pool completely
|
|
384
|
+
while not self._pool.empty():
|
|
385
|
+
try:
|
|
386
|
+
self._pool.get_nowait()
|
|
387
|
+
except queue.Empty:
|
|
388
|
+
break
|
|
389
|
+
|
|
390
|
+
# Reset internal structures - leave pool empty for testing
|
|
391
|
+
self._all_conns.clear()
|
|
392
|
+
self._pool = queue.Queue(self.max_size)
|
|
393
|
+
self._wait_count = 0
|
|
394
|
+
self._is_closed = True
|
|
395
|
+
|
|
396
|
+
logger.debug("All SQLite connections closed and pool reset")
|
|
397
|
+
|
|
398
|
+
def _atexit_cleanup(self):
|
|
399
|
+
try:
|
|
400
|
+
self.close_all()
|
|
401
|
+
logger.debug("SQLiteConnectionPool auto-cleanup complete")
|
|
402
|
+
except Exception as e:
|
|
403
|
+
logger.warning(f"Atexit cleanup failed: {e}")
|
|
@@ -0,0 +1,225 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: sql3-lite-saver
|
|
3
|
+
Version: 0.99.0
|
|
4
|
+
Summary: A Jedi-approved SQLite connection pool with WAL, retries, and multi-thread/process safety.
|
|
5
|
+
Author: apresence
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/apresence/sql3-lite-saver
|
|
8
|
+
Project-URL: Repository, https://github.com/apresence/sql3-lite-saver
|
|
9
|
+
Project-URL: Issues, https://github.com/apresence/sql3-lite-saver/issues
|
|
10
|
+
Project-URL: Changelog, https://github.com/apresence/sql3-lite-saver/blob/main/CHANGELOG.md
|
|
11
|
+
Keywords: sqlite,connection-pool,database,wal,concurrency
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Classifier: Topic :: Database
|
|
21
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
22
|
+
Requires-Python: >=3.9
|
|
23
|
+
Description-Content-Type: text/markdown
|
|
24
|
+
License-File: LICENSE
|
|
25
|
+
Provides-Extra: retry
|
|
26
|
+
Requires-Dist: tenacity>=8.0; extra == "retry"
|
|
27
|
+
Provides-Extra: dev
|
|
28
|
+
Requires-Dist: ruff>=0.5; extra == "dev"
|
|
29
|
+
Requires-Dist: pytest>=8.0; extra == "dev"
|
|
30
|
+
Requires-Dist: twine>=5.0; extra == "dev"
|
|
31
|
+
Requires-Dist: build>=1.2.1; extra == "dev"
|
|
32
|
+
Dynamic: license-file
|
|
33
|
+
|
|
34
|
+
# sql3-lite-saver
|
|
35
|
+
|
|
36
|
+
[](https://github.com/apresence/sql3-lite-saver/actions/workflows/ci.yml)
|
|
37
|
+
[](https://opensource.org/licenses/MIT)
|
|
38
|
+
[](https://www.python.org/downloads/)
|
|
39
|
+
|
|
40
|
+
*A Jedi-approved SQLite connection pool, strong in the **WAL** side of the Force.*
|
|
41
|
+
|
|
42
|
+
> We make your Bad Batch good.
|
|
43
|
+
|
|
44
|
+
---
|
|
45
|
+
|
|
46
|
+
## Why You Need a Connection Pool
|
|
47
|
+
|
|
48
|
+
SQLite is fast and embedded, but it isn't built for many simultaneous writers.
|
|
49
|
+
Without pooling, each connection must reinitialize SQLite state, and you'll quickly see:
|
|
50
|
+
|
|
51
|
+
> `sqlite3.OperationalError: database is locked`
|
|
52
|
+
|
|
53
|
+
**sql3-lite-saver** helps by:
|
|
54
|
+
- Reusing a fixed number of open connections
|
|
55
|
+
- Enabling WAL (Write-Ahead Logging) automatically
|
|
56
|
+
- Retrying transparently with exponential backoff (optional Tenacity support)
|
|
57
|
+
- Supporting multi-threaded and multi-process workloads safely
|
|
58
|
+
|
|
59
|
+
---
|
|
60
|
+
|
|
61
|
+
## ⚠️ IMPORTANT: WAL Checkpoint Maintenance
|
|
62
|
+
|
|
63
|
+
**If you use WAL mode (enabled by default), you MUST run periodic checkpoints** or your database files can bloat over time.
|
|
64
|
+
|
|
65
|
+
Without checkpoints:
|
|
66
|
+
- WAL file can grow unbounded
|
|
67
|
+
- Read performance can steadily degrade
|
|
68
|
+
|
|
69
|
+
See the [WAL Checkpoint Management](#wal-checkpoint-management) section below for details.
|
|
70
|
+
|
|
71
|
+
---
|
|
72
|
+
|
|
73
|
+
## Installation
|
|
74
|
+
|
|
75
|
+
### From PyPI
|
|
76
|
+
|
|
77
|
+
```bash
|
|
78
|
+
pip install sql3-lite-saver
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
### With optional retry engine (Tenacity)
|
|
82
|
+
|
|
83
|
+
```bash
|
|
84
|
+
pip install sql3-lite-saver[retry]
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
### For development (editable)
|
|
88
|
+
|
|
89
|
+
```bash
|
|
90
|
+
pip install -e .[dev,retry]
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
### Extras explained
|
|
94
|
+
|
|
95
|
+
| Extra | Purpose | What's Included |
|
|
96
|
+
|------|---------|------------------|
|
|
97
|
+
| `retry` | Adds [Tenacity](https://tenacity.readthedocs.io) for advanced retry control | `tenacity>=8.0` |
|
|
98
|
+
| `dev` | Developer tools | `ruff`, `pytest` (optional), `twine`, `build` |
|
|
99
|
+
|
|
100
|
+
---
|
|
101
|
+
|
|
102
|
+
## Example
|
|
103
|
+
|
|
104
|
+
```python
|
|
105
|
+
from pathlib import Path
|
|
106
|
+
from sql3_lite_saver import SQLiteConnectionPool
|
|
107
|
+
|
|
108
|
+
pool = SQLiteConnectionPool(Path("app.db"), max_size=3)
|
|
109
|
+
|
|
110
|
+
with pool.acquire() as conn:
|
|
111
|
+
conn.execute("CREATE TABLE IF NOT EXISTS demo (id, msg)")
|
|
112
|
+
conn.execute("INSERT INTO demo VALUES (?, ?)", (1, "hello there"))
|
|
113
|
+
|
|
114
|
+
with pool.acquire() as conn:
|
|
115
|
+
print([dict(r) for r in conn.execute("SELECT * FROM demo").fetchall()])
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
---
|
|
119
|
+
|
|
120
|
+
## Advanced Retry with Tenacity
|
|
121
|
+
|
|
122
|
+
When you install Tenacity (`pip install sql3-lite-saver[retry]`), **sql3-lite-saver** can use it for retry logic.
|
|
123
|
+
You can fine-tune retry behavior with parameters:
|
|
124
|
+
|
|
125
|
+
```python
|
|
126
|
+
pool = SQLiteConnectionPool(
|
|
127
|
+
Path("app.db"),
|
|
128
|
+
enable_retry=True,
|
|
129
|
+
retry_attempts=8,
|
|
130
|
+
base_delay=0.5,
|
|
131
|
+
retry_jitter=0.25,
|
|
132
|
+
)
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
Without Tenacity installed, it falls back to built-in exponential backoff (`1s, 2s, 4s, ...` up to 60s).
|
|
136
|
+
|
|
137
|
+
---
|
|
138
|
+
|
|
139
|
+
## WAL Checkpoint Management
|
|
140
|
+
|
|
141
|
+
### Why Checkpoints Matter
|
|
142
|
+
|
|
143
|
+
When using WAL (Write-Ahead Logging), SQLite writes go to a **separate file** (`app.db-wal`) instead of the main database (`app.db`). **Without regular checkpoints**, the WAL file can grow indefinitely, causing:
|
|
144
|
+
|
|
145
|
+
- **Disk space bloat** - WAL can balloon to gigabytes
|
|
146
|
+
- **Degraded read performance** - SQLite may scan a larger WAL on reads
|
|
147
|
+
- **File system issues** - Very large WAL files can cause problems
|
|
148
|
+
|
|
149
|
+
Checkpoints transfer WAL data from `app.db-wal` back to the main `app.db` file, resetting the WAL.
|
|
150
|
+
|
|
151
|
+
**Additional database files created by WAL mode:**
|
|
152
|
+
- `app.db-wal` - Write-Ahead Log
|
|
153
|
+
- `app.db-shm` - Shared memory for coordination
|
|
154
|
+
|
|
155
|
+
### Checkpoint Modes
|
|
156
|
+
|
|
157
|
+
```python
|
|
158
|
+
from pathlib import Path
|
|
159
|
+
from sql3_lite_saver import SQLiteConnectionPool
|
|
160
|
+
|
|
161
|
+
pool = SQLiteConnectionPool(Path("app.db"))
|
|
162
|
+
|
|
163
|
+
# PASSIVE (default) - Non-blocking, checkpoint what you can
|
|
164
|
+
result = pool.checkpoint("PASSIVE")
|
|
165
|
+
|
|
166
|
+
# FULL - Checkpoint all frames, may block briefly
|
|
167
|
+
result = pool.checkpoint("FULL")
|
|
168
|
+
|
|
169
|
+
# RESTART - Like FULL, then reset WAL for reuse
|
|
170
|
+
result = pool.checkpoint("RESTART")
|
|
171
|
+
|
|
172
|
+
# TRUNCATE - Like RESTART, then shrink WAL to zero bytes (reclaim disk space)
|
|
173
|
+
result = pool.checkpoint("TRUNCATE")
|
|
174
|
+
|
|
175
|
+
print(result)
|
|
176
|
+
# {'busy': 0, 'log': 1024, 'checkpointed': 1024}
|
|
177
|
+
# busy=0 means full success, >0 means some pages couldn't be checkpointed
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
### When to Checkpoint
|
|
181
|
+
|
|
182
|
+
- **Periodic background task** - Every hour with `PASSIVE` (or `TRUNCATE` if you want to reclaim space)
|
|
183
|
+
- **Before backups** - Use `TRUNCATE` to minimize backup size
|
|
184
|
+
- **Low-traffic periods** - `TRUNCATE` during maintenance windows
|
|
185
|
+
- **On application shutdown** - Final `TRUNCATE` to clean up
|
|
186
|
+
|
|
187
|
+
### References
|
|
188
|
+
|
|
189
|
+
- [SQLite WAL Mode Documentation](https://www.sqlite.org/wal.html)
|
|
190
|
+
- [PRAGMA wal_checkpoint](https://www.sqlite.org/pragma.html#pragma_wal_checkpoint)
|
|
191
|
+
- [Performance tuning with checkpoints](https://www.sqlite.org/wal.html#performance_considerations)
|
|
192
|
+
|
|
193
|
+
---
|
|
194
|
+
|
|
195
|
+
## Testing
|
|
196
|
+
|
|
197
|
+
If you use `pytest`:
|
|
198
|
+
|
|
199
|
+
```bash
|
|
200
|
+
pytest -q
|
|
201
|
+
```
|
|
202
|
+
|
|
203
|
+
Or with the standard library `unittest`:
|
|
204
|
+
|
|
205
|
+
```bash
|
|
206
|
+
python -m unittest discover -s tests -v
|
|
207
|
+
```
|
|
208
|
+
|
|
209
|
+
---
|
|
210
|
+
|
|
211
|
+
## Developer Shortcuts
|
|
212
|
+
|
|
213
|
+
```bash
|
|
214
|
+
make install-dev # editable install with dev + retry deps
|
|
215
|
+
make install-prod # editable install (minimal)
|
|
216
|
+
make lint # run Ruff
|
|
217
|
+
make test # run tests
|
|
218
|
+
make release # build + twine upload
|
|
219
|
+
```
|
|
220
|
+
|
|
221
|
+
---
|
|
222
|
+
|
|
223
|
+
## License
|
|
224
|
+
|
|
225
|
+
MIT © 2025 [@apresence](https://github.com/apresence)
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
sql3_lite_saver/__init__.py,sha256=pE4MDs5PJiBOD7VE-GNZ53HTLXKAbGo-sBBiQH-V_Sg,171
|
|
2
|
+
sql3_lite_saver/pool.py,sha256=64ROHI34OXoGCc9xNPPA_tVCXtIzR8hf_srwPNIyFpQ,15263
|
|
3
|
+
sql3_lite_saver-0.99.0.dist-info/licenses/LICENSE,sha256=DQFG_5PpUyXHfqHn5nUsApsSy2aUZmAwxuZTRT4EpN4,1068
|
|
4
|
+
sql3_lite_saver-0.99.0.dist-info/METADATA,sha256=KL5nHjcKyh_bcRjZDkWCCNKrpkVknO5uLEFWMD4A_bw,6679
|
|
5
|
+
sql3_lite_saver-0.99.0.dist-info/WHEEL,sha256=YLJXdYXQ2FQ0Uqn2J-6iEIC-3iOey8lH3xCtvFLkd8Q,91
|
|
6
|
+
sql3_lite_saver-0.99.0.dist-info/top_level.txt,sha256=MNWQgoR6ERVy_a5KjxkiirMGLFA5XKEvIEOEGepguRA,16
|
|
7
|
+
sql3_lite_saver-0.99.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
|
|
2
|
+
MIT License
|
|
3
|
+
|
|
4
|
+
Copyright (c) 2025 A Presence
|
|
5
|
+
|
|
6
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
7
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
8
|
+
in the Software without restriction, including without limitation the rights
|
|
9
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
10
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
11
|
+
furnished to do so, subject to the following conditions:
|
|
12
|
+
|
|
13
|
+
The above copyright notice and this permission notice shall be included in all
|
|
14
|
+
copies or substantial portions of the Software.
|
|
15
|
+
|
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
17
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
18
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
19
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
20
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
21
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
22
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
sql3_lite_saver
|