dbworker 0.1.1__tar.gz → 0.1.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dbworker-0.1.1 → dbworker-0.1.2}/PKG-INFO +11 -11
- {dbworker-0.1.1 → dbworker-0.1.2}/README.md +10 -10
- {dbworker-0.1.1 → dbworker-0.1.2}/pyproject.toml +1 -1
- {dbworker-0.1.1 → dbworker-0.1.2}/src/dbworker.py +46 -10
- {dbworker-0.1.1 → dbworker-0.1.2}/LICENSE +0 -0
- {dbworker-0.1.1 → dbworker-0.1.2}/poetry.lock +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dbworker
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.2
|
|
4
4
|
Summary: Portable, single-script Python/SQLAlchemy background worker for SQLite, PostgreSQL and MySQL — no Redis, broker or enqueue
|
|
5
5
|
License: MIT
|
|
6
6
|
License-File: LICENSE
|
|
@@ -94,16 +94,16 @@ Median wall time in seconds; **bold** marks the faster backend. Each experiment
|
|
|
94
94
|
|
|
95
95
|
| Experiment | Workload | Celery wall (s) | DBWorker wall (s) | C / D | Task P95 / peak outstanding (C; D) |
|
|
96
96
|
|---|---|---:|---:|---:|---|
|
|
97
|
-
| imagededup | Build 1,000 image hashes (one bulk import) | **5.
|
|
98
|
-
| imagededup | Compare 1,000 images / 999,000 directed pairs |
|
|
99
|
-
| imagededup | Build + compare 1,000 images / 999,000 directed pairs |
|
|
100
|
-
| superset | SQL Lab: 3,000 queries / 10,000 rows | **
|
|
101
|
-
| saleor | Export: 1,000 x 256 products + 1,000 emails | **
|
|
102
|
-
| paperless_ngx | OCR: 200 scans, archive and index |
|
|
103
|
-
| posthog | 2FA: 5,000 requests / 10,000 tasks |
|
|
104
|
-
| sentry | Email: 2,000 requests / 4,000 deliveries | **60.
|
|
105
|
-
|
|
106
|
-
Run: `github-
|
|
97
|
+
| imagededup | Build 1,000 image hashes (one bulk import) | **5.781** | 6.066 | 0.95 | n/a; n/a |
|
|
98
|
+
| imagededup | Compare 1,000 images / 999,000 directed pairs | 82.087 | **77.527** | 1.06 | n/a; n/a |
|
|
99
|
+
| imagededup | Build + compare 1,000 images / 999,000 directed pairs | 91.669 | **90.453** | 1.01 | n/a; n/a |
|
|
100
|
+
| superset | SQL Lab: 3,000 queries / 10,000 rows | **130.809** | 146.255 | 0.89 | 34.381s / 1384; 41.335s / 1494 |
|
|
101
|
+
| saleor | Export: 1,000 x 256 products + 1,000 emails | **132.599** | 160.993 | 0.82 | 35.885s / 606; 48.863s / 792 |
|
|
102
|
+
| paperless_ngx | OCR: 200 scans, archive and index | 134.649 | **75.436** | 1.78 | 71.852s / 114; 47.910s / 51 |
|
|
103
|
+
| posthog | 2FA: 5,000 requests / 10,000 tasks | 115.049 | **110.402** | 1.04 | 0.109s / 15; 0.594s / 65 |
|
|
104
|
+
| sentry | Email: 2,000 requests / 4,000 deliveries | **60.263** | 60.300 | 1.00 | 0.033s / 20; 0.147s / 26 |
|
|
105
|
+
|
|
106
|
+
Run: `github-38214437264-1`. [Details and scope](doc/benchmark-results.md) · [JSON results](benchmarks/results/latest/index.json).
|
|
107
107
|
Scoped application workloads; Sentry uses historical 24.1.0. Wall time includes the fixed submission window, so a ratio near 1 does not prove equal processing capacity. Task P95 starts at publication; peak outstanding counts already published tasks. Ratios above 1 favor DBWorker; no cross-project average is computed.
|
|
108
108
|
<!-- benchmark-results:end -->
|
|
109
109
|
|
|
@@ -77,15 +77,15 @@ Median wall time in seconds; **bold** marks the faster backend. Each experiment
|
|
|
77
77
|
|
|
78
78
|
| Experiment | Workload | Celery wall (s) | DBWorker wall (s) | C / D | Task P95 / peak outstanding (C; D) |
|
|
79
79
|
|---|---|---:|---:|---:|---|
|
|
80
|
-
| imagededup | Build 1,000 image hashes (one bulk import) | **5.
|
|
81
|
-
| imagededup | Compare 1,000 images / 999,000 directed pairs |
|
|
82
|
-
| imagededup | Build + compare 1,000 images / 999,000 directed pairs |
|
|
83
|
-
| superset | SQL Lab: 3,000 queries / 10,000 rows | **
|
|
84
|
-
| saleor | Export: 1,000 x 256 products + 1,000 emails | **
|
|
85
|
-
| paperless_ngx | OCR: 200 scans, archive and index |
|
|
86
|
-
| posthog | 2FA: 5,000 requests / 10,000 tasks |
|
|
87
|
-
| sentry | Email: 2,000 requests / 4,000 deliveries | **60.
|
|
88
|
-
|
|
89
|
-
Run: `github-
|
|
80
|
+
| imagededup | Build 1,000 image hashes (one bulk import) | **5.781** | 6.066 | 0.95 | n/a; n/a |
|
|
81
|
+
| imagededup | Compare 1,000 images / 999,000 directed pairs | 82.087 | **77.527** | 1.06 | n/a; n/a |
|
|
82
|
+
| imagededup | Build + compare 1,000 images / 999,000 directed pairs | 91.669 | **90.453** | 1.01 | n/a; n/a |
|
|
83
|
+
| superset | SQL Lab: 3,000 queries / 10,000 rows | **130.809** | 146.255 | 0.89 | 34.381s / 1384; 41.335s / 1494 |
|
|
84
|
+
| saleor | Export: 1,000 x 256 products + 1,000 emails | **132.599** | 160.993 | 0.82 | 35.885s / 606; 48.863s / 792 |
|
|
85
|
+
| paperless_ngx | OCR: 200 scans, archive and index | 134.649 | **75.436** | 1.78 | 71.852s / 114; 47.910s / 51 |
|
|
86
|
+
| posthog | 2FA: 5,000 requests / 10,000 tasks | 115.049 | **110.402** | 1.04 | 0.109s / 15; 0.594s / 65 |
|
|
87
|
+
| sentry | Email: 2,000 requests / 4,000 deliveries | **60.263** | 60.300 | 1.00 | 0.033s / 20; 0.147s / 26 |
|
|
88
|
+
|
|
89
|
+
Run: `github-38214437264-1`. [Details and scope](doc/benchmark-results.md) · [JSON results](benchmarks/results/latest/index.json).
|
|
90
90
|
Scoped application workloads; Sentry uses historical 24.1.0. Wall time includes the fixed submission window, so a ratio near 1 does not prove equal processing capacity. Task P95 starts at publication; peak outstanding counts already published tasks. Ratios above 1 favor DBWorker; no cross-project average is computed.
|
|
91
91
|
<!-- benchmark-results:end -->
|
|
@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
|
|
|
4
4
|
|
|
5
5
|
[tool.poetry]
|
|
6
6
|
name = "dbworker"
|
|
7
|
-
version = "0.1.
|
|
7
|
+
version = "0.1.2"
|
|
8
8
|
description = "Portable, single-script Python/SQLAlchemy background worker for SQLite, PostgreSQL and MySQL — no Redis, broker or enqueue"
|
|
9
9
|
authors = ["Ziyang Song"]
|
|
10
10
|
license = "MIT"
|
|
@@ -6,6 +6,7 @@ import logging
|
|
|
6
6
|
import multiprocessing
|
|
7
7
|
import pickle # nosec B403 -- used only for local pre-flight serialization; this module never deserializes pickle data.
|
|
8
8
|
import re
|
|
9
|
+
import sqlite3
|
|
9
10
|
import threading
|
|
10
11
|
import time
|
|
11
12
|
import uuid
|
|
@@ -19,15 +20,17 @@ from typing import Any, TypeAlias, TypeVar, cast
|
|
|
19
20
|
from multiprocessing.util import Finalize
|
|
20
21
|
|
|
21
22
|
from sqlalchemy import Column, DateTime, Enum, ForeignKey, String, Table, Text, and_, create_engine, event, exists, inspect, insert, or_, select, tuple_, update
|
|
22
|
-
from sqlalchemy.exc import IntegrityError
|
|
23
|
+
from sqlalchemy.exc import IntegrityError, OperationalError
|
|
23
24
|
from sqlalchemy.engine import CursorResult, RowMapping, URL, make_url
|
|
24
25
|
from sqlalchemy.dialects.postgresql import insert as postgresql_insert
|
|
25
26
|
from sqlalchemy.orm import DeclarativeBase, Mapper, Session, sessionmaker
|
|
26
|
-
from sqlalchemy.sql import ColumnElement, Select
|
|
27
|
+
from sqlalchemy.sql import ColumnElement, Select, Update
|
|
27
28
|
from sqlalchemy.sql.selectable import Exists
|
|
28
29
|
|
|
29
30
|
logger = logging.getLogger(__name__)
|
|
30
31
|
|
|
32
|
+
_SQLITE_FINALIZATION_RETRY_SECONDS = 5.0
|
|
33
|
+
|
|
31
34
|
|
|
32
35
|
def now() -> datetime:
|
|
33
36
|
# Database columns use naive UTC on every supported backend.
|
|
@@ -170,6 +173,37 @@ def _execute_in_process(claim: Claim) -> Outcome:
|
|
|
170
173
|
return _execute_claim(_process_worker, claim, _process_session_factory)
|
|
171
174
|
|
|
172
175
|
|
|
176
|
+
def _retry_sqlite_completion(session: Session, completion: Update, error: OperationalError) -> CursorResult[Any]:
|
|
177
|
+
"""Wait for a writer without discarding business writes or replaying a handler."""
|
|
178
|
+
deadline = time.monotonic() + _SQLITE_FINALIZATION_RETRY_SECONDS
|
|
179
|
+
delay = 0.01
|
|
180
|
+
while True:
|
|
181
|
+
# A busy SQLite statement leaves the transaction intact. A stale WAL
|
|
182
|
+
# snapshot cannot become writable by waiting (SQLITE_BUSY_SNAPSHOT).
|
|
183
|
+
if (not isinstance(error.orig, sqlite3.OperationalError)
|
|
184
|
+
or getattr(error.orig, "sqlite_errorcode", None) not in (
|
|
185
|
+
sqlite3.SQLITE_BUSY, sqlite3.SQLITE_BUSY_RECOVERY, sqlite3.SQLITE_BUSY_TIMEOUT,
|
|
186
|
+
)
|
|
187
|
+
or error.connection_invalidated or not session.is_active):
|
|
188
|
+
raise error
|
|
189
|
+
connection = session.connection()
|
|
190
|
+
driver = connection.connection.driver_connection
|
|
191
|
+
if (connection.dialect.name != "sqlite"
|
|
192
|
+
or driver is None or not driver.in_transaction):
|
|
193
|
+
raise error
|
|
194
|
+
remaining = deadline - time.monotonic()
|
|
195
|
+
if remaining <= 0:
|
|
196
|
+
raise error
|
|
197
|
+
time.sleep(min(delay, remaining))
|
|
198
|
+
if time.monotonic() >= deadline:
|
|
199
|
+
raise error
|
|
200
|
+
delay = min(delay * 2, 0.1)
|
|
201
|
+
try:
|
|
202
|
+
return cast(CursorResult[Any], session.execute(completion))
|
|
203
|
+
except OperationalError as exc:
|
|
204
|
+
error = exc
|
|
205
|
+
|
|
206
|
+
|
|
173
207
|
def _execute_claim(worker: _Worker, claim: Claim, session_factory: sessionmaker[Session]) -> Outcome:
|
|
174
208
|
"""Run a whole handler and atomically persist its writes and outcome."""
|
|
175
209
|
with session_factory() as session:
|
|
@@ -187,14 +221,16 @@ def _execute_claim(worker: _Worker, claim: Claim, session_factory: sessionmaker[
|
|
|
187
221
|
# All handler writes, including already-flushed SQL, are still in
|
|
188
222
|
# this transaction. Rejecting ownership rolls them all back.
|
|
189
223
|
with session.no_autoflush:
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
224
|
+
completion = update(worker.table).where(
|
|
225
|
+
worker.table.c.source_id == claim.source_id,
|
|
226
|
+
worker.table.c.execution_status == ExecutionStatus.WORKING,
|
|
227
|
+
worker.table.c.claim_token == claim.token,
|
|
228
|
+
).values(execution_status=ExecutionStatus.FINISHED if isinstance(outcome, Finished) else ExecutionStatus.UNFINISHED,
|
|
229
|
+
claim_token=None, lease_expires_at=None, error=None)
|
|
230
|
+
try:
|
|
231
|
+
result = cast(CursorResult[Any], session.execute(completion))
|
|
232
|
+
except OperationalError as exc:
|
|
233
|
+
result = _retry_sqlite_completion(session, completion, exc)
|
|
198
234
|
if result.rowcount != 1:
|
|
199
235
|
raise LostClaim()
|
|
200
236
|
finally:
|
|
File without changes
|
|
File without changes
|