dbworker 0.1.0__tar.gz → 0.1.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dbworker-0.1.0 → dbworker-0.1.2}/PKG-INFO +11 -11
- {dbworker-0.1.0 → dbworker-0.1.2}/README.md +10 -10
- {dbworker-0.1.0 → dbworker-0.1.2}/pyproject.toml +1 -1
- {dbworker-0.1.0 → dbworker-0.1.2}/src/dbworker.py +111 -23
- {dbworker-0.1.0 → dbworker-0.1.2}/LICENSE +0 -0
- {dbworker-0.1.0 → dbworker-0.1.2}/poetry.lock +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dbworker
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.2
|
|
4
4
|
Summary: Portable, single-script Python/SQLAlchemy background worker for SQLite, PostgreSQL and MySQL — no Redis, broker or enqueue
|
|
5
5
|
License: MIT
|
|
6
6
|
License-File: LICENSE
|
|
@@ -94,16 +94,16 @@ Median wall time in seconds; **bold** marks the faster backend. Each experiment
|
|
|
94
94
|
|
|
95
95
|
| Experiment | Workload | Celery wall (s) | DBWorker wall (s) | C / D | Task P95 / peak outstanding (C; D) |
|
|
96
96
|
|---|---|---:|---:|---:|---|
|
|
97
|
-
| imagededup | Build 1,000 image hashes (one bulk import) | **
|
|
98
|
-
| imagededup | Compare 1,000 images / 999,000 directed pairs |
|
|
99
|
-
| imagededup | Build + compare 1,000 images / 999,000 directed pairs |
|
|
100
|
-
| superset | SQL Lab: 3,000 queries / 10,000 rows | **
|
|
101
|
-
| saleor | Export: 1,000 x 256 products + 1,000 emails | **
|
|
102
|
-
| paperless_ngx | OCR: 200 scans, archive and index |
|
|
103
|
-
| posthog | 2FA: 5,000 requests / 10,000 tasks |
|
|
104
|
-
| sentry | Email: 2,000 requests / 4,000 deliveries | 60.
|
|
105
|
-
|
|
106
|
-
Run: `github-
|
|
97
|
+
| imagededup | Build 1,000 image hashes (one bulk import) | **5.781** | 6.066 | 0.95 | n/a; n/a |
|
|
98
|
+
| imagededup | Compare 1,000 images / 999,000 directed pairs | 82.087 | **77.527** | 1.06 | n/a; n/a |
|
|
99
|
+
| imagededup | Build + compare 1,000 images / 999,000 directed pairs | 91.669 | **90.453** | 1.01 | n/a; n/a |
|
|
100
|
+
| superset | SQL Lab: 3,000 queries / 10,000 rows | **130.809** | 146.255 | 0.89 | 34.381s / 1384; 41.335s / 1494 |
|
|
101
|
+
| saleor | Export: 1,000 x 256 products + 1,000 emails | **132.599** | 160.993 | 0.82 | 35.885s / 606; 48.863s / 792 |
|
|
102
|
+
| paperless_ngx | OCR: 200 scans, archive and index | 134.649 | **75.436** | 1.78 | 71.852s / 114; 47.910s / 51 |
|
|
103
|
+
| posthog | 2FA: 5,000 requests / 10,000 tasks | 115.049 | **110.402** | 1.04 | 0.109s / 15; 0.594s / 65 |
|
|
104
|
+
| sentry | Email: 2,000 requests / 4,000 deliveries | **60.263** | 60.300 | 1.00 | 0.033s / 20; 0.147s / 26 |
|
|
105
|
+
|
|
106
|
+
Run: `github-38214437264-1`. [Details and scope](doc/benchmark-results.md) · [JSON results](benchmarks/results/latest/index.json).
|
|
107
107
|
Scoped application workloads; Sentry uses historical 24.1.0. Wall time includes the fixed submission window, so a ratio near 1 does not prove equal processing capacity. Task P95 starts at publication; peak outstanding counts already published tasks. Ratios above 1 favor DBWorker; no cross-project average is computed.
|
|
108
108
|
<!-- benchmark-results:end -->
|
|
109
109
|
|
|
@@ -77,15 +77,15 @@ Median wall time in seconds; **bold** marks the faster backend. Each experiment
|
|
|
77
77
|
|
|
78
78
|
| Experiment | Workload | Celery wall (s) | DBWorker wall (s) | C / D | Task P95 / peak outstanding (C; D) |
|
|
79
79
|
|---|---|---:|---:|---:|---|
|
|
80
|
-
| imagededup | Build 1,000 image hashes (one bulk import) | **
|
|
81
|
-
| imagededup | Compare 1,000 images / 999,000 directed pairs |
|
|
82
|
-
| imagededup | Build + compare 1,000 images / 999,000 directed pairs |
|
|
83
|
-
| superset | SQL Lab: 3,000 queries / 10,000 rows | **
|
|
84
|
-
| saleor | Export: 1,000 x 256 products + 1,000 emails | **
|
|
85
|
-
| paperless_ngx | OCR: 200 scans, archive and index |
|
|
86
|
-
| posthog | 2FA: 5,000 requests / 10,000 tasks |
|
|
87
|
-
| sentry | Email: 2,000 requests / 4,000 deliveries | 60.
|
|
88
|
-
|
|
89
|
-
Run: `github-
|
|
80
|
+
| imagededup | Build 1,000 image hashes (one bulk import) | **5.781** | 6.066 | 0.95 | n/a; n/a |
|
|
81
|
+
| imagededup | Compare 1,000 images / 999,000 directed pairs | 82.087 | **77.527** | 1.06 | n/a; n/a |
|
|
82
|
+
| imagededup | Build + compare 1,000 images / 999,000 directed pairs | 91.669 | **90.453** | 1.01 | n/a; n/a |
|
|
83
|
+
| superset | SQL Lab: 3,000 queries / 10,000 rows | **130.809** | 146.255 | 0.89 | 34.381s / 1384; 41.335s / 1494 |
|
|
84
|
+
| saleor | Export: 1,000 x 256 products + 1,000 emails | **132.599** | 160.993 | 0.82 | 35.885s / 606; 48.863s / 792 |
|
|
85
|
+
| paperless_ngx | OCR: 200 scans, archive and index | 134.649 | **75.436** | 1.78 | 71.852s / 114; 47.910s / 51 |
|
|
86
|
+
| posthog | 2FA: 5,000 requests / 10,000 tasks | 115.049 | **110.402** | 1.04 | 0.109s / 15; 0.594s / 65 |
|
|
87
|
+
| sentry | Email: 2,000 requests / 4,000 deliveries | **60.263** | 60.300 | 1.00 | 0.033s / 20; 0.147s / 26 |
|
|
88
|
+
|
|
89
|
+
Run: `github-38214437264-1`. [Details and scope](doc/benchmark-results.md) · [JSON results](benchmarks/results/latest/index.json).
|
|
90
90
|
Scoped application workloads; Sentry uses historical 24.1.0. Wall time includes the fixed submission window, so a ratio near 1 does not prove equal processing capacity. Task P95 starts at publication; peak outstanding counts already published tasks. Ratios above 1 favor DBWorker; no cross-project average is computed.
|
|
91
91
|
<!-- benchmark-results:end -->
|
|
@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
|
|
|
4
4
|
|
|
5
5
|
[tool.poetry]
|
|
6
6
|
name = "dbworker"
|
|
7
|
-
version = "0.1.
|
|
7
|
+
version = "0.1.2"
|
|
8
8
|
description = "Portable, single-script Python/SQLAlchemy background worker for SQLite, PostgreSQL and MySQL — no Redis, broker or enqueue"
|
|
9
9
|
authors = ["Ziyang Song"]
|
|
10
10
|
license = "MIT"
|
|
@@ -6,6 +6,7 @@ import logging
|
|
|
6
6
|
import multiprocessing
|
|
7
7
|
import pickle # nosec B403 -- used only for local pre-flight serialization; this module never deserializes pickle data.
|
|
8
8
|
import re
|
|
9
|
+
import sqlite3
|
|
9
10
|
import threading
|
|
10
11
|
import time
|
|
11
12
|
import uuid
|
|
@@ -19,14 +20,17 @@ from typing import Any, TypeAlias, TypeVar, cast
|
|
|
19
20
|
from multiprocessing.util import Finalize
|
|
20
21
|
|
|
21
22
|
from sqlalchemy import Column, DateTime, Enum, ForeignKey, String, Table, Text, and_, create_engine, event, exists, inspect, insert, or_, select, tuple_, update
|
|
22
|
-
from sqlalchemy.exc import IntegrityError
|
|
23
|
+
from sqlalchemy.exc import IntegrityError, OperationalError
|
|
23
24
|
from sqlalchemy.engine import CursorResult, RowMapping, URL, make_url
|
|
25
|
+
from sqlalchemy.dialects.postgresql import insert as postgresql_insert
|
|
24
26
|
from sqlalchemy.orm import DeclarativeBase, Mapper, Session, sessionmaker
|
|
25
|
-
from sqlalchemy.sql import ColumnElement, Select
|
|
27
|
+
from sqlalchemy.sql import ColumnElement, Select, Update
|
|
26
28
|
from sqlalchemy.sql.selectable import Exists
|
|
27
29
|
|
|
28
30
|
logger = logging.getLogger(__name__)
|
|
29
31
|
|
|
32
|
+
_SQLITE_FINALIZATION_RETRY_SECONDS = 5.0
|
|
33
|
+
|
|
30
34
|
|
|
31
35
|
def now() -> datetime:
|
|
32
36
|
# Database columns use naive UTC on every supported backend.
|
|
@@ -140,17 +144,27 @@ def _initialize_process(
|
|
|
140
144
|
engine_options: dict[str, Any],
|
|
141
145
|
definitions: tuple[_WorkerDefinition, ...],
|
|
142
146
|
worker_name: str,
|
|
147
|
+
ready: Any,
|
|
143
148
|
) -> None:
|
|
144
149
|
global _process_worker, _process_session_factory
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
150
|
+
try:
|
|
151
|
+
engine = create_engine(database_url, **engine_options)
|
|
152
|
+
_process_session_factory = sessionmaker(engine, expire_on_commit=False)
|
|
153
|
+
Finalize(None, engine.dispose, exitpriority=10)
|
|
154
|
+
# Rebuild registered work-table metadata locally, including dependencies used
|
|
155
|
+
# by application queries. Eligibility callbacks stay in the parent process.
|
|
156
|
+
for definition in definitions:
|
|
157
|
+
worker = _Worker(name=definition.name, source=definition.source, handler=definition.handler)
|
|
158
|
+
if definition.name == worker_name:
|
|
159
|
+
_process_worker = worker
|
|
160
|
+
ready.wait(timeout=180)
|
|
161
|
+
except BaseException:
|
|
162
|
+
ready.abort()
|
|
163
|
+
raise
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def _process_ready() -> None:
|
|
167
|
+
"""Start the executor without claiming or executing an application job."""
|
|
154
168
|
|
|
155
169
|
|
|
156
170
|
def _execute_in_process(claim: Claim) -> Outcome:
|
|
@@ -159,6 +173,37 @@ def _execute_in_process(claim: Claim) -> Outcome:
|
|
|
159
173
|
return _execute_claim(_process_worker, claim, _process_session_factory)
|
|
160
174
|
|
|
161
175
|
|
|
176
|
+
def _retry_sqlite_completion(session: Session, completion: Update, error: OperationalError) -> CursorResult[Any]:
|
|
177
|
+
"""Wait for a writer without discarding business writes or replaying a handler."""
|
|
178
|
+
deadline = time.monotonic() + _SQLITE_FINALIZATION_RETRY_SECONDS
|
|
179
|
+
delay = 0.01
|
|
180
|
+
while True:
|
|
181
|
+
# A busy SQLite statement leaves the transaction intact. A stale WAL
|
|
182
|
+
# snapshot cannot become writable by waiting (SQLITE_BUSY_SNAPSHOT).
|
|
183
|
+
if (not isinstance(error.orig, sqlite3.OperationalError)
|
|
184
|
+
or getattr(error.orig, "sqlite_errorcode", None) not in (
|
|
185
|
+
sqlite3.SQLITE_BUSY, sqlite3.SQLITE_BUSY_RECOVERY, sqlite3.SQLITE_BUSY_TIMEOUT,
|
|
186
|
+
)
|
|
187
|
+
or error.connection_invalidated or not session.is_active):
|
|
188
|
+
raise error
|
|
189
|
+
connection = session.connection()
|
|
190
|
+
driver = connection.connection.driver_connection
|
|
191
|
+
if (connection.dialect.name != "sqlite"
|
|
192
|
+
or driver is None or not driver.in_transaction):
|
|
193
|
+
raise error
|
|
194
|
+
remaining = deadline - time.monotonic()
|
|
195
|
+
if remaining <= 0:
|
|
196
|
+
raise error
|
|
197
|
+
time.sleep(min(delay, remaining))
|
|
198
|
+
if time.monotonic() >= deadline:
|
|
199
|
+
raise error
|
|
200
|
+
delay = min(delay * 2, 0.1)
|
|
201
|
+
try:
|
|
202
|
+
return cast(CursorResult[Any], session.execute(completion))
|
|
203
|
+
except OperationalError as exc:
|
|
204
|
+
error = exc
|
|
205
|
+
|
|
206
|
+
|
|
162
207
|
def _execute_claim(worker: _Worker, claim: Claim, session_factory: sessionmaker[Session]) -> Outcome:
|
|
163
208
|
"""Run a whole handler and atomically persist its writes and outcome."""
|
|
164
209
|
with session_factory() as session:
|
|
@@ -176,14 +221,16 @@ def _execute_claim(worker: _Worker, claim: Claim, session_factory: sessionmaker[
|
|
|
176
221
|
# All handler writes, including already-flushed SQL, are still in
|
|
177
222
|
# this transaction. Rejecting ownership rolls them all back.
|
|
178
223
|
with session.no_autoflush:
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
224
|
+
completion = update(worker.table).where(
|
|
225
|
+
worker.table.c.source_id == claim.source_id,
|
|
226
|
+
worker.table.c.execution_status == ExecutionStatus.WORKING,
|
|
227
|
+
worker.table.c.claim_token == claim.token,
|
|
228
|
+
).values(execution_status=ExecutionStatus.FINISHED if isinstance(outcome, Finished) else ExecutionStatus.UNFINISHED,
|
|
229
|
+
claim_token=None, lease_expires_at=None, error=None)
|
|
230
|
+
try:
|
|
231
|
+
result = cast(CursorResult[Any], session.execute(completion))
|
|
232
|
+
except OperationalError as exc:
|
|
233
|
+
result = _retry_sqlite_completion(session, completion, exc)
|
|
187
234
|
if result.rowcount != 1:
|
|
188
235
|
raise LostClaim()
|
|
189
236
|
finally:
|
|
@@ -348,6 +395,33 @@ class Coordinator:
|
|
|
348
395
|
return None
|
|
349
396
|
return Claim(source_id, token)
|
|
350
397
|
|
|
398
|
+
def _record_claims(
|
|
399
|
+
self, session: Session, worker: _Worker, source_ids: Iterable[object], timestamp: datetime,
|
|
400
|
+
) -> list[Claim]:
|
|
401
|
+
ids = list(source_ids)
|
|
402
|
+
if not ids:
|
|
403
|
+
return []
|
|
404
|
+
if session.get_bind().dialect.name != "postgresql":
|
|
405
|
+
return [claim for source_id in ids
|
|
406
|
+
if (claim := self._record_claim(session, worker, source_id, timestamp)) is not None]
|
|
407
|
+
table = worker.table
|
|
408
|
+
expires = now() + timedelta(seconds=self.lease_seconds)
|
|
409
|
+
values = [
|
|
410
|
+
dict(source_id=source_id, execution_status=ExecutionStatus.WORKING,
|
|
411
|
+
claim_token=str(uuid.uuid4()), lease_expires_at=expires, error=None)
|
|
412
|
+
for source_id in ids
|
|
413
|
+
]
|
|
414
|
+
statement = postgresql_insert(table).values(values[0] if len(values) == 1 else values)
|
|
415
|
+
statement = statement.on_conflict_do_update(
|
|
416
|
+
index_elements=[table.c.source_id],
|
|
417
|
+
set_={name: statement.excluded[name] for name in
|
|
418
|
+
("execution_status", "claim_token", "lease_expires_at", "error")},
|
|
419
|
+
where=self._available(table, timestamp),
|
|
420
|
+
).returning(table.c.source_id, table.c.claim_token)
|
|
421
|
+
tokens = {source_id: token for source_id, token in session.execute(statement).tuples()}
|
|
422
|
+
# RETURNING order is not an eligibility-order guarantee.
|
|
423
|
+
return [Claim(source_id, tokens[source_id]) for source_id in ids if source_id in tokens]
|
|
424
|
+
|
|
351
425
|
def _claim_with_db_lock(self, worker: _Worker) -> Claim | None:
|
|
352
426
|
timestamp = now()
|
|
353
427
|
with self.session_factory.begin() as session:
|
|
@@ -388,8 +462,7 @@ class Coordinator:
|
|
|
388
462
|
query = query.with_for_update(skip_locked=True, of=worker.source_table)
|
|
389
463
|
# Custom eligibility joins may return the same source more than once.
|
|
390
464
|
source_ids = dict.fromkeys(session.scalars(query))
|
|
391
|
-
claims =
|
|
392
|
-
if (claim := self._record_claim(session, worker, source_id, timestamp)) is not None]
|
|
465
|
+
claims = self._record_claims(session, worker, source_ids, timestamp)
|
|
393
466
|
if claims:
|
|
394
467
|
# Earlier records must not enter the queue with leases aged by
|
|
395
468
|
# the rest of this batch. Refresh while ownership is still locked.
|
|
@@ -482,11 +555,26 @@ class Coordinator:
|
|
|
482
555
|
raise ValueError("Handlers and models must be importable; bound handler arguments and engine options must be serializable") from exc
|
|
483
556
|
self._stop.clear()
|
|
484
557
|
for worker in self.workers.values():
|
|
558
|
+
context = multiprocessing.get_context("spawn")
|
|
559
|
+
ready = context.Barrier(worker.concurrency + 1)
|
|
485
560
|
handlers = ProcessPoolExecutor(
|
|
486
|
-
max_workers=worker.concurrency, mp_context=
|
|
561
|
+
max_workers=worker.concurrency, mp_context=context,
|
|
487
562
|
initializer=_initialize_process,
|
|
488
|
-
initargs=(self.database_url, self.engine_options, definitions, worker.name),
|
|
563
|
+
initargs=(self.database_url, self.engine_options, definitions, worker.name, ready),
|
|
489
564
|
)
|
|
565
|
+
try:
|
|
566
|
+
# Initializers hold the startup barrier, so every configured
|
|
567
|
+
# process must start before any probe can finish. These probes
|
|
568
|
+
# create no source rows, claims, or execution-ledger entries.
|
|
569
|
+
probes = [handlers.submit(_process_ready) for _ in range(worker.concurrency)]
|
|
570
|
+
ready.wait(timeout=180)
|
|
571
|
+
for probe in probes:
|
|
572
|
+
probe.result()
|
|
573
|
+
except BaseException:
|
|
574
|
+
ready.abort()
|
|
575
|
+
handlers.shutdown(wait=True, cancel_futures=True)
|
|
576
|
+
self.stop()
|
|
577
|
+
raise
|
|
490
578
|
wakeup = threading.Event()
|
|
491
579
|
self._wakeups[worker.name] = wakeup
|
|
492
580
|
thread = threading.Thread(target=self._run, args=(worker, handlers, wakeup), name=worker.name, daemon=True)
|
|
File without changes
|
|
File without changes
|