dbworker 0.0.7__tar.gz → 0.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dbworker-0.0.7 → dbworker-0.1.1}/PKG-INFO +16 -16
- {dbworker-0.0.7 → dbworker-0.1.1}/README.md +15 -15
- {dbworker-0.0.7 → dbworker-0.1.1}/pyproject.toml +1 -1
- {dbworker-0.0.7 → dbworker-0.1.1}/src/dbworker.py +106 -17
- {dbworker-0.0.7 → dbworker-0.1.1}/LICENSE +0 -0
- {dbworker-0.0.7 → dbworker-0.1.1}/poetry.lock +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dbworker
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.1.1
|
|
4
4
|
Summary: Portable, single-script Python/SQLAlchemy background worker for SQLite, PostgreSQL and MySQL — no Redis, broker or enqueue
|
|
5
5
|
License: MIT
|
|
6
6
|
License-File: LICENSE
|
|
@@ -90,20 +90,20 @@ failure handling, and configuration.
|
|
|
90
90
|
<!-- benchmark-results:start -->
|
|
91
91
|
## Benchmark Results
|
|
92
92
|
|
|
93
|
-
Median wall time in seconds; **bold** marks the faster backend. Each experiment runs in a fresh Docker container on its own GitHub-hosted Ubuntu VM.
|
|
94
|
-
|
|
95
|
-
| Experiment |
|
|
96
|
-
|
|
97
|
-
| imagededup |
|
|
98
|
-
| imagededup |
|
|
99
|
-
| imagededup |
|
|
100
|
-
| superset |
|
|
101
|
-
| saleor |
|
|
102
|
-
| paperless_ngx |
|
|
103
|
-
| posthog |
|
|
104
|
-
| sentry |
|
|
105
|
-
|
|
106
|
-
Run: `github-
|
|
107
|
-
Scoped application workloads; Sentry uses historical 24.1.0. Ratios above 1 favor DBWorker; no cross-project average is computed.
|
|
93
|
+
Median wall time in seconds; **bold** marks the faster backend. Each experiment runs in a fresh Docker container on its own GitHub-hosted Ubuntu VM. Fixed profile: 8 producers, a 60-second scheduled submission window and 8 total execution slots; image hash building uses one native bulk import. Three repetitions per backend. Actual schedule delays and backlog are recorded in JSON; this is one load point, not a maximum-capacity search.
|
|
94
|
+
|
|
95
|
+
| Experiment | Workload | Celery wall (s) | DBWorker wall (s) | C / D | Task P95 / peak outstanding (C; D) |
|
|
96
|
+
|---|---|---:|---:|---:|---|
|
|
97
|
+
| imagededup | Build 1,000 image hashes (one bulk import) | **5.760** | 7.068 | 0.81 | n/a; n/a |
|
|
98
|
+
| imagededup | Compare 1,000 images / 999,000 directed pairs | 79.744 | **75.987** | 1.05 | n/a; n/a |
|
|
99
|
+
| imagededup | Build + compare 1,000 images / 999,000 directed pairs | 90.392 | **90.100** | 1.00 | n/a; n/a |
|
|
100
|
+
| superset | SQL Lab: 3,000 queries / 10,000 rows | **146.495** | 166.948 | 0.88 | 41.240s / 1510; 47.638s / 1477 |
|
|
101
|
+
| saleor | Export: 1,000 x 256 products + 1,000 emails | **130.202** | 165.889 | 0.78 | 36.031s / 604; 56.666s / 885 |
|
|
102
|
+
| paperless_ngx | OCR: 200 scans, archive and index | 126.469 | **71.819** | 1.76 | 63.548s / 108; 44.893s / 41 |
|
|
103
|
+
| posthog | 2FA: 5,000 requests / 10,000 tasks | **118.121** | 147.444 | 0.80 | 0.113s / 15; 36.036s / 2268 |
|
|
104
|
+
| sentry | Email: 2,000 requests / 4,000 deliveries | **60.306** | 60.418 | 1.00 | 0.052s / 16; 0.517s / 161 |
|
|
105
|
+
|
|
106
|
+
Run: `github-38165895322-1`. [Details and scope](doc/benchmark-results.md) · [JSON results](benchmarks/results/latest/index.json).
|
|
107
|
+
Scoped application workloads; Sentry uses historical 24.1.0. Wall time includes the fixed submission window, so a ratio near 1 does not prove equal processing capacity. Task P95 starts at publication; peak outstanding counts already published tasks. Ratios above 1 favor DBWorker; no cross-project average is computed.
|
|
108
108
|
<!-- benchmark-results:end -->
|
|
109
109
|
|
|
@@ -73,19 +73,19 @@ failure handling, and configuration.
|
|
|
73
73
|
<!-- benchmark-results:start -->
|
|
74
74
|
## Benchmark Results
|
|
75
75
|
|
|
76
|
-
Median wall time in seconds; **bold** marks the faster backend. Each experiment runs in a fresh Docker container on its own GitHub-hosted Ubuntu VM.
|
|
77
|
-
|
|
78
|
-
| Experiment |
|
|
79
|
-
|
|
80
|
-
| imagededup |
|
|
81
|
-
| imagededup |
|
|
82
|
-
| imagededup |
|
|
83
|
-
| superset |
|
|
84
|
-
| saleor |
|
|
85
|
-
| paperless_ngx |
|
|
86
|
-
| posthog |
|
|
87
|
-
| sentry |
|
|
88
|
-
|
|
89
|
-
Run: `github-
|
|
90
|
-
Scoped application workloads; Sentry uses historical 24.1.0. Ratios above 1 favor DBWorker; no cross-project average is computed.
|
|
76
|
+
Median wall time in seconds; **bold** marks the faster backend. Each experiment runs in a fresh Docker container on its own GitHub-hosted Ubuntu VM. Fixed profile: 8 producers, a 60-second scheduled submission window and 8 total execution slots; image hash building uses one native bulk import. Three repetitions per backend. Actual schedule delays and backlog are recorded in JSON; this is one load point, not a maximum-capacity search.
|
|
77
|
+
|
|
78
|
+
| Experiment | Workload | Celery wall (s) | DBWorker wall (s) | C / D | Task P95 / peak outstanding (C; D) |
|
|
79
|
+
|---|---|---:|---:|---:|---|
|
|
80
|
+
| imagededup | Build 1,000 image hashes (one bulk import) | **5.760** | 7.068 | 0.81 | n/a; n/a |
|
|
81
|
+
| imagededup | Compare 1,000 images / 999,000 directed pairs | 79.744 | **75.987** | 1.05 | n/a; n/a |
|
|
82
|
+
| imagededup | Build + compare 1,000 images / 999,000 directed pairs | 90.392 | **90.100** | 1.00 | n/a; n/a |
|
|
83
|
+
| superset | SQL Lab: 3,000 queries / 10,000 rows | **146.495** | 166.948 | 0.88 | 41.240s / 1510; 47.638s / 1477 |
|
|
84
|
+
| saleor | Export: 1,000 x 256 products + 1,000 emails | **130.202** | 165.889 | 0.78 | 36.031s / 604; 56.666s / 885 |
|
|
85
|
+
| paperless_ngx | OCR: 200 scans, archive and index | 126.469 | **71.819** | 1.76 | 63.548s / 108; 44.893s / 41 |
|
|
86
|
+
| posthog | 2FA: 5,000 requests / 10,000 tasks | **118.121** | 147.444 | 0.80 | 0.113s / 15; 36.036s / 2268 |
|
|
87
|
+
| sentry | Email: 2,000 requests / 4,000 deliveries | **60.306** | 60.418 | 1.00 | 0.052s / 16; 0.517s / 161 |
|
|
88
|
+
|
|
89
|
+
Run: `github-38165895322-1`. [Details and scope](doc/benchmark-results.md) · [JSON results](benchmarks/results/latest/index.json).
|
|
90
|
+
Scoped application workloads; Sentry uses historical 24.1.0. Wall time includes the fixed submission window, so a ratio near 1 does not prove equal processing capacity. Task P95 starts at publication; peak outstanding counts already published tasks. Ratios above 1 favor DBWorker; no cross-project average is computed.
|
|
91
91
|
<!-- benchmark-results:end -->
|
|
@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
|
|
|
4
4
|
|
|
5
5
|
[tool.poetry]
|
|
6
6
|
name = "dbworker"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.1.1"
|
|
8
8
|
description = "Portable, single-script Python/SQLAlchemy background worker for SQLite, PostgreSQL and MySQL — no Redis, broker or enqueue"
|
|
9
9
|
authors = ["Ziyang Song"]
|
|
10
10
|
license = "MIT"
|
|
@@ -18,9 +18,10 @@ from datetime import datetime, timedelta, timezone
|
|
|
18
18
|
from typing import Any, TypeAlias, TypeVar, cast
|
|
19
19
|
from multiprocessing.util import Finalize
|
|
20
20
|
|
|
21
|
-
from sqlalchemy import Column, DateTime, Enum, ForeignKey, String, Table, Text, and_, create_engine, event, exists, inspect, insert, or_, select, update
|
|
21
|
+
from sqlalchemy import Column, DateTime, Enum, ForeignKey, String, Table, Text, and_, create_engine, event, exists, inspect, insert, or_, select, tuple_, update
|
|
22
22
|
from sqlalchemy.exc import IntegrityError
|
|
23
23
|
from sqlalchemy.engine import CursorResult, RowMapping, URL, make_url
|
|
24
|
+
from sqlalchemy.dialects.postgresql import insert as postgresql_insert
|
|
24
25
|
from sqlalchemy.orm import DeclarativeBase, Mapper, Session, sessionmaker
|
|
25
26
|
from sqlalchemy.sql import ColumnElement, Select
|
|
26
27
|
from sqlalchemy.sql.selectable import Exists
|
|
@@ -140,17 +141,27 @@ def _initialize_process(
|
|
|
140
141
|
engine_options: dict[str, Any],
|
|
141
142
|
definitions: tuple[_WorkerDefinition, ...],
|
|
142
143
|
worker_name: str,
|
|
144
|
+
ready: Any,
|
|
143
145
|
) -> None:
|
|
144
146
|
global _process_worker, _process_session_factory
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
147
|
+
try:
|
|
148
|
+
engine = create_engine(database_url, **engine_options)
|
|
149
|
+
_process_session_factory = sessionmaker(engine, expire_on_commit=False)
|
|
150
|
+
Finalize(None, engine.dispose, exitpriority=10)
|
|
151
|
+
# Rebuild registered work-table metadata locally, including dependencies used
|
|
152
|
+
# by application queries. Eligibility callbacks stay in the parent process.
|
|
153
|
+
for definition in definitions:
|
|
154
|
+
worker = _Worker(name=definition.name, source=definition.source, handler=definition.handler)
|
|
155
|
+
if definition.name == worker_name:
|
|
156
|
+
_process_worker = worker
|
|
157
|
+
ready.wait(timeout=180)
|
|
158
|
+
except BaseException:
|
|
159
|
+
ready.abort()
|
|
160
|
+
raise
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def _process_ready() -> None:
|
|
164
|
+
"""Start the executor without claiming or executing an application job."""
|
|
154
165
|
|
|
155
166
|
|
|
156
167
|
def _execute_in_process(claim: Claim) -> Outcome:
|
|
@@ -348,6 +359,33 @@ class Coordinator:
|
|
|
348
359
|
return None
|
|
349
360
|
return Claim(source_id, token)
|
|
350
361
|
|
|
362
|
+
def _record_claims(
|
|
363
|
+
self, session: Session, worker: _Worker, source_ids: Iterable[object], timestamp: datetime,
|
|
364
|
+
) -> list[Claim]:
|
|
365
|
+
ids = list(source_ids)
|
|
366
|
+
if not ids:
|
|
367
|
+
return []
|
|
368
|
+
if session.get_bind().dialect.name != "postgresql":
|
|
369
|
+
return [claim for source_id in ids
|
|
370
|
+
if (claim := self._record_claim(session, worker, source_id, timestamp)) is not None]
|
|
371
|
+
table = worker.table
|
|
372
|
+
expires = now() + timedelta(seconds=self.lease_seconds)
|
|
373
|
+
values = [
|
|
374
|
+
dict(source_id=source_id, execution_status=ExecutionStatus.WORKING,
|
|
375
|
+
claim_token=str(uuid.uuid4()), lease_expires_at=expires, error=None)
|
|
376
|
+
for source_id in ids
|
|
377
|
+
]
|
|
378
|
+
statement = postgresql_insert(table).values(values[0] if len(values) == 1 else values)
|
|
379
|
+
statement = statement.on_conflict_do_update(
|
|
380
|
+
index_elements=[table.c.source_id],
|
|
381
|
+
set_={name: statement.excluded[name] for name in
|
|
382
|
+
("execution_status", "claim_token", "lease_expires_at", "error")},
|
|
383
|
+
where=self._available(table, timestamp),
|
|
384
|
+
).returning(table.c.source_id, table.c.claim_token)
|
|
385
|
+
tokens = {source_id: token for source_id, token in session.execute(statement).tuples()}
|
|
386
|
+
# RETURNING order is not an eligibility-order guarantee.
|
|
387
|
+
return [Claim(source_id, tokens[source_id]) for source_id in ids if source_id in tokens]
|
|
388
|
+
|
|
351
389
|
def _claim_with_db_lock(self, worker: _Worker) -> Claim | None:
|
|
352
390
|
timestamp = now()
|
|
353
391
|
with self.session_factory.begin() as session:
|
|
@@ -368,6 +406,38 @@ class Coordinator:
|
|
|
368
406
|
return self._claim_with_db_lock(worker)
|
|
369
407
|
return self._claim_with_conditional_update(worker)
|
|
370
408
|
|
|
409
|
+
def _claim_many(self, worker: _Worker, limit: int) -> list[Claim]:
|
|
410
|
+
"""Commit ownership for free execution slots before dispatching work."""
|
|
411
|
+
if limit == 1:
|
|
412
|
+
claim = self.claim(worker)
|
|
413
|
+
return [] if claim is None else [claim]
|
|
414
|
+
supports_skip_locked = self._supports_skip_locked
|
|
415
|
+
timestamp = now()
|
|
416
|
+
with self.session_factory.begin() as session:
|
|
417
|
+
connection = session.connection()
|
|
418
|
+
if (connection.dialect.name == "sqlite"
|
|
419
|
+
and not connection.connection.driver_connection.in_transaction):
|
|
420
|
+
# Legacy sqlite3 transaction control does not begin on SELECT.
|
|
421
|
+
# Without an outer transaction, releasing the first insertion
|
|
422
|
+
# savepoint would commit it even if a later claim fails.
|
|
423
|
+
connection.exec_driver_sql("BEGIN IMMEDIATE")
|
|
424
|
+
query = self._candidate(worker, timestamp).limit(limit)
|
|
425
|
+
if supports_skip_locked:
|
|
426
|
+
query = query.with_for_update(skip_locked=True, of=worker.source_table)
|
|
427
|
+
# Custom eligibility joins may return the same source more than once.
|
|
428
|
+
source_ids = dict.fromkeys(session.scalars(query))
|
|
429
|
+
claims = self._record_claims(session, worker, source_ids, timestamp)
|
|
430
|
+
if claims:
|
|
431
|
+
# Earlier records must not enter the queue with leases aged by
|
|
432
|
+
# the rest of this batch. Refresh while ownership is still locked.
|
|
433
|
+
table = worker.table
|
|
434
|
+
session.execute(update(table).where(
|
|
435
|
+
table.c.execution_status == ExecutionStatus.WORKING,
|
|
436
|
+
tuple_(table.c.source_id, table.c.claim_token).in_(
|
|
437
|
+
[(claim.source_id, claim.token) for claim in claims]),
|
|
438
|
+
).values(lease_expires_at=now() + timedelta(seconds=self.lease_seconds)))
|
|
439
|
+
return claims
|
|
440
|
+
|
|
371
441
|
def renew(self, worker: _Worker, claims: Iterable[Claim]) -> None:
|
|
372
442
|
table = worker.table
|
|
373
443
|
with self.session_factory.begin() as session:
|
|
@@ -400,8 +470,11 @@ class Coordinator:
|
|
|
400
470
|
for future in list(active):
|
|
401
471
|
if future.done():
|
|
402
472
|
claim = active.pop(future)
|
|
473
|
+
# Completion can publish eligible children after an empty query.
|
|
474
|
+
next_claim_at = 0.0
|
|
403
475
|
try:
|
|
404
476
|
future.result()
|
|
477
|
+
delay = self.poll_seconds
|
|
405
478
|
except LostClaim:
|
|
406
479
|
logger.info("Discarded stale result for %s:%s", worker.name, claim.source_id)
|
|
407
480
|
except Exception as exc:
|
|
@@ -413,16 +486,17 @@ class Coordinator:
|
|
|
413
486
|
next_renewal = time.monotonic() + self.lease_seconds / 3
|
|
414
487
|
if not self._stop.is_set() and time.monotonic() >= next_claim_at:
|
|
415
488
|
while len(active) < worker.concurrency and not self._stop.is_set():
|
|
416
|
-
|
|
417
|
-
if
|
|
489
|
+
claims = self._claim_many(worker, worker.concurrency - len(active))
|
|
490
|
+
if not claims:
|
|
418
491
|
next_claim_at = time.monotonic() + delay
|
|
419
492
|
delay = min(delay * 2, self.max_poll_seconds)
|
|
420
493
|
break
|
|
421
494
|
delay = self.poll_seconds
|
|
422
495
|
next_claim_at = 0.0
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
496
|
+
for claim in claims:
|
|
497
|
+
future = handlers.submit(_execute_in_process, claim)
|
|
498
|
+
active[future] = claim
|
|
499
|
+
future.add_done_callback(lambda completed: wakeup.set())
|
|
426
500
|
deadlines: list[float] = []
|
|
427
501
|
if active:
|
|
428
502
|
deadlines.append(next_renewal)
|
|
@@ -445,11 +519,26 @@ class Coordinator:
|
|
|
445
519
|
raise ValueError("Handlers and models must be importable; bound handler arguments and engine options must be serializable") from exc
|
|
446
520
|
self._stop.clear()
|
|
447
521
|
for worker in self.workers.values():
|
|
522
|
+
context = multiprocessing.get_context("spawn")
|
|
523
|
+
ready = context.Barrier(worker.concurrency + 1)
|
|
448
524
|
handlers = ProcessPoolExecutor(
|
|
449
|
-
max_workers=worker.concurrency, mp_context=
|
|
525
|
+
max_workers=worker.concurrency, mp_context=context,
|
|
450
526
|
initializer=_initialize_process,
|
|
451
|
-
initargs=(self.database_url, self.engine_options, definitions, worker.name),
|
|
527
|
+
initargs=(self.database_url, self.engine_options, definitions, worker.name, ready),
|
|
452
528
|
)
|
|
529
|
+
try:
|
|
530
|
+
# Initializers hold the startup barrier, so every configured
|
|
531
|
+
# process must start before any probe can finish. These probes
|
|
532
|
+
# create no source rows, claims, or execution-ledger entries.
|
|
533
|
+
probes = [handlers.submit(_process_ready) for _ in range(worker.concurrency)]
|
|
534
|
+
ready.wait(timeout=180)
|
|
535
|
+
for probe in probes:
|
|
536
|
+
probe.result()
|
|
537
|
+
except BaseException:
|
|
538
|
+
ready.abort()
|
|
539
|
+
handlers.shutdown(wait=True, cancel_futures=True)
|
|
540
|
+
self.stop()
|
|
541
|
+
raise
|
|
453
542
|
wakeup = threading.Event()
|
|
454
543
|
self._wakeups[worker.name] = wakeup
|
|
455
544
|
thread = threading.Thread(target=self._run, args=(worker, handlers, wakeup), name=worker.name, daemon=True)
|
|
File without changes
|
|
File without changes
|