dbworker 0.0.7__tar.gz → 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dbworker-0.0.7 → dbworker-0.1.0}/PKG-INFO +16 -16
- {dbworker-0.0.7 → dbworker-0.1.0}/README.md +15 -15
- {dbworker-0.0.7 → dbworker-0.1.0}/pyproject.toml +1 -1
- {dbworker-0.0.7 → dbworker-0.1.0}/src/dbworker.py +43 -6
- {dbworker-0.0.7 → dbworker-0.1.0}/LICENSE +0 -0
- {dbworker-0.0.7 → dbworker-0.1.0}/poetry.lock +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dbworker
|
|
3
|
-
Version: 0.0
|
|
3
|
+
Version: 0.1.0
|
|
4
4
|
Summary: Portable, single-script Python/SQLAlchemy background worker for SQLite, PostgreSQL and MySQL — no Redis, broker or enqueue
|
|
5
5
|
License: MIT
|
|
6
6
|
License-File: LICENSE
|
|
@@ -90,20 +90,20 @@ failure handling, and configuration.
|
|
|
90
90
|
<!-- benchmark-results:start -->
|
|
91
91
|
## Benchmark Results
|
|
92
92
|
|
|
93
|
-
Median wall time in seconds; **bold** marks the faster backend. Each experiment runs in a fresh Docker container on its own GitHub-hosted Ubuntu VM.
|
|
94
|
-
|
|
95
|
-
| Experiment |
|
|
96
|
-
|
|
97
|
-
| imagededup |
|
|
98
|
-
| imagededup |
|
|
99
|
-
| imagededup |
|
|
100
|
-
| superset |
|
|
101
|
-
| saleor |
|
|
102
|
-
| paperless_ngx |
|
|
103
|
-
| posthog |
|
|
104
|
-
| sentry |
|
|
105
|
-
|
|
106
|
-
Run: `github-
|
|
107
|
-
Scoped application workloads; Sentry uses historical 24.1.0. Ratios above 1 favor DBWorker; no cross-project average is computed.
|
|
93
|
+
Median wall time in seconds; **bold** marks the faster backend. Each experiment runs in a fresh Docker container on its own GitHub-hosted Ubuntu VM. Fixed profile: 8 producers, a 60-second scheduled submission window and 8 total execution slots; image hash building uses one native bulk import. Three repetitions per backend. Actual schedule delays and backlog are recorded in JSON; this is one load point, not a maximum-capacity search.
|
|
94
|
+
|
|
95
|
+
| Experiment | Workload | Celery wall (s) | DBWorker wall (s) | C / D | Task P95 / peak outstanding (C; D) |
|
|
96
|
+
|---|---|---:|---:|---:|---|
|
|
97
|
+
| imagededup | Build 1,000 image hashes (one bulk import) | **8.510** | 8.904 | 0.96 | n/a; n/a |
|
|
98
|
+
| imagededup | Compare 1,000 images / 999,000 directed pairs | 150.199 | **106.599** | 1.41 | n/a; n/a |
|
|
99
|
+
| imagededup | Build + compare 1,000 images / 999,000 directed pairs | 139.493 | **133.968** | 1.04 | n/a; n/a |
|
|
100
|
+
| superset | SQL Lab: 3,000 queries / 10,000 rows | **150.445** | 165.700 | 0.91 | 42.066s / 1550; 49.179s / 1615 |
|
|
101
|
+
| saleor | Export: 1,000 x 256 products + 1,000 emails | **109.182** | 138.540 | 0.79 | 24.518s / 495; 37.793s / 757 |
|
|
102
|
+
| paperless_ngx | OCR: 200 scans, archive and index | 105.615 | **60.843** | 1.74 | 44.666s / 90; 7.350s / 22 |
|
|
103
|
+
| posthog | 2FA: 5,000 requests / 10,000 tasks | **210.518** | 330.952 | 0.64 | 0.276s / 16; 102.777s / 4451 |
|
|
104
|
+
| sentry | Email: 2,000 requests / 4,000 deliveries | 60.303 | **60.157** | 1.00 | 0.054s / 16; 1.094s / 172 |
|
|
105
|
+
|
|
106
|
+
Run: `github-37987827539-1`. [Details and scope](doc/benchmark-results.md) · [JSON results](benchmarks/results/latest/index.json).
|
|
107
|
+
Scoped application workloads; Sentry uses historical 24.1.0. Wall time includes the fixed submission window, so a ratio near 1 does not prove equal processing capacity. Task P95 starts at publication; peak outstanding counts already published tasks. Ratios above 1 favor DBWorker; no cross-project average is computed.
|
|
108
108
|
<!-- benchmark-results:end -->
|
|
109
109
|
|
|
@@ -73,19 +73,19 @@ failure handling, and configuration.
|
|
|
73
73
|
<!-- benchmark-results:start -->
|
|
74
74
|
## Benchmark Results
|
|
75
75
|
|
|
76
|
-
Median wall time in seconds; **bold** marks the faster backend. Each experiment runs in a fresh Docker container on its own GitHub-hosted Ubuntu VM.
|
|
77
|
-
|
|
78
|
-
| Experiment |
|
|
79
|
-
|
|
80
|
-
| imagededup |
|
|
81
|
-
| imagededup |
|
|
82
|
-
| imagededup |
|
|
83
|
-
| superset |
|
|
84
|
-
| saleor |
|
|
85
|
-
| paperless_ngx |
|
|
86
|
-
| posthog |
|
|
87
|
-
| sentry |
|
|
88
|
-
|
|
89
|
-
Run: `github-
|
|
90
|
-
Scoped application workloads; Sentry uses historical 24.1.0. Ratios above 1 favor DBWorker; no cross-project average is computed.
|
|
76
|
+
Median wall time in seconds; **bold** marks the faster backend. Each experiment runs in a fresh Docker container on its own GitHub-hosted Ubuntu VM. Fixed profile: 8 producers, a 60-second scheduled submission window and 8 total execution slots; image hash building uses one native bulk import. Three repetitions per backend. Actual schedule delays and backlog are recorded in JSON; this is one load point, not a maximum-capacity search.
|
|
77
|
+
|
|
78
|
+
| Experiment | Workload | Celery wall (s) | DBWorker wall (s) | C / D | Task P95 / peak outstanding (C; D) |
|
|
79
|
+
|---|---|---:|---:|---:|---|
|
|
80
|
+
| imagededup | Build 1,000 image hashes (one bulk import) | **8.510** | 8.904 | 0.96 | n/a; n/a |
|
|
81
|
+
| imagededup | Compare 1,000 images / 999,000 directed pairs | 150.199 | **106.599** | 1.41 | n/a; n/a |
|
|
82
|
+
| imagededup | Build + compare 1,000 images / 999,000 directed pairs | 139.493 | **133.968** | 1.04 | n/a; n/a |
|
|
83
|
+
| superset | SQL Lab: 3,000 queries / 10,000 rows | **150.445** | 165.700 | 0.91 | 42.066s / 1550; 49.179s / 1615 |
|
|
84
|
+
| saleor | Export: 1,000 x 256 products + 1,000 emails | **109.182** | 138.540 | 0.79 | 24.518s / 495; 37.793s / 757 |
|
|
85
|
+
| paperless_ngx | OCR: 200 scans, archive and index | 105.615 | **60.843** | 1.74 | 44.666s / 90; 7.350s / 22 |
|
|
86
|
+
| posthog | 2FA: 5,000 requests / 10,000 tasks | **210.518** | 330.952 | 0.64 | 0.276s / 16; 102.777s / 4451 |
|
|
87
|
+
| sentry | Email: 2,000 requests / 4,000 deliveries | 60.303 | **60.157** | 1.00 | 0.054s / 16; 1.094s / 172 |
|
|
88
|
+
|
|
89
|
+
Run: `github-37987827539-1`. [Details and scope](doc/benchmark-results.md) · [JSON results](benchmarks/results/latest/index.json).
|
|
90
|
+
Scoped application workloads; Sentry uses historical 24.1.0. Wall time includes the fixed submission window, so a ratio near 1 does not prove equal processing capacity. Task P95 starts at publication; peak outstanding counts already published tasks. Ratios above 1 favor DBWorker; no cross-project average is computed.
|
|
91
91
|
<!-- benchmark-results:end -->
|
|
@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
|
|
|
4
4
|
|
|
5
5
|
[tool.poetry]
|
|
6
6
|
name = "dbworker"
|
|
7
|
-
version = "0.0
|
|
7
|
+
version = "0.1.0"
|
|
8
8
|
description = "Portable, single-script Python/SQLAlchemy background worker for SQLite, PostgreSQL and MySQL — no Redis, broker or enqueue"
|
|
9
9
|
authors = ["Ziyang Song"]
|
|
10
10
|
license = "MIT"
|
|
@@ -18,7 +18,7 @@ from datetime import datetime, timedelta, timezone
|
|
|
18
18
|
from typing import Any, TypeAlias, TypeVar, cast
|
|
19
19
|
from multiprocessing.util import Finalize
|
|
20
20
|
|
|
21
|
-
from sqlalchemy import Column, DateTime, Enum, ForeignKey, String, Table, Text, and_, create_engine, event, exists, inspect, insert, or_, select, update
|
|
21
|
+
from sqlalchemy import Column, DateTime, Enum, ForeignKey, String, Table, Text, and_, create_engine, event, exists, inspect, insert, or_, select, tuple_, update
|
|
22
22
|
from sqlalchemy.exc import IntegrityError
|
|
23
23
|
from sqlalchemy.engine import CursorResult, RowMapping, URL, make_url
|
|
24
24
|
from sqlalchemy.orm import DeclarativeBase, Mapper, Session, sessionmaker
|
|
@@ -368,6 +368,39 @@ class Coordinator:
|
|
|
368
368
|
return self._claim_with_db_lock(worker)
|
|
369
369
|
return self._claim_with_conditional_update(worker)
|
|
370
370
|
|
|
371
|
+
def _claim_many(self, worker: _Worker, limit: int) -> list[Claim]:
|
|
372
|
+
"""Commit ownership for free execution slots before dispatching work."""
|
|
373
|
+
if limit == 1:
|
|
374
|
+
claim = self.claim(worker)
|
|
375
|
+
return [] if claim is None else [claim]
|
|
376
|
+
supports_skip_locked = self._supports_skip_locked
|
|
377
|
+
timestamp = now()
|
|
378
|
+
with self.session_factory.begin() as session:
|
|
379
|
+
connection = session.connection()
|
|
380
|
+
if (connection.dialect.name == "sqlite"
|
|
381
|
+
and not connection.connection.driver_connection.in_transaction):
|
|
382
|
+
# Legacy sqlite3 transaction control does not begin on SELECT.
|
|
383
|
+
# Without an outer transaction, releasing the first insertion
|
|
384
|
+
# savepoint would commit it even if a later claim fails.
|
|
385
|
+
connection.exec_driver_sql("BEGIN IMMEDIATE")
|
|
386
|
+
query = self._candidate(worker, timestamp).limit(limit)
|
|
387
|
+
if supports_skip_locked:
|
|
388
|
+
query = query.with_for_update(skip_locked=True, of=worker.source_table)
|
|
389
|
+
# Custom eligibility joins may return the same source more than once.
|
|
390
|
+
source_ids = dict.fromkeys(session.scalars(query))
|
|
391
|
+
claims = [claim for source_id in source_ids
|
|
392
|
+
if (claim := self._record_claim(session, worker, source_id, timestamp)) is not None]
|
|
393
|
+
if claims:
|
|
394
|
+
# Earlier records must not enter the queue with leases aged by
|
|
395
|
+
# the rest of this batch. Refresh while ownership is still locked.
|
|
396
|
+
table = worker.table
|
|
397
|
+
session.execute(update(table).where(
|
|
398
|
+
table.c.execution_status == ExecutionStatus.WORKING,
|
|
399
|
+
tuple_(table.c.source_id, table.c.claim_token).in_(
|
|
400
|
+
[(claim.source_id, claim.token) for claim in claims]),
|
|
401
|
+
).values(lease_expires_at=now() + timedelta(seconds=self.lease_seconds)))
|
|
402
|
+
return claims
|
|
403
|
+
|
|
371
404
|
def renew(self, worker: _Worker, claims: Iterable[Claim]) -> None:
|
|
372
405
|
table = worker.table
|
|
373
406
|
with self.session_factory.begin() as session:
|
|
@@ -400,8 +433,11 @@ class Coordinator:
|
|
|
400
433
|
for future in list(active):
|
|
401
434
|
if future.done():
|
|
402
435
|
claim = active.pop(future)
|
|
436
|
+
# Completion can publish eligible children after an empty query.
|
|
437
|
+
next_claim_at = 0.0
|
|
403
438
|
try:
|
|
404
439
|
future.result()
|
|
440
|
+
delay = self.poll_seconds
|
|
405
441
|
except LostClaim:
|
|
406
442
|
logger.info("Discarded stale result for %s:%s", worker.name, claim.source_id)
|
|
407
443
|
except Exception as exc:
|
|
@@ -413,16 +449,17 @@ class Coordinator:
|
|
|
413
449
|
next_renewal = time.monotonic() + self.lease_seconds / 3
|
|
414
450
|
if not self._stop.is_set() and time.monotonic() >= next_claim_at:
|
|
415
451
|
while len(active) < worker.concurrency and not self._stop.is_set():
|
|
416
|
-
|
|
417
|
-
if
|
|
452
|
+
claims = self._claim_many(worker, worker.concurrency - len(active))
|
|
453
|
+
if not claims:
|
|
418
454
|
next_claim_at = time.monotonic() + delay
|
|
419
455
|
delay = min(delay * 2, self.max_poll_seconds)
|
|
420
456
|
break
|
|
421
457
|
delay = self.poll_seconds
|
|
422
458
|
next_claim_at = 0.0
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
459
|
+
for claim in claims:
|
|
460
|
+
future = handlers.submit(_execute_in_process, claim)
|
|
461
|
+
active[future] = claim
|
|
462
|
+
future.add_done_callback(lambda completed: wakeup.set())
|
|
426
463
|
deadlines: list[float] = []
|
|
427
464
|
if active:
|
|
428
465
|
deadlines.append(next_renewal)
|
|
File without changes
|
|
File without changes
|