dbworker 0.1.2__tar.gz → 0.1.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dbworker-0.1.2 → dbworker-0.1.4}/PKG-INFO +11 -11
- {dbworker-0.1.2 → dbworker-0.1.4}/README.md +10 -10
- {dbworker-0.1.2 → dbworker-0.1.4}/pyproject.toml +1 -1
- {dbworker-0.1.2 → dbworker-0.1.4}/src/dbworker.py +229 -39
- {dbworker-0.1.2 → dbworker-0.1.4}/LICENSE +0 -0
- {dbworker-0.1.2 → dbworker-0.1.4}/poetry.lock +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dbworker
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.4
|
|
4
4
|
Summary: Portable, single-script Python/SQLAlchemy background worker for SQLite, PostgreSQL and MySQL — no Redis, broker or enqueue
|
|
5
5
|
License: MIT
|
|
6
6
|
License-File: LICENSE
|
|
@@ -94,16 +94,16 @@ Median wall time in seconds; **bold** marks the faster backend. Each experiment
|
|
|
94
94
|
|
|
95
95
|
| Experiment | Workload | Celery wall (s) | DBWorker wall (s) | C / D | Task P95 / peak outstanding (C; D) |
|
|
96
96
|
|---|---|---:|---:|---:|---|
|
|
97
|
-
| imagededup | Build 1,000 image hashes (one bulk import) |
|
|
98
|
-
| imagededup | Compare 1,000 images / 999,000 directed pairs |
|
|
99
|
-
| imagededup | Build + compare 1,000 images / 999,000 directed pairs |
|
|
100
|
-
| superset | SQL Lab: 3,000 queries / 10,000 rows | **
|
|
101
|
-
| saleor | Export: 1,000 x 256 products + 1,000 emails | **
|
|
102
|
-
| paperless_ngx | OCR: 200 scans, archive and index |
|
|
103
|
-
| posthog | 2FA: 5,000 requests / 10,000 tasks |
|
|
104
|
-
| sentry | Email: 2,000 requests / 4,000 deliveries | **60.
|
|
105
|
-
|
|
106
|
-
Run: `github-
|
|
97
|
+
| imagededup | Build 1,000 image hashes (one bulk import) | 6.459 | **6.005** | 1.08 | n/a; n/a |
|
|
98
|
+
| imagededup | Compare 1,000 images / 999,000 directed pairs | 81.712 | **77.167** | 1.06 | n/a; n/a |
|
|
99
|
+
| imagededup | Build + compare 1,000 images / 999,000 directed pairs | 92.450 | **88.584** | 1.04 | n/a; n/a |
|
|
100
|
+
| superset | SQL Lab: 3,000 queries / 10,000 rows | **153.735** | 168.862 | 0.91 | 42.354s / 1462; 48.725s / 1511 |
|
|
101
|
+
| saleor | Export: 1,000 x 256 products + 1,000 emails | **137.652** | 167.635 | 0.82 | 39.572s / 630; 54.428s / 812 |
|
|
102
|
+
| paperless_ngx | OCR: 200 scans, archive and index | 142.059 | **81.849** | 1.74 | 78.973s / 118; 57.944s / 61 |
|
|
103
|
+
| posthog | 2FA: 5,000 requests / 10,000 tasks | 184.240 | **164.843** | 1.12 | 0.161s / 14; 0.179s / 30 |
|
|
104
|
+
| sentry | Email: 2,000 requests / 4,000 deliveries | **60.314** | 60.336 | 1.00 | 0.057s / 16; 0.479s / 149 |
|
|
105
|
+
|
|
106
|
+
Run: `github-38401468222-1`. [Details and scope](doc/benchmark-results.md) · [JSON results](benchmarks/results/latest/index.json).
|
|
107
107
|
Scoped application workloads; Sentry uses historical 24.1.0. Wall time includes the fixed submission window, so a ratio near 1 does not prove equal processing capacity. Task P95 starts at publication; peak outstanding counts already published tasks. Ratios above 1 favor DBWorker; no cross-project average is computed.
|
|
108
108
|
<!-- benchmark-results:end -->
|
|
109
109
|
|
|
@@ -77,15 +77,15 @@ Median wall time in seconds; **bold** marks the faster backend. Each experiment
|
|
|
77
77
|
|
|
78
78
|
| Experiment | Workload | Celery wall (s) | DBWorker wall (s) | C / D | Task P95 / peak outstanding (C; D) |
|
|
79
79
|
|---|---|---:|---:|---:|---|
|
|
80
|
-
| imagededup | Build 1,000 image hashes (one bulk import) |
|
|
81
|
-
| imagededup | Compare 1,000 images / 999,000 directed pairs |
|
|
82
|
-
| imagededup | Build + compare 1,000 images / 999,000 directed pairs |
|
|
83
|
-
| superset | SQL Lab: 3,000 queries / 10,000 rows | **
|
|
84
|
-
| saleor | Export: 1,000 x 256 products + 1,000 emails | **
|
|
85
|
-
| paperless_ngx | OCR: 200 scans, archive and index |
|
|
86
|
-
| posthog | 2FA: 5,000 requests / 10,000 tasks |
|
|
87
|
-
| sentry | Email: 2,000 requests / 4,000 deliveries | **60.
|
|
88
|
-
|
|
89
|
-
Run: `github-
|
|
80
|
+
| imagededup | Build 1,000 image hashes (one bulk import) | 6.459 | **6.005** | 1.08 | n/a; n/a |
|
|
81
|
+
| imagededup | Compare 1,000 images / 999,000 directed pairs | 81.712 | **77.167** | 1.06 | n/a; n/a |
|
|
82
|
+
| imagededup | Build + compare 1,000 images / 999,000 directed pairs | 92.450 | **88.584** | 1.04 | n/a; n/a |
|
|
83
|
+
| superset | SQL Lab: 3,000 queries / 10,000 rows | **153.735** | 168.862 | 0.91 | 42.354s / 1462; 48.725s / 1511 |
|
|
84
|
+
| saleor | Export: 1,000 x 256 products + 1,000 emails | **137.652** | 167.635 | 0.82 | 39.572s / 630; 54.428s / 812 |
|
|
85
|
+
| paperless_ngx | OCR: 200 scans, archive and index | 142.059 | **81.849** | 1.74 | 78.973s / 118; 57.944s / 61 |
|
|
86
|
+
| posthog | 2FA: 5,000 requests / 10,000 tasks | 184.240 | **164.843** | 1.12 | 0.161s / 14; 0.179s / 30 |
|
|
87
|
+
| sentry | Email: 2,000 requests / 4,000 deliveries | **60.314** | 60.336 | 1.00 | 0.057s / 16; 0.479s / 149 |
|
|
88
|
+
|
|
89
|
+
Run: `github-38401468222-1`. [Details and scope](doc/benchmark-results.md) · [JSON results](benchmarks/results/latest/index.json).
|
|
90
90
|
Scoped application workloads; Sentry uses historical 24.1.0. Wall time includes the fixed submission window, so a ratio near 1 does not prove equal processing capacity. Task P95 starts at publication; peak outstanding counts already published tasks. Ratios above 1 favor DBWorker; no cross-project average is computed.
|
|
91
91
|
<!-- benchmark-results:end -->
|
|
@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
|
|
|
4
4
|
|
|
5
5
|
[tool.poetry]
|
|
6
6
|
name = "dbworker"
|
|
7
|
-
version = "0.1.
|
|
7
|
+
version = "0.1.4"
|
|
8
8
|
description = "Portable, single-script Python/SQLAlchemy background worker for SQLite, PostgreSQL and MySQL — no Redis, broker or enqueue"
|
|
9
9
|
authors = ["Ziyang Song"]
|
|
10
10
|
license = "MIT"
|
|
@@ -4,7 +4,7 @@ from __future__ import annotations
|
|
|
4
4
|
|
|
5
5
|
import logging
|
|
6
6
|
import multiprocessing
|
|
7
|
-
import pickle # nosec B403 -- used only for local
|
|
7
|
+
import pickle # nosec B403 -- used only for local serialization checks; this module never deserializes foreign pickle data.
|
|
8
8
|
import re
|
|
9
9
|
import sqlite3
|
|
10
10
|
import threading
|
|
@@ -12,6 +12,7 @@ import time
|
|
|
12
12
|
import uuid
|
|
13
13
|
from collections.abc import Callable, Iterable
|
|
14
14
|
from concurrent.futures import Future, ProcessPoolExecutor
|
|
15
|
+
from concurrent.futures.process import BrokenProcessPool
|
|
15
16
|
from dataclasses import dataclass
|
|
16
17
|
from functools import cached_property
|
|
17
18
|
from enum import StrEnum
|
|
@@ -19,17 +20,20 @@ from datetime import datetime, timedelta, timezone
|
|
|
19
20
|
from typing import Any, TypeAlias, TypeVar, cast
|
|
20
21
|
from multiprocessing.util import Finalize
|
|
21
22
|
|
|
22
|
-
from sqlalchemy import Column, DateTime, Enum, ForeignKey, String, Table, Text, and_, create_engine, event, exists, inspect, insert, or_, select, tuple_, update
|
|
23
|
-
from sqlalchemy.exc import IntegrityError, OperationalError
|
|
23
|
+
from sqlalchemy import Column, DateTime, Enum, ForeignKey, Integer, String, Table, Text, and_, create_engine, event, exists, inspect, insert, or_, select, tuple_, update
|
|
24
|
+
from sqlalchemy.exc import DBAPIError, IntegrityError, OperationalError
|
|
24
25
|
from sqlalchemy.engine import CursorResult, RowMapping, URL, make_url
|
|
25
26
|
from sqlalchemy.dialects.postgresql import insert as postgresql_insert
|
|
26
27
|
from sqlalchemy.orm import DeclarativeBase, Mapper, Session, sessionmaker
|
|
28
|
+
from sqlalchemy.ext.compiler import compiles
|
|
29
|
+
from sqlalchemy.schema import ExecutableDDLElement
|
|
27
30
|
from sqlalchemy.sql import ColumnElement, Select, Update
|
|
28
31
|
from sqlalchemy.sql.selectable import Exists
|
|
29
32
|
|
|
30
33
|
logger = logging.getLogger(__name__)
|
|
31
34
|
|
|
32
35
|
_SQLITE_FINALIZATION_RETRY_SECONDS = 5.0
|
|
36
|
+
_MAX_RETRY_BACKOFF_SECONDS = 300.0
|
|
33
37
|
|
|
34
38
|
|
|
35
39
|
def now() -> datetime:
|
|
@@ -78,16 +82,24 @@ class _Worker:
|
|
|
78
82
|
handler: Handler,
|
|
79
83
|
eligible: Callable[[], Select[tuple[Any]]] | None = None,
|
|
80
84
|
concurrency: int = 1,
|
|
85
|
+
max_attempts: int = 3,
|
|
86
|
+
retry_backoff_seconds: float = 1.0,
|
|
81
87
|
) -> None:
|
|
82
88
|
if not re.fullmatch(r"[a-z][a-z0-9_]*", name):
|
|
83
89
|
raise ValueError("Worker name must contain lowercase letters, digits, and underscores")
|
|
84
90
|
if concurrency < 1:
|
|
85
91
|
raise ValueError("Worker concurrency must be positive")
|
|
92
|
+
if max_attempts < 1:
|
|
93
|
+
raise ValueError("Worker max_attempts must be positive")
|
|
94
|
+
if retry_backoff_seconds < 0:
|
|
95
|
+
raise ValueError("Worker retry backoff must not be negative")
|
|
86
96
|
self.name = name
|
|
87
97
|
self.source = source
|
|
88
98
|
self.handler = handler
|
|
89
99
|
self.eligible = eligible or (lambda: select(source))
|
|
90
100
|
self.concurrency = concurrency
|
|
101
|
+
self.max_attempts = max_attempts
|
|
102
|
+
self.retry_backoff_seconds = retry_backoff_seconds
|
|
91
103
|
mapper = cast(Mapper[Any], inspect(source))
|
|
92
104
|
if len(mapper.primary_key) != 1:
|
|
93
105
|
raise ValueError("Worker sources must have one primary-key column")
|
|
@@ -112,6 +124,10 @@ class _Worker:
|
|
|
112
124
|
Column("claim_token", String(36)),
|
|
113
125
|
Column("lease_expires_at", DateTime, index=True),
|
|
114
126
|
Column("error", Text),
|
|
127
|
+
# Consecutive handler failures; a committed outcome resets it.
|
|
128
|
+
Column("attempts", Integer, nullable=False, default=0, server_default="0"),
|
|
129
|
+
# A failed attempt is not claimable again before this time.
|
|
130
|
+
Column("available_at", DateTime),
|
|
115
131
|
)
|
|
116
132
|
|
|
117
133
|
def state(self, session: Session, source_id: object) -> RowMapping | None:
|
|
@@ -121,11 +137,25 @@ class _Worker:
|
|
|
121
137
|
result = cast(CursorResult[Any], session.execute(
|
|
122
138
|
update(self.table)
|
|
123
139
|
.where(self.table.c.source_id == source_id, self.table.c.execution_status == ExecutionStatus.FAILED)
|
|
124
|
-
.values(execution_status=ExecutionStatus.UNFINISHED, error=None, claim_token=None, lease_expires_at=None
|
|
140
|
+
.values(execution_status=ExecutionStatus.UNFINISHED, error=None, claim_token=None, lease_expires_at=None,
|
|
141
|
+
attempts=0, available_at=None)
|
|
125
142
|
))
|
|
126
143
|
return result.rowcount == 1
|
|
127
144
|
|
|
128
145
|
|
|
146
|
+
class _AddColumn(ExecutableDDLElement):
|
|
147
|
+
"""ALTER TABLE ... ADD COLUMN for a work table created by an earlier release."""
|
|
148
|
+
|
|
149
|
+
def __init__(self, column: Column[Any]) -> None:
|
|
150
|
+
self.column = column
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
@compiles(_AddColumn)
|
|
154
|
+
def _compile_add_column(element: _AddColumn, compiler: Any, **kw: Any) -> str:
|
|
155
|
+
return (f"ALTER TABLE {compiler.preparer.format_table(element.column.table)} "
|
|
156
|
+
f"ADD COLUMN {compiler.get_column_specification(element.column)}")
|
|
157
|
+
|
|
158
|
+
|
|
129
159
|
@dataclass(frozen=True)
|
|
130
160
|
class _WorkerDefinition:
|
|
131
161
|
"""Importable callables and model classes, never live database resources."""
|
|
@@ -170,7 +200,16 @@ def _process_ready() -> None:
|
|
|
170
200
|
def _execute_in_process(claim: Claim) -> Outcome:
|
|
171
201
|
if _process_worker is None or _process_session_factory is None:
|
|
172
202
|
raise RuntimeError("Handler process has not been initialized")
|
|
173
|
-
|
|
203
|
+
try:
|
|
204
|
+
return _execute_claim(_process_worker, claim, _process_session_factory)
|
|
205
|
+
except Exception as exc:
|
|
206
|
+
# An exception the parent cannot rebuild (for example a required
|
|
207
|
+
# multi-argument __init__) would break the whole process pool there.
|
|
208
|
+
try:
|
|
209
|
+
pickle.loads(pickle.dumps(exc)) # nosec B301 -- round-trips an object this process just created.
|
|
210
|
+
except Exception:
|
|
211
|
+
raise RuntimeError(f"{type(exc).__qualname__}: {exc}") from exc
|
|
212
|
+
raise
|
|
174
213
|
|
|
175
214
|
|
|
176
215
|
def _retry_sqlite_completion(session: Session, completion: Update, error: OperationalError) -> CursorResult[Any]:
|
|
@@ -226,7 +265,7 @@ def _execute_claim(worker: _Worker, claim: Claim, session_factory: sessionmaker[
|
|
|
226
265
|
worker.table.c.execution_status == ExecutionStatus.WORKING,
|
|
227
266
|
worker.table.c.claim_token == claim.token,
|
|
228
267
|
).values(execution_status=ExecutionStatus.FINISHED if isinstance(outcome, Finished) else ExecutionStatus.UNFINISHED,
|
|
229
|
-
claim_token=None, lease_expires_at=None, error=None)
|
|
268
|
+
claim_token=None, lease_expires_at=None, error=None, attempts=0, available_at=None)
|
|
230
269
|
try:
|
|
231
270
|
result = cast(CursorResult[Any], session.execute(completion))
|
|
232
271
|
except OperationalError as exc:
|
|
@@ -268,6 +307,7 @@ class Coordinator:
|
|
|
268
307
|
self.max_poll_seconds = max_poll_seconds
|
|
269
308
|
self.workers: dict[str, _Worker] = {}
|
|
270
309
|
self._running: dict[str, tuple[threading.Thread, ProcessPoolExecutor]] = {}
|
|
310
|
+
self._definitions: tuple[_WorkerDefinition, ...] = ()
|
|
271
311
|
self._stop = threading.Event()
|
|
272
312
|
self._wakeups: dict[str, threading.Event] = {}
|
|
273
313
|
|
|
@@ -291,9 +331,16 @@ class Coordinator:
|
|
|
291
331
|
source: type[DeclarativeBase],
|
|
292
332
|
eligible: Callable[[], Select[tuple[Any]]] | None = None,
|
|
293
333
|
concurrency: int = 1,
|
|
334
|
+
max_attempts: int = 3,
|
|
335
|
+
retry_backoff_seconds: float = 1.0,
|
|
294
336
|
) -> Callable[[_Function], _Function]:
|
|
295
337
|
"""Register a handler whose session and final commit belong to the runtime.
|
|
296
338
|
|
|
339
|
+
A handler exception rolls back that attempt. The work is retried after
|
|
340
|
+
retry_backoff_seconds (doubling per consecutive failure) and is marked
|
|
341
|
+
FAILED once max_attempts consecutive attempts have raised. Use
|
|
342
|
+
max_attempts=1 to fail on the first exception.
|
|
343
|
+
|
|
297
344
|
Registration creates worker metadata, but starts no processing. The
|
|
298
345
|
original function is returned unchanged; direct calls are not managed.
|
|
299
346
|
"""
|
|
@@ -305,6 +352,7 @@ class Coordinator:
|
|
|
305
352
|
self.workers[name] = _Worker(
|
|
306
353
|
name=name, source=source, handler=handler,
|
|
307
354
|
eligible=eligible, concurrency=concurrency,
|
|
355
|
+
max_attempts=max_attempts, retry_backoff_seconds=retry_backoff_seconds,
|
|
308
356
|
)
|
|
309
357
|
return handler
|
|
310
358
|
|
|
@@ -337,13 +385,38 @@ class Coordinator:
|
|
|
337
385
|
).correlate_except(table)
|
|
338
386
|
|
|
339
387
|
def create_worker_tables(self) -> None:
|
|
388
|
+
"""Create missing work tables and add columns introduced by later releases."""
|
|
340
389
|
with self.session_factory() as session:
|
|
341
|
-
|
|
342
|
-
|
|
390
|
+
bind = session.get_bind()
|
|
391
|
+
for worker in self.workers.values():
|
|
392
|
+
table = worker.table
|
|
393
|
+
table.create(bind, checkfirst=True)
|
|
394
|
+
|
|
395
|
+
def present(column: Column[Any]) -> bool:
|
|
396
|
+
# Probe with ordinary SQL rather than reflection, so the session's
|
|
397
|
+
# schema translation and search path apply as they do to claims.
|
|
398
|
+
try:
|
|
399
|
+
with self.session_factory() as session:
|
|
400
|
+
session.execute(select(column).limit(0))
|
|
401
|
+
return True
|
|
402
|
+
except DBAPIError:
|
|
403
|
+
return False
|
|
404
|
+
|
|
405
|
+
for column in (table.c.attempts, table.c.available_at):
|
|
406
|
+
if present(column):
|
|
407
|
+
continue
|
|
408
|
+
try:
|
|
409
|
+
with self.session_factory.begin() as session:
|
|
410
|
+
session.connection().execute(_AddColumn(column))
|
|
411
|
+
except DBAPIError:
|
|
412
|
+
# Another service may have added it while starting up.
|
|
413
|
+
if not present(column):
|
|
414
|
+
raise
|
|
343
415
|
|
|
344
416
|
def _available(self, table: Table, timestamp: datetime) -> ColumnElement[bool]:
|
|
345
417
|
return or_(
|
|
346
|
-
table.c.execution_status == ExecutionStatus.UNFINISHED,
|
|
418
|
+
and_(table.c.execution_status == ExecutionStatus.UNFINISHED,
|
|
419
|
+
or_(table.c.available_at.is_(None), table.c.available_at <= timestamp)),
|
|
347
420
|
and_(table.c.execution_status == ExecutionStatus.WORKING, table.c.lease_expires_at < timestamp),
|
|
348
421
|
)
|
|
349
422
|
|
|
@@ -484,21 +557,101 @@ class Coordinator:
|
|
|
484
557
|
).values(lease_expires_at=now() + timedelta(seconds=self.lease_seconds)))
|
|
485
558
|
|
|
486
559
|
def _fail(self, worker: _Worker, claim: Claim, error: Exception) -> None:
|
|
560
|
+
"""Record a handler exception: schedule a retry, or FAILED when attempts run out."""
|
|
487
561
|
table = worker.table
|
|
562
|
+
owned = (table.c.source_id == claim.source_id, table.c.execution_status == ExecutionStatus.WORKING,
|
|
563
|
+
table.c.claim_token == claim.token)
|
|
488
564
|
with self.session_factory.begin() as session:
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
565
|
+
# Only the claim owner changes this row, so read-then-write is safe;
|
|
566
|
+
# a replaced claim matches neither statement.
|
|
567
|
+
attempts = session.scalar(select(table.c.attempts).where(*owned))
|
|
568
|
+
if attempts is None:
|
|
569
|
+
return
|
|
570
|
+
attempts += 1
|
|
571
|
+
retry = attempts < worker.max_attempts
|
|
572
|
+
delay = min(worker.retry_backoff_seconds * 2 ** (attempts - 1), _MAX_RETRY_BACKOFF_SECONDS)
|
|
573
|
+
session.execute(update(table).where(*owned).values(
|
|
574
|
+
execution_status=ExecutionStatus.UNFINISHED if retry else ExecutionStatus.FAILED,
|
|
575
|
+
error=str(error), claim_token=None, lease_expires_at=None, attempts=attempts,
|
|
576
|
+
available_at=now() + timedelta(seconds=delay) if retry else None,
|
|
577
|
+
))
|
|
578
|
+
if retry:
|
|
579
|
+
logger.warning("Worker %s will retry %s in %.1fs (attempt %d of %d failed)",
|
|
580
|
+
worker.name, claim.source_id, delay, attempts, worker.max_attempts)
|
|
581
|
+
|
|
582
|
+
def _release(self, worker: _Worker, claims: Iterable[Claim], error: Exception) -> None:
|
|
583
|
+
"""Requeue work whose handler process was lost, without consuming an attempt.
|
|
584
|
+
|
|
585
|
+
A broken pool does not say which handler caused it, so no claim is blamed.
|
|
586
|
+
"""
|
|
587
|
+
table = worker.table
|
|
588
|
+
try:
|
|
589
|
+
with self.session_factory.begin() as session:
|
|
590
|
+
for claim in claims:
|
|
591
|
+
session.execute(update(table).where(
|
|
592
|
+
table.c.source_id == claim.source_id, table.c.execution_status == ExecutionStatus.WORKING,
|
|
593
|
+
table.c.claim_token == claim.token,
|
|
594
|
+
).values(execution_status=ExecutionStatus.UNFINISHED, error=str(error), claim_token=None,
|
|
595
|
+
lease_expires_at=None,
|
|
596
|
+
available_at=now() + timedelta(seconds=worker.retry_backoff_seconds)))
|
|
597
|
+
except Exception:
|
|
598
|
+
logger.exception("Could not requeue work for %s; it becomes claimable when its lease expires", worker.name)
|
|
599
|
+
|
|
600
|
+
def _start_pool(self, worker: _Worker) -> ProcessPoolExecutor:
|
|
601
|
+
context = multiprocessing.get_context("spawn")
|
|
602
|
+
ready = context.Barrier(worker.concurrency + 1)
|
|
603
|
+
handlers = ProcessPoolExecutor(
|
|
604
|
+
max_workers=worker.concurrency, mp_context=context,
|
|
605
|
+
initializer=_initialize_process,
|
|
606
|
+
initargs=(self.database_url, self.engine_options, self._definitions, worker.name, ready),
|
|
607
|
+
)
|
|
608
|
+
try:
|
|
609
|
+
# Initializers hold the startup barrier, so every configured
|
|
610
|
+
# process must start before any probe can finish. These probes
|
|
611
|
+
# create no source rows, claims, or execution-ledger entries.
|
|
612
|
+
probes = [handlers.submit(_process_ready) for _ in range(worker.concurrency)]
|
|
613
|
+
ready.wait(timeout=180)
|
|
614
|
+
for probe in probes:
|
|
615
|
+
probe.result()
|
|
616
|
+
except BaseException:
|
|
617
|
+
ready.abort()
|
|
618
|
+
handlers.shutdown(wait=True, cancel_futures=True)
|
|
619
|
+
raise
|
|
620
|
+
return handlers
|
|
621
|
+
|
|
622
|
+
def _replace_pool(self, worker: _Worker, broken: ProcessPoolExecutor) -> ProcessPoolExecutor:
|
|
623
|
+
"""Start fresh handler processes after the executor lost one."""
|
|
624
|
+
broken.shutdown(wait=False, cancel_futures=True)
|
|
625
|
+
handlers = self._start_pool(worker)
|
|
626
|
+
thread, _ = self._running[worker.name]
|
|
627
|
+
self._running[worker.name] = (thread, handlers)
|
|
628
|
+
return handlers
|
|
493
629
|
|
|
494
630
|
def _run(
|
|
495
631
|
self, worker: _Worker, handlers: ProcessPoolExecutor,
|
|
496
632
|
wakeup: threading.Event,
|
|
497
633
|
) -> None:
|
|
634
|
+
# No database or process error may end this loop: a dead scheduler
|
|
635
|
+
# stops claiming and renewing while the coordinator still looks alive.
|
|
498
636
|
active: dict[Future[Outcome], Claim] = {}
|
|
499
637
|
next_renewal = time.monotonic()
|
|
500
638
|
next_claim_at = 0.0
|
|
501
639
|
delay = self.poll_seconds
|
|
640
|
+
# A lost handler process breaks the pool and fails every in-flight
|
|
641
|
+
# future without saying which handler caused it. Work lost together is
|
|
642
|
+
# requeued unblamed, then run one at a time (see suspects) so that a
|
|
643
|
+
# handler which keeps killing its process is charged alone.
|
|
644
|
+
broken: BrokenProcessPool | None = None
|
|
645
|
+
lost: list[Claim] = []
|
|
646
|
+
suspects: dict[object, float] = {}
|
|
647
|
+
|
|
648
|
+
def record_failure(claim: Claim, error: Exception) -> None:
|
|
649
|
+
try:
|
|
650
|
+
self._fail(worker, claim, error)
|
|
651
|
+
except Exception:
|
|
652
|
+
logger.exception("Could not record the failure of %s:%s; it becomes claimable "
|
|
653
|
+
"when its lease expires", worker.name, claim.source_id)
|
|
654
|
+
|
|
502
655
|
while not self._stop.is_set() or active:
|
|
503
656
|
# Clear before inspecting futures so a completion during this iteration
|
|
504
657
|
# still wakes the wait below.
|
|
@@ -513,30 +666,79 @@ class Coordinator:
|
|
|
513
666
|
delay = self.poll_seconds
|
|
514
667
|
except LostClaim:
|
|
515
668
|
logger.info("Discarded stale result for %s:%s", worker.name, claim.source_id)
|
|
669
|
+
except BrokenProcessPool as exc:
|
|
670
|
+
broken = exc
|
|
671
|
+
lost.append(claim)
|
|
672
|
+
continue
|
|
516
673
|
except Exception as exc:
|
|
517
674
|
logger.exception("Worker %s failed for %s", worker.name, claim.source_id)
|
|
518
|
-
|
|
675
|
+
record_failure(claim, exc)
|
|
676
|
+
suspects.pop(claim.source_id, None)
|
|
677
|
+
if broken is not None and lost and not active:
|
|
678
|
+
if len(lost) == 1:
|
|
679
|
+
logger.error("Worker %s lost its handler process while running only %s",
|
|
680
|
+
worker.name, lost[0].source_id)
|
|
681
|
+
record_failure(lost[0], broken)
|
|
682
|
+
suspects.pop(lost[0].source_id, None)
|
|
683
|
+
else:
|
|
684
|
+
logger.warning("Worker %s lost a handler process while running %s; requeueing them "
|
|
685
|
+
"to run one at a time", worker.name, [claim.source_id for claim in lost])
|
|
686
|
+
self._release(worker, lost, broken)
|
|
687
|
+
expiry = time.monotonic() + self.lease_seconds + worker.retry_backoff_seconds
|
|
688
|
+
suspects.update((claim.source_id, expiry) for claim in lost)
|
|
689
|
+
lost = []
|
|
519
690
|
if time.monotonic() >= next_renewal:
|
|
520
|
-
if active:
|
|
521
|
-
self.renew(worker, active.values())
|
|
522
691
|
next_renewal = time.monotonic() + self.lease_seconds / 3
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
692
|
+
if active:
|
|
693
|
+
try:
|
|
694
|
+
self.renew(worker, active.values())
|
|
695
|
+
except Exception:
|
|
696
|
+
logger.exception("Error renewing leases for %s; retrying", worker.name)
|
|
697
|
+
next_renewal = time.monotonic() + self.lease_seconds / 10
|
|
698
|
+
# Suspects not seen again (no longer eligible, or claimed elsewhere) are forgotten.
|
|
699
|
+
for source_id in [source_id for source_id, expiry in suspects.items() if time.monotonic() >= expiry]:
|
|
700
|
+
del suspects[source_id]
|
|
701
|
+
capacity = 1 if suspects else worker.concurrency
|
|
702
|
+
# A broken pool is replaced once its remaining futures have reported.
|
|
703
|
+
claiming = not self._stop.is_set() and not (broken is not None and active)
|
|
704
|
+
if claiming and time.monotonic() >= next_claim_at:
|
|
705
|
+
if broken is not None:
|
|
706
|
+
try:
|
|
707
|
+
handlers = self._replace_pool(worker, handlers)
|
|
708
|
+
broken = None
|
|
709
|
+
logger.warning("Restarted handler processes for %s", worker.name)
|
|
710
|
+
except Exception:
|
|
711
|
+
logger.exception("Could not restart handler processes for %s; retrying after backoff",
|
|
712
|
+
worker.name)
|
|
713
|
+
next_claim_at = time.monotonic() + delay
|
|
714
|
+
delay = min(delay * 2, self.max_poll_seconds)
|
|
715
|
+
while broken is None and len(active) < capacity and not self._stop.is_set():
|
|
716
|
+
try:
|
|
717
|
+
claims = self._claim_many(worker, capacity - len(active))
|
|
718
|
+
except Exception:
|
|
719
|
+
logger.exception("Error claiming work for %s; retrying after backoff", worker.name)
|
|
720
|
+
claims = []
|
|
526
721
|
if not claims:
|
|
527
722
|
next_claim_at = time.monotonic() + delay
|
|
528
723
|
delay = min(delay * 2, self.max_poll_seconds)
|
|
529
724
|
break
|
|
530
725
|
delay = self.poll_seconds
|
|
531
726
|
next_claim_at = 0.0
|
|
532
|
-
for claim in claims:
|
|
533
|
-
|
|
727
|
+
for index, claim in enumerate(claims):
|
|
728
|
+
try:
|
|
729
|
+
future = handlers.submit(_execute_in_process, claim)
|
|
730
|
+
except BrokenProcessPool as exc:
|
|
731
|
+
logger.warning("Handler process pool for %s is broken; requeueing undispatched work",
|
|
732
|
+
worker.name)
|
|
733
|
+
broken = exc
|
|
734
|
+
self._release(worker, claims[index:], exc)
|
|
735
|
+
break
|
|
534
736
|
active[future] = claim
|
|
535
737
|
future.add_done_callback(lambda completed: wakeup.set())
|
|
536
738
|
deadlines: list[float] = []
|
|
537
739
|
if active:
|
|
538
740
|
deadlines.append(next_renewal)
|
|
539
|
-
if not self._stop.is_set() and len(active) <
|
|
741
|
+
if not self._stop.is_set() and not (broken is not None and active) and len(active) < capacity:
|
|
540
742
|
deadlines.append(next_claim_at)
|
|
541
743
|
if not deadlines:
|
|
542
744
|
break
|
|
@@ -553,26 +755,12 @@ class Coordinator:
|
|
|
553
755
|
pickle.dumps((self.database_url, self.engine_options, definitions))
|
|
554
756
|
except (TypeError, AttributeError, pickle.PicklingError) as exc:
|
|
555
757
|
raise ValueError("Handlers and models must be importable; bound handler arguments and engine options must be serializable") from exc
|
|
758
|
+
self._definitions = definitions
|
|
556
759
|
self._stop.clear()
|
|
557
760
|
for worker in self.workers.values():
|
|
558
|
-
context = multiprocessing.get_context("spawn")
|
|
559
|
-
ready = context.Barrier(worker.concurrency + 1)
|
|
560
|
-
handlers = ProcessPoolExecutor(
|
|
561
|
-
max_workers=worker.concurrency, mp_context=context,
|
|
562
|
-
initializer=_initialize_process,
|
|
563
|
-
initargs=(self.database_url, self.engine_options, definitions, worker.name, ready),
|
|
564
|
-
)
|
|
565
761
|
try:
|
|
566
|
-
|
|
567
|
-
# process must start before any probe can finish. These probes
|
|
568
|
-
# create no source rows, claims, or execution-ledger entries.
|
|
569
|
-
probes = [handlers.submit(_process_ready) for _ in range(worker.concurrency)]
|
|
570
|
-
ready.wait(timeout=180)
|
|
571
|
-
for probe in probes:
|
|
572
|
-
probe.result()
|
|
762
|
+
handlers = self._start_pool(worker)
|
|
573
763
|
except BaseException:
|
|
574
|
-
ready.abort()
|
|
575
|
-
handlers.shutdown(wait=True, cancel_futures=True)
|
|
576
764
|
self.stop()
|
|
577
765
|
raise
|
|
578
766
|
wakeup = threading.Event()
|
|
@@ -585,8 +773,10 @@ class Coordinator:
|
|
|
585
773
|
self._stop.set()
|
|
586
774
|
for wakeup in self._wakeups.values():
|
|
587
775
|
wakeup.set()
|
|
588
|
-
for thread,
|
|
776
|
+
for thread, _ in list(self._running.values()):
|
|
589
777
|
thread.join()
|
|
778
|
+
# A scheduler may have replaced its pool; shut down the current one.
|
|
779
|
+
for _, handlers in self._running.values():
|
|
590
780
|
handlers.shutdown(wait=True)
|
|
591
781
|
self._running.clear()
|
|
592
782
|
self._wakeups.clear()
|
|
File without changes
|
|
File without changes
|