dbworker 0.1.2__tar.gz → 0.1.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dbworker
3
- Version: 0.1.2
3
+ Version: 0.1.4
4
4
  Summary: Portable, single-script Python/SQLAlchemy background worker for SQLite, PostgreSQL and MySQL — no Redis, broker or enqueue
5
5
  License: MIT
6
6
  License-File: LICENSE
@@ -94,16 +94,16 @@ Median wall time in seconds; **bold** marks the faster backend. Each experiment
94
94
 
95
95
  | Experiment | Workload | Celery wall (s) | DBWorker wall (s) | C / D | Task P95 / peak outstanding (C; D) |
96
96
  |---|---|---:|---:|---:|---|
97
- | imagededup | Build 1,000 image hashes (one bulk import) | **5.781** | 6.066 | 0.95 | n/a; n/a |
98
- | imagededup | Compare 1,000 images / 999,000 directed pairs | 82.087 | **77.527** | 1.06 | n/a; n/a |
99
- | imagededup | Build + compare 1,000 images / 999,000 directed pairs | 91.669 | **90.453** | 1.01 | n/a; n/a |
100
- | superset | SQL Lab: 3,000 queries / 10,000 rows | **130.809** | 146.255 | 0.89 | 34.381s / 1384; 41.335s / 1494 |
101
- | saleor | Export: 1,000 x 256 products + 1,000 emails | **132.599** | 160.993 | 0.82 | 35.885s / 606; 48.863s / 792 |
102
- | paperless_ngx | OCR: 200 scans, archive and index | 134.649 | **75.436** | 1.78 | 71.852s / 114; 47.910s / 51 |
103
- | posthog | 2FA: 5,000 requests / 10,000 tasks | 115.049 | **110.402** | 1.04 | 0.109s / 15; 0.594s / 65 |
104
- | sentry | Email: 2,000 requests / 4,000 deliveries | **60.263** | 60.300 | 1.00 | 0.033s / 20; 0.147s / 26 |
105
-
106
- Run: `github-38214437264-1`. [Details and scope](doc/benchmark-results.md) · [JSON results](benchmarks/results/latest/index.json).
97
+ | imagededup | Build 1,000 image hashes (one bulk import) | 6.459 | **6.005** | 1.08 | n/a; n/a |
98
+ | imagededup | Compare 1,000 images / 999,000 directed pairs | 81.712 | **77.167** | 1.06 | n/a; n/a |
99
+ | imagededup | Build + compare 1,000 images / 999,000 directed pairs | 92.450 | **88.584** | 1.04 | n/a; n/a |
100
+ | superset | SQL Lab: 3,000 queries / 10,000 rows | **153.735** | 168.862 | 0.91 | 42.354s / 1462; 48.725s / 1511 |
101
+ | saleor | Export: 1,000 x 256 products + 1,000 emails | **137.652** | 167.635 | 0.82 | 39.572s / 630; 54.428s / 812 |
102
+ | paperless_ngx | OCR: 200 scans, archive and index | 142.059 | **81.849** | 1.74 | 78.973s / 118; 57.944s / 61 |
103
+ | posthog | 2FA: 5,000 requests / 10,000 tasks | 184.240 | **164.843** | 1.12 | 0.161s / 14; 0.179s / 30 |
104
+ | sentry | Email: 2,000 requests / 4,000 deliveries | **60.314** | 60.336 | 1.00 | 0.057s / 16; 0.479s / 149 |
105
+
106
+ Run: `github-38401468222-1`. [Details and scope](doc/benchmark-results.md) · [JSON results](benchmarks/results/latest/index.json).
107
107
  Scoped application workloads; Sentry uses historical 24.1.0. Wall time includes the fixed submission window, so a ratio near 1 does not prove equal processing capacity. Task P95 starts at publication; peak outstanding counts already published tasks. Ratios above 1 favor DBWorker; no cross-project average is computed.
108
108
  <!-- benchmark-results:end -->
109
109
 
@@ -77,15 +77,15 @@ Median wall time in seconds; **bold** marks the faster backend. Each experiment
77
77
 
78
78
  | Experiment | Workload | Celery wall (s) | DBWorker wall (s) | C / D | Task P95 / peak outstanding (C; D) |
79
79
  |---|---|---:|---:|---:|---|
80
- | imagededup | Build 1,000 image hashes (one bulk import) | **5.781** | 6.066 | 0.95 | n/a; n/a |
81
- | imagededup | Compare 1,000 images / 999,000 directed pairs | 82.087 | **77.527** | 1.06 | n/a; n/a |
82
- | imagededup | Build + compare 1,000 images / 999,000 directed pairs | 91.669 | **90.453** | 1.01 | n/a; n/a |
83
- | superset | SQL Lab: 3,000 queries / 10,000 rows | **130.809** | 146.255 | 0.89 | 34.381s / 1384; 41.335s / 1494 |
84
- | saleor | Export: 1,000 x 256 products + 1,000 emails | **132.599** | 160.993 | 0.82 | 35.885s / 606; 48.863s / 792 |
85
- | paperless_ngx | OCR: 200 scans, archive and index | 134.649 | **75.436** | 1.78 | 71.852s / 114; 47.910s / 51 |
86
- | posthog | 2FA: 5,000 requests / 10,000 tasks | 115.049 | **110.402** | 1.04 | 0.109s / 15; 0.594s / 65 |
87
- | sentry | Email: 2,000 requests / 4,000 deliveries | **60.263** | 60.300 | 1.00 | 0.033s / 20; 0.147s / 26 |
88
-
89
- Run: `github-38214437264-1`. [Details and scope](doc/benchmark-results.md) · [JSON results](benchmarks/results/latest/index.json).
80
+ | imagededup | Build 1,000 image hashes (one bulk import) | 6.459 | **6.005** | 1.08 | n/a; n/a |
81
+ | imagededup | Compare 1,000 images / 999,000 directed pairs | 81.712 | **77.167** | 1.06 | n/a; n/a |
82
+ | imagededup | Build + compare 1,000 images / 999,000 directed pairs | 92.450 | **88.584** | 1.04 | n/a; n/a |
83
+ | superset | SQL Lab: 3,000 queries / 10,000 rows | **153.735** | 168.862 | 0.91 | 42.354s / 1462; 48.725s / 1511 |
84
+ | saleor | Export: 1,000 x 256 products + 1,000 emails | **137.652** | 167.635 | 0.82 | 39.572s / 630; 54.428s / 812 |
85
+ | paperless_ngx | OCR: 200 scans, archive and index | 142.059 | **81.849** | 1.74 | 78.973s / 118; 57.944s / 61 |
86
+ | posthog | 2FA: 5,000 requests / 10,000 tasks | 184.240 | **164.843** | 1.12 | 0.161s / 14; 0.179s / 30 |
87
+ | sentry | Email: 2,000 requests / 4,000 deliveries | **60.314** | 60.336 | 1.00 | 0.057s / 16; 0.479s / 149 |
88
+
89
+ Run: `github-38401468222-1`. [Details and scope](doc/benchmark-results.md) · [JSON results](benchmarks/results/latest/index.json).
90
90
  Scoped application workloads; Sentry uses historical 24.1.0. Wall time includes the fixed submission window, so a ratio near 1 does not prove equal processing capacity. Task P95 starts at publication; peak outstanding counts already published tasks. Ratios above 1 favor DBWorker; no cross-project average is computed.
91
91
  <!-- benchmark-results:end -->
@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
4
4
 
5
5
  [tool.poetry]
6
6
  name = "dbworker"
7
- version = "0.1.2"
7
+ version = "0.1.4"
8
8
  description = "Portable, single-script Python/SQLAlchemy background worker for SQLite, PostgreSQL and MySQL — no Redis, broker or enqueue"
9
9
  authors = ["Ziyang Song"]
10
10
  license = "MIT"
@@ -4,7 +4,7 @@ from __future__ import annotations
4
4
 
5
5
  import logging
6
6
  import multiprocessing
7
- import pickle # nosec B403 -- used only for local pre-flight serialization; this module never deserializes pickle data.
7
+ import pickle # nosec B403 -- used only for local serialization checks; this module never deserializes foreign pickle data.
8
8
  import re
9
9
  import sqlite3
10
10
  import threading
@@ -12,6 +12,7 @@ import time
12
12
  import uuid
13
13
  from collections.abc import Callable, Iterable
14
14
  from concurrent.futures import Future, ProcessPoolExecutor
15
+ from concurrent.futures.process import BrokenProcessPool
15
16
  from dataclasses import dataclass
16
17
  from functools import cached_property
17
18
  from enum import StrEnum
@@ -19,17 +20,20 @@ from datetime import datetime, timedelta, timezone
19
20
  from typing import Any, TypeAlias, TypeVar, cast
20
21
  from multiprocessing.util import Finalize
21
22
 
22
- from sqlalchemy import Column, DateTime, Enum, ForeignKey, String, Table, Text, and_, create_engine, event, exists, inspect, insert, or_, select, tuple_, update
23
- from sqlalchemy.exc import IntegrityError, OperationalError
23
+ from sqlalchemy import Column, DateTime, Enum, ForeignKey, Integer, String, Table, Text, and_, create_engine, event, exists, inspect, insert, or_, select, tuple_, update
24
+ from sqlalchemy.exc import DBAPIError, IntegrityError, OperationalError
24
25
  from sqlalchemy.engine import CursorResult, RowMapping, URL, make_url
25
26
  from sqlalchemy.dialects.postgresql import insert as postgresql_insert
26
27
  from sqlalchemy.orm import DeclarativeBase, Mapper, Session, sessionmaker
28
+ from sqlalchemy.ext.compiler import compiles
29
+ from sqlalchemy.schema import ExecutableDDLElement
27
30
  from sqlalchemy.sql import ColumnElement, Select, Update
28
31
  from sqlalchemy.sql.selectable import Exists
29
32
 
30
33
  logger = logging.getLogger(__name__)
31
34
 
32
35
  _SQLITE_FINALIZATION_RETRY_SECONDS = 5.0
36
+ _MAX_RETRY_BACKOFF_SECONDS = 300.0
33
37
 
34
38
 
35
39
  def now() -> datetime:
@@ -78,16 +82,24 @@ class _Worker:
78
82
  handler: Handler,
79
83
  eligible: Callable[[], Select[tuple[Any]]] | None = None,
80
84
  concurrency: int = 1,
85
+ max_attempts: int = 3,
86
+ retry_backoff_seconds: float = 1.0,
81
87
  ) -> None:
82
88
  if not re.fullmatch(r"[a-z][a-z0-9_]*", name):
83
89
  raise ValueError("Worker name must contain lowercase letters, digits, and underscores")
84
90
  if concurrency < 1:
85
91
  raise ValueError("Worker concurrency must be positive")
92
+ if max_attempts < 1:
93
+ raise ValueError("Worker max_attempts must be positive")
94
+ if retry_backoff_seconds < 0:
95
+ raise ValueError("Worker retry backoff must not be negative")
86
96
  self.name = name
87
97
  self.source = source
88
98
  self.handler = handler
89
99
  self.eligible = eligible or (lambda: select(source))
90
100
  self.concurrency = concurrency
101
+ self.max_attempts = max_attempts
102
+ self.retry_backoff_seconds = retry_backoff_seconds
91
103
  mapper = cast(Mapper[Any], inspect(source))
92
104
  if len(mapper.primary_key) != 1:
93
105
  raise ValueError("Worker sources must have one primary-key column")
@@ -112,6 +124,10 @@ class _Worker:
112
124
  Column("claim_token", String(36)),
113
125
  Column("lease_expires_at", DateTime, index=True),
114
126
  Column("error", Text),
127
+ # Consecutive handler failures; a committed outcome resets it.
128
+ Column("attempts", Integer, nullable=False, default=0, server_default="0"),
129
+ # A failed attempt is not claimable again before this time.
130
+ Column("available_at", DateTime),
115
131
  )
116
132
 
117
133
  def state(self, session: Session, source_id: object) -> RowMapping | None:
@@ -121,11 +137,25 @@ class _Worker:
121
137
  result = cast(CursorResult[Any], session.execute(
122
138
  update(self.table)
123
139
  .where(self.table.c.source_id == source_id, self.table.c.execution_status == ExecutionStatus.FAILED)
124
- .values(execution_status=ExecutionStatus.UNFINISHED, error=None, claim_token=None, lease_expires_at=None)
140
+ .values(execution_status=ExecutionStatus.UNFINISHED, error=None, claim_token=None, lease_expires_at=None,
141
+ attempts=0, available_at=None)
125
142
  ))
126
143
  return result.rowcount == 1
127
144
 
128
145
 
146
+ class _AddColumn(ExecutableDDLElement):
147
+ """ALTER TABLE ... ADD COLUMN for a work table created by an earlier release."""
148
+
149
+ def __init__(self, column: Column[Any]) -> None:
150
+ self.column = column
151
+
152
+
153
+ @compiles(_AddColumn)
154
+ def _compile_add_column(element: _AddColumn, compiler: Any, **kw: Any) -> str:
155
+ return (f"ALTER TABLE {compiler.preparer.format_table(element.column.table)} "
156
+ f"ADD COLUMN {compiler.get_column_specification(element.column)}")
157
+
158
+
129
159
  @dataclass(frozen=True)
130
160
  class _WorkerDefinition:
131
161
  """Importable callables and model classes, never live database resources."""
@@ -170,7 +200,16 @@ def _process_ready() -> None:
170
200
  def _execute_in_process(claim: Claim) -> Outcome:
171
201
  if _process_worker is None or _process_session_factory is None:
172
202
  raise RuntimeError("Handler process has not been initialized")
173
- return _execute_claim(_process_worker, claim, _process_session_factory)
203
+ try:
204
+ return _execute_claim(_process_worker, claim, _process_session_factory)
205
+ except Exception as exc:
206
+ # An exception the parent cannot rebuild (for example a required
207
+ # multi-argument __init__) would break the whole process pool there.
208
+ try:
209
+ pickle.loads(pickle.dumps(exc)) # nosec B301 -- round-trips an object this process just created.
210
+ except Exception:
211
+ raise RuntimeError(f"{type(exc).__qualname__}: {exc}") from exc
212
+ raise
174
213
 
175
214
 
176
215
  def _retry_sqlite_completion(session: Session, completion: Update, error: OperationalError) -> CursorResult[Any]:
@@ -226,7 +265,7 @@ def _execute_claim(worker: _Worker, claim: Claim, session_factory: sessionmaker[
226
265
  worker.table.c.execution_status == ExecutionStatus.WORKING,
227
266
  worker.table.c.claim_token == claim.token,
228
267
  ).values(execution_status=ExecutionStatus.FINISHED if isinstance(outcome, Finished) else ExecutionStatus.UNFINISHED,
229
- claim_token=None, lease_expires_at=None, error=None)
268
+ claim_token=None, lease_expires_at=None, error=None, attempts=0, available_at=None)
230
269
  try:
231
270
  result = cast(CursorResult[Any], session.execute(completion))
232
271
  except OperationalError as exc:
@@ -268,6 +307,7 @@ class Coordinator:
268
307
  self.max_poll_seconds = max_poll_seconds
269
308
  self.workers: dict[str, _Worker] = {}
270
309
  self._running: dict[str, tuple[threading.Thread, ProcessPoolExecutor]] = {}
310
+ self._definitions: tuple[_WorkerDefinition, ...] = ()
271
311
  self._stop = threading.Event()
272
312
  self._wakeups: dict[str, threading.Event] = {}
273
313
 
@@ -291,9 +331,16 @@ class Coordinator:
291
331
  source: type[DeclarativeBase],
292
332
  eligible: Callable[[], Select[tuple[Any]]] | None = None,
293
333
  concurrency: int = 1,
334
+ max_attempts: int = 3,
335
+ retry_backoff_seconds: float = 1.0,
294
336
  ) -> Callable[[_Function], _Function]:
295
337
  """Register a handler whose session and final commit belong to the runtime.
296
338
 
339
+ A handler exception rolls back that attempt. The work is retried after
340
+ retry_backoff_seconds (doubling per consecutive failure) and is marked
341
+ FAILED once max_attempts consecutive attempts have raised. Use
342
+ max_attempts=1 to fail on the first exception.
343
+
297
344
  Registration creates worker metadata, but starts no processing. The
298
345
  original function is returned unchanged; direct calls are not managed.
299
346
  """
@@ -305,6 +352,7 @@ class Coordinator:
305
352
  self.workers[name] = _Worker(
306
353
  name=name, source=source, handler=handler,
307
354
  eligible=eligible, concurrency=concurrency,
355
+ max_attempts=max_attempts, retry_backoff_seconds=retry_backoff_seconds,
308
356
  )
309
357
  return handler
310
358
 
@@ -337,13 +385,38 @@ class Coordinator:
337
385
  ).correlate_except(table)
338
386
 
339
387
  def create_worker_tables(self) -> None:
388
+ """Create missing work tables and add columns introduced by later releases."""
340
389
  with self.session_factory() as session:
341
- for worker in self.workers.values():
342
- worker.table.create(session.get_bind(), checkfirst=True)
390
+ bind = session.get_bind()
391
+ for worker in self.workers.values():
392
+ table = worker.table
393
+ table.create(bind, checkfirst=True)
394
+
395
+ def present(column: Column[Any]) -> bool:
396
+ # Probe with ordinary SQL rather than reflection, so the session's
397
+ # schema translation and search path apply as they do to claims.
398
+ try:
399
+ with self.session_factory() as session:
400
+ session.execute(select(column).limit(0))
401
+ return True
402
+ except DBAPIError:
403
+ return False
404
+
405
+ for column in (table.c.attempts, table.c.available_at):
406
+ if present(column):
407
+ continue
408
+ try:
409
+ with self.session_factory.begin() as session:
410
+ session.connection().execute(_AddColumn(column))
411
+ except DBAPIError:
412
+ # Another service may have added it while starting up.
413
+ if not present(column):
414
+ raise
343
415
 
344
416
  def _available(self, table: Table, timestamp: datetime) -> ColumnElement[bool]:
345
417
  return or_(
346
- table.c.execution_status == ExecutionStatus.UNFINISHED,
418
+ and_(table.c.execution_status == ExecutionStatus.UNFINISHED,
419
+ or_(table.c.available_at.is_(None), table.c.available_at <= timestamp)),
347
420
  and_(table.c.execution_status == ExecutionStatus.WORKING, table.c.lease_expires_at < timestamp),
348
421
  )
349
422
 
@@ -484,21 +557,101 @@ class Coordinator:
484
557
  ).values(lease_expires_at=now() + timedelta(seconds=self.lease_seconds)))
485
558
 
486
559
  def _fail(self, worker: _Worker, claim: Claim, error: Exception) -> None:
560
+ """Record a handler exception: schedule a retry, or FAILED when attempts run out."""
487
561
  table = worker.table
562
+ owned = (table.c.source_id == claim.source_id, table.c.execution_status == ExecutionStatus.WORKING,
563
+ table.c.claim_token == claim.token)
488
564
  with self.session_factory.begin() as session:
489
- session.execute(update(table).where(
490
- table.c.source_id == claim.source_id, table.c.execution_status == ExecutionStatus.WORKING,
491
- table.c.claim_token == claim.token,
492
- ).values(execution_status=ExecutionStatus.FAILED, error=str(error), claim_token=None, lease_expires_at=None))
565
+ # Only the claim owner changes this row, so read-then-write is safe;
566
+ # a replaced claim matches neither statement.
567
+ attempts = session.scalar(select(table.c.attempts).where(*owned))
568
+ if attempts is None:
569
+ return
570
+ attempts += 1
571
+ retry = attempts < worker.max_attempts
572
+ delay = min(worker.retry_backoff_seconds * 2 ** (attempts - 1), _MAX_RETRY_BACKOFF_SECONDS)
573
+ session.execute(update(table).where(*owned).values(
574
+ execution_status=ExecutionStatus.UNFINISHED if retry else ExecutionStatus.FAILED,
575
+ error=str(error), claim_token=None, lease_expires_at=None, attempts=attempts,
576
+ available_at=now() + timedelta(seconds=delay) if retry else None,
577
+ ))
578
+ if retry:
579
+ logger.warning("Worker %s will retry %s in %.1fs (attempt %d of %d failed)",
580
+ worker.name, claim.source_id, delay, attempts, worker.max_attempts)
581
+
582
+ def _release(self, worker: _Worker, claims: Iterable[Claim], error: Exception) -> None:
583
+ """Requeue work whose handler process was lost, without consuming an attempt.
584
+
585
+ A broken pool does not say which handler caused it, so no claim is blamed.
586
+ """
587
+ table = worker.table
588
+ try:
589
+ with self.session_factory.begin() as session:
590
+ for claim in claims:
591
+ session.execute(update(table).where(
592
+ table.c.source_id == claim.source_id, table.c.execution_status == ExecutionStatus.WORKING,
593
+ table.c.claim_token == claim.token,
594
+ ).values(execution_status=ExecutionStatus.UNFINISHED, error=str(error), claim_token=None,
595
+ lease_expires_at=None,
596
+ available_at=now() + timedelta(seconds=worker.retry_backoff_seconds)))
597
+ except Exception:
598
+ logger.exception("Could not requeue work for %s; it becomes claimable when its lease expires", worker.name)
599
+
600
+ def _start_pool(self, worker: _Worker) -> ProcessPoolExecutor:
601
+ context = multiprocessing.get_context("spawn")
602
+ ready = context.Barrier(worker.concurrency + 1)
603
+ handlers = ProcessPoolExecutor(
604
+ max_workers=worker.concurrency, mp_context=context,
605
+ initializer=_initialize_process,
606
+ initargs=(self.database_url, self.engine_options, self._definitions, worker.name, ready),
607
+ )
608
+ try:
609
+ # Initializers hold the startup barrier, so every configured
610
+ # process must start before any probe can finish. These probes
611
+ # create no source rows, claims, or execution-ledger entries.
612
+ probes = [handlers.submit(_process_ready) for _ in range(worker.concurrency)]
613
+ ready.wait(timeout=180)
614
+ for probe in probes:
615
+ probe.result()
616
+ except BaseException:
617
+ ready.abort()
618
+ handlers.shutdown(wait=True, cancel_futures=True)
619
+ raise
620
+ return handlers
621
+
622
+ def _replace_pool(self, worker: _Worker, broken: ProcessPoolExecutor) -> ProcessPoolExecutor:
623
+ """Start fresh handler processes after the executor lost one."""
624
+ broken.shutdown(wait=False, cancel_futures=True)
625
+ handlers = self._start_pool(worker)
626
+ thread, _ = self._running[worker.name]
627
+ self._running[worker.name] = (thread, handlers)
628
+ return handlers
493
629
 
494
630
  def _run(
495
631
  self, worker: _Worker, handlers: ProcessPoolExecutor,
496
632
  wakeup: threading.Event,
497
633
  ) -> None:
634
+ # No database or process error may end this loop: a dead scheduler
635
+ # stops claiming and renewing while the coordinator still looks alive.
498
636
  active: dict[Future[Outcome], Claim] = {}
499
637
  next_renewal = time.monotonic()
500
638
  next_claim_at = 0.0
501
639
  delay = self.poll_seconds
640
+ # A lost handler process breaks the pool and fails every in-flight
641
+ # future without saying which handler caused it. Work lost together is
642
+ # requeued unblamed, then run one at a time (see suspects) so that a
643
+ # handler which keeps killing its process is charged alone.
644
+ broken: BrokenProcessPool | None = None
645
+ lost: list[Claim] = []
646
+ suspects: dict[object, float] = {}
647
+
648
+ def record_failure(claim: Claim, error: Exception) -> None:
649
+ try:
650
+ self._fail(worker, claim, error)
651
+ except Exception:
652
+ logger.exception("Could not record the failure of %s:%s; it becomes claimable "
653
+ "when its lease expires", worker.name, claim.source_id)
654
+
502
655
  while not self._stop.is_set() or active:
503
656
  # Clear before inspecting futures so a completion during this iteration
504
657
  # still wakes the wait below.
@@ -513,30 +666,79 @@ class Coordinator:
513
666
  delay = self.poll_seconds
514
667
  except LostClaim:
515
668
  logger.info("Discarded stale result for %s:%s", worker.name, claim.source_id)
669
+ except BrokenProcessPool as exc:
670
+ broken = exc
671
+ lost.append(claim)
672
+ continue
516
673
  except Exception as exc:
517
674
  logger.exception("Worker %s failed for %s", worker.name, claim.source_id)
518
- self._fail(worker, claim, exc)
675
+ record_failure(claim, exc)
676
+ suspects.pop(claim.source_id, None)
677
+ if broken is not None and lost and not active:
678
+ if len(lost) == 1:
679
+ logger.error("Worker %s lost its handler process while running only %s",
680
+ worker.name, lost[0].source_id)
681
+ record_failure(lost[0], broken)
682
+ suspects.pop(lost[0].source_id, None)
683
+ else:
684
+ logger.warning("Worker %s lost a handler process while running %s; requeueing them "
685
+ "to run one at a time", worker.name, [claim.source_id for claim in lost])
686
+ self._release(worker, lost, broken)
687
+ expiry = time.monotonic() + self.lease_seconds + worker.retry_backoff_seconds
688
+ suspects.update((claim.source_id, expiry) for claim in lost)
689
+ lost = []
519
690
  if time.monotonic() >= next_renewal:
520
- if active:
521
- self.renew(worker, active.values())
522
691
  next_renewal = time.monotonic() + self.lease_seconds / 3
523
- if not self._stop.is_set() and time.monotonic() >= next_claim_at:
524
- while len(active) < worker.concurrency and not self._stop.is_set():
525
- claims = self._claim_many(worker, worker.concurrency - len(active))
692
+ if active:
693
+ try:
694
+ self.renew(worker, active.values())
695
+ except Exception:
696
+ logger.exception("Error renewing leases for %s; retrying", worker.name)
697
+ next_renewal = time.monotonic() + self.lease_seconds / 10
698
+ # Suspects not seen again (no longer eligible, or claimed elsewhere) are forgotten.
699
+ for source_id in [source_id for source_id, expiry in suspects.items() if time.monotonic() >= expiry]:
700
+ del suspects[source_id]
701
+ capacity = 1 if suspects else worker.concurrency
702
+ # A broken pool is replaced once its remaining futures have reported.
703
+ claiming = not self._stop.is_set() and not (broken is not None and active)
704
+ if claiming and time.monotonic() >= next_claim_at:
705
+ if broken is not None:
706
+ try:
707
+ handlers = self._replace_pool(worker, handlers)
708
+ broken = None
709
+ logger.warning("Restarted handler processes for %s", worker.name)
710
+ except Exception:
711
+ logger.exception("Could not restart handler processes for %s; retrying after backoff",
712
+ worker.name)
713
+ next_claim_at = time.monotonic() + delay
714
+ delay = min(delay * 2, self.max_poll_seconds)
715
+ while broken is None and len(active) < capacity and not self._stop.is_set():
716
+ try:
717
+ claims = self._claim_many(worker, capacity - len(active))
718
+ except Exception:
719
+ logger.exception("Error claiming work for %s; retrying after backoff", worker.name)
720
+ claims = []
526
721
  if not claims:
527
722
  next_claim_at = time.monotonic() + delay
528
723
  delay = min(delay * 2, self.max_poll_seconds)
529
724
  break
530
725
  delay = self.poll_seconds
531
726
  next_claim_at = 0.0
532
- for claim in claims:
533
- future = handlers.submit(_execute_in_process, claim)
727
+ for index, claim in enumerate(claims):
728
+ try:
729
+ future = handlers.submit(_execute_in_process, claim)
730
+ except BrokenProcessPool as exc:
731
+ logger.warning("Handler process pool for %s is broken; requeueing undispatched work",
732
+ worker.name)
733
+ broken = exc
734
+ self._release(worker, claims[index:], exc)
735
+ break
534
736
  active[future] = claim
535
737
  future.add_done_callback(lambda completed: wakeup.set())
536
738
  deadlines: list[float] = []
537
739
  if active:
538
740
  deadlines.append(next_renewal)
539
- if not self._stop.is_set() and len(active) < worker.concurrency:
741
+ if not self._stop.is_set() and not (broken is not None and active) and len(active) < capacity:
540
742
  deadlines.append(next_claim_at)
541
743
  if not deadlines:
542
744
  break
@@ -553,26 +755,12 @@ class Coordinator:
553
755
  pickle.dumps((self.database_url, self.engine_options, definitions))
554
756
  except (TypeError, AttributeError, pickle.PicklingError) as exc:
555
757
  raise ValueError("Handlers and models must be importable; bound handler arguments and engine options must be serializable") from exc
758
+ self._definitions = definitions
556
759
  self._stop.clear()
557
760
  for worker in self.workers.values():
558
- context = multiprocessing.get_context("spawn")
559
- ready = context.Barrier(worker.concurrency + 1)
560
- handlers = ProcessPoolExecutor(
561
- max_workers=worker.concurrency, mp_context=context,
562
- initializer=_initialize_process,
563
- initargs=(self.database_url, self.engine_options, definitions, worker.name, ready),
564
- )
565
761
  try:
566
- # Initializers hold the startup barrier, so every configured
567
- # process must start before any probe can finish. These probes
568
- # create no source rows, claims, or execution-ledger entries.
569
- probes = [handlers.submit(_process_ready) for _ in range(worker.concurrency)]
570
- ready.wait(timeout=180)
571
- for probe in probes:
572
- probe.result()
762
+ handlers = self._start_pool(worker)
573
763
  except BaseException:
574
- ready.abort()
575
- handlers.shutdown(wait=True, cancel_futures=True)
576
764
  self.stop()
577
765
  raise
578
766
  wakeup = threading.Event()
@@ -585,8 +773,10 @@ class Coordinator:
585
773
  self._stop.set()
586
774
  for wakeup in self._wakeups.values():
587
775
  wakeup.set()
588
- for thread, handlers in self._running.values():
776
+ for thread, _ in list(self._running.values()):
589
777
  thread.join()
778
+ # A scheduler may have replaced its pool; shut down the current one.
779
+ for _, handlers in self._running.values():
590
780
  handlers.shutdown(wait=True)
591
781
  self._running.clear()
592
782
  self._wakeups.clear()
File without changes
File without changes