ergon-framework-python 0.3.0__py3-none-any.whl → 0.3.2__py3-none-any.whl

ergon/__init__.py

@@ -10,4 +10,4 @@ __all__ = [
     "manager",
 ]
 
-__version__ = "0.3.0"
+__version__ = "0.3.2"

ergon/connector/ergon_platform/channels/connector.py

@@ -208,6 +208,10 @@ class ErgonPlatformChannelsConnector(Connector):
         if requeue and transaction.id:
             self._seen_event_ids.discard(transaction.id)
 
+    def release_transaction(self, transaction: Transaction) -> None:
+        """Give back a fetched but unstarted inbox event (worker shutdown): requeued with no delay."""
+        self.nack_transaction(transaction, requeue=True, delay_seconds=0)
+
     def download_attachments(
         self,
         transaction: Transaction,

ergon/connector/ergon_platform/workflows/connector.py

@@ -159,6 +159,13 @@ class ErgonPlatformConnector(Connector):
             **fields,
         )
 
+    def release_transaction(self, transaction: Transaction) -> None:
+        """Give back a transaction that was fetched but never started (worker shutdown).
+
+        A plain release: no failure reason, no cooldown, so it is picked up again right away.
+        """
+        self.release_item(transaction.id)
+
     def nack_transaction(
         self,
         transaction: Transaction,

ergon/connector/nylas/connector.py

@@ -119,5 +119,10 @@ class NylasConnector(Connector):
             return
         logger.debug("nack_transaction is a no-op for Nylas; message %s remains available for refetch", transaction.id)
 
+    def release_transaction(self, transaction: Transaction) -> None:
+        """Give back a fetched but unstarted message (worker shutdown): unchanged, refetchable."""
+        self.service.reset_pagination()
+        logger.debug("Nylas message %s released unchanged", transaction.id)
+
     def close(self) -> None:
         self.service.close()

ergon/connector/outlook/connector.py

@@ -194,5 +194,10 @@ class OutlookGraphConnector(Connector):
             self.service.reset_pagination()
             logger.debug("Outlook message %s was made available for refetch", transaction.id)
 
+    def release_transaction(self, transaction: Transaction) -> None:
+        """Give back a fetched but unstarted message (worker shutdown): unchanged, refetchable."""
+        self.service.reset_pagination()
+        logger.debug("Outlook message %s released unchanged", transaction.id)
+
     def close(self) -> None:
         self.service.close()

ergon/task/helpers.py

@@ -309,12 +309,17 @@ def run_fn(
     return run()
 
 
+_STOP_POLL_SECONDS = 0.25
+
+
 def multithread_execute(
     submissions: Iterable[Callable[[], futures.Future]],
     *,
     concurrency: int,
     limit: Optional[int] = None,
     timeout: Optional[float] = None,
+    stop: Optional[Callable[[], bool]] = None,
+    on_stop: Optional[Callable[[], None]] = None,
 ) -> int:
     """
     Execute submitted callables with bounded concurrency and refill.
@@ -322,6 +327,8 @@ def multithread_execute(
     - submissions yields callables that RETURN a Future when called
     - this function manages in-flight futures only
     - no domain semantics, no retries, no tracing
+    - ``stop`` is polled while futures are in flight; once it returns True nothing more is
+      submitted, ``on_stop`` runs once right away, and the in-flight futures are still awaited
 
     Returns:
         Number of completed submissions
@@ -329,6 +336,10 @@ def multithread_execute(
 
     in_flight: dict[futures.Future, float] = {}
     completed = 0
+    stopping = False
+    wait_timeout = timeout
+    if stop is not None:
+        wait_timeout = _STOP_POLL_SECONDS if timeout is None else min(timeout, _STOP_POLL_SECONDS)
 
     submit_iter = iter(submissions)
 
@@ -363,7 +374,7 @@ def multithread_execute(
         done, _ = futures.wait(
             in_flight.keys(),
             return_when=futures.FIRST_COMPLETED,
-            timeout=timeout,
+            timeout=wait_timeout,
         )
 
         # ----------------------------
@@ -391,6 +402,14 @@ def multithread_execute(
                     in_flight.pop(fut)
                     completed += 1
 
+        if not stopping and stop is not None and stop():
+            stopping = True
+            logger.info("Stop requested; waiting for %d in-flight submission(s), submitting no more", len(in_flight))
+            if on_stop is not None:
+                on_stop()
+        if stopping:
+            continue
+
         # ============================================================
         # REFILL
         # ============================================================

ergon/task/mixins/consumer.py

@@ -10,7 +10,7 @@ from async_timeout import timeout as timeout_after
 from opentelemetry import context as otel_context
 
 from ... import connector, telemetry
-from .. import base, exceptions, helpers, liveness, policies, utils
+from .. import base, exceptions, helpers, liveness, policies, shutdown, utils
 from . import metrics as mixin_metrics
 from . import producer
 
@@ -304,6 +304,38 @@ class ConsumerMixin(ABC):
             return next(iter(self.connectors.values()))
         raise ValueError("Multiple connectors configured; specify one in policy")
 
+    # =====================================================================
+    # SHUTDOWN HAND-BACK
+    # =====================================================================
+    def _hand_back_unstarted(self, conn, transactions: List[connector.Transaction]) -> None:
+        """Give back fetched transactions that were never started because the process is stopping.
+
+        ``release_transaction`` (no verdict) when the connector has it, else
+        ``nack_transaction(requeue=True)``. Never raises: the stop must go on, and a
+        transaction that cannot be handed back returns when its claim or lease expires.
+        """
+        release = getattr(conn, "release_transaction", None)
+        nack = getattr(conn, "nack_transaction", None)
+        for tr in transactions:
+            try:
+                if callable(release):
+                    release(tr)
+                elif callable(nack):
+                    nack(tr, requeue=True)
+                else:
+                    logger.warning(
+                        "Connector %s cannot hand back transaction %s; it returns when its claim or lease expires",
+                        conn.__class__.__name__,
+                        tr.id,
+                    )
+                    continue
+                logger.info("Handed back unstarted transaction %s", tr.id)
+            except Exception:
+                logger.exception(
+                    "Could not hand back transaction %s; it returns when its claim or lease expires",
+                    tr.id,
+                )
+
     # =====================================================================
     # PUBLIC CONSUME LOOP
     # =====================================================================
@@ -338,6 +370,10 @@ class ConsumerMixin(ABC):
                 )
 
             while True:
+                if shutdown.is_requested():
+                    logger.info("Shutdown requested; consume loop stops before the next fetch")
+                    break
+
                 batch_number += 1
 
                 # -------------------------
@@ -345,6 +381,9 @@ class ConsumerMixin(ABC):
                 # -------------------------
                 logger.debug(f"Fetching transactions batch with fetch policy: {policy.fetch.model_dump_json(indent=2)}")
                 success, result = self._handle_fetch(conn, policy.fetch)
+                if not success and shutdown.is_requested():
+                    logger.warning(f"Fetch failed while stopping; nothing was fetched, stopping → {result}")
+                    break
                 if not success:
                     logger.error(f"Fetch failed → {result}")
                     executor.shutdown(wait=False)
@@ -354,6 +393,14 @@ class ConsumerMixin(ABC):
 
                 transactions = result
 
+                if transactions and shutdown.is_requested():
+                    logger.info(
+                        "Shutdown requested during fetch; handing back %d unstarted transaction(s)",
+                        len(transactions),
+                    )
+                    self._hand_back_unstarted(conn, list(transactions))
+                    break
+
                 # -------------------------
                 # EMPTY QUEUE HANDLING
                 # -------------------------
@@ -375,6 +422,7 @@ class ConsumerMixin(ABC):
                         policy.fetch.empty.backoff_multiplier,
                         policy.fetch.empty.backoff_cap,
                         empty_count,
+                        interruptible=True,
                     )
                     empty_count += 1
                     continue
@@ -417,11 +465,24 @@ class ConsumerMixin(ABC):
                         "streaming": policy.loop.streaming,
                     },
                 ):
+                    not_started = list(transactions)
 
                     def submissions():
-                        for tr in transactions:
+                        while not_started and not shutdown.is_requested():
+                            tr = not_started.pop(0)
                             yield lambda tr=tr: submit_start_processing(tr, policy)
 
+                    def hand_back_not_started():
+                        remaining = list(not_started)
+                        not_started.clear()
+                        if remaining:
+                            logger.info(
+                                "Shutdown requested; handing back %d unstarted transaction(s) of this batch, "
+                                "in-flight ones will finish",
+                                len(remaining),
+                            )
+                            self._hand_back_unstarted(conn, remaining)
+
                     logger.debug(
                         f"Submitting {len(transactions)} transactions for processing "
                         f"with concurrency policy: {policy.loop.concurrency.model_dump_json(indent=2)}."
@@ -432,10 +493,17 @@ class ConsumerMixin(ABC):
                         concurrency=policy.loop.concurrency.value,
                         limit=policy.loop.limit,
                         timeout=policy.transaction_runtime.timeout,
+                        stop=shutdown.is_requested,
+                        on_stop=hand_back_not_started,
                     )
 
                 processed += count
 
+                if shutdown.is_requested():
+                    hand_back_not_started()
+                    logger.info("Shutdown requested; consume loop stops after the in-flight transactions")
+                    break
+
                 if policy.loop.limit and processed >= policy.loop.limit:
                     break
 
@@ -446,6 +514,7 @@ class ConsumerMixin(ABC):
                         multiplier=policy.fetch.batch.interval.backoff_multiplier,
                         cap=policy.fetch.batch.interval.backoff_cap,
                         attempt=0,
+                        interruptible=True,
                     )
 
             executor.shutdown()

ergon/task/runner.py

@@ -1,6 +1,8 @@
 import asyncio
 import inspect
 import logging as stdlib_logging
+import multiprocessing
+import multiprocessing.connection
 import os
 import signal
 import threading
@@ -8,13 +10,13 @@ import time
 import traceback
 import uuid
 from collections.abc import Callable
-from concurrent.futures import ProcessPoolExecutor
 from datetime import datetime
 from enum import IntEnum
 from typing import Any, Literal
 
 from ..connector import Transaction
 from ..telemetry import logging, metrics, tracing
+from . import shutdown
 from .base import (
     BaseAsyncTask,
     BaseTask,
@@ -38,63 +40,24 @@ class ExitCode(IntEnum):
 
 
 # =============================================================
-# SHUTDOWN STATE (PROCESS-LOCAL)
+# SHUTDOWN STATE (PROCESS-LOCAL, see ``shutdown``)
 # =============================================================
 
-_shutdown_event = threading.Event()
-_shutdown_signal: int | None = None
-_shutdown_callbacks: set[Callable[[], None]] = set()
-_shutdown_callbacks_lock = threading.Lock()
 _supervisor_logger = stdlib_logging.getLogger(__name__)
 
-
-def _signal_handler(signum, frame):
-    global _shutdown_signal
-    _shutdown_signal = signum
-    _shutdown_event.set()
-    with _shutdown_callbacks_lock:
-        callbacks = tuple(_shutdown_callbacks)
-    for callback in callbacks:
-        try:
-            callback()
-        except Exception:
-            _supervisor_logger.exception("Async shutdown callback failed")
-
-
-def _install_signal_handlers():
-    signal.signal(signal.SIGINT, _signal_handler)
-    signal.signal(signal.SIGTERM, _signal_handler)
-
-
-def _reset_shutdown_state() -> None:
-    global _shutdown_signal
-    _shutdown_event.clear()
-    _shutdown_signal = None
-    with _shutdown_callbacks_lock:
-        _shutdown_callbacks.clear()
-
-
-def _register_shutdown_callback(callback: Callable[[], None]) -> None:
-    with _shutdown_callbacks_lock:
-        _shutdown_callbacks.add(callback)
-        shutdown_requested = _shutdown_event.is_set()
-    if shutdown_requested:
-        callback()
-
-
-def _unregister_shutdown_callback(callback: Callable[[], None]) -> None:
-    with _shutdown_callbacks_lock:
-        _shutdown_callbacks.discard(callback)
-
-
-def is_shutdown_requested() -> bool:
-    return _shutdown_event.is_set()
+_signal_handler = shutdown._handle_signal
+_install_signal_handlers = shutdown.install_handlers
+_reset_shutdown_state = shutdown.reset
+_register_shutdown_callback = shutdown.register_callback
+_unregister_shutdown_callback = shutdown.unregister_callback
+is_shutdown_requested = shutdown.is_requested
 
 
 def get_shutdown_exit_code() -> ExitCode:
-    if _shutdown_signal == signal.SIGINT:
+    requested = shutdown.requested_signal()
+    if requested == signal.SIGINT:
         return ExitCode.SIGINT
-    if _shutdown_signal == signal.SIGTERM:
+    if requested == signal.SIGTERM:
         return ExitCode.SIGTERM
     return ExitCode.ERROR
 
@@ -497,6 +460,103 @@ def __run_task_sync(
                     instance.exit()
 
 
+def _run_worker_sync(config: TaskConfig, mode: Literal["task", "transaction"] = "task", *args, **kwargs):
+    """Worker-process entrypoint: cooperative handlers, no escalation, keeps an early stop request."""
+    shutdown.prepare_worker()
+    if shutdown.is_requested():
+        _supervisor_logger.warning("Stop requested before worker %s started; not running it", kwargs.get("worker_id"))
+        return None
+    return __run_task_sync(config, mode, *args, **kwargs)
+
+
+def _worker_signal_relays(worker_pids: Callable[[], list[int]]) -> tuple[Callable[[], None], Callable[[], None]]:
+    """(forward the parent's signal to the workers, SIGKILL the workers).
+
+    ``worker_pids`` is read on every call, so workers started after registration count.
+    ``kill_workers`` runs right before the process exits, so it does not log.
+    """
+
+    def _send(signum: int) -> list[tuple[int, OSError]]:
+        failures = []
+        for pid in worker_pids():
+            try:
+                os.kill(pid, signum)
+            except ProcessLookupError:
+                continue
+            except OSError as exc:
+                failures.append((pid, exc))
+        return failures
+
+    def forward_shutdown() -> None:
+        for pid, exc in _send(shutdown.requested_signal() or signal.SIGTERM):
+            _supervisor_logger.error("Could not signal worker process %s: %s", pid, exc)
+
+    def kill_workers() -> None:
+        _send(signal.SIGKILL)
+
+    return forward_shutdown, kill_workers
+
+
+def _start_worker(config: TaskConfig, mode: Literal["task", "transaction"], args: tuple, worker_kwargs: dict):
+    process = multiprocessing.Process(
+        target=_run_worker_sync,
+        args=(config, mode, *args),
+        kwargs=worker_kwargs,
+        name=f"{config.name}-worker-{worker_kwargs['worker_id']}",
+    )
+    process.start()
+    return process
+
+
+def _run_workers(config: TaskConfig, mode: Literal["task", "transaction"], args: tuple, kwargs: dict) -> bool:
+    """Run ``max_workers`` worker processes to completion; True when one of them failed.
+
+    Plain processes, not a pool: a worker whose task returns simply exits. A worker that
+    fails or dies stops its siblings cooperatively, as a broken pool used to.
+    """
+    processes: list[multiprocessing.Process] = []
+    has_error = False
+
+    def alive_pids() -> list[int]:
+        return [p.pid for p in processes if p.pid is not None and p.exitcode is None]
+
+    forward_shutdown, kill_workers = _worker_signal_relays(alive_pids)
+    shutdown.register_callback(forward_shutdown)
+    shutdown.register_escalation_callback(kill_workers)
+    try:
+        # Workers inherit the blocked mask, so a signal that arrives while they start
+        # stays pending until ``prepare_worker`` installed their handlers.
+        with shutdown.blocked_signals():
+            for worker_id in range(config.max_workers):
+                if is_shutdown_requested():
+                    break
+                worker_kwargs = {**kwargs, "worker_id": worker_id, "total_workers": config.max_workers}
+                processes.append(_start_worker(config, mode, args, worker_kwargs))
+        if is_shutdown_requested():
+            forward_shutdown()
+
+        running = list(processes)
+        while running:
+            multiprocessing.connection.wait([p.sentinel for p in running], timeout=1.0)
+            for p in [p for p in running if p.exitcode is not None]:
+                running.remove(p)
+                p.join()
+                if p.exitcode == 0:
+                    continue
+                has_error = True
+                _supervisor_logger.error("Worker process %s (pid %s) exited with %s", p.name, p.pid, p.exitcode)
+                for sibling in running:
+                    if sibling.pid is not None:
+                        try:
+                            os.kill(sibling.pid, signal.SIGTERM)
+                        except ProcessLookupError:
+                            pass
+    finally:
+        shutdown.unregister_callback(forward_shutdown)
+        shutdown.unregister_escalation_callback(kill_workers)
+    return has_error
+
+
 # =============================================================
 # PUBLIC API — RUNNER
 # =============================================================
@@ -515,9 +575,10 @@ def run_task(
     Returns POSIX-compatible exit code.
     """
 
-    _reset_shutdown_state()
-    _install_signal_handlers()
     is_async = issubclass(config.task, BaseAsyncTask)  # type: ignore[arg-type]
+    _reset_shutdown_state()
+    # Async: every signal re-cancels and the liveness supervisor owns the hard deadline.
+    _install_signal_handlers(escalate=not is_async)
 
     # ---------------------------------------------------------
     # SINGLE PROCESS
@@ -552,24 +613,7 @@ def run_task(
     if is_async:
         raise RuntimeError("Async tasks cannot be executed with multiple processes. Use debug=True or max_workers=1.")
 
-    has_error = False
-
-    with ProcessPoolExecutor(max_workers=config.max_workers) as executor:
-        futures = []
-        for worker_id in range(config.max_workers):
-            worker_kwargs = {
-                **kwargs,
-                "worker_id": worker_id,
-                "total_workers": config.max_workers,
-            }
-            futures.append(executor.submit(__run_task_sync, config, mode, *args, **worker_kwargs))
-
-        for f in futures:
-            try:
-                f.result()
-            except Exception:
-                traceback.print_exc()
-                has_error = True
+    has_error = _run_workers(config, mode, args, kwargs)
 
     if is_shutdown_requested():
         return int(get_shutdown_exit_code())

ergon/task/shutdown.py

@@ -0,0 +1,275 @@
+"""Process-wide shutdown state shared by the runner, the consume loops and the backoff sleeps.
+
+The runner installs the signal handlers; everything else only reads this module, which
+imports nothing from ``ergon.task`` so ``utils`` and the mixins can use it without a cycle.
+
+First SIGINT/SIGTERM: a cooperative stop. Sync consume loops stop fetching, hand back
+fetched-but-unstarted transactions and let the in-flight ones finish. With escalation on
+(sync runner), a second signal while the stop is in progress exits at once with
+``128 + signum``, so an operator is never stuck waiting for a long transaction.
+
+How a signal is processed:
+
+- The kernel may deliver it to any thread that does not block it, and Python runs the
+  Python-level handler only on the main thread, at its next bytecode. A main thread asleep
+  in a lock (``Future.result()``, ``Event.wait``) would never notice. So the runner owns
+  ``signal.set_wakeup_fd``: CPython's C handler writes the signal number to a pipe from
+  whichever thread received it, and a watcher thread acts on it.
+- The watcher is the only place that counts signals, sets the ``Event``, logs, runs the
+  callbacks and escalates. The Python handler only flips a flag so the main thread sees
+  the stop right away; it never takes a lock, because the main thread may hold one
+  (``threading.Event``, logging handlers, buffered stdio) when it runs.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+import signal
+import threading
+from collections.abc import Callable
+
+logger = logging.getLogger(__name__)
+
+_HANDLED_SIGNALS = (signal.SIGINT, signal.SIGTERM)
+
+_requested = False
+_signal: int | None = None
+_seen = False
+_event = threading.Event()
+
+# Copy-on-write tuples: the watcher reads them without a lock.
+_callbacks: tuple[Callable[[], None], ...] = ()
+_escalation_callbacks: tuple[Callable[[], None], ...] = ()
+_registry_lock = threading.Lock()
+
+_handlers_installed = False
+_escalate = True
+
+_wakeup_read: int | None = None
+_wakeup_write: int | None = None
+_wakeup_installed = False
+_watcher: threading.Thread | None = None
+
+
+def _handle_signal(signum, frame) -> None:
+    global _requested, _signal
+    if _signal is None:
+        _signal = signum
+    _requested = True
+    if not _wakeup_installed:
+        # Direct call (tests), or a forked child that has not run ``prepare_worker`` yet.
+        _on_signal(signum)
+
+
+def _on_signal(signum: int) -> None:
+    global _requested, _signal, _seen
+    if _seen and _handlers_installed and _escalate:
+        _escalate_now(signum)
+        return
+    _seen = True
+    if _signal is None:
+        _signal = signum
+    _requested = True
+    _event.set()
+    logger.warning(
+        "%s received; finishing in-flight work before exiting%s",
+        signal.Signals(signum).name,
+        " (send it again to exit now)" if _handlers_installed and _escalate else "",
+    )
+    for callback in _callbacks:
+        try:
+            callback()
+        except Exception:
+            logger.exception("Shutdown callback failed")
+
+
+def _escalate_now(signum: int) -> None:
+    try:
+        os.write(2, f"Second {signal.Signals(signum).name} during shutdown; exiting immediately\n".encode())
+    except OSError:
+        pass
+    for callback in _escalation_callbacks:
+        try:
+            callback()
+        except Exception:
+            pass
+    os._exit(128 + signum)
+
+
+def _watch(read_fd: int) -> None:
+    while True:
+        try:
+            data = os.read(read_fd, 64)
+        except OSError:
+            return
+        if not data:
+            return
+        for signum in data:
+            if signum in _HANDLED_SIGNALS:
+                _on_signal(signum)
+
+
+def _start_watcher() -> None:
+    global _wakeup_read, _wakeup_write, _watcher
+    if _watcher is not None and _watcher.is_alive():
+        return
+    read_fd, write_fd = os.pipe()
+    os.set_blocking(write_fd, False)
+    _wakeup_read, _wakeup_write = read_fd, write_fd
+    _watcher = threading.Thread(target=_watch, args=(read_fd,), name="ergon-shutdown-watcher", daemon=True)
+    _watcher.start()
+
+
+def install_handlers(*, escalate: bool = True) -> None:
+    """Route SIGINT/SIGTERM to the cooperative stop. Main thread only (``signal`` rule).
+
+    ``escalate=False`` keeps every signal cooperative. Worker processes use it because
+    systemd's ``KillMode=control-group`` signals every process and the parent forwards
+    its own signal too; the parent escalates for them (``register_escalation_callback``).
+    The async runner uses it as well: its supervisor owns the hard deadline.
+    """
+    global _handlers_installed, _escalate, _wakeup_installed
+    _escalate = escalate
+    _start_watcher()
+    assert _wakeup_write is not None
+    for signum in _HANDLED_SIGNALS:
+        signal.signal(signum, _handle_signal)
+    signal.set_wakeup_fd(_wakeup_write, warn_on_full_buffer=False)
+    _wakeup_installed = True
+    _handlers_installed = True
+
+
+def restore_default_handlers() -> None:
+    global _handlers_installed, _wakeup_installed
+    signal.set_wakeup_fd(-1)
+    _wakeup_installed = False
+    signal.signal(signal.SIGINT, signal.default_int_handler)
+    signal.signal(signal.SIGTERM, signal.SIG_DFL)
+    _handlers_installed = False
+
+
+def reset() -> None:
+    """Forget a previous stop request and all callbacks (start of a run, tests)."""
+    global _requested, _signal, _seen, _callbacks, _escalation_callbacks
+    _requested = False
+    _signal = None
+    _seen = False
+    _event.clear()
+    with _registry_lock:
+        _callbacks = ()
+        _escalation_callbacks = ()
+
+
+def prepare_worker() -> None:
+    """Enter a worker process of the multiprocess runner.
+
+    Keeps a stop request that already arrived (inherited through fork, or delivered
+    while signals were blocked), drops the parent's callbacks and never escalates.
+    """
+    global _callbacks, _escalation_callbacks
+    with _registry_lock:
+        _callbacks = ()
+        _escalation_callbacks = ()
+    install_handlers(escalate=False)
+    if _requested:
+        _event.set()
+    unblock_signals()
+
+
+def _after_fork_in_child() -> None:
+    global _escalate, _callbacks, _escalation_callbacks, _registry_lock
+    global _wakeup_read, _wakeup_write, _wakeup_installed, _watcher
+    # The parent's watcher thread does not exist here, and the parent's callbacks
+    # (forward to workers, SIGKILL workers) must never run in a child.
+    _escalate = False
+    _callbacks = ()
+    _escalation_callbacks = ()
+    _registry_lock = threading.Lock()
+    _wakeup_installed = False
+    try:
+        signal.set_wakeup_fd(-1)
+    except ValueError:
+        # Forked from a non-main thread: the C handler may still write to the old pipe,
+        # so leave it open rather than let the fd number be reused.
+        pass
+    else:
+        for fd in (_wakeup_read, _wakeup_write):
+            if fd is not None:
+                try:
+                    os.close(fd)
+                except OSError:
+                    pass
+    _wakeup_read = _wakeup_write = None
+    _watcher = None
+
+
+if hasattr(os, "register_at_fork"):
+    os.register_at_fork(after_in_child=_after_fork_in_child)
+
+
+def is_requested() -> bool:
+    return _requested
+
+
+def requested_signal() -> int | None:
+    return _signal
+
+
+def wait(timeout: float) -> bool:
+    """Sleep up to ``timeout`` seconds; return True as soon as a shutdown is requested."""
+    if _requested:
+        return True
+    return _event.wait(timeout) or _requested
+
+
+def register_callback(callback: Callable[[], None]) -> None:
+    """Run ``callback`` on every stop request; immediately when a stop was already requested."""
+    global _callbacks
+    with _registry_lock:
+        _callbacks = (*_callbacks, callback)
+    if _requested:
+        callback()
+
+
+def unregister_callback(callback: Callable[[], None]) -> None:
+    global _callbacks
+    with _registry_lock:
+        _callbacks = tuple(c for c in _callbacks if c is not callback)
+
+
+def register_escalation_callback(callback: Callable[[], None]) -> None:
+    """Run ``callback`` right before a second signal ends the process (keep it to ``os.kill``-like work)."""
+    global _escalation_callbacks
+    with _registry_lock:
+        _escalation_callbacks = (*_escalation_callbacks, callback)
+
+
+def unregister_escalation_callback(callback: Callable[[], None]) -> None:
+    global _escalation_callbacks
+    with _registry_lock:
+        _escalation_callbacks = tuple(c for c in _escalation_callbacks if c is not callback)
+
+
+def blocked_signals():
+    """Context manager that defers SIGINT/SIGTERM for the calling thread (POSIX), e.g. while forking workers.
+
+    A child process inherits the mask, so a worker cannot be killed by a signal that
+    arrives before it installed its handlers; ``prepare_worker`` unblocks it.
+    """
+    return _BlockedSignals()
+
+
+class _BlockedSignals:
+    def __enter__(self):
+        if hasattr(signal, "pthread_sigmask"):
+            signal.pthread_sigmask(signal.SIG_BLOCK, _HANDLED_SIGNALS)
+        return self
+
+    def __exit__(self, *exc) -> None:
+        unblock_signals()
+
+
+def unblock_signals() -> None:
+    if hasattr(signal, "pthread_sigmask"):
+        signal.pthread_sigmask(signal.SIG_UNBLOCK, _HANDLED_SIGNALS)

ergon/task/utils.py

@@ -6,6 +6,7 @@ import time
 from datetime import datetime
 
 from .. import telemetry
+from . import shutdown
 
 logger = logging.getLogger(__name__)
 tracer = telemetry.tracing.get_tracer(__name__)
@@ -42,14 +43,24 @@ def compute_backoff(backoff: float, multiplier: float, cap: float, attempt: int)
         return computed_delay
 
 
-def backoff(backoff: float, multiplier: float, cap: float, attempt: int):
-    """Blocking sleep with computed backoff."""
+def backoff(backoff: float, multiplier: float, cap: float, attempt: int, *, interruptible: bool = False):
+    """Blocking sleep with computed backoff.
+
+    ``interruptible=True`` returns as soon as a shutdown is requested. Only idle waits use
+    it (empty queue, batch interval): a retry sleep inside a transaction must not turn
+    into an immediate retry just because the process is stopping.
+    """
     delay = compute_backoff(backoff, multiplier, cap, attempt)
     if delay > 0:
         with tracer.start_as_current_span("sleep", attributes={"delay": delay}):
             estimated_wake_time_iso = _get_wake_time_iso(delay)
             logger.info(f"Sleeping for {delay} seconds until {estimated_wake_time_iso}")
-            time.sleep(delay)
+            if interruptible:
+                if shutdown.wait(delay):
+                    logger.info("Woke up early: shutdown requested")
+                    return
+            else:
+                time.sleep(delay)
             logger.info(f"Woke up from {delay} second{'' if delay == 1 else 's'} sleep")
 
 

{ergon_framework_python-0.3.0.dist-info → ergon_framework_python-0.3.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ergon-framework-python
-Version: 0.3.0
+Version: 0.3.2
 Summary: Ergon internal task-oriented project bootstrapper
 Author-email: Ergondata Technologies <anza.vossos@protonmail.com>
 License-Expression: MIT
@@ -29,7 +29,7 @@ Requires-Dist: aio-pika>=9.0.0; extra == "all"
 Requires-Dist: aiobotocore>=2.13.0; extra == "all"
 Requires-Dist: asyncpg>=0.29.0; extra == "all"
 Requires-Dist: boto3>=1.35.0; extra == "all"
-Requires-Dist: ergon-platform-sdk<0.6,>=0.5.1; extra == "all"
+Requires-Dist: ergon-platform-sdk<1,>=0.5.1; extra == "all"
 Requires-Dist: httpx>=0.28.0; extra == "all"
 Requires-Dist: msal>=1.37.0; extra == "all"
 Requires-Dist: nylas>=6.0.0; extra == "all"
@@ -37,7 +37,7 @@ Requires-Dist: openpyxl>=3.1.5; extra == "all"
 Requires-Dist: pika>=1.3.0; extra == "all"
 Requires-Dist: requests>=2.32.5; extra == "all"
 Provides-Extra: ergon-platform
-Requires-Dist: ergon-platform-sdk<0.6,>=0.5.1; extra == "ergon-platform"
+Requires-Dist: ergon-platform-sdk<1,>=0.5.1; extra == "ergon-platform"
 Requires-Dist: httpx>=0.28.0; extra == "ergon-platform"
 Provides-Extra: excel
 Requires-Dist: openpyxl>=3.1.5; extra == "excel"

{ergon_framework_python-0.3.0.dist-info → ergon_framework_python-0.3.2.dist-info}/RECORD

@@ -1,4 +1,4 @@
-ergon/__init__.py,sha256=LI3pDbBIuX3yMUrCAeP5z6Kw96CA5xpjWKcYhSzA8Y4,218
+ergon/__init__.py,sha256=KTd2qcgHy8oQQydByvjgW4Prc3-gviPXsE4yscZ262s,218
 ergon/cli.py,sha256=CF8PvOjIguR18a657aEBEyhFXUoSVtSRVXv5PfnuWtA,6021
 ergon/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ergon/bootstrap/src/__project__/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -35,7 +35,7 @@ ergon/connector/ergon_platform/utils.py,sha256=eZU8PHwarpz--EWgz74MkfPLD2AbKp5pm
 ergon/connector/ergon_platform/channels/__init__.py,sha256=wEpy-8p1mwQNZmosO17hOZ3DIPisX3P14T6WVbyMeBo,1480
 ergon/connector/ergon_platform/channels/adapters.py,sha256=LoylPshBCMi5R32Ms-IHe_H2rCO4QFrUWMvscRWkmQ8,8123
 ergon/connector/ergon_platform/channels/async_connector.py,sha256=iKqj-I_2k6DPRd-r0FZ7li659L0Rf3k5--lT3VCBd-8,12853
-ergon/connector/ergon_platform/channels/connector.py,sha256=UEgT7so-CWEVPUmxhBzkX4WnxC5PJVlhs-fDd1o1qYk,11637
+ergon/connector/ergon_platform/channels/connector.py,sha256=yJTNiFHDx_aXtLrAwKUCGyHdwMG_OeHT9IvIOm25uCA,11884
 ergon/connector/ergon_platform/channels/models.py,sha256=00WTLbYdputWd9FHy9D5h3X4AAiAIBeYlBeH5cGY6vA,18292
 ergon/connector/ergon_platform/channels/services/__init__.py,sha256=EQBc2Sc2mL3gwitd1cPL2wpYnio7C7nHzAA1ml45UzQ,1091
 ergon/connector/ergon_platform/channels/services/activity.py,sha256=SxN3y2VZDulPxiqy6XNDipVvhCk9uDzsIeSXs9Hd6WA,9756
@@ -48,7 +48,7 @@ ergon/connector/ergon_platform/workflows/__init__.py,sha256=W5TDhYmldZqfQRJ4G8si
 ergon/connector/ergon_platform/workflows/_mode.py,sha256=ycgCprhZrJzLH67nxkM0hWIM85_0pmiOpXbHSd50gso,3309
 ergon/connector/ergon_platform/workflows/_operations.py,sha256=bOLc1XS3XGzp1OswwNPy6dViRigSyMdOI_SVnnFmJV8,13377
 ergon/connector/ergon_platform/workflows/async_connector.py,sha256=wrzDri3F04BD_-W-aoDNZly1XIsQrNhygELjBv9-Dng,12341
-ergon/connector/ergon_platform/workflows/connector.py,sha256=Y0Vl1EEfcyydanEK8hDuDV6DGdHw5en2Aua4YKQX6QI,11466
+ergon/connector/ergon_platform/workflows/connector.py,sha256=pRC1AkMseNrfDupyBAEeKXvzbllfyqocbvblHL9uhwk,11774
 ergon/connector/ergon_platform/workflows/models.py,sha256=EsYilk19iqhy1cpOTtBx120lvXufMgGFH-sTWZFGchI,5063
 ergon/connector/ergon_platform/workflows/utils.py,sha256=---WoYGjyAqKLl-OWOqpkvFDK8HCeiApmz47nJ3z958,8779
 ergon/connector/excel/__init__.py,sha256=YHrcMRwW96DtvRs2CNpKK5GW3lJXZSdClLUWCDMgzsA,1013
@@ -60,14 +60,14 @@ ergon/connector/nylas/async_auth_service.py,sha256=sEMvoSCDjRFUSZgmN39L4e5lI2mqh
 ergon/connector/nylas/async_connector.py,sha256=VXSPY31iYJYn1KPmmxqwhfKi1OxwHXl94xYpXx9oNbY,4774
 ergon/connector/nylas/async_service.py,sha256=lkxWEAkaMCKD2X7E9ijyuE8CjnR3ITdB1sH3lj1wAFM,4797
 ergon/connector/nylas/auth_service.py,sha256=f2FguCE5AqqInig1su4Z-rMMoh6zXn9UY7M2eb3aYs0,1835
-ergon/connector/nylas/connector.py,sha256=4mIRdaQ6YH6j1gt4htAVeXVoKvZMYses6M3d2G6GCAs,4525
+ergon/connector/nylas/connector.py,sha256=kIpn4yvrGr6ceFJad0m4MxgPXyjvv97MwCTS5KckQLU,4810
 ergon/connector/nylas/models.py,sha256=dqPG5xhM5ZBFKN1hPW0US7lDztSaV-CZwwIherrdDkA,10230
 ergon/connector/nylas/service.py,sha256=kj0zY0cvBWkqlne_gZyw4fFDDiIJ5ACB4PX7vYksE3Q,11525
 ergon/connector/nylas/utils.py,sha256=jP-_hslfSWQgS4Sh5FPOKKLnX-n68m38mkuHOcRVaFk,4715
 ergon/connector/outlook/__init__.py,sha256=lYLgvXtKTv-jxMyJaeGhd6MO_FWnd0N-1nxRGOmXLXs,1737
 ergon/connector/outlook/async_connector.py,sha256=aV60SXsFM9rDmbNItdnvpTNYqFy0iqqbvaxXfO9sOGM,7545
 ergon/connector/outlook/async_service.py,sha256=uC3ldsS8d-yN5inY6qA5-NOyWRaeAc3mxcX-hFslz6M,5618
-ergon/connector/outlook/connector.py,sha256=jX8kGV6glQ1z5cF9Spg5V1fXlYHaslcT7Q9xpuWEJlE,7190
+ergon/connector/outlook/connector.py,sha256=G8mHK_VItYPLr1GN2aZAbtDmAlmqmlKFsmSITXvJDzY,7477
 ergon/connector/outlook/models.py,sha256=zIww_QBqSSnkWv3c13pdtMr97-LwQZn1S4eD5OWXRS0,11219
 ergon/connector/outlook/service.py,sha256=TdkTroNeQxZ-Sh5mcETRFkqb3OGEZuAKJtQfXCcgTNk,15695
 ergon/connector/outlook/utils.py,sha256=--IXPNJAG-Bxe4ATFaT0aAKMznMazAOlMpjRMke3KMQ,4102
@@ -100,14 +100,15 @@ ergon/service/service.py,sha256=X4XaeHTKokkz7TGqR0VaNM1nUQSytR91Zukxj1k48X4,384
 ergon/task/__init__.py,sha256=aXHg8flbZuQ5tFx_0kDjvhqmU0mJyXqcZoRmZK6Dl0s,406
 ergon/task/base.py,sha256=hmg1C3i-nEgeNv9q6nz5tGS1HOk-tNZdfrzOZygdNnU,6770
 ergon/task/exceptions.py,sha256=3bObJAkA9u34XgBXYdOUmrBtxPRN0sXM4Z-R-xMATRI,7043
-ergon/task/helpers.py,sha256=Fzb2WsP--5f54VRo_oaPH3Hq9kIYoykfEDOhpd_QHXY,22879
+ergon/task/helpers.py,sha256=7YZpNfs9urTlqlWAuMVzGd4WvmkBXEhxQuCsgLbjMU8,23675
 ergon/task/liveness.py,sha256=TsytIfmi2ua_x3KEPjTdx4WbqHk0WXTqeCSiQUBKPsE,1314
 ergon/task/manager.py,sha256=3Xr8RPRjtCEoupMeyYb19uK0TQhoVehFlBZd8ZFZ-YA,2972
 ergon/task/policies.py,sha256=3x1SV-yt3FXNgmnATdQdpRtsDtlskjx582SpbC6UH6A,7358
-ergon/task/runner.py,sha256=xUaMSO6V_iOq6QJdiyW1IA5kfWuwQjh1IJl6HUh06hY,20344
-ergon/task/utils.py,sha256=OmlP2AhLM32V8O9OMky6DdSvxwmMNn9x2LTguf0y-M0,2501
+ergon/task/runner.py,sha256=L-ExZmzUVqOuGaFPrxhhH_kt8M8yYTFhFbAOR2diXAw,23017
+ergon/task/shutdown.py,sha256=LMm9x2xjGeazVDpSWYwjUGtivk1CvR9s3rd3I9046W8,9173
+ergon/task/utils.py,sha256=lm5hWBQgLJnRILadMwEBTQP8jEu3wZMIau3ATP3fjvs,2995
 ergon/task/mixins/__init__.py,sha256=6A8vNgdjaQYNJvymWNF-kBN2oTaQBYRLctS3Sq1w_-Y,320
-ergon/task/mixins/consumer.py,sha256=LrCSbekQos5GU0xIY3F5_jUd69L09JV6FY_hMy2eol4,52467
+ergon/task/mixins/consumer.py,sha256=JDAMIZbLAHxd4VRJ2wmJW3t0ham2rFZcVoOn48tL_yY,55837
 ergon/task/mixins/metrics.py,sha256=4bavwo8koWiN5ADD8t5qd2BVeVcgxo6ddzq-xOybeTg,13159
 ergon/task/mixins/producer.py,sha256=xENTZdZ7t0dpw5YsreQT20iTycSaj1EA4ny-60u-SWM,20427
 ergon/telemetry/__init__.py,sha256=8waeIAHEx6tMXWfw-zOEKgjrg7V8S1poG070MdAmi_8,100
@@ -117,9 +118,9 @@ ergon/telemetry/metrics.py,sha256=0T_CuNw9P5AG1U9k1-bNp1g7McyiNLUIt_zKyLOtrnA,28
 ergon/telemetry/tracing.py,sha256=_ktiZWAgB1DU-uN91Mzr0iYNXxbCN21LnKiRo5SOJxY,4454
 ergon/utils/__init__.py,sha256=M0_VCjMN7jvSPlRZLZHeFKTHAl1zu3BnAhjrzJjDfiI,57
 ergon/utils/env.py,sha256=lji4xENvfHqg2GTGnzG95VKpxxM4f3DHIKKP61xQG08,621
-ergon_framework_python-0.3.0.dist-info/licenses/LICENSE,sha256=bWCIhHzaadz_ecLdCdtwyMafeQ17s2d7_FSSuSuEcDs,1079
-ergon_framework_python-0.3.0.dist-info/METADATA,sha256=4jpSXaxJHbORTHYo64JiURQUqS2GcseCnLz9xuw8T8A,18381
-ergon_framework_python-0.3.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
-ergon_framework_python-0.3.0.dist-info/entry_points.txt,sha256=reZ934txb5OrtcNBgTGqCsXa1biUDQWubD2a9fGZynE,42
-ergon_framework_python-0.3.0.dist-info/top_level.txt,sha256=KSPt6N7RzsjU7PRAalWQ9Cul7YTUxbo84yAJ-y2vLm8,6
-ergon_framework_python-0.3.0.dist-info/RECORD,,
+ergon_framework_python-0.3.2.dist-info/licenses/LICENSE,sha256=bWCIhHzaadz_ecLdCdtwyMafeQ17s2d7_FSSuSuEcDs,1079
+ergon_framework_python-0.3.2.dist-info/METADATA,sha256=i2N5giC7tvqYI_JRFxopnfoDYQgadQCQ2X-lacx7t3c,18377
+ergon_framework_python-0.3.2.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
+ergon_framework_python-0.3.2.dist-info/entry_points.txt,sha256=reZ934txb5OrtcNBgTGqCsXa1biUDQWubD2a9fGZynE,42
+ergon_framework_python-0.3.2.dist-info/top_level.txt,sha256=KSPt6N7RzsjU7PRAalWQ9Cul7YTUxbo84yAJ-y2vLm8,6
+ergon_framework_python-0.3.2.dist-info/RECORD,,