vortexpy 6.2.4__tar.gz → 6.2.6__tar.gz

{vortexpy-6.2.4 → vortexpy-6.2.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: vortexpy
-Version: 6.2.4
+Version: 6.2.6
 Summary: Flying Ion's observable, routable, data serialisation and transport code.
 Author-email: Flying Ion Pty Ltd <contact@flyingion.com>
 License: MIT

{vortexpy-6.2.4 → vortexpy-6.2.6}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "vortexpy"
-version = "6.2.4"
+version = "6.2.6"
 license = {text = "MIT"}
 authors = [{name = "Flying Ion Pty Ltd", email = "contact@flyingion.com"}]
 description = "Flying Ion's observable, routable, data serialisation and transport code."

{vortexpy-6.2.4 → vortexpy-6.2.6}/vortex/DeferUtil.py

@@ -16,6 +16,7 @@ import twisted
 from twisted.internet import reactor
 from twisted.internet.defer import (
     maybeDeferred,
+    CancelledError,
     Deferred,
     inlineCallbacks,
     ensureDeferred,
@@ -239,6 +240,17 @@ def deferToThreadWrapWithLogger(
                     )
 
             def handleFailure(failure):
+                if failure.check(CancelledError):
+                    # Cancellation is designed control flow (for example a
+                    # limitConcurrency timeout quarantining the thread), not
+                    # a threaded call failure.
+                    loggerArg.debug(
+                        "Threaded call cancelled in %s.%s",
+                        funcToWrap.__module__,
+                        funcToWrap.__qualname__,
+                    )
+                    return True if consumeError else failure
+
                 if logException:
                     return vortexLogFailure(
                         failure, loggerArg, consumeError=consumeError
@@ -253,6 +265,10 @@ def deferToThreadWrapWithLogger(
                 return True if consumeError else failure
 
             d: Deferred[_R] = deferToThread(timedFuncToRun, *args, **kwargs)
+            # Marks this Deferred as threaded for limitConcurrency: a timed
+            # out threaded call cannot be killed, so the limiter accumulates
+            # marked Deferreds in its quarantine registry.
+            d._isVortexThreadedCall = True  # type: ignore[attr-defined]
             d.addCallback(checkForInlineCallbackYield)
             d.addErrback(handleFailure)
             return d

{vortexpy-6.2.4 → vortexpy-6.2.6}/vortex/VortexServer.py

@@ -27,6 +27,7 @@ from zope.interface.declarations import implementer
 from .DeferUtil import isMainThread
 from .DeferUtil import vortexLogFailure
 from .VortexSession import IVortexSession
+from .VortexSession import VortexTcpSession
 from .PayloadEnvelope import PayloadEnvelope
 from .PayloadEnvelope import VortexMsgList
 from .PayloadFilterKeys import rapuiServerEcho
@@ -394,3 +395,23 @@ class VortexSession(object):
 
 
 registerAdapter(VortexSession, Session, VortexSessionI)
+
+
+def _tcpSessionVortexSessionIAdapter(session: VortexTcpSession) -> VortexSession:
+    """Adapt a :class:`VortexTcpSession` to :class:`VortexSessionI`.
+
+    Twisted ``Session`` instances cache adapters through
+    ``Componentized.__conform__``; the TCP stub is not a ``Componentized``,
+    so the adapter is cached on the stub's own component store — without
+    this, every adaptation would return a fresh adapter with an empty
+    ``connections`` dict and ``connectionOpened`` registrations would be
+    lost.
+    """
+    adapter = session.getComponent(VortexSessionI)
+    if adapter is None:
+        adapter = VortexSession(session)
+        session.setComponent(VortexSessionI, adapter)
+    return adapter
+
+
+registerAdapter(_tcpSessionVortexSessionIAdapter, VortexTcpSession, VortexSessionI)

{vortexpy-6.2.4 → vortexpy-6.2.6}/vortex/VortexServer_test.py

@@ -14,6 +14,7 @@ from vortex.VortexServer import (
     VortexSessionI,
 )
 from vortex.VortexServerWebsocket import VortexWebsocketServerProtocol
+from vortex.VortexSession import VortexTcpSession
 
 
 class _MockServerConnection:
@@ -217,6 +218,43 @@ class VortexSessionAdapterTimeoutTest(SynchronousTestCase):
         self.assertEqual(session.sessionTimeout, HEART_BEAT_TIMEOUT)
 
 
+class VortexTcpSessionAdapterTest(SynchronousTestCase):
+    def testTcpStubAdaptsToVortexSessionI(self) -> None:
+        stub = VortexTcpSession(ip="1.2.3.4", vortexName="remote")
+
+        adapter = VortexSessionI(stub)
+
+        self.assertEqual(adapter.connections, {})
+
+    def testTcpStubAdaptationIsCached(self) -> None:
+        stub = VortexTcpSession(ip="1.2.3.4", vortexName="remote")
+
+        first = VortexSessionI(stub)
+        first.connections["uuid-a"] = object()
+
+        self.assertIs(VortexSessionI(stub), first)
+        self.assertIn("uuid-a", VortexSessionI(stub).connections)
+
+    def testTcpStubTimeoutRaisedToHeartbeatFloor(self) -> None:
+        stub = VortexTcpSession(ip="1.2.3.4", vortexName="remote")
+        stub.sessionTimeout = 5
+
+        VortexSessionI(stub)
+
+        self.assertEqual(stub.sessionTimeout, HEART_BEAT_TIMEOUT)
+
+    def testConnectionOpenedAndClosedWithTcpStub(self) -> None:
+        vortexServer = VortexServer("test-server")
+        stub = VortexTcpSession(ip="1.2.3.4", vortexName="remote")
+        conn = _MockServerConnection(vortexServer, "uuid-a", "remote", stub)
+
+        vortexServer.connectionOpened(stub, conn)
+        self.assertIn("uuid-a", VortexSessionI(stub).connections)
+
+        vortexServer.connectionClosed(conn)
+        self.assertNotIn("uuid-a", VortexSessionI(stub).connections)
+
+
 class VortexWebsocketServerProtocolConnectionLostTest(SynchronousTestCase):
     def testConnectionLostTwiceDoesNotRaise(self) -> None:
         vortex_server = VortexServer("test-server")

{vortexpy-6.2.4 → vortexpy-6.2.6}/vortex/VortexSession.py

@@ -65,12 +65,17 @@ class VortexTcpSession:
 
     TCP connections have no HTTP session.  This stub exposes the subset of
     the Twisted ``Session`` API that vortex code uses (``uid``, ``touch``,
-    ``notifyOnExpire``, ``expire``, ``getComponent``, ``setComponent``) so
-    the rest of the codebase can treat all sessions uniformly.
+    ``notifyOnExpire``, ``expire``, ``getComponent``, ``setComponent``,
+    ``sessionTimeout``) so the rest of the codebase can treat all sessions
+    uniformly.
 
     A :class:`VortexSession` is attached immediately on construction.
     """
 
+    # Mirrors twisted.web.server.Session.sessionTimeout; the
+    # VortexSessionI adapter factory in VortexServer reads and raises it.
+    sessionTimeout = 900
+
     def __init__(self, ip: str, vortexName: str):
         self.uid: bytes = uuid4().bytes
         self._expireCallbacks = []

{vortexpy-6.2.4 → vortexpy-6.2.6}/vortex/VortexUtil.py

@@ -7,6 +7,7 @@
 # Support : contact@flyingion.com
 
 import json
+from collections import deque
 from collections import namedtuple
 from datetime import datetime
 from datetime import timedelta
@@ -20,10 +21,10 @@ from uuid import uuid4
 
 import pytz
 from twisted.internet import reactor
+from twisted.internet.defer import CancelledError
 from twisted.internet.defer import Deferred
 from twisted.internet.defer import fail
 from twisted.internet.defer import maybeDeferred
-from twisted.python.failure import Failure
 
 from vortex.PayloadEnvelope import PayloadEnvelope
 from vortex.PayloadEnvelope import VortexMsgList
@@ -37,6 +38,20 @@ _LimitConcurrencyArgsTuple = namedtuple(
     ("deferred", "funcSelf", "args", "kwargs", "queuedTime"),
 )
 
+_LimitConcurrencyQuarantinedCallTuple = namedtuple(
+    "_LimitConcurrencyQuarantinedCallTuple",
+    ("methodQualname", "callKey", "startDate", "timeoutDate", "deferred"),
+)
+
+
+class LimitConcurrencyTimeoutError(Exception):
+    """Limit Concurrency Timeout Error
+
+    Raised on the caller's Deferred when a call started by
+    L{limitConcurrency} exceeds C{killOrQuarantineCallsLongerThanSeconds}
+    and is cancelled.
+    """
+
 
 def _dedupProcessorCallKeys(
     payloadEnvelope: PayloadEnvelope,
@@ -56,6 +71,7 @@ def limitConcurrency(
     concurrency: int,
     logTaskTimeGreaterThanSeconds=5.0,
     deduplicateBasedOnKwArgsKey: Optional[Callable] = None,
+    killOrQuarantineCallsLongerThanSeconds: Optional[float] = None,
 ):
     """Limit Concurrency to a call, decorator
 
@@ -66,6 +82,17 @@ def limitConcurrency(
         of strings. If this argument is provided,
         then duplicate calls with the same values for these arguments will
         be deduplicated. Duplicate calls will use the newest set of arguments.
+    :param killOrQuarantineCallsLongerThanSeconds: When set, any started call
+        still running after this many seconds is cancelled: @inlineCallbacks
+        callees are killed (CancelledError reaches the generator), threaded
+        callees are quarantined (the caller's Deferred is released while the
+        thread runs on). An error is logged, the limiter's alarm count is
+        incremented in TwistedRuntimeMetrics, the caller's Deferred
+        errbacks with LimitConcurrencyTimeoutError. Timed out threaded
+        calls (deferToThreadWrapWithLogger Deferreds) are accumulated with
+        their Deferred reference in the limiter's bounded _quarantinedCalls
+        deque for manhole inspection; killed @inlineCallbacks callees die
+        and are not registered.
 
     # As of python 3.6, dicts are ordered.
 
@@ -97,6 +124,8 @@ def limitConcurrency(
             self._callQueue: dict[Any, _LimitConcurrencyArgsTuple] = {}
             self._runningCount = 0
             self._processedCount = 0
+            self._callTimeoutCount = 0
+            self._quarantinedCalls: deque = deque(maxlen=50)
             self._lastLogTime = None
             self._lastQueueProcessingDelta = None
             TwistedRuntimeMetrics._collector()._registerLimitConcurrencyLimiter(
@@ -146,12 +175,87 @@ def limitConcurrency(
                 )
                 d = maybeDeferred(callWithArgs)
                 d.addBoth(self._callComplete, now, queueWaitMs)
+
+                if killOrQuarantineCallsLongerThanSeconds is not None and not d.called:
+                    self._utilArmCallTimeout(
+                        d, killOrQuarantineCallsLongerThanSeconds, nextKey, now
+                    )
+
                 d.chainDeferred(args.deferred)
 
             self._logQueueSize()
             self._f.running = self._runningCount
             self._f.waiting = len(self._callQueue)
 
+        def _utilArmCallTimeout(self, d, killSeconds, callKey, startDate):
+            def onTimeout(d=d, callKey=callKey, startDate=startDate):
+                if d.called:
+                    return
+
+                # A deferToThreadWrapWithLogger-marked Deferred means the
+                # callee is a thread that cannot be killed: it is
+                # quarantined, and the Deferred is accumulated in the
+                # bounded registry for manhole inspection. maybeDeferred
+                # passes a threaded callee's Deferred through as d; an
+                # @inlineCallbacks callee pauses d on it as d.result.
+                # Killed @inlineCallbacks callees die and are not registered.
+                if getattr(d, "_isVortexThreadedCall", False):
+                    quarantinedDeferred = d
+                else:
+                    awaitedDeferred = getattr(d, "result", None)
+                    quarantinedDeferred = (
+                        awaitedDeferred
+                        if getattr(awaitedDeferred, "_isVortexThreadedCall", False)
+                        else None
+                    )
+
+                self._callTimeoutCount += 1
+                if quarantinedDeferred is not None:
+                    self._quarantinedCalls.append(
+                        _LimitConcurrencyQuarantinedCallTuple(
+                            methodQualname=self._methodQualname,
+                            callKey=callKey,
+                            startDate=startDate,
+                            timeoutDate=datetime.now(pytz.utc),
+                            deferred=quarantinedDeferred,
+                        )
+                    )
+                TwistedRuntimeMetrics._collector()._recordLimitConcurrencyCallTimeout(
+                    self._methodQualname
+                )
+                decoratorLogger.error(
+                    "%s call exceeded killOrQuarantineCallsLongerThanSeconds"
+                    " of %ss (key=%s, timeout count=%s),"
+                    " killing or quarantining the call",
+                    self._methodQualname,
+                    killSeconds,
+                    callKey,
+                    self._callTimeoutCount,
+                )
+                d.cancel()
+
+            timeoutCall = reactor.callLater(killSeconds, onTimeout)
+
+            def cancelTimeoutCall(result):
+                if timeoutCall.active():
+                    timeoutCall.cancel()
+                return result
+
+            def convertCancelledToTimeout(failure):
+                # The watchdog's own `called` flag marks whether this
+                # cancellation came from it: DelayedCall.called is only set
+                # once onTimeout begins running.
+                if not timeoutCall.called:
+                    return failure
+                failure.trap(CancelledError)
+                raise LimitConcurrencyTimeoutError(
+                    "%s call exceeded killOrQuarantineCallsLongerThanSeconds"
+                    " of %ss" % (self._methodQualname, killSeconds)
+                )
+
+            d.addBoth(cancelTimeoutCall)
+            d.addErrback(convertCancelledToTimeout)
+
         def call(self, funcSelf, *args, **kwargs):
             if deduplicateBasedOnKwArgsKey:
                 key = deduplicateBasedOnKwArgsKey(*args, **kwargs)

vortexpy-6.2.6/vortex/VortexUtil_test.py

@@ -0,0 +1,292 @@
+# Copyright (c) 2016-2026 Flying Ion Pty Ltd
+# This software is open source; the MIT license applies.
+# See LICENSE for the full license text.
+#
+# Website : https://flyingion.com
+# Support : contact@flyingion.com
+
+from __future__ import annotations
+
+import gc
+import logging
+import threading
+from collections.abc import Generator
+
+import pytest
+from pytest_twisted import ensureDeferred  # type: ignore[import-untyped]
+from twisted.internet import threads
+from twisted.internet.defer import CancelledError
+from twisted.internet.defer import Deferred
+from twisted.internet.defer import inlineCallbacks
+from twisted.internet.task import Clock
+from twisted.python.failure import Failure
+
+from vortex import VortexUtil
+from vortex.DeferUtil import deferToThreadWrapWithLogger
+from vortex.VortexUtil import LimitConcurrencyTimeoutError
+from vortex.VortexUtil import limitConcurrency
+from vortex.twisted_runtime_metrics import TwistedRuntimeMetrics
+from vortex.twisted_runtime_metrics_test import (
+    utilResetTwistedRuntimeMetricsSingletonForTests,
+)
+
+logger = logging.getLogger(__name__)
+
+_KILL_SECONDS = 10.0
+
+
+@pytest.fixture(autouse=True)
+def resetMetricsSingleton() -> Generator[None, None, None]:
+    utilResetTwistedRuntimeMetricsSingletonForTests()
+    yield
+    utilResetTwistedRuntimeMetricsSingletonForTests()
+
+
+@pytest.fixture
+def clock(monkeypatch: pytest.MonkeyPatch) -> Clock:
+    clock = Clock()
+    monkeypatch.setattr(VortexUtil, "reactor", clock)
+    return clock
+
+
+class _SwitchableWorker:
+    """One decorated method: parks forever or completes, by flag."""
+
+    def __init__(self) -> None:
+        self.behaviour = "stuck"
+        self.cancelledCount = 0
+
+    @limitConcurrency(logger, 1, killOrQuarantineCallsLongerThanSeconds=_KILL_SECONDS)
+    @inlineCallbacks
+    def work(self) -> Generator[Deferred, object, str]:
+        if self.behaviour == "fast":
+            return "done"
+
+        try:
+            yield Deferred()  # Never fires, the call is stuck
+        except CancelledError:
+            self.cancelledCount += 1
+            raise
+        return "unreachable"
+
+
+class _ConcurrencyTwoWorker:
+    def __init__(self) -> None:
+        self.cancelledCount = 0
+
+    @limitConcurrency(logger, 2, killOrQuarantineCallsLongerThanSeconds=_KILL_SECONDS)
+    @inlineCallbacks
+    def work(self) -> Generator[Deferred, object, str]:
+        try:
+            yield Deferred()  # Never fires, the call is stuck
+        except CancelledError:
+            self.cancelledCount += 1
+            raise
+        return "unreachable"
+
+
+class _NoTimeoutWorker:
+    @limitConcurrency(logger, 1)
+    @inlineCallbacks
+    def work(self) -> Generator[Deferred, object, str]:
+        yield Deferred()  # Never fires, the call is stuck
+        return "unreachable"
+
+
+class _ThreadedWorker:
+    """A limiter-wrapped call that blocks in a real threadpool thread."""
+
+    def __init__(self) -> None:
+        self.behaviour = "stuck"
+        self.threadStarted = threading.Event()
+        self.threadRelease = threading.Event()
+        self.threadFinished = threading.Event()
+        self.threadCompleted = False
+
+    @limitConcurrency(logger, 1, killOrQuarantineCallsLongerThanSeconds=_KILL_SECONDS)
+    def work(self) -> Deferred:
+        return self._threadedWork()
+
+    @deferToThreadWrapWithLogger(logger)
+    def _threadedWork(self) -> str:
+        if self.behaviour == "fast":
+            return "done"
+
+        self.threadStarted.set()
+        self.threadRelease.wait(timeout=30)
+        self.threadCompleted = True
+        self.threadFinished.set()
+        return "thread-done"
+
+
+def _callerFailure(caller: Deferred) -> Failure:
+    assert caller.called, "caller Deferred must have fired"
+    assert isinstance(caller.result, Failure), (
+        f"caller must errback, got {caller.result!r}"
+    )
+    return caller.result
+
+
+def test_stuck_call_times_out_kills_generator_and_frees_slot(clock) -> None:
+    worker = _SwitchableWorker()
+
+    caller1 = worker.work()
+    assert not caller1.called, "stuck call must not fire before the timeout"
+
+    clock.advance(_KILL_SECONDS)
+
+    failure = _callerFailure(caller1)
+    failure.trap(LimitConcurrencyTimeoutError)
+    assert worker.cancelledCount == 1, (
+        "CancelledError must reach the inlineCallbacks generator"
+    )
+
+    worker.behaviour = "fast"
+    caller2 = worker.work()
+    assert caller2.called and caller2.result == "done", (
+        "the timed out call must free its concurrency slot"
+    )
+
+    snapshot = TwistedRuntimeMetrics.snapshotAndResetWindow()
+    assert snapshot.limitConcurrencyCallTimeoutCount == 1, (
+        "one call timeout must be recorded in the alarm metrics"
+    )
+    assert snapshot.limitConcurrencyLastCallTimeoutMethodName is not None
+    assert snapshot.limitConcurrencyLastCallTimeoutMethodName.endswith("work")
+
+
+def test_fast_call_is_not_timed_out(clock) -> None:
+    worker = _SwitchableWorker()
+    worker.behaviour = "fast"
+
+    caller = worker.work()
+    clock.advance(_KILL_SECONDS * 2)
+
+    assert caller.called and caller.result == "done", (
+        "fast call must complete normally and cancel its watchdog"
+    )
+
+    snapshot = TwistedRuntimeMetrics.snapshotAndResetWindow()
+    assert snapshot.limitConcurrencyCallTimeoutCount == 0, (
+        "no timeout may be recorded for a fast call"
+    )
+
+
+def test_timeouts_are_per_call_at_concurrency_two(clock) -> None:
+    worker = _ConcurrencyTwoWorker()
+
+    caller1 = worker.work()
+    caller2 = worker.work()
+    clock.advance(_KILL_SECONDS)
+
+    _callerFailure(caller1).trap(LimitConcurrencyTimeoutError)
+    _callerFailure(caller2).trap(LimitConcurrencyTimeoutError)
+    assert worker.cancelledCount == 2, "each started call must have its own watchdog"
+
+    snapshot = TwistedRuntimeMetrics.snapshotAndResetWindow()
+    assert snapshot.limitConcurrencyCallTimeoutCount == 2
+
+
+def test_no_timeout_when_parameter_not_set(clock) -> None:
+    worker = _NoTimeoutWorker()
+
+    caller = worker.work()
+    clock.advance(1000)
+
+    assert not caller.called, (
+        "without killOrQuarantineCallsLongerThanSeconds, calls must not be timed out"
+    )
+
+    snapshot = TwistedRuntimeMetrics.snapshotAndResetWindow()
+    assert snapshot.limitConcurrencyCallTimeoutCount == 0
+
+
+def test_timed_out_call_is_registered_for_manhole(clock) -> None:
+    # A function-local class gets its own limiter Wrap, isolated from the
+    # module-level workers' shared Wraps.
+    class _ManholeWorker:
+        @limitConcurrency(
+            logger, 1, killOrQuarantineCallsLongerThanSeconds=_KILL_SECONDS
+        )
+        @inlineCallbacks
+        def work(self) -> Generator[Deferred, object, str]:
+            yield Deferred()  # Never fires, the call is stuck
+            return "unreachable"
+
+    worker = _ManholeWorker()
+
+    caller1 = worker.work()
+    clock.advance(_KILL_SECONDS)
+    _callerFailure(caller1).trap(LimitConcurrencyTimeoutError)
+
+    # Reach the limiter the way manhole inspection does in production.
+    wraps = [
+        o
+        for o in gc.get_objects()
+        if type(o).__name__ == "Wrap"
+        and hasattr(o, "_callQueue")
+        and any(
+            entry.methodQualname.endswith("_ManholeWorker.work")
+            for entry in o._quarantinedCalls
+        )
+    ]
+    assert wraps == [], (
+        "a killed inlineCallbacks call must not be quarantine-registered;"
+        " only timed out threaded calls accumulate"
+    )
+
+
+@pytest.mark.twisted
+@ensureDeferred
+async def test_threaded_call_is_quarantined_not_killed(clock) -> None:
+    worker = _ThreadedWorker()
+
+    caller1 = worker.work()
+    await threads.deferToThread(worker.threadStarted.wait, 5)
+
+    clock.advance(_KILL_SECONDS)
+
+    with pytest.raises(LimitConcurrencyTimeoutError):
+        await caller1
+
+    worker.behaviour = "fast"
+    result = await worker.work()
+    assert result == "done", (
+        "a quarantined call must be moved out of the limited count:"
+        " the freed slot serves the next call while the thread runs on"
+    )
+
+    assert worker.threadCompleted is False, (
+        "the quarantined thread must still be blocked; it cannot be killed"
+    )
+
+    worker.threadRelease.set()
+    await threads.deferToThread(worker.threadFinished.wait, 5)
+    assert worker.threadCompleted is True, (
+        "the late thread result must be discarded without AlreadyCalledError"
+    )
+
+    # The timed out threaded Deferred is accumulated in the limiter's
+    # quarantine registry, reachable the way manhole inspection works.
+    wraps = [
+        o
+        for o in gc.get_objects()
+        if type(o).__name__ == "Wrap"
+        and hasattr(o, "_callQueue")
+        and any(
+            entry.methodQualname.endswith("_ThreadedWorker.work")
+            for entry in o._quarantinedCalls
+        )
+    ]
+    assert len(wraps) == 1, "one limiter must hold the quarantined thread"
+
+    entry = wraps[0]._quarantinedCalls[0]
+    assert entry.startDate <= entry.timeoutDate, (
+        "quarantine entry must carry the call start and timeout dates"
+    )
+    assert getattr(entry.deferred, "_isVortexThreadedCall", False), (
+        "the registry must hold the threaded Deferred itself"
+    )
+
+    snapshot = TwistedRuntimeMetrics.snapshotAndResetWindow()
+    assert snapshot.limitConcurrencyCallTimeoutCount == 1

{vortexpy-6.2.4 → vortexpy-6.2.6}/vortex/__init__.py

@@ -6,4 +6,4 @@
 # Website : https://flyingion.com
 # Support : contact@flyingion.com
 
-__version__ = '6.2.4'
+__version__ = '6.2.6'

{vortexpy-6.2.4 → vortexpy-6.2.6}/vortex/handler/TupleDataObservableHandler.py

@@ -271,7 +271,14 @@ class TupleDataObservableHandler:
         if d:
             d.addErrback(vortexLogFailure, logger, consumeError=True)
 
-    @limitConcurrency(logger, 25, deduplicateBasedOnKwArgsKey=_dedupProcessorCallKeys)
+    @limitConcurrency(
+        logger,
+        25,
+        deduplicateBasedOnKwArgsKey=_dedupProcessorCallKeys,
+        # Legitimate slow queries run to ~2 minutes; 300s bounds slot
+        # starvation to 5 minutes without killing healthy calls.
+        killOrQuarantineCallsLongerThanSeconds=300.0,
+    )
     def _processLimitedDecorated(
         self,
         payloadEnvelope: PayloadEnvelope,
@@ -394,10 +401,32 @@ class TupleDataObservableHandler:
                 vortexUuid
             ] = accessControlContext
 
-        vortexMsg = yield self._createVortexMsg(
-            payloadEnvelope.filt,
-            tupleSelector,
-        )
+        try:
+            vortexMsg = yield self._createVortexMsg(
+                payloadEnvelope.filt,
+                tupleSelector,
+            )
+        except Exception as e:
+            # The caller is waiting on a PayloadResponse; without an error
+            # response it waits the full client timeout for every failure.
+            logger.error(
+                "Observable:%s provider failed for TupleSelector = %s,"
+                " sending error response",
+                self._observableName,
+                tsStr,
+            )
+            logger.exception(e)
+            # DatabaseError strings include the full SQL and parameters;
+            # the leading characters carry the actual error code.
+            errorText = f"{e.__class__.__name__}: {e}"[:500]
+            errorEnvelope = PayloadEnvelope(
+                filt=payloadEnvelope.filt,
+                result=f"Tuple provider failed on the server: {errorText}",
+            )
+            errorVortexMsg = yield errorEnvelope.toVortexMsgDefer()
+            yield sendResponse(errorVortexMsg)
+            return
+
         try:
             yield sendResponse(vortexMsg)
         except Exception as e: