vortexpy 4.3.10__tar.gz → 4.3.12__tar.gz

{vortexpy-4.3.10 → vortexpy-4.3.12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: vortexpy
-Version: 4.3.10
+Version: 4.3.12
 Summary: Flying Ion's observable, routable, data serialisation and transport code.
 Author-email: Flying Ion Pty Ltd <contact@flyingion.com>
 License: MIT

{vortexpy-4.3.10 → vortexpy-4.3.12}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "vortexpy"
-version = "4.3.10"
+version = "4.3.12"
 license = {text = "MIT"}
 authors = [{name = "Flying Ion Pty Ltd", email = "contact@flyingion.com"}]
 description = "Flying Ion's observable, routable, data serialisation and transport code."

{vortexpy-4.3.10 → vortexpy-4.3.12}/vortex/DeferUtil.py

@@ -16,6 +16,7 @@ import twisted
 from twisted.internet import reactor
 from twisted.internet.defer import (
     maybeDeferred,
+    CancelledError,
     Deferred,
     inlineCallbacks,
     ensureDeferred,
@@ -239,6 +240,17 @@ def deferToThreadWrapWithLogger(
                     )
 
             def handleFailure(failure):
+                if failure.check(CancelledError):
+                    # Cancellation is designed control flow (for example a
+                    # limitConcurrency timeout quarantining the thread), not
+                    # a threaded call failure.
+                    loggerArg.debug(
+                        "Threaded call cancelled in %s.%s",
+                        funcToWrap.__module__,
+                        funcToWrap.__qualname__,
+                    )
+                    return True if consumeError else failure
+
                 if logException:
                     return vortexLogFailure(
                         failure, loggerArg, consumeError=consumeError
@@ -253,6 +265,10 @@ def deferToThreadWrapWithLogger(
                 return True if consumeError else failure
 
             d: Deferred[_R] = deferToThread(timedFuncToRun, *args, **kwargs)
+            # Marks this Deferred as threaded for limitConcurrency: a timed
+            # out threaded call cannot be killed, so the limiter accumulates
+            # marked Deferreds in its quarantine registry.
+            d._isVortexThreadedCall = True  # type: ignore[attr-defined]
             d.addCallback(checkForInlineCallbackYield)
             d.addErrback(handleFailure)
             return d

{vortexpy-4.3.10 → vortexpy-4.3.12}/vortex/PayloadResponse.py

@@ -16,6 +16,8 @@ from uuid import uuid4
 import pytz
 from twisted.internet import reactor
 from twisted.internet.defer import Deferred
+from twisted.internet.defer import TimeoutError
+from twisted.internet.interfaces import IDelayedCall
 from twisted.python.failure import Failure
 
 from vortex.PayloadEndpoint import PayloadEndpoint
@@ -68,6 +70,7 @@ class PayloadResponse(Deferred):
         timeout: Optional[float] = None,
         resultCheck=True,
         logTimeoutException=True,
+        retries: Optional[int] = None,
     ) -> None:
         """Constructor
 
@@ -83,9 +86,16 @@ class PayloadResponse(Deferred):
                     it errback will be called.
         :param logTimeoutException: When True, payload timeouts log at ERROR.
                     When False, timeouts log at DEBUG with neutral wording.
+        :param retries: The number of times to resend the payload envelope
+                    when the response times out; None sends once with no
+                    retries. Requires destVortexName or destVortexUuid.
         """
         Deferred.__init__(self)
 
+        assert retries is None or (destVortexName or destVortexUuid), (
+            "retries requires destVortexName or destVortexUuid to resend"
+        )
+
         if not timeout:
             timeout = self.TIMEOUT
 
@@ -99,6 +109,11 @@ class PayloadResponse(Deferred):
         payloadEnvelope.filt[self.MESSAGE_ID_KEY] = self._messageId
         self._filt = copy(payloadEnvelope.filt)
         self._destVortexName = destVortexName
+        self._destVortexUuid = destVortexUuid
+        self._payloadEnvelope = payloadEnvelope
+        self._timeout = timeout
+        self._retries: Optional[int] = retries
+        self._timeoutDelayedCall: Optional[IDelayedCall] = None
 
         self._status = self.PROCESSING
         self._date = datetime.now(pytz.utc)
@@ -107,23 +122,49 @@ class PayloadResponse(Deferred):
             self._filt, self._process, requiredPermissions=frozenset()
         )
 
-        if destVortexName or destVortexUuid:
-            d: Deferred = payloadEnvelope.toVortexMsgDefer()
-            d.addCallback(
-                VortexFactory.sendVortexMsg,
-                destVortexName=destVortexName,
-                destVortexUuid=destVortexUuid,
-            )
-            d.addErrback(self.errback)
+        self._sendVortexMsg()
 
         try:
             raise Exception()
         except:
             self._stack = sys.exc_info()[2]
 
-        # noinspection PyTypeChecker
-        self.addTimeout(timeout, reactor)
-        self.addErrback(self._timedOut)
+        if retries is None:
+            # noinspection PyTypeChecker
+            self.addTimeout(timeout, reactor)
+            self.addErrback(self._timedOut)
+        else:
+            self._armTimeout()
+
+    def _sendVortexMsg(self) -> None:
+        if not (self._destVortexName or self._destVortexUuid):
+            return
+
+        d: Deferred = self._payloadEnvelope.toVortexMsgDefer()
+        d.addCallback(
+            VortexFactory.sendVortexMsg,
+            destVortexName=self._destVortexName,
+            destVortexUuid=self._destVortexUuid,
+        )
+        d.addErrback(self.errback)
+
+    def _armTimeout(self) -> None:
+        self._timeoutDelayedCall = reactor.callLater(  # type: ignore[attr-defined]  # pyright: ignore[reportAttributeAccessIssue]
+            self._timeout, self._onTimeout
+        )
+
+    def _onTimeout(self) -> None:
+        self._timeoutDelayedCall = None
+
+        if self._retries:
+            self._retries -= 1
+            self._sendVortexMsg()
+            self._armTimeout()
+            return
+
+        failure = Failure(TimeoutError("timed out"))
+        self._timedOut(failure)
+        self.errback(failure)
 
     @classmethod
     def isResponsePayloadEnvelope(cls, payloadEnvelope: PayloadEnvelope):
@@ -163,6 +204,12 @@ class PayloadResponse(Deferred):
             self._endpoint.shutdown()
             self._endpoint = None
 
+        if self._timeoutDelayedCall is not None:
+            # zope Interface methods omit self, which static analysis cannot bind
+            if self._timeoutDelayedCall.active():  # pyright: ignore[reportAttributeAccessIssue]
+                self._timeoutDelayedCall.cancel()  # pyright: ignore[reportAttributeAccessIssue]
+            self._timeoutDelayedCall = None
+
         if self._destVortexName and vortexName != self._destVortexName:
             logger.debug(
                 "Received response from a vortex other than the dest vortex, "
@@ -176,14 +223,10 @@ class PayloadResponse(Deferred):
             logger.error("Received response after timeout for %s" % self._filt)
             return
 
-        if self._resultCheck and not payloadEnvelope.result in (None, True):
+        if self._resultCheck and payloadEnvelope.result not in (None, True):
             self._status = self.FAILED
             self.errback(
-                Failure(
-                    Exception(payloadEnvelope.result).with_traceback(
-                        self._stack
-                    )
-                )
+                Failure(Exception(payloadEnvelope.result).with_traceback(self._stack))
             )
 
         else:

vortexpy-4.3.12/vortex/PayloadResponse_test.py

@@ -0,0 +1,161 @@
+import unittest
+from unittest.mock import MagicMock, patch
+
+from twisted.internet.defer import TimeoutError
+from twisted.internet.defer import succeed
+from twisted.python.failure import Failure
+
+from vortex.PayloadEnvelope import PayloadEnvelope
+from vortex.PayloadResponse import PayloadResponse
+
+
+class _FakeDelayedCall:
+    def __init__(self, callback) -> None:
+        self.callback = callback
+        self.cancelled = False
+
+    def active(self) -> bool:
+        return not self.cancelled
+
+    def cancel(self) -> None:
+        self.cancelled = True
+
+    def fire(self) -> None:
+        self.callback()
+
+
+class _FakeReactor:
+    def __init__(self) -> None:
+        self.delayedCalls: list[_FakeDelayedCall] = []
+
+    def callLater(self, _seconds: float, callback) -> _FakeDelayedCall:
+        call = _FakeDelayedCall(callback)
+        self.delayedCalls.append(call)
+        return call
+
+
+class PayloadResponseLoggingTest(unittest.TestCase):
+    @patch("vortex.PayloadResponse.VortexFactory.sendVortexMsg")
+    @patch("vortex.PayloadResponse.logger")
+    def testTimeoutLogsDebugWhenLogTimeoutExceptionFalse(
+        self, mockLogger: MagicMock, _mockSend: MagicMock
+    ) -> None:
+        pr = PayloadResponse(
+            PayloadEnvelope(filt={"key": "example"}),
+            timeout=30.0,
+            logTimeoutException=False,
+        )
+        pr._timedOut(Failure(TimeoutError("timed out")))
+        mockLogger.error.assert_not_called()
+        mockLogger.debug.assert_called_once()
+
+    @patch("vortex.PayloadResponse.VortexFactory.sendVortexMsg")
+    @patch("vortex.PayloadResponse.logger")
+    def testTimeoutLogsErrorWhenLogTimeoutExceptionTrue(
+        self, mockLogger: MagicMock, _mockSend: MagicMock
+    ) -> None:
+        pr = PayloadResponse(
+            PayloadEnvelope(filt={"key": "example"}),
+            timeout=30.0,
+            logTimeoutException=True,
+        )
+        pr._timedOut(Failure(TimeoutError("timed out")))
+        mockLogger.debug.assert_not_called()
+        mockLogger.error.assert_called_once()
+
+
+class PayloadResponseRetryTest(unittest.TestCase):
+    def setUp(self) -> None:
+        self.fakeReactor = _FakeReactor()
+        patch("vortex.PayloadResponse.reactor", self.fakeReactor).start()
+        self.mockSend: MagicMock = patch(
+            "vortex.PayloadResponse.VortexFactory.sendVortexMsg"
+        ).start()
+        patch(
+            "vortex.PayloadEnvelope.PayloadEnvelope.toVortexMsgDefer",
+            return_value=succeed(b"{}"),
+        ).start()
+        self.addCleanup(patch.stopall)
+
+    def testTimeoutResendsPayloadAndSucceedsOnLateResponse(self) -> None:
+        pr = PayloadResponse(
+            PayloadEnvelope(filt={"key": "example"}),
+            destVortexName="destVortex",
+            timeout=30.0,
+            retries=1,
+        )
+
+        self.assertEqual(self.mockSend.call_count, 1)
+        self.assertEqual(len(self.fakeReactor.delayedCalls), 1)
+
+        # First attempt times out, the payload is resent and re-armed
+        self.fakeReactor.delayedCalls[0].fire()
+
+        self.assertEqual(self.mockSend.call_count, 2)
+        self.assertEqual(len(self.fakeReactor.delayedCalls), 2)
+        self.assertEqual(pr.status, PayloadResponse.PROCESSING)
+
+        # The resent payload receives a response
+        results = []
+        pr.addCallback(results.append)
+        response = PayloadEnvelope(filt={"key": "example"})
+        pr._process(response, "vortexUuid", "destVortex")
+
+        self.assertEqual(pr.status, PayloadResponse.SUCCESS)
+        self.assertEqual(results, [response])
+        self.assertTrue(self.fakeReactor.delayedCalls[1].cancelled)
+
+    def testExhaustedRetriesErrbackWithTimedOut(self) -> None:
+        pr = PayloadResponse(
+            PayloadEnvelope(filt={"key": "example"}),
+            destVortexName="destVortex",
+            timeout=30.0,
+            retries=1,
+        )
+
+        failures = []
+        pr.addErrback(failures.append)
+
+        self.fakeReactor.delayedCalls[0].fire()
+        self.assertEqual(self.mockSend.call_count, 2)
+        self.assertEqual(pr.status, PayloadResponse.PROCESSING)
+
+        self.fakeReactor.delayedCalls[1].fire()
+
+        self.assertEqual(self.mockSend.call_count, 2)
+        self.assertEqual(pr.status, PayloadResponse.TIMED_OUT)
+        self.assertEqual(len(failures), 1)
+        self.assertIsInstance(failures[0].value, TimeoutError)
+
+    def testRetriesRequiresDestinationVortex(self) -> None:
+        with self.assertRaises(AssertionError):
+            PayloadResponse(
+                PayloadEnvelope(filt={"key": "example"}),
+                timeout=30.0,
+                retries=1,
+            )
+
+    def testDefaultTimeoutPathDoesNotResend(self) -> None:
+        pr = PayloadResponse(
+            PayloadEnvelope(filt={"key": "example"}),
+            destVortexName="destVortex",
+            timeout=30.0,
+        )
+
+        failures = []
+        pr.addErrback(failures.append)
+
+        self.assertEqual(self.mockSend.call_count, 1)
+        self.assertEqual(len(self.fakeReactor.delayedCalls), 1)
+
+        # addTimeout cancels the deferred, which errbacks with TimeoutError
+        self.fakeReactor.delayedCalls[0].fire()
+
+        self.assertEqual(self.mockSend.call_count, 1)
+        self.assertEqual(pr.status, PayloadResponse.TIMED_OUT)
+        self.assertEqual(len(failures), 1)
+        self.assertIsInstance(failures[0].value, TimeoutError)
+
+
+if __name__ == "__main__":
+    unittest.main()

{vortexpy-4.3.10 → vortexpy-4.3.12}/vortex/VortexUtil.py

@@ -7,6 +7,7 @@
 # Support : contact@flyingion.com
 
 import json
+from collections import deque
 from collections import namedtuple
 from datetime import datetime
 from datetime import timedelta
@@ -20,10 +21,10 @@ from uuid import uuid4
 
 import pytz
 from twisted.internet import reactor
+from twisted.internet.defer import CancelledError
 from twisted.internet.defer import Deferred
 from twisted.internet.defer import fail
 from twisted.internet.defer import maybeDeferred
-from twisted.python.failure import Failure
 
 from vortex.PayloadEnvelope import PayloadEnvelope
 from vortex.PayloadEnvelope import VortexMsgList
@@ -37,6 +38,20 @@ _LimitConcurrencyArgsTuple = namedtuple(
     ("deferred", "funcSelf", "args", "kwargs", "queuedTime"),
 )
 
+_LimitConcurrencyQuarantinedCallTuple = namedtuple(
+    "_LimitConcurrencyQuarantinedCallTuple",
+    ("methodQualname", "callKey", "startDate", "timeoutDate", "deferred"),
+)
+
+
+class LimitConcurrencyTimeoutError(Exception):
+    """Limit Concurrency Timeout Error
+
+    Raised on the caller's Deferred when a call started by
+    L{limitConcurrency} exceeds C{killOrQuarantineCallsLongerThanSeconds}
+    and is cancelled.
+    """
+
 
 def _dedupProcessorCallKeys(
     payloadEnvelope: PayloadEnvelope,
@@ -56,6 +71,7 @@ def limitConcurrency(
     concurrency: int,
     logTaskTimeGreaterThanSeconds=5.0,
     deduplicateBasedOnKwArgsKey: Optional[Callable] = None,
+    killOrQuarantineCallsLongerThanSeconds: Optional[float] = None,
 ):
     """Limit Concurrency to a call, decorator
 
@@ -66,6 +82,17 @@ def limitConcurrency(
         of strings. If this argument is provided,
         then duplicate calls with the same values for these arguments will
         be deduplicated. Duplicate calls will use the newest set of arguments.
+    :param killOrQuarantineCallsLongerThanSeconds: When set, any started call
+        still running after this many seconds is cancelled: @inlineCallbacks
+        callees are killed (CancelledError reaches the generator), threaded
+        callees are quarantined (the caller's Deferred is released while the
+        thread runs on). An error is logged, the limiter's alarm count is
+        incremented in TwistedRuntimeMetrics, the caller's Deferred
+        errbacks with LimitConcurrencyTimeoutError. Timed out threaded
+        calls (deferToThreadWrapWithLogger Deferreds) are accumulated with
+        their Deferred reference in the limiter's bounded _quarantinedCalls
+        deque for manhole inspection; killed @inlineCallbacks callees die
+        and are not registered.
 
     # As of python 3.6, dicts are ordered.
 
@@ -97,6 +124,8 @@ def limitConcurrency(
             self._callQueue: dict[Any, _LimitConcurrencyArgsTuple] = {}
             self._runningCount = 0
             self._processedCount = 0
+            self._callTimeoutCount = 0
+            self._quarantinedCalls: deque = deque(maxlen=50)
             self._lastLogTime = None
             self._lastQueueProcessingDelta = None
             TwistedRuntimeMetrics._collector()._registerLimitConcurrencyLimiter(
@@ -146,12 +175,87 @@ def limitConcurrency(
                 )
                 d = maybeDeferred(callWithArgs)
                 d.addBoth(self._callComplete, now, queueWaitMs)
+
+                if killOrQuarantineCallsLongerThanSeconds is not None and not d.called:
+                    self._utilArmCallTimeout(
+                        d, killOrQuarantineCallsLongerThanSeconds, nextKey, now
+                    )
+
                 d.chainDeferred(args.deferred)
 
             self._logQueueSize()
             self._f.running = self._runningCount
             self._f.waiting = len(self._callQueue)
 
+        def _utilArmCallTimeout(self, d, killSeconds, callKey, startDate):
+            def onTimeout(d=d, callKey=callKey, startDate=startDate):
+                if d.called:
+                    return
+
+                # A deferToThreadWrapWithLogger-marked Deferred means the
+                # callee is a thread that cannot be killed: it is
+                # quarantined, and the Deferred is accumulated in the
+                # bounded registry for manhole inspection. maybeDeferred
+                # passes a threaded callee's Deferred through as d; an
+                # @inlineCallbacks callee pauses d on it as d.result.
+                # Killed @inlineCallbacks callees die and are not registered.
+                if getattr(d, "_isVortexThreadedCall", False):
+                    quarantinedDeferred = d
+                else:
+                    awaitedDeferred = getattr(d, "result", None)
+                    quarantinedDeferred = (
+                        awaitedDeferred
+                        if getattr(awaitedDeferred, "_isVortexThreadedCall", False)
+                        else None
+                    )
+
+                self._callTimeoutCount += 1
+                if quarantinedDeferred is not None:
+                    self._quarantinedCalls.append(
+                        _LimitConcurrencyQuarantinedCallTuple(
+                            methodQualname=self._methodQualname,
+                            callKey=callKey,
+                            startDate=startDate,
+                            timeoutDate=datetime.now(pytz.utc),
+                            deferred=quarantinedDeferred,
+                        )
+                    )
+                TwistedRuntimeMetrics._collector()._recordLimitConcurrencyCallTimeout(
+                    self._methodQualname
+                )
+                decoratorLogger.error(
+                    "%s call exceeded killOrQuarantineCallsLongerThanSeconds"
+                    " of %ss (key=%s, timeout count=%s),"
+                    " killing or quarantining the call",
+                    self._methodQualname,
+                    killSeconds,
+                    callKey,
+                    self._callTimeoutCount,
+                )
+                d.cancel()
+
+            timeoutCall = reactor.callLater(killSeconds, onTimeout)
+
+            def cancelTimeoutCall(result):
+                if timeoutCall.active():
+                    timeoutCall.cancel()
+                return result
+
+            def convertCancelledToTimeout(failure):
+                # The watchdog's own `called` flag marks whether this
+                # cancellation came from it: DelayedCall.called is only set
+                # once onTimeout begins running.
+                if not timeoutCall.called:
+                    return failure
+                failure.trap(CancelledError)
+                raise LimitConcurrencyTimeoutError(
+                    "%s call exceeded killOrQuarantineCallsLongerThanSeconds"
+                    " of %ss" % (self._methodQualname, killSeconds)
+                )
+
+            d.addBoth(cancelTimeoutCall)
+            d.addErrback(convertCancelledToTimeout)
+
         def call(self, funcSelf, *args, **kwargs):
             if deduplicateBasedOnKwArgsKey:
                 key = deduplicateBasedOnKwArgsKey(*args, **kwargs)