vortexpy 4.0.23__py3-none-any.whl

vortex/handler/TupleActionProcessorProxy.py

@@ -0,0 +1,218 @@
+import logging
+from copy import copy
+from typing import Optional
+from typing import Union
+
+from twisted.internet.defer import Deferred
+from twisted.internet.defer import TimeoutError
+from twisted.internet.defer import inlineCallbacks
+from twisted.python.failure import Failure
+
+from vortex.DeferUtil import vortexLogFailure
+from vortex.PayloadEndpoint import PayloadEndpoint
+from vortex.PayloadEnvelope import PayloadEnvelope
+from vortex.PayloadResponse import PayloadResponse
+from vortex.VortexABC import SendVortexMsgResponseCallable
+from vortex.VortexFactory import VortexFactory
+from vortex.VortexUtil import limitConcurrency
+from vortex.handler.TupleActionProcessor import TupleActionProcessor
+from vortex.handler.TupleActionProcessor import TupleActionProcessorDelegateABC
+
+logger = logging.getLogger(__name__)
+
+
+class TupleActionProcessorProxy:
+    """Tuple Data Observable Proxy Handler
+
+    This class proxies the TupleActions onto another destination, giving the ability
+    to pass through one services into another. EG, from a client facing python service
+    to a server backend.
+    """
+
+    _DEBUG_LOGGING = False
+
+    def __init__(
+        self,
+        tupleActionProcessorName,
+        proxyToVortexName: str,
+        additionalFilt=None,
+        acceptOnlyFromVortex: Optional[Union[str, tuple]] = None,
+    ) -> None:
+        """Constructor
+
+        :param tupleActionProcessorName: The name of this and the other action handler
+        :param proxyToVortexName: The vortex dest name to proxy requests to
+        :param additionalFilt: Any additional filter keys that are required
+        """
+        self._proxyToVortexName = proxyToVortexName
+        self._tupleActionProcessorName = tupleActionProcessorName
+        self._delegateProcessor = TupleActionProcessor(
+            tupleActionProcessorName=tupleActionProcessorName,
+            usedForProxy__=True,
+        )
+
+        self._filt = dict(
+            name=tupleActionProcessorName, key="tupleActionProcessorName"
+        )
+
+        if additionalFilt:
+            self._filt.update(additionalFilt)
+
+        self._endpoint = PayloadEndpoint(
+            self._filt,
+            self._processLimited,
+            acceptOnlyFromVortex=acceptOnlyFromVortex,
+        )
+
+    def shutdown(self):
+        self._endpoint.shutdown()
+
+    def setDelegate(
+        self, tupleName: str, processor: TupleActionProcessorDelegateABC
+    ):
+        """Add Tuple Action Processor Delegate
+
+        :param tupleName: The tuple name to process actions for.
+        :param processor: The processor to use for processing this tuple name.
+
+        """
+        self._delegateProcessor.setDelegate(tupleName, processor)
+
+    def _processLimited(self, *args, **kwargs):
+        d = self._processLimitedDecorated(*args, **kwargs)
+        if d:
+            d.addErrback(vortexLogFailure, logger, consumeError=True)
+
+    @limitConcurrency(logger, 20)
+    def _processLimitedDecorated(self, *args, **kwargs):
+        return self._process(*args, **kwargs)
+
+    @inlineCallbacks
+    def _process(
+        self,
+        payloadEnvelope: PayloadEnvelope,
+        vortexName: str,
+        vortexUuid: str,
+        sendResponse: SendVortexMsgResponseCallable,
+        **kwargs,
+    ) -> None:
+        # Ignore responses from the backend, these are handled by PayloadResponse
+        if vortexName == self._proxyToVortexName:
+            return
+
+        if not VortexFactory.isVortexUuidOnline(vortexUuid):
+            logger.debug(
+                "Vortex %s is no longer online, skipping reply", vortexUuid
+            )
+            return
+
+        # logger.debug(
+        #     "Vortex %s is STILL online, continuing to reply", vortexUuid
+        # )
+
+        # Shortcut the logic, so that we don't decode the payload unless we need to.
+        if not self._delegateProcessor.delegateCount:
+            yield self._processForProxy(
+                payloadEnvelope, vortexName, sendResponse
+            )
+            return
+
+        # If we have local processors, then work out if this tupleAction is meant for
+        # the local processor.
+        payload = yield payloadEnvelope.decodePayloadDefer()
+
+        assert (
+            len(payload.tuples) == 1
+        ), "TupleActionProcessor:%s Expected 1 tuples, received %s" % (
+            self._tupleActionProcessorName,
+            len(payload.tuples),
+        )
+
+        tupleAction = payload.tuples[0]
+
+        if self._delegateProcessor.hasDelegate(tupleAction.tupleName()):
+            self._delegateProcessor._processTupleAction(
+                payloadEnvelope.filt, sendResponse, tupleAction
+            )
+            return
+
+        # Else, Just send it on to the delegate we're proxying for (the backend)
+        yield self._processForProxy(payloadEnvelope, vortexName, sendResponse)
+
+    @inlineCallbacks
+    def _processForProxy(
+        self,
+        payloadEnvelope: PayloadEnvelope,
+        vortexName: str,
+        sendResponse: SendVortexMsgResponseCallable,
+        **kwargs,
+    ):
+        # Keep a copy of the incoming filt, in case they are using PayloadResponse
+        responseFilt = copy(payloadEnvelope.filt)
+
+        # Track the response, log an error if it fails
+        # 5 Seconds is long enough.
+        # VortexJS defaults to 10s, so we have some room for round trip time.
+        pr = PayloadResponse(
+            payloadEnvelope,
+            timeout=PayloadResponse.TIMEOUT - 5,  # 5 seconds less
+            resultCheck=False,
+            logTimeoutError=False,
+        )
+
+        # This is not a lambda, so that it can have a breakpoint
+        def reply(payloadEnvelope: PayloadEnvelope):
+            payloadEnvelope.filt = responseFilt
+            d: Deferred = payloadEnvelope.toVortexMsgDefer()
+            d.addCallback(sendResponse)
+            return d
+
+        pr.addCallback(reply)
+
+        if self._DEBUG_LOGGING:
+            pr.addCallback(
+                lambda _: logger.debug(
+                    "Received action response from server for %s",
+                    payloadEnvelope.filt,
+                )
+            )
+        pr.addErrback(self.__handlePrFailure, payloadEnvelope, sendResponse)
+
+        vortexMsg = yield payloadEnvelope.toVortexMsgDefer()
+        try:
+            yield VortexFactory.sendVortexMsg(
+                vortexMsgs=vortexMsg, destVortexName=self._proxyToVortexName
+            )
+        except Exception as e:
+            logger.exception(e)
+
+    @inlineCallbacks
+    def __handlePrFailure(
+        self,
+        f: Failure,
+        payloadEnvelope: PayloadEnvelope,
+        sendResponse: SendVortexMsgResponseCallable,
+    ):
+        payload = yield payloadEnvelope.decodePayloadDefer()
+        action = payload.tuples[0]
+        if f.check(TimeoutError):
+            logger.error(
+                "Received no response from\nprocessor %s\naction %s",
+                self._filt,
+                action,
+            )
+        else:
+            logger.error(
+                "Unexpected error, %s\nprocessor %s\naction %s",
+                f,
+                self._filt,
+                action,
+            )
+
+        vortexLogFailure(f, logger)
+
+        vortexMsg = yield PayloadEnvelope(
+            filt=payloadEnvelope.filt, result=str(f.value)
+        ).toVortexMsgDefer()
+
+        sendResponse(vortexMsg)

vortex/handler/TupleActionProcessorTestHandler.py

@@ -0,0 +1,27 @@
+import logging
+
+from twisted.internet import defer
+from twisted.internet.defer import Deferred
+
+from vortex.PerformTestActionTuple import PerformTestActionTuple
+from vortex.TupleAction import TupleActionABC, TupleGenericAction
+from vortex.handler.TupleActionProcessor import (
+    TupleActionProcessor,
+    TupleActionProcessorDelegateABC,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class TestProcessor(TupleActionProcessorDelegateABC):
+    def processTupleAction(self, tupleAction: TupleActionABC) -> Deferred:
+        assert isinstance(
+            tupleAction, TupleGenericAction
+        ), "tupleAction is not TupleGenericAction"
+        if "FAIL PLEASE" == tupleAction.data:
+            raise Exception("Processing failed scenario, exception raised")
+        return defer.succeed(True)
+
+
+actionProcessor = TupleActionProcessor("vortexTestActions")
+actionProcessor.setDelegate(PerformTestActionTuple.tupleName(), TestProcessor())

vortex/handler/TupleDataActionClient.py

@@ -0,0 +1,67 @@
+import logging
+from copy import copy
+
+from twisted.internet.defer import Deferred
+
+from vortex.Payload import Payload
+from vortex.PayloadResponse import PayloadResponse
+from vortex.TupleAction import TupleActionABC
+
+logger = logging.getLogger(__name__)
+
+
+class TupleDataActionClient:
+    def __init__(
+        self,
+        destVortexName: str,
+        tupleActionProcessorName: str,
+        additionalFilt: dict = None,
+    ) -> None:
+        """Constructor
+
+        :param destVortexName: The name of the destination vortex to send to.
+
+        :param tupleActionProcessorName: The name of this observable
+
+        :param additionalFilt: Any additional filter keys that are required
+        """
+        self._destVortexName = destVortexName
+        self._filt = dict(
+            name=tupleActionProcessorName, key="tupleActionProcessorName"
+        )
+        if additionalFilt:
+            self._filt.update(additionalFilt)
+
+    def pushAction(
+        self, tupleAction: TupleActionABC, additionalFilt=None
+    ) -> Deferred:
+        """Push Action
+
+        This pushes the action, either locally or to the server, depending on the
+        implementation.
+
+        If pushed locally, the promise will resolve when the action has been saved.
+        If pushed directly to the server, the promise will resolve when the server has
+        responded.
+
+        :param tupleAction The tuple action to send to the remote end
+        :param additionalFilt: Any extra filt that may be required to send with the
+                payload.
+        """
+        filt = copy(self._filt)
+        if additionalFilt:
+            filt.update(additionalFilt)
+
+        d = Payload(filt=filt, tuples=[tupleAction]).makePayloadEnvelopeDefer()
+        d.addCallback(
+            lambda payloadEnvelope: PayloadResponse(
+                payloadEnvelope, destVortexName=self._destVortexName
+            )
+        )
+
+        # Convert the data to TupleAction
+        d.addCallback(
+            lambda payloadEnvelope: payloadEnvelope.decodePayloadDefer()
+        )
+        d.addCallback(lambda payload: payload.tuples)
+        return d

vortex/handler/TupleDataObservableCache.py

@@ -0,0 +1,201 @@
+import logging
+import typing
+import weakref
+from abc import abstractmethod, ABCMeta
+from collections import defaultdict
+from datetime import datetime, timedelta
+from typing import Optional, Dict, Set
+
+import pytz
+from reactivex.subject import Subject
+from twisted.internet import task
+
+from vortex.DeferUtil import vortexLogFailure
+from vortex.PayloadEnvelope import PayloadEnvelope
+from vortex.TupleSelector import TupleSelector
+from vortex.VortexFactory import VortexFactory
+
+logger = logging.getLogger(__name__)
+
+
+class _CachedSubscribedData:
+    """Cached Subscribed Data
+
+    The client will now cache
+    """
+
+    TEARDOWN_WAIT = 120  # 2 minutes in seconds
+
+    __memoryLoggingRefs = None
+    __memoryLoggingEnabled = False
+
+    @classmethod
+    def setupMemoryLogging(cls) -> None:
+        cls.__memoryLoggingRefs = []
+        cls.__memoryLoggingEnabled = True
+
+    @classmethod
+    def memoryLoggingDump(
+        cls, top=10, over=100
+    ) -> typing.List[typing.Tuple[str, int, int]]:
+        if not cls.__memoryLoggingRefs:
+            return []
+
+        # Filter out expired items
+        cls.__memoryLoggingRefs = list(
+            filter(lambda o: o(), cls.__memoryLoggingRefs)
+        )
+
+        cachesByTupleType = defaultdict(list)
+        for cacheRef in cls.__memoryLoggingRefs:
+            cache = cacheRef()
+            if not cache:
+                continue
+            cachesByTupleType[cache.tupleSelector.name].append(
+                len(cache.encodedPayload)
+            )
+
+        results = []
+        for tupleName, lengths in cachesByTupleType.items():
+            results.append((tupleName, len(lengths), sum(lengths)))
+
+        data = sorted(results, key=lambda x: x[2], reverse=True)
+
+        return list(filter(lambda x: x[2] >= over, data))[:top]
+
+    def __init__(
+        self, tupleSelector: TupleSelector, cacheEnabled: bool = True
+    ) -> None:
+        if _CachedSubscribedData.__memoryLoggingEnabled:
+            _CachedSubscribedData.__memoryLoggingRefs.append(weakref.ref(self))
+
+        self.tupleSelector: TupleSelector = tupleSelector
+        self.vortexUuids: Set[str] = set()
+        self.tearDownDate: Optional[datetime] = None
+        self.encodedPayload: bytes = None
+
+        #: Is the cache enabled
+        # HINT: Once it's turned off for a tupleSelector, it remains off.
+        # Disabling it can also be done from VortexJS
+        self.cacheEnabled = cacheEnabled
+
+        #: Last Server Payload Date
+        # If the server has responded with a payload, this is the date in the payload
+        # @type {Date | null}
+        self.lastServerPayloadDate: Optional[datetime] = None
+
+        self.subject = Subject()
+
+    def markForTearDown(self) -> None:
+        if self.tearDownDate is not None:
+            return
+        self.tearDownDate = datetime.now(pytz.utc) + timedelta(
+            seconds=self.TEARDOWN_WAIT
+        )
+
+    def resetTearDown(self) -> None:
+        self.tearDownDate = None
+
+    def isReadyForTearDown(self) -> bool:
+        return (
+            self.tearDownDate is not None
+            and self.tearDownDate <= datetime.now(pytz.utc)
+        )
+
+
+class TupleDataObservableCache(metaclass=ABCMeta):
+    __CHECK_PERIOD = 30  # seconds
+
+    def __init__(self) -> None:
+        self.__cache: Dict[str, _CachedSubscribedData] = {}
+
+    def shutdown(self):
+        if not self.__pollLoopingCall:
+            raise Exception("This has already been shutdown")
+
+        self.__cache = {}
+        self.__pollLoopingCall.stop()
+        self.__pollLoopingCall = None
+
+        for cache in self.__cache.values():
+            cache.subject.dispose()
+
+        self.__cache = {}
+
+    def start(self):
+
+        self.__pollLoopingCall = task.LoopingCall(self.__cacheCheck)
+
+        d = self.__pollLoopingCall.start(self.__CHECK_PERIOD, now=False)
+        d.addErrback(vortexLogFailure, logger, consumeError=True)
+
+    def _tupleSelectors(self) -> typing.List[TupleSelector]:
+        tupleSelectors = []
+        for key in self.__cache:
+            tupleSelectors.append(TupleSelector.fromJsonStr(key))
+        return tupleSelectors
+
+    ## ----- Implement local observable
+
+    def __cacheCheck(self):
+        currentVortexUuids = set(VortexFactory.getRemoteVortexUuids())
+
+        for ts, cache in list(self.__cache.items()):
+            cache.vortexUuids = cache.vortexUuids & currentVortexUuids
+
+            if cache.vortexUuids or cache.subject.observers:
+                cache.resetTearDown()
+
+            elif cache.isReadyForTearDown():
+                logger.debug("Cleaning cache for %s", cache.tupleSelector)
+                self._sendUnsubscribeToServer(cache.tupleSelector)
+                del self.__cache[ts]
+
+            else:
+                cache.markForTearDown()
+
+    def _getCache(
+        self, tupleSelector: TupleSelector
+    ) -> Optional[_CachedSubscribedData]:
+        return self.__cache.get(tupleSelector.toJsonStr())
+
+    def _hasTupleSelector(self, tupleSelectorAny: TupleSelector) -> bool:
+        return tupleSelectorAny.toJsonStr() in self.__cache
+
+    def _makeCache(self, tupleSelector: TupleSelector) -> _CachedSubscribedData:
+        tsStr = tupleSelector.toJsonStr()
+        if tsStr in self.__cache:
+            return self.__cache[tsStr]
+
+        cache = _CachedSubscribedData(tupleSelector)
+        self.__cache[tsStr] = cache
+        return cache
+
+    def _updateCache(
+        self, payloadEnvelope: PayloadEnvelope
+    ) -> typing.Tuple[_CachedSubscribedData, bool]:
+        """Update Cache
+
+        Update the cache if it requires it
+
+        :returns a tuple of (cache, requiredUpdate)
+
+        """
+        tupleSelector = payloadEnvelope.filt["tupleSelector"]
+
+        cache = self._getCache(tupleSelector)
+        if not cache:
+            cache = self._makeCache(tupleSelector)
+
+        if cache.lastServerPayloadDate is not None:
+            if payloadEnvelope.date < cache.lastServerPayloadDate:
+                return cache, False
+
+        cache.lastServerPayloadDate = payloadEnvelope.date
+        cache.encodedPayload = payloadEnvelope.encodedPayload
+
+        return cache, True
+
+    @abstractmethod
+    def _sendUnsubscribeToServer(self, tupleSelector: TupleSelector):
+        pass