vortexpy 4.0.23__py3-none-any.whl

vortex/handler/TupleDataObservableHandler.py

@@ -0,0 +1,371 @@
+import logging
+from abc import ABCMeta
+from abc import abstractmethod
+from collections import namedtuple
+from typing import List
+from typing import Optional
+from typing import Union
+
+from twisted.internet import reactor
+from twisted.internet.defer import Deferred
+from twisted.internet.defer import fail
+from twisted.internet.defer import inlineCallbacks
+from twisted.internet.defer import succeed
+from twisted.python import failure
+
+from vortex.DeferUtil import vortexLogFailure
+from vortex.PayloadEndpoint import PayloadEndpoint
+from vortex.PayloadEnvelope import PayloadEnvelope
+from vortex.TupleSelector import TupleSelector
+from vortex.VortexABC import SendVortexMsgResponseCallable
+from vortex.VortexFactory import VortexFactory
+from vortex.VortexUtil import limitConcurrency, _dedupProcessorCallKeys
+
+logger = logging.getLogger(__name__)
+
+
+class TupleSelectorUpdateMapperABC(metaclass=ABCMeta):
+    @abstractmethod
+    def mapTupleSelector(
+        self,
+        triggerTupleSelector: TupleSelector,
+        allTupleSelectors: List[TupleSelector],
+    ) -> List[TupleSelector]:
+        """Map Tuple Selector
+
+        Implementing this class provides some intelligence to the Observable
+        to trigger a refresh of related data to the triggerTupleSelector.
+
+        :param triggerTupleSelector: The TupleSelector that has triggered the update.
+        :param allTupleSelectors: All TupleSelectors being observed from this observable.
+        """
+
+
+class TuplesProviderABC(metaclass=ABCMeta):
+    @abstractmethod
+    def makeVortexMsg(
+        self, filt: dict, tupleSelector: TupleSelector
+    ) -> Union[Deferred, bytes]:
+        """Make Vortex Msg
+
+        The method generates the vortexMsg for the vortex to send.
+
+        :param filt: The filt for the payload.
+        :param tupleSelector: The tuple selector us used to determing which tuples to
+        send back,
+        """
+
+
+_ObserverDetails = namedtuple(
+    "_ObserverDetails", ["vortexUuid", "observerName"]
+)
+
+
+class _ObserverData:
+    def __init__(self, tupleSelector: TupleSelector):
+        self.tupleSelector = tupleSelector
+        self.observers = set()
+
+
+class TupleDataObservableHandler:
+    def __init__(
+        self,
+        observableName,
+        additionalFilt=None,
+        subscriptionsEnabled=True,
+        acceptOnlyFromVortex: Optional[Union[str, tuple]] = None,
+    ):
+        """Constructor
+
+        :param observableName: The name of this observable
+
+        :param additionalFilt: Any additional filter keys that are required
+
+        :param subscriptionsEnabled: Should subscriptions be enabled (default)
+
+        :param acceptOnlyFromVortex: Accept requests only from this vortex,
+            The vortex can be str or tuple of str, or None to accept from any.
+
+        """
+        self._observableName = observableName
+        self._subscriptionsEnabled = subscriptionsEnabled
+        self._filt = dict(name=observableName, key="tupleDataObservable")
+        if additionalFilt:
+            self._filt.update(additionalFilt)
+
+        self._endpoint = PayloadEndpoint(
+            self._filt,
+            self._processLimited,
+            acceptOnlyFromVortex=acceptOnlyFromVortex,
+        )
+
+        self._observerDataByTupleSelector = {}
+
+        self._tupleProvidersByTupleName = {}
+        self._tupleSelectorUpdateMappers = []
+
+    def addTupleProvider(self, tupleName, provider: TuplesProviderABC):
+        """Add Tuple Provider"""
+        assert (
+            not tupleName in self._tupleProvidersByTupleName
+        ), "Observable:%s, Tuple name %s is already registered" % (
+            self._observableName,
+            tupleName,
+        )
+
+        assert isinstance(provider, TuplesProviderABC), (
+            "Observable:%s, provider must be an instance of TuplesProviderABC"
+            % self._observableName
+        )
+
+        self._tupleProvidersByTupleName[tupleName] = provider
+
+    def addTupleSelectorUpdateMapper(
+        self, mapper: TupleSelectorUpdateMapperABC
+    ):
+        """Add Tuple Selector Update Mapper
+
+        This mapper will be called every time a tuple selector is notified of an update.
+
+        """
+        self._tupleSelectorUpdateMappers.append(mapper)
+
+    def hasTupleProvider(self, tupleName: str):
+        return tupleName in self._tupleProvidersByTupleName
+
+    def hasTupleSubscribers(self, tupleSelector: TupleSelector):
+        return tupleSelector.toJsonStr() in self._observerDataByTupleSelector
+
+    def tupleSubscribers(self) -> List[TupleSelector]:
+        # This is a bit costly
+        return [
+            TupleSelector.fromJsonStr(jsonStr)
+            for jsonStr in self._observerDataByTupleSelector
+        ]
+
+    def shutdown(self):
+        self._endpoint.shutdown()
+        self._tupleSelectorUpdateMappers = []
+        self._tupleProvidersByTupleName = {}
+        self._observerDataByTupleSelector = {}
+
+    @inlineCallbacks
+    def _createVortexMsg(self, filt, tupleSelector: TupleSelector) -> Deferred:
+        tupleProvider = self._tupleProvidersByTupleName.get(tupleSelector.name)
+        assert (
+            tupleProvider
+        ), "Observable:%s, No providers registered for tupleName %s" % (
+            self._observableName,
+            tupleSelector.name,
+        )
+        try:
+            vortexMsg = yield self._customMaybeDeferred(
+                tupleProvider.makeVortexMsg, filt, tupleSelector
+            )
+        except Exception as e:
+            logger.error(
+                f"createVortexMsg"
+                f" TupleSelector = {tupleSelector.toJsonStr()},"
+                f" ObservableFilt = {self._filt}"
+            )
+            logger.exception(e)
+            raise
+        if not vortexMsg:
+            msg = f"TupleDataProvider did not return VortexPY message {tupleSelector}"
+            logger.error(msg)
+            raise Exception(msg)
+        return vortexMsg
+
+    def _processLimited(self, *args, **kwargs):
+        d = self._processLimitedDecorated(*args, **kwargs)
+        if d:
+            d.addErrback(vortexLogFailure, logger, consumeError=True)
+
+    @limitConcurrency(
+        logger, 25, deduplicateBasedOnKwArgsKey=_dedupProcessorCallKeys
+    )
+    def _processLimitedDecorated(self, *args, **kwargs):
+        return self._process(*args, **kwargs)
+
+    @inlineCallbacks
+    def _process(
+        self,
+        payloadEnvelope: PayloadEnvelope,
+        vortexUuid: str,
+        sendResponse: SendVortexMsgResponseCallable,
+        **kwargs,
+    ):
+        if not VortexFactory.isVortexUuidOnline(vortexUuid):
+            logger.debug(
+                "Vortex %s is no longer online, skipping reply", vortexUuid
+            )
+            return
+
+        # logger.debug(
+        #     "Vortex %s is STILL online, continuing to reply", vortexUuid
+        # )
+
+        tupleSelector = payloadEnvelope.filt["tupleSelector"]
+        tsStr = tupleSelector.toJsonStr()
+
+        observerDetails = _ObserverDetails(
+            vortexUuid, payloadEnvelope.filt.get("observerName")
+        )
+
+        # Handle unsubscribe
+        if payloadEnvelope.filt.get("unsubscribe"):
+            if tsStr in self._observerDataByTupleSelector:
+                try:
+                    self._observerDataByTupleSelector[tsStr].observers.remove(
+                        observerDetails
+                    )
+
+                except KeyError:
+                    pass
+
+                if not self._observerDataByTupleSelector[tsStr].observers:
+                    del self._observerDataByTupleSelector[tsStr]
+
+            return
+
+        # Add support for just getting data, no subscription.
+        if self._subscriptionsEnabled and payloadEnvelope.filt.get(
+            "subscribe", True
+        ):
+            if tsStr not in self._observerDataByTupleSelector:
+                self._observerDataByTupleSelector[tsStr] = _ObserverData(
+                    tupleSelector
+                )
+
+            self._observerDataByTupleSelector[tsStr].observers.add(
+                observerDetails
+            )
+
+        vortexMsg = yield self._createVortexMsg(
+            payloadEnvelope.filt, tupleSelector
+        )
+        try:
+            yield sendResponse(vortexMsg)
+        except Exception as e:
+            logger.error(
+                f"_process"
+                f" TupleSelector = {tupleSelector.toJsonStr()},"
+                f" ObservableFilt = {self._filt}"
+            )
+            logger.exception(e)
+
+    def _getMappedTupleSelectors(
+        self, tupleSelector: TupleSelector
+    ) -> List[TupleSelector]:
+        # Get all tuple selectors
+        allTupleSelectors = [
+            data.tupleSelector
+            for data in self._observerDataByTupleSelector.values()
+        ]
+
+        # Create a dict so we end up with only unique ones
+        tupleSelectorByStr = {tupleSelector.toJsonStr(): tupleSelector}
+
+        # Run through the mappers
+        for mapper in self._tupleSelectorUpdateMappers:
+            mappedSelectors = mapper.mapTupleSelector(
+                tupleSelector, allTupleSelectors
+            )
+            if mappedSelectors:
+                tupleSelectorByStr.update(
+                    {ts.toJsonStr(): ts for ts in mappedSelectors}
+                )
+
+        # Return a list of tuple selectors
+        return list(tupleSelectorByStr.values())
+
+    def notifyOfTupleUpdate(self, tupleSelector: TupleSelector) -> None:
+        """Notify Of Tuple Update
+
+        This method tells the observable that an update has occurred and that it should
+        send new data to its observers.
+
+        Tuple selectors should be identical to the data being observed.
+
+        :param tupleSelector: A tuple selector that describes the scope of the update.
+        :returns: None
+
+        """
+        tupleSelectors = self._getMappedTupleSelectors(tupleSelector)
+        reactor.callLater(0, self._notifyOfTupleUpdateInMain, tupleSelectors)
+
+    def notifyOfTupleUpdateForTuple(self, tupleName: str) -> None:
+        """Notify of Tuple Update for Tuple
+
+        Like the above notification method, this method will aquire new data and
+        send it to observers.
+
+        .. note:: Calling this will not trigger the TupleSelector Update Mappers as
+                  this is an earlier soloution.
+
+                  For all new code, use notifyOfTupleUpdate() and define mappers.
+
+        """
+        tupleSelectors = []
+        for data in self._observerDataByTupleSelector.values():
+            if data.tupleSelector.name == tupleName:
+                tupleSelectors.append(data.tupleSelector)
+
+        reactor.callLater(0, self._notifyOfTupleUpdateInMain, tupleSelectors)
+
+    @inlineCallbacks
+    def _notifyOfTupleUpdateInMain(self, tupleSelectors: List[TupleSelector]):
+        for tupleSelector in tupleSelectors:
+            yield self._notifyOfTupleUpdateInMainOne(tupleSelector)
+
+    @inlineCallbacks
+    def _notifyOfTupleUpdateInMainOne(self, tupleSelector: TupleSelector):
+        tsStr = tupleSelector.toJsonStr()
+        if tsStr not in self._observerDataByTupleSelector:
+            return
+
+        # Filter out the offline observables
+        onlineUuids = set(VortexFactory.getRemoteVortexUuids())
+        observers = self._observerDataByTupleSelector[tsStr].observers
+        for od in list(observers):
+            if od.vortexUuid not in onlineUuids:
+                observers.remove(od)
+
+        # Get / update the list of observing UUIDs
+        if not observers:
+            del self._observerDataByTupleSelector[tsStr]
+            return
+
+        # Create the vortexMsg
+        filt = dict(tupleSelector=tupleSelector)
+        filt.update(self._filt)
+        vortexMsg = yield self._createVortexMsg(filt, tupleSelector)
+
+        # We can have multiple Observable clients on the one vortex, so make sure
+        # we only send one message for these.
+        destVortexUuids = set([od.vortexUuid for od in observers])
+
+        # Send the vortex messages
+        for destVortexUuid in destVortexUuids:
+            d = VortexFactory.sendVortexMsg(
+                vortexMsgs=vortexMsg, destVortexUuid=destVortexUuid
+            )
+            d.addErrback(vortexLogFailure, logger)
+
+    def _customMaybeDeferred(self, f, *args, **kw):
+        try:
+            result = f(*args, **kw)
+        except Exception as e:
+            logger.error(
+                "TupleDataObservableHandler:%s TupleProvider failed",
+                self._observableName,
+            )
+            logger.exception(e)
+            return fail(failure.Failure(e))
+
+        if isinstance(result, Deferred):
+            return result
+        elif isinstance(result, failure.Failure):
+            return fail(result)
+        else:
+            return succeed(result)

vortex/handler/TupleDataObservableProxyHandler.py

@@ -0,0 +1,340 @@
+import logging
+from copy import copy
+
+from twisted.internet import reactor
+from twisted.internet.defer import TimeoutError
+from twisted.python.failure import Failure
+
+from vortex.DeferUtil import deferToThreadWrapWithLogger
+from vortex.DeferUtil import vortexLogFailure
+from vortex.PayloadEndpoint import PayloadEndpoint
+from vortex.PayloadEnvelope import PayloadEnvelope
+from vortex.PayloadResponse import PayloadResponse
+from vortex.TupleSelector import TupleSelector
+from vortex.VortexABC import SendVortexMsgResponseCallable
+from vortex.VortexFactory import NoVortexException
+from vortex.VortexFactory import VortexFactory
+from vortex.VortexUtil import limitConcurrency, _dedupProcessorCallKeys
+from vortex.handler.TupleDataObservableCache import TupleDataObservableCache
+from vortex.handler.TupleDataObservableHandler import TupleDataObservableHandler
+from vortex.handler.TupleDataObservableHandler import TuplesProviderABC
+
+logger = logging.getLogger(__name__)
+
+
+class TupleDataObservableProxyHandler(TupleDataObservableCache):
+    __CHECK_PERIOD = 30  # seconds
+
+    def __init__(
+        self,
+        observableName,
+        proxyToVortexName: str,
+        additionalFilt=None,
+        subscriptionsEnabled=True,
+        observerName="default",
+    ) -> None:
+        """Constructor
+
+        :param observableName: The name of this and the other observable
+        :param proxyToVortexName: The vortex dest name to proxy requests to
+        :param additionalFilt: Any additional filter keys that are required
+        :param subscriptionsEnabled: Should subscriptions be enabled (default)
+        :param observerName: We can clash with other observers, so where there are
+        multiple observers on the one vortex, they should use different names.
+        """
+        TupleDataObservableCache.__init__(self)
+
+        self._proxyToVortexName = proxyToVortexName
+        self._subscriptionsEnabled = subscriptionsEnabled
+        self._observerName = observerName
+        self._filt = dict(name=observableName, key="tupleDataObservable")
+        if additionalFilt:
+            self._filt.update(additionalFilt)
+
+        # Create the local observable, this allows local tuple providers
+        # The rest are proxied on to the backend
+        self._localObservableHandler = TupleDataObservableHandler(
+            observableName,
+            additionalFilt=additionalFilt,
+            subscriptionsEnabled=subscriptionsEnabled,
+        )
+        # Shutdown the local observables' endpoint, we don't want it
+        # listening its self
+        self._localObservableHandler.shutdown()
+
+        # Finally, Setup our endpoint
+        self._endpoint = PayloadEndpoint(self._filt, self._process)
+
+        TupleDataObservableCache.start(self)
+
+    def shutdown(self):
+        self._endpoint.shutdown()
+        TupleDataObservableCache.shutdown(self)
+
+    ## ----- Implement local observable
+
+    def addTupleProvider(self, tupleName, provider: TuplesProviderABC):
+        """Add Tuple Provider
+
+        Adds a tuple provider to the local observable.
+
+        All other requests are proxied on
+
+        """
+        self._localObservableHandler.addTupleProvider(
+            tupleName, provider=provider
+        )
+
+    def notifyOfTupleUpdate(self, tupleSelector: TupleSelector) -> None:
+        """Notify of Tuple Update
+
+        Notifies the local observable that tuples have been updated
+
+        """
+        if not self._localObservableHandler.hasTupleProvider(
+            tupleSelector.name
+        ):
+            raise Exception(
+                "Local observable doesn't have tuple provider for %s"
+                " registered, Proxy is : %s" % (tupleSelector.name, self._filt)
+            )
+
+        self._localObservableHandler.notifyOfTupleUpdate(tupleSelector)
+
+    def _process(
+        self,
+        payloadEnvelope: PayloadEnvelope,
+        vortexUuid: str,
+        vortexName: str,
+        sendResponse: SendVortexMsgResponseCallable,
+        **kwargs,
+    ):
+
+        if vortexName == self._proxyToVortexName:
+            return self._processUpdateFromBackend(payloadEnvelope)
+
+        else:
+            d = self._processSubscribeFromFrontend(
+                payloadEnvelope, vortexUuid, sendResponse
+            )
+            if d:
+                d.addErrback(vortexLogFailure, logger, consumeError=True)
+
+    @limitConcurrency(
+        logger, 10, deduplicateBasedOnKwArgsKey=_dedupProcessorCallKeys
+    )
+    def _processSubscribeFromFrontend(
+        self,
+        payloadEnvelope: PayloadEnvelope,
+        vortexUuid: str,
+        sendResponse: SendVortexMsgResponseCallable,
+    ):
+
+        tupleSelector: TupleSelector = payloadEnvelope.filt["tupleSelector"]
+
+        # If the local observable provides this tuple, then use that instead
+        if self._localObservableHandler.hasTupleProvider(tupleSelector.name):
+            return self._localObservableHandler._process(
+                payloadEnvelope=payloadEnvelope,
+                vortexUuid=vortexUuid,
+                sendResponse=sendResponse,
+            )
+
+        if not VortexFactory.isVortexUuidOnline(vortexUuid):
+            logger.debug(
+                "Vortex %s is no longer online, skipping reply", vortexUuid
+            )
+            return
+
+        # logger.debug(
+        #     "Vortex %s is STILL online, continuing to reply", vortexUuid
+        # )
+
+        # Add support for just getting data, no subscription.
+        if payloadEnvelope.filt.get("unsubscribe", True):
+            return self._handleUnsubscribe(tupleSelector, vortexUuid)
+
+        elif (
+            payloadEnvelope.filt.get("subscribe", True)
+            and self._subscriptionsEnabled
+        ):
+            return self._handleSubscribe(
+                payloadEnvelope, tupleSelector, sendResponse, vortexUuid
+            )
+
+        else:
+            return self._handlePoll(
+                payloadEnvelope, tupleSelector, sendResponse
+            )
+
+    def _handleUnsubscribe(self, tupleSelector: TupleSelector, vortexUuid: str):
+
+        if not self._hasTupleSelector(tupleSelector):
+            return
+
+        cache = self._getCache(tupleSelector)
+        try:
+            cache.vortexUuids.remove(vortexUuid)
+        except KeyError:
+            pass
+
+    def _handleSubscribe(
+        self,
+        payloadEnvelope: PayloadEnvelope,
+        tupleSelector: TupleSelector,
+        sendResponse: SendVortexMsgResponseCallable,
+        vortexUuid: str,
+    ):
+
+        # Add support for just getting data, no subscription.
+        cache = self._getCache(tupleSelector)
+        if (
+            cache
+            and cache.lastServerPayloadDate is not None
+            and cache.cacheEnabled
+        ):
+            respPayloadEnvelope = PayloadEnvelope(
+                filt=payloadEnvelope.filt,
+                encodedPayload=cache.encodedPayload,
+                date=cache.lastServerPayloadDate,
+            )
+
+            d = respPayloadEnvelope.toVortexMsgDefer()
+            d.addCallback(sendResponse)
+            d.addErrback(vortexLogFailure, logger, consumeError=True)
+
+        elif cache:
+            self._sendRequestToServer(payloadEnvelope)
+
+        else:
+            cache = self._makeCache(tupleSelector)
+            self._sendRequestToServer(payloadEnvelope)
+
+        cache.vortexUuids.add(vortexUuid)
+        # Allow the cache to be disabled
+        cache.cacheEnabled = (
+            cache.cacheEnabled
+            and not payloadEnvelope.filt.get("disableCache", False)
+        )
+
+    def _sendRequestToServer(self, payloadEnvelope: PayloadEnvelope):
+        payloadEnvelope.filt["observerName"] = self._observerName
+        d = VortexFactory.sendVortexMsg(
+            vortexMsgs=payloadEnvelope.toVortexMsg(),
+            destVortexName=self._proxyToVortexName,
+        )
+        d.addErrback(vortexLogFailure, logger, consumeError=True)
+
+    def _sendUnsubscribeToServer(self, tupleSelector: TupleSelector):
+        payloadEnvelope = PayloadEnvelope()
+        payloadEnvelope.filt["tupleSelector"] = tupleSelector
+        payloadEnvelope.filt["unsubscribe"] = True
+        self._sendRequestToServer(payloadEnvelope)
+
+    def _handlePoll(
+        self,
+        payloadEnvelope: PayloadEnvelope,
+        tupleSelector: TupleSelector,
+        sendResponse: SendVortexMsgResponseCallable,
+    ):
+
+        useCache = payloadEnvelope.filt.get("useCache", True)
+
+        # Keep a copy of the incoming filt, in case they are using PayloadResponse
+        responseFilt = copy(payloadEnvelope.filt)
+
+        # Restore the original payload filt (PayloadResponse) and send it back
+        def reply(payload):
+            payload.filt = responseFilt
+            d = payload.toVortexMsgDefer()
+            d.addCallback(sendResponse)
+            d.addErrback(vortexLogFailure, logger, consumeError=True)
+            # logger.debug("Received response from observable")
+
+        if useCache:
+            cache = self._getCache(tupleSelector)
+            if (
+                cache
+                and cache.lastServerPayloadDate is not None
+                and cache.cacheEnabled
+            ):
+                payloadEnvelope.encodedPayload = cache.encodedPayload
+                payloadEnvelope.date = cache.lastServerPayloadDate
+                reply(payloadEnvelope)
+                return
+
+        # Track the response, log an error if it fails
+        # 5 Seconds is long enough
+        pr = PayloadResponse(
+            payloadEnvelope,
+            timeout=PayloadResponse.TIMEOUT - 5,  # 5 seconds less
+            logTimeoutError=False,
+        )
+
+        pr.addErrback(self._handlePrFailure, tupleSelector)
+        pr.addErrback(vortexLogFailure, logger, consumeError=True)
+        pr.addCallback(reply)
+
+        self._sendRequestToServer(payloadEnvelope)
+
+    def _handlePrFailure(self, f: Failure, tupleSelector):
+        if f.check(TimeoutError):
+            logger.error(
+                "Received no response from\nobservable %s\ntuple selector %s",
+                self._filt,
+                tupleSelector.toJsonStr(),
+            )
+        else:
+            logger.error(
+                "Unexpected error, %s\nobservable %s\ntuple selector %s",
+                f,
+                self._filt,
+                tupleSelector.toJsonStr(),
+            )
+
+    @deferToThreadWrapWithLogger(logger)
+    def _processUpdateFromBackend(self, payloadEnvelope: PayloadEnvelope):
+
+        tupleSelector: TupleSelector = payloadEnvelope.filt["tupleSelector"]
+
+        if not self._hasTupleSelector(tupleSelector):
+            return
+
+        cache, requiredUpdate = self._updateCache(payloadEnvelope)
+        if not requiredUpdate:
+            return
+
+        # Get / update the list of observing UUIDs
+        observingUuids = (
+            cache.vortexUuids & VortexFactory.getRemoteVortexUuids()
+        )
+
+        if not observingUuids:
+            return
+
+        # Create the vortexMsg
+        vortexMsg = payloadEnvelope.toVortexMsg()
+
+        # Send the vortex messages
+        for vortexUuid in observingUuids:
+            reactor.callLater(
+                0,
+                self._sendVortexMsgInReactor,
+                vortexMsg=vortexMsg,
+                destVortexUuid=vortexUuid,
+            )
+
+    def _sendVortexMsgInReactor(self, vortexMsg, destVortexUuid):
+        # Send the vortex messages
+        d = VortexFactory.sendVortexMsg(
+            vortexMsgs=vortexMsg, destVortexUuid=destVortexUuid
+        )
+        d.addErrback(self._sendErrback)
+
+    def _sendErrback(self, failure):
+
+        if failure.check(NoVortexException):
+            logger.debug(str(failure.value))
+            return
+
+        vortexLogFailure(failure, logger)