vortexpy 4.0.23__py3-none-any.whl

vortex/PayloadEndpoint.py

@@ -0,0 +1,216 @@
+"""
+ * Created by Synerty Pty Ltd
+ *
+ * This software is open source, the MIT license applies.
+ *
+ * Website : http://www.synerty.com
+ * Support : support@synerty.com
+"""
+import inspect
+import logging
+import types
+import weakref
+from copy import copy
+from typing import Callable, Optional, Dict, Union
+
+from vortex.PayloadEnvelope import PayloadEnvelope
+from vortex.PayloadIO import PayloadIO
+from vortex.VortexABC import SendVortexMsgResponseCallable
+
+logger = logging.getLogger(__name__)
+
+
+# TODO: typings needs to support keyword args before tbis will work.
+# PayloadEndpointProcessCallable = Callable[
+#     [
+#         ...
+#         # PayloadEnvelope,  # payloadEnvelope
+#         # str,  # vortexUuid
+#         # str,  # vortexName
+#         # Any,  # httpSession
+#         # SendVortexMsgResponseCallable  # sendResponse
+#     ],
+#     Optional[Deferred]  # Return
+# ]
+
+
+class PayloadEndpoint:
+    """
+    The payload endpoint is responsible for matching payloads filters against
+    filters defined in the endpoint. If the end point filters are within the
+    payload filter then the payload will be passed to the supplied callable.
+    """
+
+    PERMITTER = None
+
+    def __init__(
+        self,
+        filt: Dict,
+        callable_,
+        acceptOnlyFromVortex: Optional[Union[str, tuple]] = None,
+        ignoreFromVortex: Optional[Union[str, tuple]] = None,
+    ) -> None:
+        """
+        :param filt: The filter to match against payloads
+        :param callable_: This will be called and passed the payload if it matches
+        :param acceptOnlyFromVortex: Accept payloads only from this vortex,
+            The vortex can be str or tuple of str, or None to accept from any.
+        """
+        self._acceptOnlyFromVortex = acceptOnlyFromVortex
+        if isinstance(self._acceptOnlyFromVortex, str):
+            self._acceptOnlyFromVortex = (self._acceptOnlyFromVortex,)
+
+        self._ignoreFromVortex = ignoreFromVortex
+        if isinstance(self._ignoreFromVortex, str):
+            self._ignoreFromVortex = (self._ignoreFromVortex,)
+
+        self._keyCheck(filt)
+
+        self._wref: Callable[[], Optional[Callable]] = None
+        if isinstance(callable_, types.FunctionType):
+            w = None
+            if hasattr(callable_, "_endpointWeakClass"):
+                w = callable_._endpointWeakClass
+
+            else:
+
+                class W:
+                    def __init__(self, callable_):
+                        self._callable = callable_
+                        self._callable._endpointWeakClass = self
+
+                    def __call__(self, payload):
+                        self._callable(payload)
+
+                w = W(callable_)
+
+            self._wref = weakref.ref(w)
+
+        else:
+            weakObject = weakref.ref(callable_.__self__)
+            weakMethod = weakref.ref(callable_.__func__)
+
+            if callable_.__func__.__name__ == "func":
+                logger.warning(
+                    "The registered callback is 'func' this is likely"
+                    " a method wrapped in @deferToThreadWrapWithLogger, %s",
+                    filt,
+                )
+
+            def getCallable():
+                obj = weakObject()
+                func = weakMethod()
+                if obj and func:
+                    return getattr(obj, func.__name__)
+                return None
+
+            self._wref = getCallable
+
+        self._filt = filt
+        PayloadIO().add(self)
+
+    def _keyCheck(self, filt):
+        if not "key" in filt:
+            e = Exception(
+                "There is no 'key' in the payload filt"
+                ", There must be one for routing"
+            )
+            logger.exception(e)
+            raise e
+
+    @property
+    def filt(self):
+        return copy(self._filt)
+
+    def check(self, payloadEnvelope: PayloadEnvelope, vortexName: str) -> bool:
+
+        # Filter for the backend vortexes.
+        if self._ignoreFromVortex and vortexName in self._ignoreFromVortex:
+            return False
+
+        # Filter for the backend vortexes.
+        if (
+            self._acceptOnlyFromVortex
+            and vortexName not in self._acceptOnlyFromVortex
+        ):
+            return False
+
+        def removeUnhashable(filt):
+            items = set()
+            for key, value in list(filt.items()):
+                # We can compare an array of primitives
+                if isinstance(value, list):
+                    value = tuple(sorted(value))
+
+                # We don't compare complex structures
+                if isinstance(value, dict):
+                    continue
+
+                if inspect.isclass(value) and (
+                    issubclass(value, dict) or issubclass(value, list)
+                ):
+                    raise Exception(
+                        "Class type passed instead of an instance"
+                        " key:%s, value:%s" % (key, value)
+                    )
+
+                items.add((key, value))
+            return items
+
+        theirFilt = removeUnhashable(payloadEnvelope.filt)
+        ourFilt = removeUnhashable(self._filt)
+
+        return set(ourFilt).issubset(theirFilt)
+
+    def process(
+        self,
+        payloadEnvelope: PayloadEnvelope,
+        vortexUuid: str,
+        vortexName: str,
+        httpSession,
+        sendResponse: SendVortexMsgResponseCallable,
+    ):
+
+        if not self.check(payloadEnvelope, vortexName):
+            return
+
+        if self.PERMITTER and not self.PERMITTER.check(
+            payloadEnvelope, httpSession
+        ):
+            logger.debug("Permission failed for %s", payloadEnvelope.filt)
+            return
+
+        callable_ = self._wref()
+        if callable_:
+            return callable_(
+                payloadEnvelope=payloadEnvelope,
+                vortexUuid=vortexUuid,
+                vortexName=vortexName,
+                httpSession=httpSession,
+                sendResponse=sendResponse,
+            )
+        else:
+            PayloadIO().remove(self)
+
+    def _callableExpired(self, expiredCallable):
+        pass
+        # PayloadIO().remove(self)
+
+    def __repr__(self):
+        callable_ = self._wref()
+        if callable_:
+            try:
+                callbackStr = "%s.%s" % (
+                    callable_.__self__.__class__.__name__,
+                    callable_.__name__,
+                )
+            except Exception as e:
+                callbackStr = str(e)
+        else:
+            callbackStr = "None"
+
+        s = "Payload id=%s\nfilt=%s\ncallback=%s"
+        return s % (id(self), self._filt, callbackStr)
+
+    def shutdown(self):
+        PayloadIO().remove(self)

vortex/PayloadEndpointTest.py

@@ -0,0 +1,164 @@
+"""
+ * Created by Synerty Pty Ltd
+ *
+ * This software is open source, the MIT license applies.
+ *
+ * Website : http://www.synerty.com
+ * Support : support@synerty.com
+"""
+
+from twisted.trial import unittest
+
+from vortex.Payload import Payload
+from vortex.PayloadEndpoint import PayloadEndpoint
+from vortex.PayloadEnvelope import PayloadEnvelope
+from vortex.PayloadIO import PayloadIO
+
+
+class PayloadEndpointPyTestAssignPayload:
+    def __init__(self, unitTest):
+        self._ut = unitTest
+
+    def process(self, payloadEnvelope: PayloadEnvelope, **kwargs):
+        self._ut.deliveredPayload = payloadEnvelope
+
+
+class PayloadEndpointPyTest(unittest.TestCase):
+    def setUp(self):
+        PayloadIO._instance = None
+
+        self.deliveredPayloadEnvelope = None
+
+    def _testBuild(self, plFilt, epFilt):
+        payload = Payload()
+        payload.filt.update(plFilt)
+        for x in range(6):
+            payload.filt["%s" % x] = x
+
+        def processPayload(payloadEnvelope: PayloadEnvelope, **kwargs):
+            self.deliveredPayloadEnvelope = payloadEnvelope
+
+        self._keepFuncInScope = processPayload
+
+        PayloadEndpoint(epFilt, processPayload)
+
+        PayloadIO().process(
+            payloadEnvelope=payload.makePayloadEnvelope(),
+            vortexUuid="test",
+            vortexName="test",
+            httpSession=None,
+            sendResponse=lambda x: x,
+        )
+
+        return payload
+
+    def testFiltMatches(self):
+        plFilt = {"key": "unittest", "This matches": 555}
+        epFilt = {"key": "unittest", "This matches": 555}
+
+        payload = self._testBuild(plFilt, epFilt)
+
+        self.assertEqual(
+            payload,
+            self.deliveredPayloadEnvelope,
+            "PayloadIO/PayloadEndpoint delivery error",
+        )
+
+    def testFiltValueUnmatched(self):
+        plFilt = {"key": "unittest", "This matches": 555}
+        epFilt = {"key": "unittest", "This matches": 0}
+
+        self._testBuild(plFilt, epFilt)
+
+        self.assertEqual(
+            self.deliveredPayloadEnvelope,
+            None,
+            "PayloadIO/PayloadEndpoint unmatched value test error",
+        )
+
+    def testFiltKeyUnmatched(self):
+        plFilt = {"key": "unittest", "This matches": 555}
+        epFilt = {"key": "unittest", "This doesnt matches": 555}
+
+        self._testBuild(plFilt, epFilt)
+
+        self.assertEqual(
+            self.deliveredPayloadEnvelope,
+            None,
+            "PayloadIO/PayloadEndpoint unmatched value test error",
+        )
+
+    def testFunctionGoesOutOfScope(self):
+        filt = {"key": "unittest", "This matches": 555}
+
+        payload = Payload()
+        payload.filt = filt
+
+        def subScope():
+            def outOfScopeFunc(
+                payloadEnvelope: PayloadEnvelope, *args, **kwargs
+            ):
+                self.deliveredPayloadEnvelope = payloadEnvelope
+
+            PayloadEndpoint(filt, outOfScopeFunc)
+
+        PayloadIO().process(
+            payloadEnvelope=payload.makePayloadEnvelope(),
+            vortexUuid="test",
+            vortexName="test",
+            httpSession=None,
+            sendResponse=lambda x: x,
+        )
+
+        self.assertEqual(
+            self.deliveredPayloadEnvelope,
+            None,
+            "PayloadIO/PayloadEndpoint unmatched value test error",
+        )
+
+    def testClassGoesOutOdScope(self):
+        filt = {"key": "unittest", "This matches": 555}
+
+        payload = Payload()
+        payload.filt = filt
+
+        def subScope():
+            inst = PayloadEndpointPyTestAssignPayload(self)
+            PayloadEndpoint(filt, inst.process)
+
+        PayloadIO().process(
+            payloadEnvelope=payload.makePayloadEnvelope(),
+            vortexUuid="test",
+            vortexName="test",
+            httpSession=None,
+            sendResponse=lambda x: x,
+        )
+
+        self.assertEqual(
+            self.deliveredPayloadEnvelope,
+            None,
+            "PayloadIO/PayloadEndpoint unmatched value test error",
+        )
+
+    def testClassStaysInScope(self):
+        filt = {"key": "unittest", "This matches": 555}
+
+        payload = Payload()
+        payload.filt = filt
+
+        inst = PayloadEndpointPyTestAssignPayload(self)
+        PayloadEndpoint(filt, inst.process)
+
+        PayloadIO().process(
+            payloadEnvelope=payload.makePayloadEnvelope(),
+            vortexUuid="test",
+            vortexName="test",
+            httpSession=None,
+            sendResponse=lambda x: x,
+        )
+
+        self.assertEqual(
+            self.deliveredPayloadEnvelope,
+            payload,
+            "PayloadIO/PayloadEndpoint unmatched value test error",
+        )

vortex/PayloadEnvelope.py

@@ -0,0 +1,166 @@
+"""
+ * Created by Synerty Pty Ltd
+ *
+ * This software is open source, the MIT license applies.
+ *
+ * Website : http://www.synerty.com
+ * Support : support@synerty.com
+"""
+
+import base64
+import json
+import logging
+from base64 import b64decode
+from base64 import b64encode
+from datetime import datetime
+from typing import Any
+from typing import Dict
+from typing import List
+from typing import Optional
+from typing import Union
+
+import pytz
+
+from .DeferUtil import deferToThreadWrapWithLogger
+from .DeferUtil import noMainThread
+from .Jsonable import Jsonable
+from .Payload import Payload
+from .SerialiseUtil import T_RAPUI_PAYLOAD_ENVELOPE
+
+logger = logging.getLogger(__name__)
+
+PayloadEnvelopeList = List["PayloadEnvelope"]
+VortexMsgList = List[bytes]
+
+
+class NoPayloadException(Exception):
+    pass
+
+
+class PayloadEnvelope(Jsonable):
+    """Payload
+    This object represents a hierarchy of data transferred between client and server
+    """
+
+    __fieldNames__ = ["filt", "data", "result", "date"]
+    __rapuiSerialiseType__ = T_RAPUI_PAYLOAD_ENVELOPE
+
+    vortexUuidKey = "__vortexUuid__"
+    vortexNameKey = "__vortexName__"
+
+    def __init__(
+        self,
+        filt: Optional[Dict] = None,
+        encodedPayload: Optional[str] = None,
+        result=None,
+        date: Optional[datetime] = None,
+        data: Optional[Any] = None,
+    ) -> None:
+        Jsonable.__init__(self)
+        """Constructor"""
+        assert not (
+            data and encodedPayload
+        ), "You can not pass encodedData and encodedPayload"
+
+        self.filt: Dict = {} if filt is None else filt
+        self.data: Union[Any, str, None] = data if data else encodedPayload
+        self.result = result
+        self.date: datetime = date if date else datetime.now(pytz.utc)
+
+        if isinstance(filt, str):
+            self.filt = {self.filt: None}
+
+    # -------------------------------------------
+    # PayloadEnvelope Methods
+
+    @property
+    def encodedPayload(self) -> Optional[str]:
+        # self.data could be from VortexJS/Typescript where we allow `str` data
+        assert self.data is None or isinstance(
+            self.data, str
+        ), "Encoded payload is not None or a string"
+        return self.data
+
+    @encodedPayload.setter
+    def encodedPayload(self, val: Optional[str]):
+        # This value should only be set from users of VortexPy and
+        # Payload.toEncodedPayload so we only allow str or None
+        assert val is None or isinstance(
+            val, str
+        ), "Encoded payload is not None or a string"
+        self.data = val
+
+    def isEmpty(self):
+        return (
+            (
+                not self.filt
+                or (
+                    self.vortexNameKey in self.filt
+                    and self.vortexUuidKey in self.filt
+                    and len(self.filt) == 2
+                )
+            )
+            and not self.encodedPayload
+            and not self.result
+        )
+
+    # -------------------------------------------
+    # Decode Helper Methods
+
+    def decodePayload(self) -> Payload:
+        if not self.encodedPayload:
+            if self.result:
+                logger.debug(
+                    "encodedPayload is None, but maybe there was an error: %s",
+                    self.result,
+                )
+            raise NoPayloadException()
+
+        noMainThread()
+        return Payload().fromEncodedPayload(self.encodedPayload)
+
+    @deferToThreadWrapWithLogger(logger)
+    def decodePayloadDefer(self) -> "Payload":
+        return self.decodePayload()
+
+    # -------------------------------------------
+    # JSON Related methods
+    def _fromJson(self, jsonStr):
+        jsonDict = json.loads(jsonStr)
+
+        assert jsonDict[Jsonable.JSON_CLASS_TYPE] == self.__rapuiSerialiseType__
+        return self.fromJsonDict(jsonDict)
+
+    def _toJson(self) -> str:
+        return json.dumps(self.toJsonDict())
+
+    # -------------------------------------------
+    # VortexServer Message Methods
+    def toVortexMsg(self, base64Encode=False) -> bytes:
+        jsonStr = self._toJson().encode()
+        if base64Encode:
+            return b64encode(jsonStr)
+        return jsonStr
+
+    @deferToThreadWrapWithLogger(logger)
+    def toVortexMsgDefer(self, base64Encode=False) -> bytes:
+        return self.toVortexMsg(base64Encode=base64Encode)
+
+    def fromVortexMsg(
+        self,
+        vortexMsg: bytes,
+    ) -> "PayloadEnvelope":
+        if vortexMsg.startswith(b"{"):
+            jsonStr = vortexMsg.decode()
+        else:
+            jsonStr = b64decode(vortexMsg).decode()
+        return self._fromJson(jsonStr)
+
+    @deferToThreadWrapWithLogger(logger)
+    def fromVortexMsgDefer(self, vortexMsg: bytes) -> "PayloadEnvelope":
+        return self.fromVortexMsg(vortexMsg)
+
+    @classmethod
+    @deferToThreadWrapWithLogger(logger)
+    def base64EncodeDefer(cls, vortexMsg) -> bytes:
+        return base64.b64encode(vortexMsg)

vortex/PayloadFilterKeys.py

@@ -0,0 +1,15 @@
+"""
+ * Created by Synerty Pty Ltd
+ *
+ * This software is open source, the MIT license applies.
+ *
+ * Website : http://www.synerty.com
+ * Support : support@synerty.com
+"""
+
+rapuiServerEcho = "rapuiServerEcho"
+rapuiClientEcho = "rapuiClientEcho"
+rapuiVortexUuid = "rapuiVortexUuid"
+
+plIdKey = "id"
+plDeleteKey = "delete"

vortex/PayloadIO.py

@@ -0,0 +1,152 @@
+"""
+ * Created by Synerty Pty Ltd
+ *
+ * This software is open source, the MIT license applies.
+ *
+ * Website : http://www.synerty.com
+ * Support : support@synerty.com
+"""
+import logging
+from datetime import datetime
+
+import pytz
+from twisted.internet import reactor
+from twisted.internet.defer import Deferred
+from twisted.python.failure import Failure
+
+from vortex.DeferUtil import vortexLogFailure
+from vortex.PayloadEnvelope import PayloadEnvelope
+from vortex.VortexABC import SendVortexMsgResponseCallable
+
+logger = logging.getLogger(name="PayloadIO")
+
+
+class PayloadIO(object):
+    """
+    PayloadIO, Processes payloads received from the vortex and distributes
+    them to where they need to go.
+    """
+
+    # Singleton
+    _instance = None
+
+    def __new__(cls):
+        if not cls._instance:
+            cls._instance = super(PayloadIO, cls).__new__(cls)
+            cls._instance.__singleton_init__()
+        return cls._instance
+
+    def __singleton_init__(self):
+        self._endpoints = set()
+
+    def remove(self, endpoint):
+        if endpoint in self._endpoints:
+            self._endpoints.remove(endpoint)
+
+    def add(self, endpoint):
+        self._endpoints.add(endpoint)
+
+    @property
+    def endpoints(self):
+        """Endpoints property
+
+        @:return a copy of the list of endpoints
+        """
+        return list(self._endpoints)
+
+    def process(
+        self,
+        payloadEnvelope: PayloadEnvelope,
+        vortexUuid: str,
+        vortexName: str,
+        httpSession,
+        sendResponse: SendVortexMsgResponseCallable,
+    ):
+        from vortex.VortexFactory import VortexFactory
+
+        # noinspection PyProtectedMember
+        if VortexFactory._DEBUG_LOGGING:
+            logger.debug(
+                "Received payloadEnvelope from %s:%s for %s",
+                vortexName,
+                vortexUuid,
+                payloadEnvelope.filt,
+            )
+
+        immutableEndpoints = list(self._endpoints)
+        for endpoint in immutableEndpoints:
+            if not endpoint.check(payloadEnvelope, vortexName):
+                continue
+
+            reactor.callLater(
+                0,
+                self._processLater,
+                endpoint,
+                payloadEnvelope,
+                vortexUuid,
+                vortexName,
+                httpSession,
+                sendResponse,
+            )
+
+    def _processLater(
+        self,
+        endpoint,
+        payloadEnvelope: PayloadEnvelope,
+        vortexUuid: str,
+        vortexName: str,
+        httpSession,
+        sendResponse: SendVortexMsgResponseCallable,
+    ):
+        startDate = datetime.now(pytz.utc)
+
+        def respondToException(failure):
+            """Respond To Exception
+            Putting the exception into a failure messes with the stack, hence the
+            common function
+            """
+            try:
+                sendResponse(
+                    PayloadEnvelope(
+                        filt=payloadEnvelope.filt,
+                        result=str(failure.getTraceback()),
+                    ).toVortexMsg()
+                )
+            except Exception as e:
+                logger.exception(e)
+
+            vortexLogFailure(failure, logger)
+            logger.error(payloadEnvelope.filt)
+
+        def callback(value, blocking=False):
+            # A blocking call taking more than 100ms is BAD
+            # Otherwise a call taking more than a 1s is just poor performance.
+            secondsTaken = (datetime.now(pytz.utc) - startDate).total_seconds()
+            if secondsTaken > 0.1 and blocking or secondsTaken > 2.0:
+                func = logger.critical if blocking else logger.debug
+                func(
+                    "%s Payload endpoint took %s\npayload.filt=%s\n%s",
+                    "BLOCKING " if blocking else "",
+                    secondsTaken,
+                    payloadEnvelope.filt,
+                    endpoint,
+                )
+
+        try:
+            d = endpoint.process(
+                payloadEnvelope,
+                vortexUuid,
+                vortexName,
+                httpSession,
+                sendResponse,
+            )
+            if isinstance(d, Deferred):
+                d.addCallback(callback)
+                d.addErrback(respondToException)
+            elif hasattr(d, "addCallback"):
+                raise Exception("isinstance FAILED")
+            else:
+                callback(True, blocking=True)
+
+        except Exception as e:
+            respondToException(Failure(e))