vortexpy 4.0.23__py3-none-any.whl

vortex/PayloadPriority.py

@@ -0,0 +1,11 @@
+"""
+ * Created by Synerty Pty Ltd
+ *
+ * This software is open source, the MIT license applies.
+ *
+ * Website : http://www.synerty.com
+ * Support : support@synerty.com
+"""
+
+RPC_PRIORITY = 10
+DEFAULT_PRIORITY = 100

vortex/PayloadResponse.py

@@ -0,0 +1,173 @@
+import logging
+import sys
+from copy import copy
+from datetime import datetime
+from typing import Optional
+from uuid import uuid4
+
+import pytz
+from twisted.internet import reactor
+from twisted.internet.defer import Deferred
+from twisted.python.failure import Failure
+
+from vortex.PayloadEndpoint import PayloadEndpoint
+from vortex.PayloadEnvelope import PayloadEnvelope
+from vortex.VortexFactory import VortexFactory
+
+logger = logging.getLogger(__name__)
+
+
+class PayloadResponse(Deferred):
+    """Payload Response
+
+    This class is used to catch responses from a sent payload.
+    If the remote end is going to send back a payload, with the same filt, this class can
+    be used to capture this can and call it's call back.
+
+    If the response is not received within the timeout, the errback is called.
+
+    Here is some example usage.
+
+    ::
+
+        logger = logging.getLogger("Example Payload Response")
+        payload = Payload(filt={"rapuiServerEcho":True})
+        responseDeferred = PayloadResponse(payload)
+        VortexFactory.sendVortexMsg(payload.toVortexMsg())
+        logger.info("Payload Sent")
+
+        responseDeferred.addCallbacks(logger.info, logger.error)
+
+    """
+
+    MESSAGE_ID_KEY = "PayloadResponse.messageId"
+
+    PROCESSING = "Processing"
+    # NO_ENDPOINT = "No Endpoint"
+    FAILED = "Failed"
+    SUCCESS = "Success"
+    TIMED_OUT = "Timed Out"
+
+    TIMEOUT: float = 30.00
+
+    __SEQ = 1
+
+    def __init__(
+        self,
+        payloadEnvelope: PayloadEnvelope,
+        destVortexName: Optional[str] = None,
+        destVortexUuid: Optional[str] = None,
+        timeout: Optional[float] = None,
+        resultCheck=True,
+        logTimeoutError=True,
+    ) -> None:
+        """Constructor
+
+        Tag and optionally send a payload.
+
+        The timeout starts as soon as the constructor is called.
+
+        :param payloadEnvelope The payloadEnvelope to send to the remote and, and wait for a response for
+        :param destVortexName The name of the vortex to send to.
+        :param destVortexUuid The UUID of the vortex to send a payload to.
+        :param timeout The timeout to wait for a response
+        :param resultCheck Should the response payload.result be checked, if it fails
+                    it errback will be called.
+        """
+        Deferred.__init__(self)
+
+        if not timeout:
+            timeout = self.TIMEOUT
+
+        self._resultCheck = resultCheck
+        self._logTimeoutError = logTimeoutError
+
+        # uuid4 can have collisions
+        self._messageId = str(uuid4()) + str(PayloadResponse.__SEQ)
+        PayloadResponse.__SEQ += 1
+
+        payloadEnvelope.filt[self.MESSAGE_ID_KEY] = self._messageId
+        self._filt = copy(payloadEnvelope.filt)
+        self._destVortexName = destVortexName
+
+        self._status = self.PROCESSING
+        self._date = datetime.now(pytz.utc)
+
+        self._endpoint = PayloadEndpoint(self._filt, self._process)
+
+        if destVortexName or destVortexUuid:
+            d: Deferred = payloadEnvelope.toVortexMsgDefer()
+            d.addCallback(
+                VortexFactory.sendVortexMsg,
+                destVortexName=destVortexName,
+                destVortexUuid=destVortexUuid,
+            )
+            d.addErrback(self.errback)
+
+        try:
+            raise Exception()
+        except:
+            self._stack = sys.exc_info()[2]
+
+        # noinspection PyTypeChecker
+        self.addTimeout(timeout, reactor)
+        self.addErrback(self._timedOut)
+
+    @classmethod
+    def isResponsePayloadEnvelope(cls, payloadEnvelope: PayloadEnvelope):
+        """Is Response Payload Envelope
+
+        The PayloadResponse tags the payloads, so it expects a unique message back.
+
+        :returns: True if this payload has been tagged by a PayloadResponse class
+        """
+        return cls.MESSAGE_ID_KEY in payloadEnvelope.filt
+
+    @property
+    def status(self):
+        return self._status
+
+    def _timedOut(self, failure: Failure):
+        if self._endpoint:
+            self._endpoint.shutdown()
+            self._endpoint = None
+
+        if self._logTimeoutError:
+            logger.error("Timed out for payload %s", self._filt)
+        self._status = self.TIMED_OUT
+        return failure
+
+    def _process(self, payloadEnvelope: PayloadEnvelope, vortexName, **kwargs):
+        if self._endpoint:
+            self._endpoint.shutdown()
+            self._endpoint = None
+
+        if self._destVortexName and vortexName != self._destVortexName:
+            logger.debug(
+                "Received response from a vortex other than the dest vortex, "
+                "Expected %s, Received %s",
+                self._destVortexName,
+                vortexName,
+            )
+            return
+
+        if self.called:
+            logger.error("Received response after timeout for %s" % self._filt)
+            return
+
+        if self._resultCheck and not payloadEnvelope.result in (None, True):
+            self._status = self.FAILED
+            self.errback(
+                Failure(
+                    Exception(payloadEnvelope.result).with_traceback(
+                        self._stack
+                    )
+                )
+            )
+
+        else:
+            self._status = self.SUCCESS
+            self.callback(payloadEnvelope)
+
+        # Return self, so that PayloadIO knows this is a asyncronous method
+        return self

vortex/PayloadResponseTest.py

@@ -0,0 +1,39 @@
+from twisted.internet import defer
+from twisted.trial import unittest
+
+from vortex.Payload import Payload
+from vortex.PayloadIO import PayloadIO
+from vortex.PayloadResponse import PayloadResponse
+
+
+class PayloadResponseTest(unittest.TestCase):
+    def testSuccesCb(self):
+        payload = Payload(filt={"key": "PayloadResponseTest"})
+
+        payloadReponse = PayloadResponse(payload)
+        payloadReponse.addCallback(lambda pl: self.assertIs(pl, payload))
+        payloadReponse.addErrback(lambda pl: self.assertTrue(False))
+
+        PayloadIO().process(payload, "", "", None, lambda _: None)
+
+        return payloadReponse
+
+    def testTimeoutErrback(self):
+        payload = Payload(filt={"key": "PayloadResponseTest"})
+
+        payloadReponse = PayloadResponse(payload, timeout=3)
+        payloadReponse.addCallback(lambda pl: self.assertFalse(True))
+        self.assertFailure(payloadReponse, defer.TimeoutError)
+
+        # PayloadIO().process(payload)
+
+        return payloadReponse
+
+    def testSerialiseDeserialise(self):
+        payload = Payload(filt={"key": "PayloadResponseTest"})
+
+        d = PayloadResponse(payload, timeout=0)
+        d.addErrback(lambda _: True)  # Handle the errback
+
+        vortexMsg = payload.toVortexMsg()
+        Payload().fromVortexMsg(vortexMsg)

vortex/PayloadTest.py

@@ -0,0 +1,136 @@
+"""
+ * Created by Synerty Pty Ltd
+ *
+ * This software is open source, the MIT license applies.
+ *
+ * Website : http://www.synerty.com
+ * Support : support@synerty.com
+"""
+import gc
+
+from twisted.trial import unittest
+
+from vortex.Payload import Payload
+from vortex.PayloadFilterKeys import rapuiClientEcho
+from vortex.Tuple import TupleHash
+from vortex.VortexTcpMemoryLeakPayloadTest import MemoryCheckerTestMixin
+
+
+def makeTestPayloadA(tupleDataSize=10 * 1024):
+    payload = Payload()
+    payload.filt[rapuiClientEcho] = None
+    payload.tuples = ["12345678" * int(tupleDataSize / 8)]
+    return payload
+
+
+class PayloadPyTest(unittest.TestCase, MemoryCheckerTestMixin):
+    PERF_TEST_COUNT = 10000
+
+    def setUp(self):
+        pass
+
+    def tearDown(self):
+        pass
+
+    # @profile
+    def _makeStr(self, size):
+        _ = ["12345678" * int(size / 8)]
+        del _
+        gc.collect()
+
+    # @profile
+    def _payloadEncodeDecode(self, size):
+        origPayload = makeTestPayloadA(size)
+        origPayload.date = None
+
+        encodedPayload = origPayload.toEncodedPayload()
+        payload = Payload().fromEncodedPayload(encodedPayload)
+
+        self.assertEqual(payload.tuples[0], origPayload.tuples[0])
+
+    def testMakeStr_100m_x100(self):
+        self._memCheckFunction(self._makeStr, 1000, 100 * 1024**2)
+
+    def testPayloadToFromEncodedPayload_10k_x1000(self):
+        self._memCheckFunction(self._payloadEncodeDecode, 1000, 10 * 1024)
+
+    def testPayloadToFromEncodedPayload_100m_x100(self):
+        self._memCheckFunction(self._payloadEncodeDecode, 100, 10 * 1024**2)
+
+    def testPayloadToFromEncodedPayload_1m_x1000(self):
+        self._memCheckFunction(self._payloadEncodeDecode, 1000, 1 * 1024**2)
+
+    def testPayloadToFromEncodedPayload10mb(self):
+        # Create Payload
+        origPayload = makeTestPayloadA()
+        origPayload.tuples = ["1234567890" * 1024**2]
+
+        encodedPayload = origPayload.toEncodedPayload()
+        payload = Payload().fromEncodedPayload(encodedPayload)
+
+        self.assertEqual(payload.tuples[0], origPayload.tuples[0])
+
+    def testPayloadToFromEncodedPayload(self):
+        # Create Payload
+        origPayload = makeTestPayloadA()
+
+        # To VortexServer Message
+        encodedPayload = origPayload.toEncodedPayload()
+        # print encodedPayload
+
+        payload = Payload().fromEncodedPayload(encodedPayload)
+
+        self.assertEqual(TupleHash(payload), TupleHash(payload))
+        self.assertEqual(TupleHash(origPayload), TupleHash(origPayload))
+
+        self.assertEqual(
+            TupleHash(origPayload),
+            TupleHash(payload),
+            "Payload serialise -> deserialize error",
+        )
+
+    def testPayloadToFromJson(self):
+        # Create Payload
+        origPayload = makeTestPayloadA()
+
+        # To JSON
+        jsonStr = origPayload._toJson()
+        # print encodedPayload
+        # print(jsonStr)
+
+        payload = Payload()._fromJson(jsonStr)
+
+        self.assertEqual(TupleHash(payload), TupleHash(payload))
+        self.assertEqual(TupleHash(origPayload), TupleHash(origPayload))
+
+        self.assertEqual(
+            TupleHash(origPayload),
+            TupleHash(payload),
+            "Payload serialise -> deserialise error",
+        )
+
+    def testPayloadJsonPerformanceSingle(self):
+        origPayload = makeTestPayloadA()
+        for _ in range(self.PERF_TEST_COUNT):
+            origPayload.tuples.append(makeTestPayloadA())
+        jsonStr = origPayload._toJson()
+        payload = Payload()._fromJson(jsonStr)
+
+    def testPayloadEncodedPayloadPerformanceSingle(self):
+        origPayload = makeTestPayloadA()
+        for x in range(self.PERF_TEST_COUNT):
+            origPayload.tuples.append(makeTestPayloadA())
+        encodedPayload = origPayload.toEncodedPayload()
+        payload = Payload().fromEncodedPayload(encodedPayload)
+
+    def testPayloadJsonPerformanceMultiple(self):
+        origPayload = makeTestPayloadA()
+        for _ in range(self.PERF_TEST_COUNT):
+            jsonStr = origPayload._toJson()
+            payload = Payload()._fromJson(jsonStr)
+
+    def testPayloadEncodedPayloadPerformanceMultiple(self):
+        origPayload = makeTestPayloadA()
+        for _ in range(self.PERF_TEST_COUNT):
+            encodedPayload = origPayload.toEncodedPayload()
+            payload = Payload().fromEncodedPayload(encodedPayload)

vortex/PerformTestActionTuple.py

@@ -0,0 +1,30 @@
+"""
+ *
+ *  Copyright Synerty Pty Ltd 2013
+ *
+ *  This software is proprietary, you are not free to copy
+ *  or redistribute this code in any format.
+ *
+ *  All rights to this software are reserved by
+ *  Synerty Pty Ltd
+ *
+ * Website : http://www.synerty.com
+ * Support : support@synerty.com
+ *
+"""
+
+from vortex.Tuple import TupleField, Tuple, addTupleType
+
+
+@addTupleType
+class PerformTestActionTuple(Tuple):
+    """Perform Test Action Tuple
+
+    This tuple is used for testing the action code.
+
+    """
+
+    __tupleType__ = "synerty.vortex.PerformTestActionTuple"
+    actionDataInt = TupleField(typingType=int)
+    actionDataUnicode = TupleField(typingType=str)
+    failProcessing = TupleField(typingType=bool)

vortex/SerialiseUtil.py

@@ -0,0 +1,273 @@
+"""
+ * Created by Synerty Pty Ltd
+ *
+ * This software is open source, the MIT license applies.
+ *
+ * Website : http://www.synerty.com
+ * Support : support@synerty.com
+"""
+
+import decimal
+import logging
+from base64 import b64encode, b64decode
+from collections import defaultdict
+from datetime import datetime, date
+
+import pytz
+from sqlalchemy.orm.collections import InstrumentedList
+
+logger = logging.getLogger(__name__)
+
+try:
+    from geoalchemy2.elements import WKBElement
+
+except ImportError:
+
+    class WKBElement:
+        def __init__(self):
+            raise NotImplementedError("")
+
+
+try:
+    from geoalchemy2.elements import WKBElement
+
+except ImportError:
+
+    class WKBElement:
+        def __init__(self):
+            raise NotImplementedError("")
+
+
+###############################################################################
+# Utility functions
+###############################################################################
+
+T_RAPUI_TUPLE = "rt"
+T_RAPUI_PAYLOAD = "rp"
+T_RAPUI_PAYLOAD_ENVELOPE = "rpe"
+T_GENERIC_CLASS = "gen"  # NOT SUPPORTED
+T_FLOAT = "float"
+T_INT = "int"
+T_STR = "str"
+T_BYTES = "bytes"
+T_BOOL = "bool"
+T_DATETIME = "datetime"
+T_DICT = "dict"
+T_LIST = "list"
+T_WKB = "wkb"  # Well Known Binary, a part of the GEO library
+
+V_NULL = "null"
+V_TRUE = "1"
+V_FALSE = "0"
+
+ISO8601_ORA = "YYYY-MM-DD HH24:MI:SS.FF6"
+
+# This will print a date using TO_CHAR(), in the format fromStr() below expects
+ISO8601_ORA_WITH_TZ = "YYYY-MM-DD HH24:MI:SS.FF6TZHTZM"
+ISO8601 = "%Y-%m-%d %H:%M:%S.%f%z"
+ISO8601_JS_REGEXP = r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}[Z]{1}"
+ISO8601_JS_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"
+ISO8601_REGEXP = r"^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{6}[+-]\d{4}$"
+ISO8601_NOTZ = "%Y-%m-%d %H:%M:%S.%f"
+
+TYPE_MAP_PY_TO_RAPUI = {
+    decimal.Decimal: T_FLOAT,
+    float: T_FLOAT,
+    str: T_STR,
+    bytes: T_BYTES,
+    int: T_INT,
+    bool: T_BOOL,
+    datetime: T_DATETIME,
+    date: T_DATETIME,
+    dict: T_DICT,
+    defaultdict: T_DICT,
+    list: T_LIST,
+    set: T_LIST,
+    tuple: T_LIST,
+    WKBElement: T_WKB,
+    InstrumentedList: T_LIST,
+}
+
+
+def className(o):
+    cls = o if o.__class__ == type else o.__class__
+    return str(cls).split("'")[1]
+
+
+def convertFromShape(shapelyShape):
+    from shapely.geometry.polygon import Polygon
+
+    if isinstance(shapelyShape, Polygon):
+        coords = shapelyShape.exterior.coords
+    else:
+        coords = shapelyShape.coords
+    return [{"x": i[0], "y": i[1]} for i in coords]
+
+
+def convertFromWkbElement(value):
+    from geoalchemy2.shape import to_shape
+
+    return convertFromShape(to_shape(value))
+
+
+def serialiseWkb(wkb: WKBElement) -> str:
+    from geoalchemy2.shape import to_shape
+    import shapely
+
+    return b64encode(shapely.wkb.dumps(to_shape(wkb))).decode()
+
+
+def deserialiseWkb(wkbB64Data: str) -> WKBElement:
+    from geoalchemy2.shape import from_shape
+    import shapely
+
+    return from_shape(shapely.wkb.loads(b64decode(wkbB64Data)))
+
+
+TYPE_MAP_RAPUI_TO_PY = {
+    T_FLOAT: float,
+    T_STR: str,
+    T_BYTES: bytes,
+    T_INT: int,
+    T_BOOL: bool,
+    T_DATETIME: datetime,
+    T_DICT: dict,
+    T_LIST: list,
+    T_WKB: deserialiseWkb,
+}
+
+
+def decimalToStr(dec):
+    """Return string representation of the number in scientific notation.
+
+    Captures all of the information in the underlying representation.
+    """
+
+    if str(dec) == "0E-10":
+        return "0"
+
+    sign = ["", "-"][dec._sign]
+    if dec._is_special:
+        if dec._exp == "F":
+            return sign + "Infinity"
+        elif dec._exp == "n":
+            return sign + "NaN" + dec._int
+        else:  # dec._exp == 'N'
+            return sign + "sNaN" + dec._int
+
+    # number of digits of dec._int to left of decimal point
+    leftdigits = dec._exp + len(dec._int)
+
+    # no exponent required
+    dotplace = leftdigits
+
+    if dotplace <= 0:
+        intpart = "0"
+        fracpart = "." + "0" * (-dotplace) + dec._int
+    elif dotplace >= len(dec._int):
+        intpart = dec._int + "0" * (dotplace - len(dec._int))
+        fracpart = ""
+    else:
+        intpart = dec._int[:dotplace]
+        fracpart = "." + dec._int[dotplace:]
+
+    if leftdigits == dotplace:
+        exp = ""
+    else:
+        exp = "E%+d" % (leftdigits - dotplace)
+
+    s = sign + intpart + fracpart + exp
+    if "." in s:
+        s = s.rstrip("0").rstrip(".")
+    assert len(s)
+    return s
+
+
+def toStr(obj) -> str:
+    if isinstance(obj, WKBElement):
+        # noinspection PyTypeChecker
+        return serialiseWkb(obj)
+
+    if isinstance(obj, decimal.Decimal):
+        return decimalToStr(obj)
+
+    if isinstance(obj, datetime):
+        return obj.strftime(ISO8601)
+
+    if isinstance(obj, bool):
+        return V_TRUE if obj else V_FALSE
+
+    if isinstance(obj, bytes):
+        return b64encode(obj).decode()  # to make it a str
+
+    if isinstance(obj, str):
+        return obj
+
+    try:
+        return str(obj)
+    except:
+        logger.critical("Failed to convert the following to string")
+        logger.critical(obj)
+        raise
+
+
+def fromStr(val: str, typeName: str):
+    if typeName == T_DATETIME:
+        # TODO: Check with ISO8601_JS_REGEXP
+        if len(val) == len("2017-06-06T09:40:37.097Z"):
+            return datetime.strptime(val, ISO8601_JS_FORMAT).replace(
+                tzinfo=pytz.UTC
+            )
+
+        if len(val) <= len("2017-06-06 09:40:37.097068"):
+            logger.warning(
+                "Parsing date time with no timezone: %s Setting TZ to UTC",
+                val,
+            )
+            return datetime.strptime(val, ISO8601_NOTZ).replace(tzinfo=pytz.UTC)
+
+        return datetime.strptime(val, ISO8601)
+
+    if typeName == T_BOOL:
+        return val == V_TRUE
+
+    if typeName == T_BYTES:
+        return b64decode(val)
+
+    if typeName in (T_STR):
+        return val
+
+    pyVal = fromRapuiType(typeName)(val)
+
+    if typeName == T_FLOAT and pyVal.is_integer():
+        pyVal = int(pyVal)
+
+    return pyVal
+
+
+def fromRapuiType(typeName: str):
+    return TYPE_MAP_RAPUI_TO_PY[typeName]
+
+
+def toRapuiType(value):
+    if value is None:
+        return V_NULL
+
+    from .Tuple import Tuple
+
+    if isinstance(value, Tuple):
+        return T_RAPUI_TUPLE
+
+    from .Payload import Payload
+
+    if isinstance(value, Payload):
+        return T_RAPUI_PAYLOAD
+
+    from .PayloadEnvelope import PayloadEnvelope
+
+    if isinstance(value, PayloadEnvelope):
+        return T_RAPUI_PAYLOAD_ENVELOPE
+
+    valueClass = value.__class__
+
+    return TYPE_MAP_PY_TO_RAPUI[valueClass]