vortexpy 4.0.23__py3-none-any.whl

vortex/DataWrapTuple.py

@@ -0,0 +1,27 @@
+"""
+ * Created by Synerty Pty Ltd
+ *
+ * This software is open source, the MIT license applies.
+ *
+ * Website : http://www.synerty.com
+ * Support : support@synerty.com
+"""
+from .Tuple import Tuple, TupleField, addTupleType
+
+
+@addTupleType
+class DataWrapTuple(Tuple):
+    """Data Wrap Tuple
+
+    This tuple wraps a peice of data for use in RapUI.
+
+    Attributes:
+      name         What ever you want to use it for
+      data         The data
+
+    """
+
+    __tupleType__ = "c.s.r.datawraptuple"
+
+    name = TupleField(None)
+    data = TupleField(None)

vortex/DeferUtil.py

@@ -0,0 +1,224 @@
+import inspect
+import logging
+
+import twisted
+from twisted.internet import reactor
+from twisted.internet.defer import (
+    maybeDeferred,
+    Deferred,
+    inlineCallbacks,
+    ensureDeferred,
+)
+from twisted.internet.threads import deferToThread
+
+logger = logging.getLogger(__name__)
+
+
+def callMethodLater(func):
+    def wrapper(*args, **kwargs):
+        return reactor.callLater(0, func, *args, **kwargs)
+
+    return wrapper
+
+
+def nonConcurrentMethod(method):
+    def call(*args, **kwargs):
+        if call.running:
+            return
+        call.running = True
+        try:
+            return method(*args, **kwargs)
+        finally:
+            call.running = False
+
+    call.running = False
+
+    return call
+
+
+def maybeDeferredWrap(funcToWrap):
+    """Maybe Deferred Wrap
+
+    A decorator that ensures a function will return a deferred.
+
+    """
+
+    def func(*args, **kwargs):
+        return maybeDeferred(funcToWrap, *args, **kwargs)
+
+    return func
+
+
+def ensureDeferredWrap(funcToWrap):
+    """Ensured Deferred Wrap
+
+    A decorator that converts asyncio functions to ones that
+     return Twisted Deferred instead.
+
+    This keeps all the async processing in Twisteds reactive design while providing
+     support of the await/async syntax (and asyncio processing)
+
+
+    """
+
+    def func(*args, **kwargs):
+        return ensureDeferred(funcToWrap(*args, **kwargs))
+
+    return func
+
+
+def vortexLogFailure(failure, loggerArg, consumeError=False, successValue=True):
+    """
+    Log a Twisted Failure instance using the provided logger.
+
+    Args:
+        failure: Twisted Failure instance to log
+        loggerArg: Logger instance to use for logging
+        consumeError: If True, returns successValue instead of the failure
+        successValue: Value to return if consumeError is True
+
+    Returns:
+        The original failure or successValue depending on consumeError
+    """
+    try:
+        if not hasattr(failure, "_vortexFailureLogged"):
+            exc_info = (
+                failure.type,
+                failure.value,
+                failure.getTracebackObject(),
+            )
+
+            # Log with full exception info for better tracebacks
+            loggerArg.exception(str(failure.value), exc_info=exc_info)
+
+            # Mark as logged to prevent duplicate logging
+            failure._vortexFailureLogged = True
+
+        return successValue if consumeError else failure
+
+    except Exception as e:
+        # If something goes wrong in our logging logic, log that error
+        loggerArg.exception("Error while logging failure: %s", str(e))
+        return failure
+
+
+def vortexLogAndConsumeFailure(failure, loggerArg, successValue=True):
+    return vortexLogFailure(
+        failure, loggerArg, consumeError=True, successValue=successValue
+    )
+
+
+printFailure = vortexLogFailure
+
+
+def vortexInlineCallbacksLogAndConsumeFailure(loggerArg):
+    """Vortex InlineCallbacks Log and Consume Failure
+
+    This is exactly the same as @inlineCallbacks decorator, except it will log
+    and consume any deferred failures that are thrown.
+    """
+
+    def wrapper(funcToWrap):
+        funcIcb = inlineCallbacks(funcToWrap)
+
+        def called(*args, **kwargs):
+            d = funcIcb(*args, **kwargs)
+            d.addErrback(vortexLogAndConsumeFailure, loggerArg)
+            return d
+
+        return called
+
+    return wrapper
+
+
+class YieldInDeferToThreadException(Exception):
+    pass
+
+
+def deferToThreadWrapWithLogger(
+    logger, consumeError=False, checkMainThread=True
+):
+    """Defer To Thread Wrap With Logger
+
+    This method is a decorator used to send blocking methods to threads easily.
+
+    :param logger: An instance from logging.getLogger(..)
+    :param consumeError: Should errors occuring in the thread be consumed or errback'd
+    :param checkMainThread: Don't use this parameter, it's only needed during reactor
+                startup when the reactors thread isn't properly set and the check
+                fails.
+
+    :return: A deferred.
+
+    Usage: ::
+
+            import logging
+            logger = logging.getLogger(__name__)
+            @deferToThreadWrapWithLogger(logger)
+            def myFunction(arg1, kw=True):
+                pass
+
+
+    """
+    assert isinstance(
+        logger, logging.Logger
+    ), """Usage:
+    import logging
+    logger = logging.getLogger(__name__)
+    @deferToThreadWrapWithLogger(logger)
+    def myFunction(arg1, kw=True):
+        pass
+    """
+
+    def wrapper(funcToWrap) -> Deferred:
+        def checkForInlineCallbackYield(result):
+            if inspect.isgenerator(result):
+                logger.error(
+                    "Module Name %s, Class Name %s, Function Name %s"
+                    " has a yield in it. IT SHOULD NOT.",
+                    funcToWrap.__module__,
+                    funcToWrap.__class__.__name__,
+                    funcToWrap.__name__,
+                )
+                raise YieldInDeferToThreadException()
+            return result
+
+        def func(*args, **kwargs):
+            if not twisted.python.threadable.isInIOThread() and checkMainThread:
+                logger.error(
+                    "Module Name %s, Class Name %s, Function Name %s"
+                    " is being deferred from a thread. IT SHOULD NOT.",
+                    funcToWrap.__module__,
+                    funcToWrap.__class__.__name__,
+                    funcToWrap.__name__,
+                )
+                raise Exception(
+                    "Deferring to a thread can only be done from the main thread"
+                )
+
+            d = deferToThread(funcToWrap, *args, **kwargs)
+            d.addCallback(checkForInlineCallbackYield)
+            d.addErrback(vortexLogFailure, logger, consumeError=consumeError)
+            return d
+
+        return func
+
+    return wrapper
+
+
+def isMainThread():
+    return twisted.python.threadable.isInIOThread()
+
+
+def noMainThread():
+    if isMainThread():
+        raise Exception(
+            "Blocking operations shouldn't occur in the reactors thread."
+        )
+
+
+def yesMainThread():
+    if not isMainThread():
+        raise Exception(
+            "Async operations must occur in the reactors main thread."
+        )

vortex/Jsonable.py

@@ -0,0 +1,304 @@
+"""
+ * Created by Synerty Pty Ltd
+ *
+ * This software is open source, the MIT license applies.
+ *
+ * Website : http://www.synerty.com
+ * Support : support@synerty.com
+"""
+
+import logging
+import traceback
+import typing
+from copy import copy, deepcopy
+from json import JSONEncoder
+from typing import List
+from _collections import defaultdict
+
+logger = logging.getLogger(__name__)
+
+from .SerialiseUtil import *
+
+###############################################################################
+# JSON Constants
+###############################################################################
+
+# Globally mess with the json dump separators for more compact json
+JSONEncoder.item_separator = ","
+JSONEncoder.key_separator = ":"
+
+
+class Jsonable(object):
+    """Xmlable
+    Inherit from this class if you want to serialise to XML.
+    """
+
+    __fieldNames__: List[str] = None
+    __rapuiSerialiseType__ = T_GENERIC_CLASS
+
+    __rawJonableFields__: List[str] = None
+
+    JSON_CLASS_TYPE = "_ct"
+    JSON_CLASS = "_c"
+    JSON_TUPLE_TYPE = "_c"
+    JSON_FIELD_TYPE = "_ft"
+    JSON_FIELD_DATA = "_fd"
+
+    __memoryLoggingRefs = None
+    __memoryLoggingEnabled = False
+
+    @classmethod
+    def setupMemoryLogging(cls) -> None:
+        cls.__memoryLoggingRefs = defaultdict(lambda: 0)
+        cls.__memoryLoggingEnabled = True
+
+    @classmethod
+    def memoryLoggingDump(
+        cls, top=10, over=100
+    ) -> List[typing.Tuple[str, int]]:
+        if not Jsonable.__memoryLoggingRefs:
+            return []
+
+        data = sorted(
+            Jsonable.__memoryLoggingRefs.items(),
+            key=lambda x: x[1],
+            reverse=True,
+        )
+
+        return list(filter(lambda x: x[1] >= over, data))[:top]
+
+    def __memLoggingKey(self):
+        if getattr(self, "__rapuiSerialiseType__") == T_RAPUI_TUPLE:
+            key = "Tuple: " + getattr(self, "__tupleType__")
+
+        elif getattr(self, "__rapuiSerialiseType__") == T_RAPUI_PAYLOAD:
+            assert hasattr(self, "filt"), "Payload is missing filt"
+            if "key" in self.filt:
+                key = "Payload: key=" + self.filt["key"]
+            else:
+                key = "Payload: " + str(self.filt)
+
+        else:
+            key = "Jsonable: " + getattr(self, "__rapuiSerialiseType__")
+
+        return key
+
+    def __init__(self):
+        if Jsonable.__memoryLoggingEnabled:
+            Jsonable.__memoryLoggingRefs[self.__memLoggingKey()] += 1
+
+    def __del__(self):
+        if Jsonable.__memoryLoggingEnabled:
+            Jsonable.__memoryLoggingRefs[self.__memLoggingKey()] -= 1
+
+    def __isRawJsonableField(self, name: str) -> bool:
+        if not (name and self.__rawJonableFields__):
+            return False
+        return name in self.__rawJonableFields__
+
+    def toJsonField(self, value, jsonDict=None, name=None):
+        """To Json Field"""
+        from .Tuple import TupleField
+
+        if isinstance(value, TupleField):
+            value = (
+                None if value.defaultValue == None else copy(value.defaultValue)
+            )
+
+        try:
+            valueType = V_NULL if value is None else toRapuiType(value)
+        except KeyError as e:
+            raise KeyError(
+                "%s field name is %s, type is %s", e, name, value.__class__
+            )
+
+        # Payloads and Tuples
+        if self.__isRawJsonableField(name):
+            # This sho
+            convertedValue = deepcopy(value)
+
+        elif isinstance(value, Jsonable):
+            convertedValue = value.toJsonDict()
+
+        elif isinstance(value, dict):
+            convertedValue = {}
+            for key, value in list(value.items()):
+                self.toJsonField(value, convertedValue, key)
+
+        elif isinstance(value, (set, list, tuple)):
+            convertedValue = []
+            for val in value:
+                convertedValue.append(self.toJsonField(val))
+
+        # Decimals are detected as floats for some reason, so convert it here
+        elif valueType == T_FLOAT:
+            convertedValue = float(value)
+
+        elif valueType in (T_INT, T_BOOL, T_STR):
+            convertedValue = value
+
+        elif valueType == V_NULL:
+            convertedValue = None
+
+        else:
+            convertedValue = toStr(value)
+
+        # Non standard values need a dict to store their value type attributes
+        # Create a sub dict that contains the value and type
+        if valueType not in (
+            T_FLOAT,
+            T_STR,
+            V_NULL,
+            T_BOOL,
+            T_LIST,
+            T_DICT,
+        ) and not isinstance(value, Jsonable):
+            convertedValue = {
+                Jsonable.JSON_FIELD_TYPE: valueType,
+                Jsonable.JSON_FIELD_DATA: convertedValue,
+            }
+
+        if name and jsonDict is not None:
+            # Now assign the value and it's data type if applicable
+            if not toRapuiType(name) in (T_STR, T_INT, T_FLOAT):
+                raise Exception(
+                    "name=%s, type=%s, is not an allowed dict key",
+                    name,
+                    toRapuiType(name),
+                )
+
+            jsonDict[name] = convertedValue
+
+        return convertedValue
+
+    # -----------------------------------------------------------------------------
+    def fromJsonField(self, value, valueType=None):
+        # Single Value
+        if (
+            valueType == V_NULL or value is None
+        ):  # V_NULL will never be set in toJsonField
+            return None
+
+        if valueType == T_INT:
+            return int(value)
+
+        if isinstance(value, dict) and Jsonable.JSON_CLASS_TYPE in value:
+            valueType = value[Jsonable.JSON_CLASS_TYPE]
+
+        # JSON handles these types natively,
+        # if there is no type then these are the right types
+        if valueType is None:
+            valueType = toRapuiType(value)
+            if valueType in (T_BOOL, T_FLOAT, T_INT, T_STR):
+                return value
+
+        # Non standard values need a dict to store their value type attributes, decode these
+        if isinstance(value, dict) and Jsonable.JSON_FIELD_TYPE in value:
+            return self.fromJsonField(
+                value[Jsonable.JSON_FIELD_DATA], value[Jsonable.JSON_FIELD_TYPE]
+            )
+
+        # Tuple
+        if valueType == T_RAPUI_TUPLE:
+            if not Jsonable.JSON_TUPLE_TYPE in value:
+                raise Exception(
+                    "Tuple is missing type field '_c',"
+                    " if this is a javascript tuple,"
+                    " ensure it's constructor calls super()"
+                )
+
+            tupleType = value[Jsonable.JSON_TUPLE_TYPE]
+
+            from .Tuple import TUPLE_TYPES_BY_NAME
+
+            if not tupleType in TUPLE_TYPES_BY_NAME:
+                raise Exception(
+                    "Tuple type |%s| not registered within this program.",
+                    tupleType,
+                )
+
+            try:
+                from .Tuple import TUPLE_TYPES_BY_NAME
+
+                return TUPLE_TYPES_BY_NAME[tupleType]().fromJsonDict(value)
+
+            except Exception as e:
+                logger.critical(traceback.format_exc())
+                raise Exception("%s for tuple type %s" % (str(e), tupleType))
+
+        # Handle the case of payloads within payloads
+        if valueType == T_RAPUI_PAYLOAD:
+            from .Payload import Payload
+
+            return Payload().fromJsonDict(value)
+
+        if valueType == T_RAPUI_PAYLOAD_ENVELOPE:
+            from .PayloadEnvelope import PayloadEnvelope
+
+            return PayloadEnvelope().fromJsonDict(value)
+
+        if valueType == T_GENERIC_CLASS:
+            # OTHER JSONABLES GO HERE, INSPECT THE tuple_TYPE
+            # jsonClass = value[JSON_CLASS]
+            raise NotImplementedError()
+            # import importlib
+            # MyClass = getattr(importlib.import_module("module.submodule"), "Klass")
+            # instance = MyClass()
+
+        if valueType == T_DICT:
+            # Dict
+            restoredDict = {}
+            for subName, subVal in list(value.items()):
+                try:
+                    subNamStr = subName
+                    subName = float(subNamStr)
+                    subName = int(subNamStr)
+                except ValueError:
+                    pass
+                restoredDict[subName] = self.fromJsonField(subVal)
+
+            return restoredDict
+
+        if valueType == T_LIST:
+            restoredList = []
+            for subVal in value:
+                restoredList.append(self.fromJsonField(subVal))
+
+            return restoredList
+
+        return fromStr(value, valueType)
+
+    def toJsonDict(self):
+
+        fieldNames = self.__fieldNames__
+
+        jsonDict = {Jsonable.JSON_CLASS_TYPE: self.__rapuiSerialiseType__}
+
+        if hasattr(self, "tupleName"):
+            jsonDict[Jsonable.JSON_TUPLE_TYPE] = self.tupleName()
+
+        else:
+            jsonDict[Jsonable.JSON_CLASS] = className(self)
+
+        for name in fieldNames:
+            data = getattr(self, name)
+            self.toJsonField(data, jsonDict, name)
+
+        return jsonDict
+
+    def fromJsonDict(self, jsonDict):
+        """From Xml
+        Returns and instance of this object populated with data from the json dict
+        """
+
+        # Use the fromJsonField code to convert all the values
+        for name, value in list(jsonDict.items()):
+            if name.startswith("_"):
+                continue
+
+            if self.__isRawJsonableField(name):
+                setattr(self, name, value)
+            else:
+                setattr(self, name, self.fromJsonField(value))
+
+        return self

vortex/Payload.py

@@ -0,0 +1,115 @@
+"""
+ * Created by Synerty Pty Ltd
+ *
+ * This software is open source, the MIT license applies.
+ *
+ * Website : http://www.synerty.com
+ * Support : support@synerty.com
+"""
+import logging
+import zlib
+from base64 import b64encode, b64decode
+from datetime import datetime
+from typing import List, Optional, Dict
+
+import pytz
+import json
+
+from vortex.DeferUtil import deferToThreadWrapWithLogger, noMainThread
+from .Jsonable import Jsonable
+from .SerialiseUtil import T_RAPUI_PAYLOAD
+
+logger = logging.getLogger(__name__)
+
+PayloadList = List["Payload"]
+EncodedPayloadList = List[bytes]
+
+
+class Payload(Jsonable):
+    """Payload
+    This object represents a hierarchy of data transferred between client and server
+    """
+
+    __fieldNames__ = ["filt", "tuples", "date"]
+    __rapuiSerialiseType__ = T_RAPUI_PAYLOAD
+
+    def __init__(self, filt: Optional[Dict] = None, tuples=None) -> None:
+        """Constructor"""
+        super().__init__()
+        self.filt = {} if filt is None else filt
+        self.tuples = [] if tuples is None else tuples
+        self.date = datetime.now(pytz.utc)
+
+        if isinstance(filt, str):
+            self.filt = {self.filt: None}
+
+        if not isinstance(self.tuples, list):
+            self.tuples = [self.tuples]
+
+    # -------------------------------------------
+    # PayloadEnvelopes helper messages
+
+    def makePayloadEnvelope(self, result=None, compressionLevel: int = 9):
+        from .PayloadEnvelope import PayloadEnvelope
+
+        noMainThread()
+        encodedSelf = self.toEncodedPayload(compressionLevel=compressionLevel)
+        return PayloadEnvelope(
+            self.filt, encodedPayload=encodedSelf, date=self.date, result=result
+        )
+
+    @deferToThreadWrapWithLogger(logger)
+    def makePayloadEnvelopeDefer(self, result=None, compressionLevel: int = 9):
+        return self.makePayloadEnvelope(
+            result=result, compressionLevel=compressionLevel
+        )
+
+    # -------------------------------------------
+    # VortexMsg helper messages
+
+    def makePayloadEnvelopeVortexMsg(
+        self, result=None, compressionLevel: int = 9
+    ):
+        noMainThread()
+        return self.makePayloadEnvelope(
+            result=result, compressionLevel=compressionLevel
+        ).toVortexMsg()
+
+    @deferToThreadWrapWithLogger(logger)
+    def makePayloadEnvelopeVortexMsgDefer(
+        self, result=None, compressionLevel: int = 9
+    ):
+        return self.makePayloadEnvelopeVortexMsg(
+            result=result, compressionLevel=compressionLevel
+        )
+
+    # -------------------------------------------
+    # JSON Related methods
+    def _fromJson(self, jsonStr):
+        jsonDict = json.loads(jsonStr)
+
+        assert jsonDict[Jsonable.JSON_CLASS_TYPE] == self.__rapuiSerialiseType__
+        return self.fromJsonDict(jsonDict)
+
+    def _toJson(self) -> str:
+        return json.dumps(self.toJsonDict())
+
+    # -------------------------------------------
+    # VortexServer Message Methods
+    def toEncodedPayload(self, compressionLevel: int = 9) -> str:
+        jsonStr = self._toJson()
+        return b64encode(
+            zlib.compress(jsonStr.encode("UTF-8"), compressionLevel)
+        ).decode()
+
+    @deferToThreadWrapWithLogger(logger)
+    def toEncodedPayloadDefer(self, compressionLevel: int = 9) -> str:
+        return self.toEncodedPayload(compressionLevel=compressionLevel)
+
+    def fromEncodedPayload(self, encodedPayload: str):
+        jsonStr = zlib.decompress(b64decode(encodedPayload))
+        return self._fromJson(jsonStr)
+
+    @deferToThreadWrapWithLogger(logger)
+    def fromEncodedPayloadDefer(self, encodedPayload: str):
+        return self.fromEncodedPayload(encodedPayload)