vortexpy 4.0.23__py3-none-any.whl

vortex/VortexTest.py

@@ -0,0 +1,22 @@
+"""
+ * Created by Synerty Pty Ltd
+ *
+ * This software is open source, the MIT license applies.
+ *
+ * Website : http://www.synerty.com
+ * Support : support@synerty.com
+"""
+
+from twisted.trial import unittest
+
+from run_test_vortex_server import setupVortexServer
+from vortex.VortexClientHttp import VortexClientHttp
+
+
+class VortexTest(unittest.TestCase):
+    def setUp(self):
+        self.port = setupVortexServer()
+
+    def testVortexConnect(self):
+        client = VortexClientHttp("unittest")
+        return client.connect("127.0.0.1", self.port)

vortex/VortexUtil.py

@@ -0,0 +1,314 @@
+import json
+from collections import namedtuple
+from datetime import datetime
+from datetime import timedelta
+from typing import Any
+from typing import Callable
+from typing import Dict
+from typing import Optional
+from typing import Set
+from typing import Union
+from uuid import uuid4
+
+import pytz
+from twisted.internet import reactor
+from twisted.internet.defer import Deferred
+from twisted.internet.defer import fail
+from twisted.internet.defer import maybeDeferred
+from twisted.python.failure import Failure
+
+from vortex.PayloadEnvelope import PayloadEnvelope
+from vortex.PayloadEnvelope import VortexMsgList
+from vortex.Tuple import Tuple
+from vortex.Tuple import TupleField
+from vortex.Tuple import addTupleType
+
+_LimitConcurrencyArgsTuple = namedtuple(
+    "_LimitConcurrencyArgsTuple",
+    ("deferred", "funcSelf", "args", "kwargs", "queuedTime"),
+)
+
+
+def _dedupProcessorCallKeys(
+    payloadEnvelope: PayloadEnvelope,
+    vortexUuid: str,
+    *args,
+    **kwargs,
+):
+    filt = payloadEnvelope.filt.copy()
+    from vortex.PayloadResponse import PayloadResponse
+
+    filt.pop(PayloadResponse.MESSAGE_ID_KEY, None)
+    return vortexUuid, str(filt)
+
+
+def limitConcurrency(
+    decoratorLogger,
+    concurrency: int,
+    logTaskTimeGreaterThanSeconds=5.0,
+    deduplicateBasedOnKwArgsKey: Optional[Callable] = None,
+):
+    """Limit Concurrency to a call, decorator
+
+    :param decoratorLogger: The logger to log messages for.
+    :param concurrency: The number of concurrent calls
+    :param logTaskTimeGreaterThanSeconds: If a task takes longer then this
+    :param deduplicateBasedOnKwArgsKey: This is a callable that returns a tuple
+        of strings. If this argument is provided,
+        then duplicate calls with the same values for these arguments will
+        be deduplicated. Duplicate calls will use the newest set of arguments.
+
+    # As of python 3.6, dicts are ordered.
+
+    >>> x={}
+    >>> x[1]=None
+    >>> x
+    {1: None}
+    >>> x[0]=None
+    >>> x
+    {1: None, 0: None}
+    >>> x[123]=None
+    >>> x
+    {1: None, 0: None, 123: None}
+    # Updating a item in the dict retains the same position
+    >>> x[0]=None
+    >>> x
+    {1: None, 0: None, 123: None}
+
+    # Poping the first off can be done with an iterator.
+    >>> next(iter(x.keys()))
+    1
+
+    """
+
+    class Wrap:
+        def __init__(self, f):
+            self._f = f
+            self._callQueue: dict[Any, _LimitConcurrencyArgsTuple] = {}
+            self._runningCount = 0
+            self._processedCount = 0
+            self._lastLogTime = None
+            self._lastQueueProcessingDelta = None
+
+        def _callComplete(self, result, callStartTime):
+            self._runningCount -= 1
+            self._processedCount += 1
+            # decoratorLogger.debug("Queued task completed %s", str(self._f))
+
+            processingTime = datetime.now(pytz.utc) - callStartTime
+            if logTaskTimeGreaterThanSeconds < processingTime.total_seconds():
+                decoratorLogger.debug(
+                    "%s call took %s",
+                    self._f.__name__,
+                    processingTime,
+                )
+
+            reactor.callLater(0, self._processNext)
+            return result
+
+        def _processNext(self):
+            now = datetime.now(pytz.utc)
+            while self._runningCount < concurrency and self._callQueue:
+                # decoratorLogger.debug("Queueing new task %s", str(self._f))
+                self._runningCount += 1
+
+                nextKey = next(iter(self._callQueue.keys()))
+
+                args = self._callQueue.pop(nextKey)
+
+                self._lastQueueProcessingDelta = now - args.queuedTime
+
+                callWithArgs = lambda: self._f(
+                    args.funcSelf, *args.args, **args.kwargs
+                )
+                d = maybeDeferred(callWithArgs)
+                d.addBoth(self._callComplete, now)
+                d.chainDeferred(args.deferred)
+
+            self._logQueueSize()
+            self._f.running = self._runningCount
+            self._f.waiting = len(self._callQueue)
+
+        def call(self, funcSelf, *args, **kwargs):
+            if deduplicateBasedOnKwArgsKey:
+                key = deduplicateBasedOnKwArgsKey(*args, **kwargs)
+            else:
+                key = uuid4()
+
+            if key in self._callQueue:
+                decoratorLogger.debug(
+                    "%s call was deduplicated for key %s",
+                    self._f.__name__,
+                    str(key),
+                )
+
+            d = Deferred()
+            self._callQueue[key] = _LimitConcurrencyArgsTuple(
+                d, funcSelf, args, kwargs, datetime.now(pytz.utc)
+            )
+            self._processNext()
+            return d
+
+        def _logQueueSize(self):
+            if (
+                self._lastLogTime
+                and (datetime.now(pytz.utc) - self._lastLogTime).total_seconds()
+                < 5.0
+            ):
+                return
+
+            deltaTime = self._lastQueueProcessingDelta
+            queueSize = len(self._callQueue)
+
+            def logIt(logFunc):
+                self._lastLogTime = datetime.now(pytz.utc)
+                logFunc(
+                    "%s last call was queued for %s, queue size %s,"
+                    " processed %s since last log message",
+                    self._f.__name__,
+                    deltaTime,
+                    queueSize,
+                    self._processedCount,
+                )
+                self._processedCount = 0
+
+            if 15.0 < deltaTime.total_seconds() or 2000 < queueSize:
+                logIt(decoratorLogger.warning)
+
+            elif 5.0 < deltaTime.total_seconds() or 200 < queueSize:
+                logIt(decoratorLogger.debug)
+
+    def wrap(f):
+        wrapC = Wrap(f)
+
+        def wrapInner(funcSelf, *args, **kwargs):
+            return wrapC.call(funcSelf, *args, **kwargs)
+
+        return wrapInner
+
+    return wrap
+
+
+@addTupleType
+class _DebounceArgsTuple(Tuple):
+    __tupleType__ = __name__ + "._DebounceArgsTuple"
+    args = TupleField()
+    kwargs = TupleField()
+
+
+def debounceCall(debounceSeconds: float):
+    """Call a debounced function that delays invoking wrapped function until after wait n seconds
+
+    :param debounceSeconds: cooling down peroid of n seconds
+    :return: the wrapped function
+    """
+
+    class Wrap:
+        _lastCleanup = datetime.now(pytz.utc)
+        _cleanupDelta = timedelta(minutes=1, seconds=debounceSeconds)
+
+        _debounceTimeDelta = timedelta(seconds=debounceSeconds)
+
+        _calls: Dict[str, datetime] = {}
+        _callsQueued: Set[tuple] = set()
+
+        def __init__(self, f):
+            self._f = f
+
+        def __cleanup(self):
+            now = datetime.now(pytz.utc)
+            # Cleanup the old values
+            if self._cleanupDelta < now - self._lastCleanup:
+                self._lastCleanup = now
+
+                for key, val in list(self._calls.items()):
+                    if (
+                        self._cleanupDelta < now - val
+                        and key not in self._callsQueued
+                    ):
+                        self._calls.pop(key)
+
+        def __wrapArgsAndKwargs(self, *args, **kwargs):
+            if not args and not kwargs:
+                return ""
+            hashedArgs = _DebounceArgsTuple(
+                args=args, kwargs=kwargs
+            ).toJsonDict()
+            return json.dumps(hashedArgs)
+
+        def call(self, funcSelf, *args, **kwargs):
+            now = datetime.now(pytz.utc)
+
+            # Create a hash of the args
+            hashedArgs = self.__wrapArgsAndKwargs(args, kwargs)
+            self.__cleanup()
+
+            # If this method has never been called before, call it
+            lastCallDate = self._calls.get(hashedArgs)
+            if not lastCallDate:
+                self._calls[hashedArgs] = now
+                return self._f(funcSelf, *args, **kwargs)
+
+            # Work out how long it's been since this method has been called
+            timeSinceLastCall = now - lastCallDate
+
+            # If this method is overdue for a call, call it
+            if timedelta(seconds=debounceSeconds) < timeSinceLastCall:
+                # If this call was queued, remove it
+                if hashedArgs in self._callsQueued:
+                    self._callsQueued.remove(hashedArgs)
+                self._calls[hashedArgs] = now
+                return self._f(funcSelf, *args, **kwargs)
+
+            timeToNextCall = (
+                timedelta(seconds=debounceSeconds) - timeSinceLastCall
+            )
+
+            # If not, schedule this method for call after the debounce
+            # but only if there isn't already a call queued
+            if hashedArgs not in self._callsQueued:
+                self._callsQueued.add(hashedArgs)
+                reactor.callLater(
+                    timeToNextCall.total_seconds() + 0.1,
+                    self.call,
+                    funcSelf,
+                    *args,
+                    **kwargs,
+                )
+
+                return
+
+            return
+
+    def wrap(f):
+        wrapC = Wrap(f)
+
+        def wrapInner(funcSelf, *args, **kwargs):
+            return wrapC.call(funcSelf, *args, **kwargs)
+
+        return wrapInner
+
+    return wrap
+
+
+def logLargeMessages(
+    logger, vortexMsgs: Union[VortexMsgList, bytes], destVortexUuid: str
+):
+    def logIt(vortexMsg):
+        # Log anything larger than 128kb
+        length = len(vortexMsg)
+        if length < 512 * 1024:
+            return
+
+        logger.debug(
+            "Sending vortexMsg of size %s to vortex %s",
+            "{:,}kB".format(int(length / 1024)),
+            destVortexUuid,
+        )
+
+    if isinstance(vortexMsgs, bytes):
+        logIt(vortexMsgs)
+
+    else:
+        for msg in vortexMsgs:
+            logIt(msg)

vortex/VortexWritePushProducer.py

@@ -0,0 +1,321 @@
+"""
+ * Created by Synerty Pty Ltd
+ *
+ * This software is open source, the MIT license applies.
+ *
+ * Website : http://www.synerty.com
+ * Support : support@synerty.com
+"""
+import logging
+import typing
+import weakref
+from collections import deque, defaultdict
+from typing import Callable, Deque, Dict
+
+from twisted.internet.interfaces import IPushProducer
+from zope.interface import implementer
+
+from vortex.DeferUtil import nonConcurrentMethod
+from vortex.PayloadPriority import DEFAULT_PRIORITY
+
+logger = logging.getLogger(name=__name__)
+
+
+def _format_size(size):
+    for unit in ("B", "KiB", "MiB", "GiB", "TiB"):
+        if abs(size) < 100 and unit != "B":
+            # 3 digits (xx.x UNIT)
+            return "%.1f %s" % (size, unit)
+        if abs(size) < 10 * 1024 or unit == "TiB":
+            # 4 or 5 digits (xxxx UNIT)
+            return "%.0f %s" % (size, unit)
+        size /= 1024
+
+
+@implementer(IPushProducer)
+class VortexWritePushProducer:
+    WARNING_DATA_LENGTH = 50 * 1024 * 1024
+    ERROR_DATA_LENGTH = 500 * 1024 * 1024
+    WRITE_CHUNK_SIZE = 128 * 1024
+
+    __memoryLoggingRefs = None
+    __memoryLoggingEnabled = False
+    _DEBUG_LOGGING = False
+
+    @classmethod
+    def setupMemoryLogging(cls) -> None:
+        cls.__memoryLoggingRefs = []
+        cls.__memoryLoggingEnabled = True
+
+    @classmethod
+    def memoryLoggingDump(
+        cls, top=10, msgs=1
+    ) -> typing.List[typing.Tuple[str, int, int]]:
+        if not cls.__memoryLoggingRefs:
+            return []
+
+        # Filter out expired items
+        cls.__memoryLoggingRefs = list(
+            filter(lambda o: o(), cls.__memoryLoggingRefs)
+        )
+
+        results = []
+        for producerRef in cls.__memoryLoggingRefs:
+            producer = producerRef()
+            if not producer:
+                continue
+
+            queueCount = sum(
+                [len(q) for q in producer._queueByPriority.values()]
+            )
+
+            results.append(
+                (
+                    producer._remoteVortexName,
+                    queueCount,
+                    producer._queuedDataLen,
+                )
+            )
+
+        data = sorted(results, key=lambda x: x[2], reverse=True)
+
+        return list(filter(lambda x: x[2] >= msgs, data))[:top]
+
+    def __init__(
+        self,
+        transport,
+        stopProducingCallback: Callable,
+        remoteVortexName: str = "Pending",
+        writeWholeFrames=False,
+        terminateFrameWithDot=True,
+        splitFrames=False,
+    ):
+        """
+
+        :param splitFrames: Split frames that come in if they are larger than 128kb.
+        This allows better use of ram, as keeping a 1mb to ?mb bytes object around
+        is bad for memory management and memory fragmentation.
+        """
+        if VortexWritePushProducer.__memoryLoggingEnabled:
+            VortexWritePushProducer.__memoryLoggingRefs.append(
+                weakref.ref(self)
+            )
+
+        self._transport = transport
+        self._stopProducingCallback = stopProducingCallback
+        self._remoteVortexName = remoteVortexName
+        self._writeWholeFrames = writeWholeFrames
+        self._terminateFrameWithDot = terminateFrameWithDot
+        self._splitFrames = splitFrames
+
+        self._paused = False
+        self._pausedDeferred = None
+        self._writingInProgress = False
+        self._writingFrameInProgress = False
+        self._closed = False
+        self._queuedDataLen = 0
+        self._queueByPriority: Dict[int, Deque] = defaultdict(deque)
+
+        self._currentlyWritingData: typing.Optional[bytes] = None
+        self._currentlyWritingDataOffset: int = 0
+
+    def setRemoteVortexName(self, remoteVortexName: str):
+        self._remoteVortexName = remoteVortexName
+
+    @property
+    def _canContinue(self):
+        return (
+            not self._paused
+            and not self._startWritingFrame.running
+            and [q for q in self._queueByPriority.values() if q]
+        )
+
+    @nonConcurrentMethod
+    def _startWriting(self):
+        if self._currentlyWritingData is not None:
+            self._startWritingFrame()
+
+        while self._canContinue:
+            # Send the messages in order of priority
+            for priority in sorted(self._queueByPriority):
+                if not self._canContinue:
+                    return
+
+                queue = self._queueByPriority[priority]
+                if not queue:
+                    continue
+
+                data = queue.popleft()
+                preLen = self._queuedDataLen
+                self._queuedDataLen -= len(data)
+
+                if self._queuedDataLen < self.WARNING_DATA_LENGTH < preLen:
+                    logger.info(
+                        "%s: Data Queue memory high warning - returned to normal : %s",
+                        self._remoteVortexName,
+                        _format_size(self._queuedDataLen),
+                    )
+
+                if self._queuedDataLen < self.ERROR_DATA_LENGTH < preLen:
+                    logger.info(
+                        "%s: Data Queue memory high error - returned to warning : %s",
+                        self._remoteVortexName,
+                        _format_size(self._queuedDataLen),
+                    )
+
+                if self._writeWholeFrames:
+                    self._transport.write(data)
+                    if self._terminateFrameWithDot:
+                        self._transport.write(b".")
+
+                else:
+                    # Use a memoryview for fast slicing.
+                    self._currentlyWritingData = data
+                    self._currentlyWritingDataOffset = 0
+                    self._startWritingFrame()
+
+    @nonConcurrentMethod
+    def _startWritingFrame(self):
+        # ---------------
+        # Write in progress logic.
+        # We should only have one write loop at a time
+        if self._writingFrameInProgress:
+            return
+        self._writingFrameInProgress = True
+
+        data = self._currentlyWritingData
+        offset = self._currentlyWritingDataOffset
+        while not self._paused and offset < len(data):
+            if self._DEBUG_LOGGING:
+                logger.debug(
+                    "%s: Producer _startWritingFrame called,"
+                    " wrote %s to %s of %s",
+                    self._remoteVortexName,
+                    offset,
+                    min(offset + self.WRITE_CHUNK_SIZE, len(data)),
+                    len(data),
+                )
+            self._transport.write(data[offset : offset + self.WRITE_CHUNK_SIZE])
+            offset += self.WRITE_CHUNK_SIZE
+
+        if len(data) <= offset:
+            self._currentlyWritingDataOffset = 0
+            self._currentlyWritingData = None
+            if self._terminateFrameWithDot:
+                self._transport.write(b".")
+
+        else:
+            self._currentlyWritingDataOffset = offset
+
+        self._writingFrameInProgress = False
+
+    def pauseProducing(self):
+        """
+        Pause producing data.
+
+        Tells a producer that it has produced too much data to process for
+        the time being, and to stop until C{resumeProducing()} is called.
+        """
+        self._paused = True
+        if self._DEBUG_LOGGING:
+            logger.debug(
+                "%s: Producer paused, data len = %s",
+                self._remoteVortexName,
+                self._queuedDataLen,
+            )
+
+    def resumeProducing(self):
+        """
+        Resume producing data.
+
+        This tells a producer to re-add itself to the main loop and produce
+        more data for its consumer.
+        """
+        self._paused = False
+        if self._DEBUG_LOGGING:
+            logger.debug(
+                "%s: Producer resumed, data len = %s",
+                self._remoteVortexName,
+                self._queuedDataLen,
+            )
+        self._startWriting()
+
+    def stopProducing(self):
+        """
+        Stop producing data.
+
+        This tells a producer that its consumer has died, so it must stop
+        producing data for good.
+        """
+        if self._stopProducingCallback:
+            self._stopProducingCallback()
+            self._stopProducingCallback = None
+
+    def write(self, data: bytes, priority: int = DEFAULT_PRIORITY):
+        assert not self._closed
+
+        # Since write actually queues data and does not write, we will split
+        # the data if required so we are storing smaller memory objects.
+        if self._splitFrames and self.WRITE_CHUNK_SIZE < len(data):
+            for offset in range(0, len(data), self.WRITE_CHUNK_SIZE):
+
+                if self._DEBUG_LOGGING:
+                    logger.debug(
+                        "%s: Producer write, splitting frames,"
+                        " split %s to %s of %s",
+                        self._remoteVortexName,
+                        offset,
+                        min(offset + self.WRITE_CHUNK_SIZE, len(data)),
+                        len(data),
+                    )
+                self.write(data[offset : offset + self.WRITE_CHUNK_SIZE])
+            return
+
+        if self._DEBUG_LOGGING:
+            logger.debug(
+                "%s: Producer write called, data len = %s",
+                self._remoteVortexName,
+                len(data),
+            )
+
+        preLen = self._queuedDataLen
+        self._queuedDataLen += len(data)
+
+        # Queue the data up in chunks
+        self._queueByPriority[priority].append(data)
+
+        if preLen < self.WARNING_DATA_LENGTH < self._queuedDataLen:
+            logger.warning(
+                "%s: Data Queue memory high warning : %s",
+                self._remoteVortexName,
+                _format_size(self._queuedDataLen),
+            )
+
+        if preLen < self.ERROR_DATA_LENGTH < self._queuedDataLen:
+            logger.error(
+                "%s: Data Queue memory high error : %s",
+                self._remoteVortexName,
+                _format_size(self._queuedDataLen),
+            )
+
+        self._startWriting()
+
+    def writeSequence(
+        self, data: typing.Iterable[bytes], priority: int = DEFAULT_PRIORITY
+    ) -> None:
+
+        if self._DEBUG_LOGGING:
+            logger.debug(
+                "%s: Producer writeSequence called",
+                self._remoteVortexName,
+            )
+
+        for seq in data:
+            self.write(seq, priority=priority)
+
+    def close(self):
+        self._closed = True
+        self._queuedDataLen = 0
+        self._queueByPriority = defaultdict(deque)
+        self._currentlyWritingData = None
+        self._currentlyWritingDataOffset = 0

vortex/__init__.py

@@ -0,0 +1,10 @@
+"""
+ * Created by Synerty Pty Ltd
+ *
+ * This software is open source, the MIT license applies.
+ *
+ * Website : http://www.synerty.com
+ * Support : support@synerty.com
+"""
+
+__version__ = '4.0.23'