vortexpy 4.0.23__py3-none-any.whl

vortex/VortexPayloadProtocol.py

@@ -0,0 +1,180 @@
+"""
+ * Created by Synerty Pty Ltd
+ *
+ * This software is open source, the MIT license applies.
+ *
+ * Website : http://www.synerty.com
+ * Support : support@synerty.com
+"""
+
+import logging
+from abc import ABCMeta, abstractmethod
+from collections import deque
+
+from twisted.internet import reactor
+from twisted.internet.defer import inlineCallbacks
+from twisted.internet.error import ConnectionDone, ConnectionLost
+from twisted.internet.protocol import Protocol, connectionDone
+
+from vortex.DeferUtil import nonConcurrentMethod
+from vortex.PayloadEnvelope import PayloadEnvelope
+from vortex.PayloadIO import PayloadIO
+
+logger = logging.getLogger(name=__name__)
+
+
+class VortexPayloadProtocol(Protocol, metaclass=ABCMeta):
+    def __init__(self, logger):
+        self._serverVortexUuid = None
+        self._serverVortexName = None
+        self._logger = logger
+
+        self._data = b""
+        self._vortexMsgsQueue = deque()
+
+    @abstractmethod
+    def _beat(self):
+        """
+        EG :
+        if self._vortexClient:
+            self._vortexClient._beat()
+        """
+
+    @abstractmethod
+    def _nameAndUuidReceived(self, name, uuid):
+        """
+        EG :
+        if self._vortexClient:
+            self._vortexClient._setNameAndUuid(name=self._serverVortexName,
+                                               uuid=self._serverVortexUuid)
+        """
+
+    @abstractmethod
+    def _createResponseSenderCallable(self):
+        """
+        EG :
+        def sendResponse(vortexMsgs: Union[VortexMsgList, bytes]):
+            return self._vortexClient.sendVortexMsg(vortexMsgs=vortexMsgs)
+        """
+
+    def vortexMsgReceived(self, vortexMsg):
+        """Vortex Msg Received
+
+        For protocols where framing is used, we don't need to process messages
+        with dots on the end, for those protocols, call this method to process
+        the whole message.
+        """
+        self._data = b""
+        self._beat()
+        if vortexMsg == b".":
+            return
+
+        self._vortexMsgsQueue.append(vortexMsg)
+
+        if self._vortexMsgsQueue and not self._processVortexMsgs.running:
+            reactor.callLater(0, self._processVortexMsgs)
+
+    def dataReceived(self, bytesIn):
+        """Data Received
+
+        This method is used for protocols that don't have framing. We use a
+        '.' to delineate messages.
+
+        """
+        if bytesIn.startswith(b"<"):
+            raise Exception("Not Logged In")
+
+        self._data += bytesIn
+        self._beat()
+        self._processData()
+
+    def connectionLost(self, reason=connectionDone):
+        reasonFailure = reason
+        self._processData()
+        if reasonFailure.check(ConnectionDone):
+            self._logger.info(
+                "Connection closed by other end (it may be shutting down)"
+            )
+
+        elif isinstance(reasonFailure.value, ConnectionLost):
+            self._logger.info(
+                "Connection to other end lost (We may be shutting down)"
+            )
+
+        else:
+            self._logger.error("Closed with error")
+            try:
+                self._logger.exception(reason.getErrorMessage())
+                self._logger.exception(reason.getTraceback())
+            except:
+                self._logger.exception(reasonFailure.value)
+
+    def _processData(self):
+        def getNextChunkIter():
+            try:
+                while True:
+                    yield self._data.index(b".")
+            except ValueError:
+                # There is no '.' in it, wait for more data.
+                return
+
+        if self._data:
+            for nextChunk in getNextChunkIter():
+                vortexMsg = self._data[:nextChunk]
+                self._data = self._data[nextChunk + 1 :]
+
+                # If we get two heartbeats in a row, this will be false
+                if len(vortexMsg):
+                    self._vortexMsgsQueue.append(vortexMsg)
+
+        if self._vortexMsgsQueue and not self._processVortexMsgs.running:
+            reactor.callLater(0, self._processVortexMsgs)
+
+    @inlineCallbacks
+    @nonConcurrentMethod
+    def _processVortexMsgs(self):
+        while self._vortexMsgsQueue:
+            vortexMsg = self._vortexMsgsQueue.popleft()
+
+            try:
+                payloadEnvelope = yield PayloadEnvelope().fromVortexMsgDefer(
+                    vortexMsg
+                )
+
+                if payloadEnvelope.isEmpty():
+                    self._processServerInfoPayload(payloadEnvelope)
+                else:
+                    self._deliverPayload(payloadEnvelope)
+
+            except Exception as e:
+                print(vortexMsg)
+                print(e)
+                self._logger.exception(e)
+                raise
+
+    def _processServerInfoPayload(self, payload):
+        """Process Server Info Payload
+
+        The first payload a server sends to the client contains information about it's
+        self.
+
+        """
+
+        if PayloadEnvelope.vortexUuidKey in payload.filt:
+            self._serverVortexUuid = payload.filt[PayloadEnvelope.vortexUuidKey]
+
+        if PayloadEnvelope.vortexNameKey in payload.filt:
+            self._serverVortexName = payload.filt[PayloadEnvelope.vortexNameKey]
+
+        self._nameAndUuidReceived(
+            name=self._serverVortexName, uuid=self._serverVortexUuid
+        )
+
+    def _deliverPayload(self, payloadEnvelope):
+        PayloadIO().process(
+            payloadEnvelope,
+            vortexUuid=self._serverVortexUuid,
+            vortexName=self._serverVortexName,
+            httpSession=None,
+            sendResponse=self._createResponseSenderCallable(),
+        )

vortex/VortexServer.py

@@ -0,0 +1,367 @@
+"""
+ * Created by Synerty Pty Ltd
+ *
+ * This software is open source, the MIT license applies.
+ *
+ * Website : http://www.synerty.com
+ * Support : support@synerty.com
+"""
+
+import logging
+import uuid
+from typing import Any
+from typing import Dict
+from typing import Optional
+from typing import Union
+from weakref import WeakValueDictionary
+
+from twisted.internet import reactor
+from twisted.internet import task
+from twisted.internet.defer import Deferred
+from twisted.internet.defer import inlineCallbacks
+from twisted.python.components import registerAdapter
+from twisted.web.server import Session
+from zope.interface import Attribute
+from zope.interface import Interface
+from zope.interface.declarations import implementer
+
+from .DeferUtil import isMainThread
+from .DeferUtil import vortexLogFailure
+from .PayloadEnvelope import PayloadEnvelope
+from .PayloadEnvelope import VortexMsgList
+from .PayloadFilterKeys import rapuiServerEcho
+from .PayloadIO import PayloadIO
+from .PayloadPriority import DEFAULT_PRIORITY
+from .VortexABC import VortexABC
+from .VortexABC import VortexInfo
+from .VortexUtil import logLargeMessages
+
+logger = logging.getLogger(__name__)
+
+""" ---------------------------------------------------------------------------
+VortexServer
+"""
+
+HEART_BEAT_PERIOD = 10.0
+HEART_BEAT_TIMEOUT = 180.0
+
+
+class VortexServer(VortexABC):
+    """VortexServer
+    The static instance of the controller
+    """
+
+    _DEBUG_LOGGING = False
+
+    def __init__(self, name: str, requiresBase64Encoding=True) -> None:
+        # Simple initialisations up the top
+        self._name = name
+        self._requiresBase64Encoding = requiresBase64Encoding
+        self._uuid = str(uuid.uuid4())
+        self._shutdown = False
+
+        # Store all our sessions
+        self._httpSessionsBySessionUuid: WeakValueDictionary = (
+            WeakValueDictionary()
+        )
+        self._connectionByVortexUuid: Dict[str, Any] = {}
+        self._remoteVortexInfo: tuple[VortexInfo] = tuple()
+
+    def name(self):
+        return self._name
+
+    def uuid(self):
+        return self._uuid
+
+    @property
+    def requiresBase64Encoding(self):
+        return self._requiresBase64Encoding
+
+    @property
+    def localVortexInfo(self) -> VortexInfo:
+        return VortexInfo(name=self._name, uuid=self._uuid)
+
+    @property
+    def remoteVortexInfo(self) -> tuple[VortexInfo]:
+        return self._remoteVortexInfo
+
+    def _connectionsChanged(
+        self, vortexClosedNames: list[str], nowOnline: bool
+    ):
+        vortexInfos = []
+
+        for conn in self._connectionByVortexUuid.values():
+            vortexInfos.append(
+                VortexInfo(
+                    name=conn.remoteVortexName, uuid=conn.remoteVortexUuid
+                )
+            )
+
+        self._remoteVortexInfo = tuple(vortexInfos)
+
+        from vortex.VortexFactory import VortexFactory
+
+        VortexFactory.connectionChanged()
+
+        for vortexName in vortexClosedNames:
+            # noinspection PyProtectedMember
+            VortexFactory._notifyOfVortexStatusChange(
+                vortexName, online=nowOnline
+            )
+
+    @property
+    def connections(self) -> Dict[str, Any]:
+        return self._connectionByVortexUuid
+
+    def isShutdown(self):
+        return self._shutdown
+
+    def shutdown(self):
+        self._shutdown = True
+
+        for sess in list(self._httpSessionsBySessionUuid.values()):
+            sess.expire()
+
+        for conn in list(self._connectionByVortexUuid.values()):
+            conn.close()
+
+    def vortexUuids(self):
+        return list(self._connectionByVortexUuid.keys())
+
+    def isVortexAlive(self, vortexUuid):
+        return vortexUuid in self._connectionByVortexUuid
+
+    def vortexClientIpPort(self, vortexUuid):
+        connection = self._connectionByVortexUuid.get(vortexUuid)
+        if not connection:
+            return None
+
+        return "%s:%s" % (connection.ip, connection.port)
+
+    def connectionOpened(self, httpSession, vortexConnection):
+        if self._DEBUG_LOGGING:
+            logger.debug("VortexServer - connectionOpened")
+
+        vortexUuid = vortexConnection.remoteVortexUuid
+        assert vortexUuid
+
+        if httpSession:
+            # If this is a new session, Make sure we have an expire callback on it
+            if httpSession.uid not in self._httpSessionsBySessionUuid:
+                self._httpSessionsBySessionUuid[httpSession.uid] = httpSession
+                httpSession.notifyOnExpire(
+                    lambda: self._sessionExpired(httpSession.uid)
+                )
+
+            # Update the connection dict in the sessions object
+            httpSessionConns = VortexSessionI(httpSession).connections
+            httpSessionConns[vortexUuid] = vortexConnection
+
+        # Get the old connection if it exists
+        if vortexUuid in self._connectionByVortexUuid:
+            self._connectionByVortexUuid[vortexUuid].close()
+
+        # Update the _connectionsByvortexUuid
+        self._connectionByVortexUuid[vortexUuid] = vortexConnection
+        self._connectionsChanged(
+            [vortexConnection.remoteVortexName], nowOnline=True
+        )
+
+    def connectionClosed(self, conn):
+        if self._DEBUG_LOGGING:
+            logger.debug("VortexServer - connectionClosed")
+
+        vortexUuid = conn.remoteVortexUuid
+
+        if conn.httpSessionUuid in self._httpSessionsBySessionUuid:
+            session = self._httpSessionsBySessionUuid[conn.httpSessionUuid]
+            assert session
+
+            # cleanup _sessionsBySessionUuid
+            conns = VortexSessionI(session).connections
+            if conns[vortexUuid] == conn:
+                del conns[vortexUuid]
+
+        # cleanup _connectionsByvortexUuid
+        vortexClosedNames = []
+        if conn.remoteVortexUuid in self._connectionByVortexUuid:
+            if self._connectionByVortexUuid[conn.remoteVortexUuid] == conn:
+                del self._connectionByVortexUuid[conn.remoteVortexUuid]
+                vortexClosedNames.append(conn.remoteVortexName)
+
+        self._connectionsChanged(vortexClosedNames, nowOnline=False)
+
+    def _sessionExpired(self, httpSessionUuid):
+        logger.debug(
+            "VortexServer - _sessionExpired, Session %s has expired"
+            % httpSessionUuid
+        )
+
+        # cleanup _sessionsBySessionUuid
+        del self._httpSessionsBySessionUuid[httpSessionUuid]
+
+        # cleanup _connectionsByvortexUuid
+        vortexClosedNames = []
+        for vortexUuid, conn in list(self._connectionByVortexUuid.items()):
+            if conn.httpSessionUuid == httpSessionUuid:
+                del self._connectionByVortexUuid[vortexUuid]
+                vortexClosedNames.append(conn.remoteVortexName)
+
+        self._connectionsChanged(vortexClosedNames, nowOnline=False)
+
+    def payloadReveived(self, httpSession, vortexUuid, vortexName, payload):
+        # print "VortexServer - payloadReveived"
+
+        if rapuiServerEcho in payload.filt:
+            payload.filt.pop(rapuiServerEcho)
+            self.sendVortexMsg(payload.toVortexMsg(), vortexUuid)
+
+        def sendResponse(
+            vortexMsg: bytes, priority: int = DEFAULT_PRIORITY
+        ) -> Deferred:
+            """Send Back
+
+            Sends a response back to where this payload come from.
+
+            """
+            return self.sendVortexMsg(
+                vortexMsg, vortexUuid=vortexUuid, priority=priority
+            )
+
+        PayloadIO().process(
+            payload,
+            vortexUuid=vortexUuid,
+            vortexName=vortexName,
+            httpSession=httpSession,
+            sendResponse=sendResponse,
+        )
+
+    def sendVortexMsg(
+        self,
+        vortexMsgs: Union[VortexMsgList, bytes, None] = None,
+        vortexUuid: Optional[str] = None,
+        priority: int = DEFAULT_PRIORITY,
+    ):
+        """Send Vortex Msg
+
+        Sends the vortex message to any conencted clients with vortexUuid.
+        Or broadcast it to all connected vortex clients if it's None
+
+        :param vortexMsgs: The vortex message to send
+        :param vortexUuid: The vortexUuid of the client to send to.
+        """
+        if vortexMsgs is None:
+            vortexMsgs = [PayloadEnvelope().toVortexMsg()]
+
+        logLargeMessages(logger, vortexMsgs, vortexUuid)
+
+        if isMainThread():
+            return self._sendVortexMsgLater(
+                vortexMsgs, vortexUuid=vortexUuid, priority=priority
+            )
+
+        return task.deferLater(
+            reactor,
+            0,
+            self._sendVortexMsgLater,
+            vortexMsgs,
+            vortexUuid=vortexUuid,
+            priority=priority,
+        )
+
+    @inlineCallbacks
+    def _sendVortexMsgLater(
+        self,
+        vortexMsgs: Union[VortexMsgList, bytes],
+        vortexUuid: Optional[str],
+        priority: int,
+    ):
+        """Send the message.
+
+        Send it later,
+        This also means it doesn't matter what thread this is called from
+
+        """
+        from vortex.VortexFactory import VortexFactory
+
+        yield None
+
+        if not isinstance(vortexMsgs, list):
+            vortexMsgs = [vortexMsgs]
+
+        # Deliver locally
+        if vortexUuid == self._uuid:
+            for vortexMsg in vortexMsgs:
+
+                def sendResponse(vortexMsg_, priority_=DEFAULT_PRIORITY):
+                    self._sendVortexMsgLater(vortexMsg_, self._uuid, priority_)
+
+                def cb(payloadEnvelope: PayloadEnvelope) -> None:
+                    PayloadIO().process(
+                        payloadEnvelope,
+                        vortexUuid=self._uuid,
+                        vortexName=self._name,
+                        httpSession=None,
+                        sendResponse=sendResponse,
+                    )
+
+                d = PayloadEnvelope.fromVortexMsgDefer(vortexMsg)
+                d.addCallback(cb)
+                d.addErrback(vortexLogFailure, logger, consumeError=True)
+
+            return
+
+        # If any transports require base64 encoding, then encode them all
+        if self.requiresBase64Encoding:
+            for index, vortexMsg in enumerate(vortexMsgs):
+                if vortexMsg.startswith(b"{"):
+                    vortexMsg = yield PayloadEnvelope.base64EncodeDefer(
+                        vortexMsg
+                    )
+                    vortexMsgs[index] = vortexMsg + b"."
+
+        if vortexUuid is None:
+            conns = self._connectionByVortexUuid.values()
+        elif vortexUuid in self._connectionByVortexUuid:
+            conns = [self._connectionByVortexUuid[vortexUuid]]
+        else:
+            logger.debug(
+                "Vortex %s offline, not sending message of size %s",
+                vortexUuid,
+                ", ".join(["{:,}".format(len(m)) for m in vortexMsgs]),
+            )
+            return False
+
+        for conn in conns:
+            for vortexMsg in vortexMsgs:
+
+                # noinspection PyProtectedMember
+                if VortexFactory._DEBUG_LOGGING:
+                    logger.debug(
+                        "Sending vortexMsg to %s", conn.remoteVortexUuid
+                    )
+
+                conn.write(vortexMsg, priority)
+
+        return True
+
+
+""" ---------------------------------------------------------------------------
+VortexServer Session
+"""
+
+
+class VortexSessionI(Interface):
+    connections: Dict = Attribute(
+        "VortexServer connections for this session, by window uuid"
+    )
+
+
+@implementer(VortexSessionI)
+class VortexSession(object):
+    def __init__(self, session):
+        session.sessionTimeout = HEART_BEAT_TIMEOUT
+        self.connections = {}
+
+
+registerAdapter(VortexSession, Session, VortexSessionI)

vortex/VortexServerConnection.py

@@ -0,0 +1,127 @@
+"""
+ * Created by Synerty Pty Ltd
+ *
+ * This software is open source, the MIT license applies.
+ *
+ * Website : http://www.synerty.com
+ * Support : support@synerty.com
+"""
+import logging
+from datetime import datetime
+
+import pytz
+from twisted.internet import task
+
+from .PayloadPriority import DEFAULT_PRIORITY
+from .VortexConnectionABC import VortexConnectionABC
+from .VortexServer import VortexServer, HEART_BEAT_PERIOD, HEART_BEAT_TIMEOUT
+from .VortexWritePushProducer import VortexWritePushProducer
+
+logger = logging.getLogger(name=__name__)
+
+
+class VortexServerConnection(VortexConnectionABC):
+    def __init__(
+        self,
+        vortexServer: VortexServer,
+        remoteVortexUuid: str,
+        remoteVortexName: str,
+        httpSession,
+        transport,
+        addr,
+    ) -> None:
+        VortexConnectionABC.__init__(
+            self,
+            logger,
+            vortexServer,
+            remoteVortexUuid=remoteVortexUuid,
+            remoteVortexName=remoteVortexName,
+            httpSessionUuid=httpSession,
+        )
+
+        self._lastHeartBeatTime = datetime.now(pytz.utc)
+        self._lastHeartBeatCheckTime = datetime.now(pytz.utc)
+
+        self._transport = transport
+        self._addr = addr
+
+        # Start our heart beat
+        self._beatLoopingCall = task.LoopingCall(self._beat)
+        d = self._beatLoopingCall.start(HEART_BEAT_PERIOD, now=False)
+        d.addErrback(lambda f: logger.exception(f.value))
+
+        self._producer = None
+
+        # Register the producer if there isn't one already.
+        # The websocket server already has one.
+        if not self._transport.producer:
+            self._producer = VortexWritePushProducer(
+                transport, lambda: self.close(), remoteVortexName
+            )
+
+            transport.registerProducer(self._producer, True)
+
+    def beatReceived(self):
+        self._lastHeartBeatTime = datetime.now(pytz.utc)
+
+    def _beat(self):
+        # If we're closed, do nothing
+        if self._closed:
+            if self._beatLoopingCall.running:
+                self._beatLoopingCall.stop()
+            return
+
+        beatTimeout = (
+            datetime.now(pytz.utc) - self._lastHeartBeatTime
+        ).seconds > HEART_BEAT_TIMEOUT
+
+        # If we've been asleep, then make note of that (VM suspended)
+        checkTimout = (
+            datetime.now(pytz.utc) - self._lastHeartBeatCheckTime
+        ).seconds > HEART_BEAT_TIMEOUT
+
+        # Mark that we've just checked it
+        self._lastHeartBeatCheckTime = datetime.now(pytz.utc)
+
+        if checkTimout:
+            self._lastHeartBeatTime = datetime.now(pytz.utc)
+            return
+
+        # If we havn't heard from the client, then close the connection
+        if beatTimeout:
+            self._beatLoopingCall.stop()
+            self._timedOut = True
+            self.close()
+            return
+
+        self._write(b".", DEFAULT_PRIORITY)
+
+    @property
+    def ip(self):
+        return self._addr.host
+
+    @property
+    def port(self):
+        return self._addr.port
+
+    def write(self, payloadVortexStr: bytes, priority: int = DEFAULT_PRIORITY):
+        assert not self._closed
+        self._write(payloadVortexStr, priority)
+
+    def _write(self, payloadVortexStr: bytes, priority: int):
+        if self._producer:
+            self._producer.write(payloadVortexStr, priority)
+        else:
+            self._transport.write(payloadVortexStr)
+
+    def close(self):
+        if self._beatLoopingCall.running:
+            self._beatLoopingCall.stop()
+
+        self._transport.loseConnection()
+
+    def transportClosed(self):
+        if self._producer and hasattr(self._producer, "close"):
+            self._producer.close()
+        self._producer = None
+        VortexConnectionABC.close(self)