vortexpy 4.0.23__py3-none-any.whl

vortex/handler/TupleDataObservableTestHandler.py

@@ -0,0 +1,60 @@
+import logging
+from datetime import datetime
+
+import pytz
+from twisted.internet import task
+
+from vortex.DeferUtil import deferToThreadWrapWithLogger
+from vortex.Payload import Payload
+from vortex.TestTuple import TestTuple
+from vortex.TupleDataForTest import makeTestTupleData
+from vortex.TupleSelector import TupleSelector
+from vortex.handler.TupleDataObservableHandler import (
+    TuplesProviderABC,
+    TupleDataObservableHandler,
+)
+
+logger = logging.getLogger(__name__)
+testTuples1Selector = TupleSelector("testTuples1", {"count": 4})
+
+testTuples2Selector = TupleSelector("testTuples2", {"count": 7})
+
+
+class TestTupleProvider(TuplesProviderABC):
+    @deferToThreadWrapWithLogger(logger)
+    def makeVortexMsg(self, filt: dict, tupleSelector: TupleSelector) -> bytes:
+        count = tupleSelector.selector["count"]
+        tuples = makeTestTupleData(count)
+
+        for t in tuples:
+            t.aDate = datetime.now(pytz.utc)
+            t.aDict = tupleSelector.selector
+            t.aString = tupleSelector.name
+
+        return (
+            Payload(filt=filt, tuples=tuples)
+            .makePayloadEnvelope()
+            .toVortexMsg()
+        )
+
+
+class NotifyTestTimer:
+    @classmethod
+    def __notify(cls):
+        observableHandler.notifyOfTupleUpdate(testTuples1Selector)
+        observableHandler.notifyOfTupleUpdate(testTuples2Selector)
+
+    @classmethod
+    def startTupleUpdateNotifyer(cls):
+        cls.__loopingCall = task.LoopingCall(cls.__notify)
+        d = cls.__loopingCall.start(2)
+        d.addErrback(lambda f: logger.exception(f.value))
+        d.addCallback(
+            lambda _: logger.debug("Observable tuple updates started")
+        )
+
+
+observableHandler = TupleDataObservableHandler("vortexTestObservable")
+observableHandler.addTupleProvider(TestTuple.tupleName(), TestTupleProvider())
+observableHandler.addTupleProvider("testTuples1", TestTupleProvider())
+observableHandler.addTupleProvider("testTuples2", TestTupleProvider())

vortex/handler/TupleDataObserverClient.py

@@ -0,0 +1,158 @@
+import logging
+from copy import copy
+from typing import List
+
+from reactivex.subject import Subject
+from reactivex import operators as ops
+from twisted.internet.defer import Deferred, inlineCallbacks
+
+from vortex.DeferUtil import vortexLogFailure
+from vortex.Payload import Payload
+from vortex.PayloadEndpoint import PayloadEndpoint
+from vortex.PayloadEnvelope import PayloadEnvelope
+from vortex.PayloadResponse import PayloadResponse
+from vortex.TupleSelector import TupleSelector
+from vortex.VortexFactory import VortexFactory
+from vortex.handler.TupleDataObservableCache import TupleDataObservableCache
+
+logger = logging.getLogger(__name__)
+
+
+class TupleDataObserverClient(TupleDataObservableCache):
+    def __init__(
+        self,
+        destVortexName,
+        observableName,
+        additionalFilt=None,
+        observerName="default",
+    ):
+        """Constructor
+
+        :param observableName: The name of this observable
+        :param additionalFilt: Any additional filter keys that are required
+        :param destVortexName: The dest vortex name to send the payloads to
+
+        """
+        TupleDataObservableCache.__init__(self)
+
+        self._destVortexName = destVortexName
+        self._observableName = observableName
+        self._observerName = observerName
+
+        self._sendFilt = dict(
+            name=observableName,
+            observerName=observerName,
+            key="tupleDataObservable",
+        )
+
+        self._listenFilt = dict(name=observableName, key="tupleDataObservable")
+
+        if additionalFilt:
+            self._sendFilt.update(additionalFilt)
+            self._listenFilt.update(additionalFilt)
+
+        self._endpoint = PayloadEndpoint(
+            self._listenFilt,
+            self._receivePayload,
+            acceptOnlyFromVortex=destVortexName,
+        )
+
+        # There are no online checks for the vortex
+        # isOnlineSub = statusService.isOnline
+        # .filter(online= > online == = true)
+        # .subscribe(online= > self.vortexOnlineChanged())
+        #
+        # self.onDestroyEvent.subscribe(() = > isOnlineSub.unsubscribe())
+        VortexFactory.subscribeToVortexStatusChange(destVortexName).pipe(ops.filter(
+            lambda online: online is True
+        )).subscribe(self._vortexOnlineChanged)
+
+        TupleDataObservableCache.start(self)
+
+    def shutdown(self):
+        self._endpoint.shutdown()
+        TupleDataObservableCache.shutdown(self)
+
+    def pollForTuples(
+        self, tupleSelector: TupleSelector, logTimeoutError: bool = True
+    ) -> Deferred:
+        startFilt = copy(self._sendFilt)
+        startFilt.update({"subscribe": False, "tupleSelector": tupleSelector})
+
+        def updateCacheCallback(
+            payloadEnvelope: PayloadEnvelope,
+        ) -> PayloadEnvelope:
+            cache, _ = self._updateCache(payloadEnvelope)
+            return payloadEnvelope
+
+        pr = PayloadResponse(
+            payloadEnvelope=PayloadEnvelope(startFilt),
+            destVortexName=self._destVortexName,
+            logTimeoutError=logTimeoutError,
+        )
+        pr.addCallback(updateCacheCallback)
+        pr.addCallback(
+            lambda payloadEnvelope: payloadEnvelope.decodePayloadDefer()
+        )
+        pr.addCallback(lambda payload: payload.tuples)
+        return pr
+
+    def subscribeToTupleSelector(self, tupleSelector: TupleSelector) -> Subject:
+        cache = self._makeCache(tupleSelector)
+        self._tellServerWeWantData([tupleSelector])
+        return cache.subject
+
+    def _vortexOnlineChanged(self, *args) -> None:
+        self._tellServerWeWantData(self._tupleSelectors())
+
+    @inlineCallbacks
+    def _receivePayload(self, payloadEnvelope: PayloadEnvelope, **kwargs):
+        # If this message is for a specific observer, and it's not us, then discard it.
+        filtObserverName = payloadEnvelope.filt.get("observerName")
+        if (
+            filtObserverName is not None
+            and filtObserverName != self._observerName
+        ):
+            return
+
+        # If this message is an error response, then don't process it.
+        if payloadEnvelope.result not in (None, True):
+            logger.error(
+                "Vortex responded with error : %s" % payloadEnvelope.result
+            )
+            logger.error(str(payloadEnvelope.filt))
+            return
+
+        tupleSelector: TupleSelector = payloadEnvelope.filt["tupleSelector"]
+
+        if not self._hasTupleSelector(tupleSelector):
+            return
+
+        cache, requiredUpdate = self._updateCache(payloadEnvelope)
+        if not requiredUpdate:
+            return
+
+        payload = yield Payload().fromEncodedPayloadDefer(cache.encodedPayload)
+        cache.subject.on_next(payload.tuples)
+
+    def _tellServerWeWantData(self, tupleSelectors: List[TupleSelector]):
+        for tupleSelector in tupleSelectors:
+            self._sendRequestToServer(
+                PayloadEnvelope(
+                    {"subscribe": True, "tupleSelector": tupleSelector}
+                )
+            )
+
+    def _sendRequestToServer(self, payload):
+        payload.filt.update(self._sendFilt)
+        d = VortexFactory.sendVortexMsg(
+            vortexMsgs=payload.toVortexMsg(),
+            destVortexName=self._destVortexName,
+        )
+        d.addErrback(vortexLogFailure, logger, consumeError=True)
+
+    def _sendUnsubscribeToServer(self, tupleSelector: TupleSelector):
+        payload = PayloadEnvelope()
+        payload.filt["tupleSelector"] = tupleSelector
+        payload.filt["unsubscribe"] = True
+        self._sendRequestToServer(payload)

vortex/handler/VortexJSTupleLoaderTestHandler.py

@@ -0,0 +1,42 @@
+# import logging
+#
+# from vortex.PayloadFilterKeys import plDeleteKey
+# from vortex.test.TestTuple import TestTuple
+# from vortex.test.TupleDataForTest import makeTestTupleData
+#
+# logger = logging.getLogger(__name__)
+#
+#
+# # If there were reallying going to a DB, we'd use OrmCrudHandler
+# class VortexJSTupleLoaderTestHandler(ModelHandler):
+#     def buildModel(self, payload, **kwargs):
+#         logger.debug("Received payload with %s tuples and filt=%s",
+#                      len(payload.tuples), payload.filt)
+#
+#         data = []
+#
+#         if payload.tuples:
+#             # Return nothing if this was a delete
+#             if plDeleteKey in payload.filt:
+#                 # Return nothing, it was deleted
+#                 pass
+#
+#             else:
+#                 # Else this was a save, just update some data and return id
+#                 data = payload.tuples
+#                 for testTuple in data:
+#                     testTuple.aInt = (testTuple.aInt if testTuple.aInt else 0) + 10
+#                     testTuple.aBoolTrue = not (testTuple.aBoolTrue
+#                                                if testTuple.aBoolTrue
+#                                                else 0)
+#
+#         else:
+#             # Else this is to get new data.
+#             data = makeTestTupleData()
+#
+#         return data
+#
+#
+# __handler = VortexJSTupleLoaderTestHandler({
+#     "key": "vortex.tuple-loader.test.data"
+# })

vortex/restful/GzippedDataHttpClient.py

@@ -0,0 +1,170 @@
+import gzip
+import logging
+import struct
+from datetime import datetime
+from io import BytesIO
+from typing import Union, List, Type
+
+import pytz
+from twisted.internet import reactor
+from twisted.internet.defer import Deferred, inlineCallbacks
+from twisted.internet.defer import succeed
+from twisted.internet.protocol import Protocol
+from twisted.web._newclient import ResponseDone
+from twisted.web.client import Agent, GzipDecoder, ContentDecoderAgent
+from twisted.web.http_headers import Headers
+from twisted.web.iweb import IBodyProducer, UNKNOWN_LENGTH
+from zope.interface import implementer
+
+from vortex.Tuple import Tuple
+from vortex.restful.RestfulResource import HTTP_REQUEST
+
+logger = logging.getLogger(__name__)
+
+
+class GzippedDataHttpClient:
+    """A Restful http client that sends out a binary payload"""
+
+    DEFAULT_HEADERS = {
+        "User-Agent": ["vortex/1.0"],
+        "Content-Type": ["application/json"],
+    }
+
+    def __init__(
+        self,
+        url: Union[str, bytes],
+        payload: bytes,
+        headers: dict[str, List[str]],
+        method: HTTP_REQUEST,
+        meta: Type[Tuple],
+        isPayloadGzipped: bool = True,
+        compressed: bool = True,
+        timeout: float = 10,
+    ):
+        self._httpMethod = method.value.encode()
+        self._url = url.encode() if isinstance(url, str) else url
+        self._payload = payload
+        self._headers = {**self.DEFAULT_HEADERS, **headers}
+        self._meta = meta
+        self._compressed = (
+            compressed  # send http requests whether gzip-compressed or not
+        )
+        self._isPayloadGzipped = isPayloadGzipped
+        self._timeout = timeout  # connect timeout
+
+    @inlineCallbacks
+    def run(self) -> Type[Tuple]:
+        agent = Agent(reactor, connectTimeout=self._timeout)
+        # Add the gzip decoder
+        if self._compressed:
+            agent = ContentDecoderAgent(agent, [(b"gzip", GzipDecoder)])
+
+        binaryPayloadRequestProducer = _BinaryPayloadRequestProducer(
+            self._payload,
+            self._meta,
+            self._isPayloadGzipped,
+        )
+        # Make the web request
+        try:
+            response = yield agent.request(
+                self._httpMethod,
+                self._url,
+                Headers(self._headers),
+                binaryPayloadRequestProducer,
+            )
+            self._meta = binaryPayloadRequestProducer.meta
+        except Exception as e:
+            self._meta.exceptions.append(e)
+        finally:
+            if self._meta.exceptions:
+                return self._meta
+
+        self._meta.code = response.code
+        self._meta.version = response.version
+        self._meta.headers = {
+            k.decode(): v[0].decode()
+            for k, v in response.headers.getAllRawHeaders()
+        }
+
+        self._meta = yield self._cbResponse(response, self._meta)
+        return self._meta
+
+    def _cbResponse(self, response, meta):
+        responseProducer = _ResponseProducer(meta)
+        response.deliverBody(responseProducer)
+        return responseProducer.asyncMeta
+
+
+@implementer(IBodyProducer)
+class _BinaryPayloadRequestProducer:
+    def __init__(
+        self,
+        payload: bytes,
+        meta: Type[Tuple],
+        isPayloadGzipped: bool = True,
+    ):
+        self._meta = meta
+        self._payload = payload
+        self._isGzippedAlready = isPayloadGzipped
+        self.length = self._getGzipUncompressedSize()
+        self._gzipFile = None
+
+    @property
+    def meta(self):
+        return self._meta
+
+    def startProducing(self, consumer):
+        self._meta.requestDate = datetime.now(tz=pytz.utc)
+
+        payload = self._payload
+        if self._isGzippedAlready:
+            payload = gzip.decompress(self._payload)
+
+        try:
+            consumer.write(payload)
+        except Exception as e:
+            raise e
+
+        return succeed(None)
+
+    def pauseProducing(self):
+        pass
+
+    def stopProducing(self):
+        pass
+
+    def _getGzipUncompressedSize(self) -> int:
+        # Uncompressed size is stored in the last 4 bytes of the gzip file.
+        # We can read the binary data and convert it to an int.
+        # (This will only work for files under 4GB)
+        if not self._isGzippedAlready:
+            return len(self._payload)
+
+        with BytesIO(self._payload) as f:
+            try:
+                return struct.unpack("I", self._payload[-4:])[0]
+            except Exception as e:
+                # may be invalid format of gzip
+                logger.exception(e)
+                return UNKNOWN_LENGTH
+
+
+class _ResponseProducer(Protocol):
+    def __init__(self, meta: Type[Tuple]):
+        self._finishedDeferred = Deferred()
+        self._writeSize = 0
+        self._body = b""
+        self._meta = meta
+
+    @property
+    def asyncMeta(self):
+        return self._finishedDeferred
+
+    def dataReceived(self, data: bytes):
+        self._body += data
+
+    def connectionLost(self, reason):
+        if isinstance(reason.value, ResponseDone):
+            self._meta.responseDate = datetime.now(tz=pytz.utc)
+            self._meta.body = self._body
+            self._finishedDeferred.callback(self._meta)

vortex/restful/GzippedDataHttpClientTest.py

@@ -0,0 +1,103 @@
+import gzip
+import logging
+from datetime import datetime
+from typing import List
+
+from twisted.trial import unittest
+from twisted.internet.error import TimeoutError as TwistedTimeoutError
+
+from vortex.Tuple import Tuple, TupleField, addTupleType
+from vortex.restful.GzippedDataHttpClient import GzippedDataHttpClient
+from vortex.restful.RestfulResource import HTTP_REQUEST
+
+logger = logging.getLogger(__name__)
+
+
+@addTupleType
+class GzippedDataHttpResponseTuple(Tuple):
+    __tupleType__ = "GzippedDataHttpResponseTuple"
+
+    requestDate: datetime = TupleField()
+    responseDate: datetime = TupleField()
+    code: int = TupleField(defaultValue=0)
+    version: tuple = TupleField()  # (b'HTTP', 1, 1)
+    headers: dict[str, str] = TupleField()
+    body: bytes = TupleField()
+    exceptions: List[Exception] = TupleField(defaultValue=[])
+
+
+class GzippedDataHttpClientTest(unittest.TestCase):
+    def setUp(self):
+        self.payload = gzip.compress(b"test")
+        self.headers = {}
+        self.meta = GzippedDataHttpResponseTuple()
+
+    def testHttpPost(self):
+        client = GzippedDataHttpClient(
+            url="http://httpbin.org/post",
+            payload=self.payload,
+            headers=self.headers,
+            method=HTTP_REQUEST.POST,
+            meta=GzippedDataHttpResponseTuple(),
+            isPayloadGzipped=True,
+            compressed=True,
+        )
+        d = client.run()
+        d.addCallback(self._assertValidResponse)
+        return d
+
+    def testHttpsPost(self):
+        client = GzippedDataHttpClient(
+            url="https://httpbin.org/post",
+            payload=self.payload,
+            headers=self.headers,
+            method=HTTP_REQUEST.POST,
+            meta=GzippedDataHttpResponseTuple(),
+            isPayloadGzipped=True,
+            compressed=True,
+        )
+        d = client.run()
+        d.addCallback(self._assertValidResponse)
+        return d
+
+    def testPlaintextDataPost(self):
+        payload = gzip.decompress(self.payload)
+        client = GzippedDataHttpClient(
+            url="http://httpbin.org/post",
+            payload=payload,
+            headers=self.headers,
+            method=HTTP_REQUEST.POST,
+            meta=GzippedDataHttpResponseTuple(),
+            isPayloadGzipped=False,
+            compressed=True,
+        )
+        d = client.run()
+        d.addCallback(self._assertValidResponse)
+        return d
+
+    def testConnectTimeout(self):
+        client = GzippedDataHttpClient(
+            url="http://example.com:19839",
+            payload=self.payload,
+            headers=self.headers,
+            method=HTTP_REQUEST.POST,
+            meta=GzippedDataHttpResponseTuple(),
+            isPayloadGzipped=False,
+            compressed=True,
+            timeout=1.0,
+        )
+        d = client.run()
+        d.addCallback(self._assertTimeoutException)
+        return d
+
+    def _assertValidResponse(self, tuple_: GzippedDataHttpResponseTuple):
+        self.assertEqual(tuple_.code, 200)
+        self.assertTrue(
+            (tuple_.responseDate - tuple_.requestDate).total_seconds() > 0
+        )
+
+    def _assertTimeoutException(self, tuple_: GzippedDataHttpResponseTuple):
+        booleans = [
+            isinstance(e, TwistedTimeoutError) for e in tuple_.exceptions
+        ]
+        self.assertTrue(any(booleans))