vortexpy 4.0.23__py3-none-any.whl

vortex/restful/GzippedPayloadHttpClient.py

@@ -0,0 +1,181 @@
+import gzip
+import logging
+import struct
+from datetime import datetime
+from io import BytesIO
+from typing import Union, List, Type
+
+import pytz
+from twisted.internet import reactor
+from twisted.internet.defer import Deferred, inlineCallbacks
+from twisted.internet.defer import succeed
+from twisted.internet.protocol import Protocol
+from twisted.web._newclient import ResponseDone
+from twisted.web.client import Agent, GzipDecoder, ContentDecoderAgent
+from twisted.web.http_headers import Headers
+from twisted.web.iweb import IBodyProducer, UNKNOWN_LENGTH
+from zope.interface import implementer
+
+from vortex.Tuple import Tuple
+from vortex.restful.RestfulResource import HTTP_REQUEST
+
+logger = logging.getLogger(__name__)
+
+
+class GzippedPayloadHttpClient:
+    """A Restful http client that sends out a binary payload"""
+
+    DEFAULT_HEADERS = {
+        "User-Agent": ["synerty/1.0"],
+        "Content-Type": ["application/json"],
+    }
+
+    def __init__(
+        self,
+        url: Union[str, bytes],
+        payload: bytes,
+        headers: dict[str, List[str]],
+        method: HTTP_REQUEST,
+        meta: Type[Tuple],
+        isPayloadGzipped: bool = True,
+        compressed: bool = True,
+        timeout: float = 10,
+    ):
+        self._httpMethod = method.value.encode()
+        self._url = url.encode() if isinstance(url, str) else url
+        self._payload = payload
+        self._headers = {**self.DEFAULT_HEADERS, **headers}
+        self._meta = meta
+        self._compressed = (
+            compressed  # send http requests whether gzip-compressed or not
+        )
+        self._isPayloadGzipped = isPayloadGzipped
+        self._timeout = timeout  # connect timeout
+
+    @inlineCallbacks
+    def run(self) -> Type[Tuple]:
+        agent = Agent(reactor, connectTimeout=self._timeout)
+        # Add the gzip decoder
+        if self._compressed:
+            agent = ContentDecoderAgent(agent, [(b"gzip", GzipDecoder)])
+
+        binaryPayloadRequestProducer = _BinaryPayloadRequestProducer(
+            self._payload,
+            self._meta,
+            self._isPayloadGzipped,
+        )
+        # Make the web request
+        response = yield agent.request(
+            self._httpMethod,
+            self._url,
+            Headers(self._headers),
+            binaryPayloadRequestProducer,
+        )
+
+        self._meta = binaryPayloadRequestProducer.meta
+        self._meta.code = response.code
+        self._meta.version = response.version
+        self._meta.headers = {
+            k.decode(): v[0].decode()
+            for k, v in response.headers.getAllRawHeaders()
+        }
+
+        # Get the responseTuple data
+        responseProducer = self._cbResponse(response, self._meta)
+
+        self._meta = responseProducer.meta
+        return self._meta
+        # return GzippedPayloadHttpResponse(
+        #     code=response.code,
+        #     version=response.version,
+        #     headers={
+        #         k.decode(): v[0].decode()
+        #         for k, v in response.headers.getAllRawHeaders()
+        #     },
+        #     body=responseProducer.asyncData,
+        #     requestDate=meta.requestDate,
+        #     responseDate=meta.responseDate,
+        # )
+
+    def _cbResponse(self, response, meta):
+        responseProducer = _ResponseProducer(meta)
+        response.deliverBody(responseProducer)
+        return responseProducer
+
+
+@implementer(IBodyProducer)
+class _BinaryPayloadRequestProducer:
+    def __init__(
+        self,
+        payload: bytes,
+        meta: Type[Tuple],
+        isPayloadGzipped: bool = True,
+    ):
+        self._meta = meta
+        self._payload = payload
+        self._isGzippedAlready = isPayloadGzipped
+        self.length = self._getGzipUncompressedSize()
+        self._gzipFile = None
+
+    @property
+    def meta(self):
+        return self._meta
+
+    def startProducing(self, consumer):
+        self._meta.requestDate = datetime.now(tz=pytz.utc)
+
+        if self._isGzippedAlready:
+            try:
+                decompressed = gzip.decompress(self._payload)
+                logger.info(f"declared payload length: {self.length}")
+                logger.info(f"decompressed payload length: {len(decompressed)}")
+                consumer.write(decompressed)
+            except Exception as e:
+                raise e
+        else:
+            # uncompressed payload
+            consumer.write(self._payload)
+        return succeed(None)
+
+    def pauseProducing(self):
+        pass
+
+    def stopProducing(self):
+        pass
+
+    def _getGzipUncompressedSize(self) -> int:
+        # Uncompressed size is stored in the last 4 bytes of the gzip file.
+        # We can read the binary data and convert it to an int.
+        # (This will only work for files under 4GB)
+        with BytesIO(self._payload) as f:
+            try:
+                return struct.unpack("I", self._payload[-4:])[0]
+            except Exception as e:
+                # may be invalid format of gzip
+                logger.exception(e)
+                return UNKNOWN_LENGTH
+
+
+class _ResponseProducer(Protocol):
+    def __init__(self, meta: Type[Tuple]):
+        self._finishedDeferred = Deferred()
+        self._writeSize = 0
+        self._body = b""
+        self._meta = meta
+
+    @property
+    def asyncData(self):
+        return self._finishedDeferred
+
+    @property
+    def meta(self):
+        return self._meta
+
+    def dataReceived(self, data: bytes):
+        logger.debug(bytes)
+        self._body += data
+
+    def connectionLost(self, reason):
+        if isinstance(reason.value, ResponseDone):
+            self._meta.responseDate = datetime.now(tz=pytz.utc)
+            self._meta.body = self._body

vortex/restful/RestfulHttpClient.py

@@ -0,0 +1,107 @@
+import json
+import logging
+from typing import Union, Type, runtime_checkable
+
+from twisted.internet import reactor
+from twisted.internet.defer import Deferred, inlineCallbacks
+from twisted.internet.protocol import Protocol
+from twisted.web._newclient import ResponseDone
+from twisted.web.client import Agent, GzipDecoder, ContentDecoderAgent
+from twisted.web.http_headers import Headers
+from zope.interface import implementer
+from twisted.internet.defer import succeed
+from twisted.web.iweb import IBodyProducer
+
+from vortex.Tuple import Tuple
+from vortex.restful.RestfulResource import HTTP_REQUEST
+
+logger = logging.getLogger(__name__)
+
+
+class RestfulHttpClient:
+    def __init__(
+        self,
+        url: Union[str, bytes],
+        method: HTTP_REQUEST,
+        postTuple: Tuple,
+        ResponseTuple: Type[Tuple],
+    ):
+        self._httpMethod = method.value.encode()
+        self._url = url.encode() if isinstance(url, str) else url
+        self._postTuple = postTuple
+        self._ResponseTuple = ResponseTuple
+
+    @inlineCallbacks
+    def run(self):
+        # convert tuple to json
+        body = self._postTuple.tupleToRestfulJsonDict()
+        body = json.dumps(body).encode("utf-8")
+
+        # add http headers
+        headers = Headers(
+            {
+                "User-Agent": ["synerty/1.0"],
+                "Content-Type": ["application/json"],
+            }
+        )
+
+        # Add the gzip decoder
+        agent = ContentDecoderAgent(Agent(reactor), [(b"gzip", GzipDecoder)])
+
+        # Make the web request
+        response = yield agent.request(
+            self._httpMethod, self._url, headers, _BytesProducer(body)
+        )
+
+        # Get the response data
+        responseData = yield self._cbResponse(response)
+
+        # Convert the bytes into a tuple and return
+        return self._parseTuple(responseData)
+
+    def _cbResponse(self, response):
+        bodyDownloader = _RestfulBody()
+        response.deliverBody(bodyDownloader)
+        return bodyDownloader.asyncData
+
+    def _parseTuple(self, bytes_):
+        json_ = json.loads(bytes_.decode("utf-8"))
+        return Tuple.restfulJsonDictToTupleWithValidation(
+            json_, self._ResponseTuple
+        )
+
+
+@implementer(IBodyProducer)
+class _BytesProducer:
+    def __init__(self, body):
+        self.body = body
+        self.length = len(body)
+
+    def startProducing(self, consumer):
+        consumer.write(self.body)
+        return succeed(None)
+
+    def pauseProducing(self):
+        pass
+
+    def stopProducing(self):
+        pass
+
+
+class _RestfulBody(Protocol):
+    def __init__(self):
+        self._finishedDeferred = Deferred()
+        self._writeSize = 0
+        self._body = b""
+
+    @property
+    def asyncData(self):
+        return self._finishedDeferred
+
+    def dataReceived(self, data: bytes):
+        logger.debug(bytes)
+        self._body += data
+
+    def connectionLost(self, reason):
+        if isinstance(reason.value, ResponseDone):
+            self._finishedDeferred.callback(self._body)

vortex/restful/RestfulHttpClientTest.py

@@ -0,0 +1,60 @@
+import json
+import logging
+from typing import Annotated, List
+
+from twisted.trial import unittest
+
+from vortex.Tuple import (
+    Tuple,
+    TupleField,
+    addTupleType,
+    IntTupleFieldValidator,
+    TupleFieldValidatorABC,
+)
+from vortex.restful.RestfulHttpClient import RestfulHttpClient
+from vortex.restful.RestfulResource import HTTP_REQUEST
+
+logger = logging.getLogger(__name__)
+
+
+@addTupleType
+class TestTuple(Tuple):
+    __tupleType__ = "TestTuple"
+    id: int = TupleField()
+    count: Annotated[int, IntTupleFieldValidator(0, 100)] = TupleField()
+    text: str = TupleField()
+
+
+@addTupleType
+class ResponseTuple(Tuple):
+    __tupleType__ = "ResponseTuple"
+    data: str = TupleField(comment="request in string")
+    json: TestTuple = TupleField(comment="request converted to jsonDict")
+
+
+class RestfulClientTest(unittest.TestCase):
+    def setUp(self):
+        self.tuple = TestTuple()
+        self.tuple.id = 10
+        self.tuple.count = 50
+        self.tuple.text = "test"
+
+    def testHttpPost(self):
+        client = RestfulHttpClient(
+            "http://httpbin.org/anything",
+            method=HTTP_REQUEST.POST,
+            postTuple=self.tuple,
+            ResponseTuple=ResponseTuple,
+        )
+        d = client.run()
+        d.addCallback(self._checkPostJSON)
+        # d.addCallback(self.print)
+        return d
+
+    def _checkPostJSON(self, tuple_):
+        self.assertTrue(isinstance(tuple_, ResponseTuple))
+        self.assertEqual(
+            tuple_.data, json.dumps(self.tuple.tupleToRestfulJsonDict())
+        )
+        self.assertEqual(tuple_.json.id, self.tuple.id)
+        self.assertEqual(tuple_.json.text, self.tuple.text)

vortex/restful/RestfulResource.py

@@ -0,0 +1,153 @@
+import json
+import logging
+from enum import Enum
+from typing import Callable
+from typing import List
+
+from twisted.internet.defer import inlineCallbacks
+from twisted.python.compat import nativeString
+from twisted.web.server import NOT_DONE_YET
+from txhttputil.site.BasicResource import BasicResource
+from vortex import Tuple
+from vortex.DeferUtil import deferToThreadWrapWithLogger
+from vortex.DeferUtil import vortexLogFailure
+
+logger = logging.getLogger(__name__)
+
+
+class HTTP_REQUEST(Enum):
+    GET = "GET"
+    POST = "POST"
+    PUT = "PUT"
+    DELETE = "DELETE"
+
+
+class _JsonResource(BasicResource):
+    isLeaf = True
+    isGzipped = True
+
+    def __init__(self, handler: Callable, TupleClass: Tuple):
+        BasicResource.__init__(self)
+
+        self._handler = handler
+        self._tupleClass = TupleClass
+
+    @deferToThreadWrapWithLogger(logger)
+    def _requestToTuple(self, request) -> Tuple:
+        req = json.load(request.content)
+        tuple_ = self._tupleClass()
+        tuple_.fromJsonDict(req)
+        return tuple_
+
+    @deferToThreadWrapWithLogger(logger)
+    def _tupleToResponse(self, tuple_: Tuple) -> bytes:
+        dict_ = tuple_.tupleToRestfulJsonDict()
+        json_ = json.dumps(dict_)
+        return json_.encode()
+
+    def _writeSuccessResponse(self, response: bytes, request):
+        request.setResponseCode(200)
+        request.setHeader("Content-Type", "application/json")
+        request.write(response)
+        request.finish()
+
+    def _writeErrorResponse(self, request):
+        request.setResponseCode(500)
+        request.setHeader("Content-Type", "application/json")
+        request.write(b'{"error":"internal error"}')
+        request.finish()
+
+    @inlineCallbacks
+    def _renderAsync(self, request):
+        tupleIn = None
+        try:
+            tupleIn = yield self._requestToTuple(request)
+            tupleOut = yield self._handler(tupleIn)
+            response = yield self._tupleToResponse(tupleOut)
+            self._writeSuccessResponse(response, request)
+
+        except Exception as e:
+            self._writeErrorResponse(request)
+            logger.debug(
+                f"Error while processing REST Tuple\n{tupleIn}\n{request.path}"
+            )
+            logger.exception(e)
+
+    def render(self, request):
+        d = self._renderAsync(request)
+        d.addErrback(vortexLogFailure, logger=logger, consumeError=True)
+
+        return NOT_DONE_YET
+
+
+class ErrorJsonResource(BasicResource):
+    isLeaf = True
+    isGzipped = True
+
+    def __init__(self, errorCode: int):
+        self._errorCode = errorCode
+
+    def render(self, request):
+        request.setResponseCode(self._errorCode)
+        return b""
+
+
+class PluginRestfulResource(BasicResource):
+    def __init__(self):
+        BasicResource.__init__(self)
+        self._registeredMethods = {}
+
+    def registerMethod(
+        self,
+        handlerFunction: Callable,
+        TupleClass: Tuple,
+        urlPrefix: bytes,
+        registeredRequestMethod: List[HTTP_REQUEST],
+    ):
+        if urlPrefix in self._registeredMethods:
+            raise ValueError(f'Route "{urlPrefix}" already exists')
+        if not callable(handlerFunction):
+            raise TypeError(f'"{str(handlerFunction)}" is not a callable')
+
+        # register handler as partial function (poor man's Template<type>)
+        # as callback function whose input is an instance of a Tuple
+        self._registeredMethods[urlPrefix] = {
+            "handler": handlerFunction,
+            "allowedMethods": set(
+                [method.value for method in registeredRequestMethod]
+            ),
+            "tupleClass": TupleClass,
+        }
+
+    def getChild(self, path, request):
+        """Get requests routed to an handler
+
+        Routes to a registered handler to process the request.
+        This function routes resources to handlers which eventually returns
+        JSON response resource in json. Everything returned here should be
+        final.
+
+        Once url prefix for a resource is matched, it checks the
+        request method is allowed. If granted, it delegates the request to the
+        registered handler to process the request. The handler should convert
+        request to a Tuple and should respond a tuple.
+
+        :param path: route url for a resource, with plugin name prefix stripped
+        :param request: twisted.web.http.Request
+        :return: twisted.web.resource.Resource
+        """
+        # route check
+        if path not in self._registeredMethods.keys():
+            return ErrorJsonResource(404)
+
+        # request method check
+        requestMethod = nativeString(request.method)
+        if requestMethod not in self._registeredMethods[path]["allowedMethods"]:
+            return ErrorJsonResource(403)
+
+        # invoke resource handler
+        TupleClass = self._registeredMethods[path]["tupleClass"]
+
+        handler = self._registeredMethods[path]["handler"]
+
+        return _JsonResource(handler, TupleClass)

vortex/restful/RestfulResourceTest.py

@@ -0,0 +1,100 @@
+import io
+import json
+import logging
+
+from vortex.restful.RestfulResource import HTTP_REQUEST
+from vortex.restful.RestfulResource import PluginRestfulResource
+from twisted.trial import unittest
+from twisted.web.test.requesthelper import DummyRequest
+from vortex.Tuple import Tuple
+from vortex.Tuple import TupleField
+from vortex.Tuple import addTupleType
+
+logger = logging.getLogger(__name__)
+
+
+@addTupleType
+class DummyTuple(Tuple):
+    __tupleType__ = "DummyTuple"
+
+    id = TupleField()
+    text = TupleField()
+
+
+class TestHandlerController:
+    def testMethod(self, tuple_: DummyTuple):
+        if hasattr(tuple_, "tupleToRestfulJsonDict"):
+            # vortexpy > 2.1.3
+            return tuple_.tupleToRestfulJsonDict()
+        else:
+            return tuple_.tupleToSmallJsonDict()
+
+
+class RestfulServerTest(unittest.TestCase):
+    def setUp(self):
+        handlerController = TestHandlerController()
+        self.pluginAResource = PluginRestfulResource()
+        self.pluginAResource.registerMethod(
+            handlerController.testMethod,
+            DummyTuple,
+            b"test",
+            [HTTP_REQUEST.GET, HTTP_REQUEST.POST],
+        )
+        self.pluginAResource.registerMethod(
+            handlerController.testMethod,
+            DummyTuple,
+            b"test/test",
+            [HTTP_REQUEST.GET, HTTP_REQUEST.POST],
+        )
+        self.pluginAResource.registerMethod(
+            handlerController.testMethod,
+            DummyTuple,
+            b"test/test/test",
+            [HTTP_REQUEST.GET, HTTP_REQUEST.POST],
+        )
+
+    def _dictToBytes(self, dictionary: dict) -> bytes:
+        return bytes(json.dumps(dictionary), "utf-8")
+
+    def _bytesToDict(self, bytes_: bytes) -> dict:
+        string = bytes_[0].decode("utf-8")
+        return json.loads(string)
+
+    def _check(self, expected: dict, actual: dict) -> bool:
+        # expected dict should be equal or a subnet of actual dict
+        return expected.items() <= actual.items()
+
+    def testValidJsonRequest(self):
+        requestDict = {"id": 1, "text": "text"}
+
+        request = DummyRequest([])
+        request.content = io.BytesIO(self._dictToBytes(requestDict))
+
+        jsonResource = self.pluginAResource.getChild(b"test", request)
+
+        jsonResource.render(request)
+        self.assertEqual(request.responseCode, 200)
+        responseDict = self._bytesToDict(request.written)
+        self.assertTrue(self._check(requestDict, responseDict))
+
+    def testInvalidJsonRequest(self):
+        request = DummyRequest([])
+        request.content = io.BytesIO(b"##invalid json}")
+
+        jsonResource = self.pluginAResource.getChild(b"test", request)
+        jsonResource.render(request)
+        self.assertEqual(request.responseCode, 500)
+
+    def testPath(self):
+        requestDict = {"id": 1, "text": "text"}
+
+        for path in [b"test", b"test/test", b"test/test/test"]:
+            request = DummyRequest([])
+            request.content = io.BytesIO(self._dictToBytes(requestDict))
+
+            jsonResource = self.pluginAResource.getChild(path, request)
+
+            jsonResource.render(request)
+            self.assertEqual(request.responseCode, 200)
+            responseDict = self._bytesToDict(request.written)
+            self.assertTrue(self._check(requestDict, responseDict))