txhttputil 1.4.9__py3-none-any.whl

txhttputil/site/FileUnderlayResource.py

@@ -0,0 +1,188 @@
+# Copyright (c) 2016-2026 Flying Ion Pty Ltd
+#
+# This software is open source; the MIT license applies.
+# See LICENSE for the full license text.
+#
+# Website : https://flyingion.com
+# Support : contact@flyingion.com
+
+import logging
+import os
+from typing import Optional, Union
+
+from twisted.web.resource import NoResource
+from twisted.web.util import Redirect
+from txhttputil.site.BasicResource import BasicResource
+from txhttputil.site.RedirectionRule import RedirectionRule
+
+logger = logging.getLogger(__name__)
+
+import mimetypes
+
+mimetypes.init()
+
+
+def get_extensions_for_type(general_type):
+    for ext in mimetypes.types_map:
+        if mimetypes.types_map[ext].split("/")[0] == general_type:
+            yield ext
+
+
+IMAGE_EXTENSIONS = list(get_extensions_for_type("image"))
+FONT_EXTENSIONS = list(get_extensions_for_type("font"))
+
+
+class SinglePageAppConfig:
+    def __init__(self, **kwargs):
+        self.indexHtmlPath = None
+
+        for kw, arg in kwargs.items():
+            if not hasattr(self, kw):
+                raise KeyError(
+                    "%s is not an argument for %s" % (kw, self.__class__)
+                )
+            setattr(self, kw, arg)
+
+
+class FileUnderlayResource(BasicResource):
+    """
+    This class resolves URLs into either a static file or a C{BasicResource}
+
+    This is a multi level search :
+    1) getChild, looking for resource in the resource tree
+    2) The staticFileUnderlay is searched.
+    3) Request fails with NoResource()
+
+    """
+
+    acceptedExtensions = [".js", ".css", ".html", ".xml"]
+    acceptedExtensions += FONT_EXTENSIONS
+    acceptedExtensions += IMAGE_EXTENSIONS
+
+    # TODO Implement accepted extensions
+
+    def __init__(self, allowedIpsList=None):
+        BasicResource.__init__(self, allowedIpsList)
+
+        self._fileSystemRoots = []
+        self._singlePageAppConfig = None
+        self._redirections = {}
+
+    def addRedirectionRule(self, redirection: Optional[RedirectionRule] = None):
+        if redirection is not None:
+            self._redirections[redirection.origin] = redirection.destination
+
+    def getFileResource(self, postPath):
+        from txhttputil.site.StaticFileResource import StaticFileResource
+
+        resourcePath = os.path.join(*postPath).decode() if postPath else None
+        filePath = self.getRealFilePath(resourcePath) if resourcePath else None
+
+        if filePath:
+            return self._gzipIfRequired(StaticFileResource(filePath))
+
+        if not resourcePath or "." not in os.path.basename(resourcePath):
+            singlePageResource = self._getSinglePageAppResource()
+            if singlePageResource:
+                return singlePageResource
+
+        return NoResource()
+
+    def addFileSystemRoot(self, fileSystemRoot: str):
+        if not os.path.isdir(fileSystemRoot):
+            raise NotADirectoryError("%s is not a directory" % fileSystemRoot)
+
+        self._fileSystemRoots.append(fileSystemRoot)
+
+    def enableSinglePageApplication(self, indexHtmlPath="index.html"):
+        self._singlePageAppConfig = SinglePageAppConfig(
+            indexHtmlPath=indexHtmlPath
+        )
+
+    def _resolveFilePathUnderRoot(
+        self, rootDir: str, resourcePath: str
+    ) -> Union[str, None]:
+        if os.path.isabs(resourcePath):
+            logger.debug(
+                "Resource path %s is absolute and not allowed",
+                resourcePath,
+            )
+            return None
+
+        rootRealPath = os.path.realpath(rootDir)
+        candidatePath = os.path.realpath(os.path.join(rootDir, resourcePath))
+
+        try:
+            withinRoot = (
+                os.path.commonpath([rootRealPath, candidatePath])
+                == rootRealPath
+            )
+        except ValueError:
+            withinRoot = False
+
+        if not withinRoot:
+            logger.debug(
+                "Resource path %s escapes root %s",
+                resourcePath,
+                rootDir,
+            )
+            return None
+
+        return candidatePath
+
+    def getRealFilePath(self, resourcePath: str) -> Union[str, None]:
+        if not resourcePath:
+            return None
+
+        for rootDir in self._fileSystemRoots[::-1]:
+            realFilePath = self._resolveFilePathUnderRoot(rootDir, resourcePath)
+            if realFilePath is None:
+                continue
+
+            if os.path.isdir(realFilePath):
+                logger.debug(
+                    "Resource path %s is a directory %s",
+                    resourcePath,
+                    realFilePath,
+                )
+                continue
+
+            if os.path.isfile(realFilePath):
+                return realFilePath
+
+        return None
+
+    def _getSinglePageAppResource(self):
+        if not self._singlePageAppConfig:
+            return None
+
+        # Try to serve a single page app
+        indexHtmlPath = self._singlePageAppConfig.indexHtmlPath
+        assert indexHtmlPath is not None, "indexHtmlPath must be set for SPA"
+        filePath = self.getRealFilePath(indexHtmlPath)
+        if filePath:
+            from txhttputil.site.StaticFileResource import StaticFileResource
+
+            return self._gzipIfRequired(StaticFileResource(filePath))
+
+        return None
+
+    def render_GET(self, request):
+        """Render Get
+
+        If we're being redered then that means we've exactly matched the resurce path.
+        IE, The path is looking for a directory, not a file.
+
+        """
+
+        singlePageAppResource = self._getSinglePageAppResource()
+        if singlePageAppResource:
+            return singlePageAppResource.render_GET(request)  # pyright: ignore[reportAttributeAccessIssue]
+
+        # Else, render no resource
+        return NoResource().render(request)
+
+    def getChild(self, path, request):
+        if request.uri in self._redirections.keys():
+            return Redirect(self._redirections[request.uri])
+        return super().getChild(path, request)

txhttputil/site/FileUnderlayResourceTest.py

@@ -0,0 +1,108 @@
+# Copyright (c) 2016-2026 Flying Ion Pty Ltd
+#
+# This software is open source; the MIT license applies.
+# See LICENSE for the full license text.
+#
+# Website : https://flyingion.com
+# Support : contact@flyingion.com
+
+import logging
+import os
+
+from twisted.internet import reactor
+from twisted.trial import unittest
+from twisted.web import server
+from twisted.web.resource import ErrorPage
+from txhttputil.downloader.HttpFileDownloader import HttpFileDownloader
+from txhttputil.site.FileUnderlayResource import FileUnderlayResource
+
+import txhttputil
+
+logger = logging.getLogger(__name__)
+
+
+LOG_FORMAT = "%(asctime)s %(levelname)s %(name)s:%(message)s"
+DATE_FORMAT = "%d-%b-%Y %H:%M:%S"
+logging.basicConfig(format=LOG_FORMAT, datefmt=DATE_FORMAT, level=logging.DEBUG)
+
+
+class FileUnderlayResourceTest(unittest.TestCase):
+    PORT = 7999
+
+    def setUp(self):
+        rootResource = FileUnderlayResource()
+
+        self.fsRoot = os.path.dirname(txhttputil.__file__)
+        rootResource.addFileSystemRoot(self.fsRoot)
+
+        logger.info("Test file system root = %s", self.fsRoot)
+
+        rootResource.putChild(
+            b"test", ErrorPage(200, "This path worked, /test", "")
+        )
+
+        self.site = reactor.listenTCP(self.PORT, server.Site(rootResource))
+
+        self.sitePath = "http://%s:%s" % ("127.0.0.1", self.site.port)
+
+    def tearDown(self):
+        d = self.site.stopListening()
+        return d
+
+    def _checkDownloadedResource(
+        self, spooledNamedTempFile, fileName=None, testContents=None
+    ):
+        contents = spooledNamedTempFile.read()
+        logger.debug(contents)
+
+        if fileName:
+            with open(fileName, "rb") as f:
+                self.assertEqual(contents, f.read())
+
+        if testContents:
+            self.assertEqual(contents, testContents)
+
+    def testDowloadUnderlayRootFile(self):
+        testContents = (
+            b"\n<html>\n  <head><title>200 - This path worked, /test</title><"
+            b"/head>\n  <body>\n    <h1>This path worked, /test</h1>\n    "
+            b"<p></p>\n  </body>\n</html>\n"
+        )
+
+        d = HttpFileDownloader(self.sitePath + "/test").run()
+        d.addCallback(self._checkDownloadedResource, testContents=testContents)
+        return d
+
+    def testDowloadUnderlayFile2(self):
+        fileName = "/login_page/LoginElement.py"
+        realPath = self.fsRoot + fileName
+
+        d = HttpFileDownloader(self.sitePath + fileName).run()
+        d.addCallback(self._checkDownloadedResource, fileName=realPath)
+        return d
+
+    def testDirectoryTraversalBlocked(self):
+        traversalPath = "/".join([".."] * 10) + "/etc/passwd"
+        d = HttpFileDownloader(self.sitePath + "/" + traversalPath).run()
+
+        def assertNotSuccessful(_result):
+            self.fail("Directory traversal request must not succeed")
+
+        def assertNotFound(failure):
+            self.assertIn("404", str(failure.value))
+
+        d.addCallbacks(
+            callback=assertNotSuccessful,
+            errback=assertNotFound,
+        )
+        return d
+
+
+if __name__ == "__main__":
+    from twisted.trial import runner, reporter
+    import sys
+
+    trialRunner = runner.TrialRunner(reporter.VerboseTextReporter)
+    suite = runner.TestLoader().loadClass(FileUnderlayResourceTest)
+    result = trialRunner.run(suite)
+    sys.exit(not result.wasSuccessful())

txhttputil/site/FileUploadRequest.py

@@ -0,0 +1,250 @@
+# Copyright (c) 2016-2026 Flying Ion Pty Ltd
+#
+# This software is open source; the MIT license applies.
+# See LICENSE for the full license text.
+#
+# Website : https://flyingion.com
+# Support : contact@flyingion.com
+
+import logging
+import os
+from urllib.parse import parse_qs
+
+from pytmpdir.spooled_named_temporary_file import (
+    SpooledNamedTemporaryFile,
+)
+
+logger = logging.getLogger(name=__name__)
+
+import time
+import cgi
+from cgi import valid_boundary, maxlen
+
+from twisted.web import server
+
+
+class FileUploadRequest(server.Request):
+    """File Upload Request
+
+    This request handlers supports the upload of large files.
+
+    """
+
+    # max amount of memory to allow any ~single~ request argument [ie: POSTed file]
+    # to take up before being flushed into a temporary file.
+    # eg:   50 users uploading 4 large files could use up to [and in excess of]
+    #       200 times value specified below.
+    # note: this value seems to be taken with a grain of salt, memory usage may spike
+    #       FAR above this value in some cases.
+    #       eg: set the memory limit to 5 MB, write 2 blocks of 4MB, mem usage will
+    #           have spiked to 8MB before the data is rolled to disk after the
+    #           second write completes.
+    memorylimit = 1024 * 1024 * 1
+
+    # enable/disable debug logging
+    do_log = False
+
+    # re-defined only for debug/logging purposes
+    def gotLength(self, length):
+        if self.do_log:
+            logger.debug(
+                "%f Headers received, Content-Length: %d", time.time(), length
+            )
+
+        return server.Request.gotLength(self, length)
+
+    # re-definition of twisted.web.server.Request.requestrecieved, the only difference
+    # is that self.parse_multipart() is used rather than cgi.parse_multipart()
+    def requestReceived(self, command, path, version):
+        """
+        Called by channel when all data has been received.
+
+        This method is not intended for users.
+
+        @type command: C{bytes}
+        @param command: The HTTP verb of this request.  This has the case
+            supplied by the client (eg, it maybe "get" rather than "GET").
+
+        @type path: C{bytes}
+        @param path: The URI of this request.
+
+        @type version: C{bytes}
+        @param version: The HTTP version of this request.
+        """
+        clength = self.content.tell()
+        self.content.seek(0, 0)
+        self.args = {}
+
+        self.method, self.uri = command, path
+        self.clientproto = version
+        x = self.uri.split(b"?", 1)
+
+        if len(x) == 1:
+            self.path = self.uri
+        else:
+            self.path, argstring = x
+            self.args = parse_qs(argstring, 1)
+
+        # Argument processing
+        args = self.args
+        ctype = self.requestHeaders.getRawHeaders(b"content-type")
+        if ctype is not None:
+            ctype = ctype[0]
+
+        if self.method == b"POST" and ctype and clength:
+            mfd = b"multipart/form-data"
+            key, pdict = self._parseHeader(ctype)
+            # This weird CONTENT-LENGTH param is required by
+            # cgi.parse_multipart() in some versions of Python 3.7+, see
+            # bpo-29979. It looks like this will be relaxed and backported, see
+            # https://github.com/python/cpython/pull/8530.
+            pdict["CONTENT-LENGTH"] = clength
+            if key == b"application/x-www-form-urlencoded":
+                args.update(parse_qs(self.content.read(), 1))
+            elif key == mfd:
+                try:
+                    if b"useLargeRequest" in args:
+                        self.content.seek(0, 0)
+                        cgiArgs = self.parse_multipart(self.content, pdict)
+
+                    else:
+                        cgiArgs = cgi.parse_multipart(self.content, pdict)
+
+                    self.args.update(
+                        {x.encode("iso-8859-1"): y for x, y in cgiArgs.items()}
+                    )
+
+                except Exception as e:
+                    # It was a bad request, or we got a signal.
+                    self.channel._respondToBadRequestAndDisconnect()
+                    if isinstance(e, (TypeError, ValueError, KeyError)):
+                        return
+                    # If it's not a userspace error from CGI, reraise
+                    raise
+
+            self.content.seek(0, 0)
+
+        self.process()
+
+    # re-definition of cgi.parse_multipart that uses a single temporary file to store
+    # data rather than storing 2 to 3 copies in various lists.
+    def parse_multipart(self, fp, pdict):
+        """Parse multipart input.
+
+        Arguments:
+        fp   : input file
+        pdict: dictionary containing other parameters of content-type header
+
+        Returns a dictionary just like parse_qs(): keys are the field names, each
+        value is a list of values for that field.  This is easy to use but not
+        much good if you are expecting megabytes to be uploaded -- in that case,
+        use the FieldStorage class instead which is much more flexible.  Note
+        that content-type is the raw, unparsed contents of the content-type
+        header.
+
+        XXX This does not parse nested multipart parts -- use FieldStorage for
+        that.
+
+        XXX This should really be subsumed by FieldStorage altogether -- no
+        point in having two implementations of the same parsing algorithm.
+        Also, FieldStorage protects itself better against certain DoS attacks
+        by limiting the size of the data read in one chunk.  The API here
+        does not support that kind of protection.  This also affects parse()
+        since it can call parse_multipart().
+
+        """
+        import http.client
+
+        boundary = b""
+        if "boundary" in pdict:
+            boundary = pdict["boundary"]
+        if not valid_boundary(boundary):
+            raise ValueError(
+                "Invalid boundary in multipart form: %r" % (boundary,)
+            )
+
+        nextpart = b"--" + boundary
+        lastpart = b"--" + boundary + b"--"
+        partdict = {}
+        terminator = b""
+
+        while terminator != lastpart:
+            bytes = -1
+            data = SpooledNamedTemporaryFile(max_size=self.memorylimit)
+            if terminator:
+                # At start of next part.  Read headers first.
+                headers = http.client.parse_headers(fp)
+                clength = headers.get("content-length")
+                if clength:
+                    try:
+                        bytes = int(clength)
+                    except ValueError:
+                        pass
+                if bytes > 0:
+                    if maxlen and bytes > maxlen:
+                        raise ValueError("Maximum content length exceeded")
+                    data.write(fp.read(bytes))
+            # Read lines until end of part.
+            while 1:
+                line = fp.readline()
+                if not line:
+                    terminator = lastpart  # End outer loop
+                    break
+                if line.startswith(b"--"):
+                    terminator = line.rstrip()
+                    if terminator in (nextpart, lastpart):
+                        break
+                data.write(line)
+            # Done with part.
+            if data.tell() == 0:
+                continue
+            if bytes < 0:
+                # if a Content-Length header was not supplied with the MIME part
+                # then the trailing line break must be removed.
+                # we have data, read the last 2 bytes
+                rewind = min(2, data.tell())
+                data.seek(-rewind, os.SEEK_END)
+                line = data.read(2)
+                if line[-2:] == b"\r\n":
+                    data.seek(-2, os.SEEK_END)
+                    data.truncate()
+                elif line[-1:] == b"\n":
+                    data.seek(-1, os.SEEK_END)
+                    data.truncate()
+
+            line = headers["content-disposition"]
+            if not line:
+                continue
+            key, params = cgi.parse_header(line)
+            if key != "form-data":
+                continue
+            if "name" in params:
+                name = params["name"]
+                # kludge in the filename
+                if "filename" in params:
+                    fname_index = name + "_filename"
+                    if fname_index in partdict:
+                        partdict[fname_index].append(params["filename"])
+                    else:
+                        partdict[fname_index] = [params["filename"]]
+            else:
+                continue
+            if name in partdict:
+                data.seek(0, 0)
+                partdict[name].append(data)
+            else:
+                data.seek(0, 0)
+                partdict[name] = [data]
+
+        fp.seek(rewind)  # restore cursor
+        return partdict
+
+    def _parseHeader(self, line):
+        # cgi.parse_header requires a str
+        key, pdict = cgi.parse_header(line.decode("charmap"))
+
+        # We want the key as bytes, and cgi.parse_multipart (which consumes
+        # pdict) expects a dict of str keys but bytes values
+        key = key.encode("charmap")
+        pdict = {x: y.encode("charmap") for x, y in pdict.items()}
+        return (key, pdict)

txhttputil/site/RedirectToHttpsResource.py

@@ -0,0 +1,43 @@
+# Copyright (c) 2016-2026 Flying Ion Pty Ltd
+#
+# This software is open source; the MIT license applies.
+# See LICENSE for the full license text.
+#
+# Website : https://flyingion.com
+# Support : contact@flyingion.com
+
+from twisted.web import resource
+from twisted.web.server import NOT_DONE_YET
+
+
+class RedirectToHttpsResource(resource.Resource):
+    """
+    I redirect to the same path at a given URL scheme
+    @param newScheme: scheme to redirect to (e.g. https)
+    """
+
+    isLeaf = 0
+    newScheme = b'https'
+
+    def __init__(self, newPort):
+        resource.Resource.__init__(self)
+        self.newPort = newPort
+
+    def render(self, request):
+        newURLPath = request.URLPath()
+
+        # TODO Double check that == gives the correct behaviour here
+        if newURLPath.scheme == self.newScheme:
+            raise ValueError("Redirect loop: we're trying to redirect"
+                             " to the same URL scheme in the request")
+
+        newURLPath.scheme = self.newScheme
+        newURLPath.netloc = b'%s:%s' % (newURLPath.netloc.split(b':')[0],
+                                        str(self.newPort).encode())
+
+        request.redirect(str(newURLPath).encode())
+        request.finish()
+        return NOT_DONE_YET
+
+    def getChild(self, name, request):
+        return self

txhttputil/site/RedirectionRule.py

@@ -0,0 +1,24 @@
+# Copyright (c) 2016-2026 Flying Ion Pty Ltd
+#
+# This software is open source; the MIT license applies.
+# See LICENSE for the full license text.
+#
+# Website : https://flyingion.com
+# Support : contact@flyingion.com
+
+class RedirectionRule:
+    def __init__(self, origin: str = "", destination: str = ""):
+        """A definition of Resource redirection rule
+        :param str from: a source uri as in twisted.web.http.Request
+        :param str to: a destination url.
+        """
+        self._origin: str = origin
+        self._destination: str = destination
+
+    @property
+    def origin(self) -> bytes:
+        return self._origin.encode()
+
+    @property
+    def destination(self) -> bytes:
+        return self._destination.encode()

txhttputil/site/ResourceCreator.py

@@ -0,0 +1,12 @@
+# Copyright (c) 2016-2026 Flying Ion Pty Ltd
+#
+# This software is open source; the MIT license applies.
+# See LICENSE for the full license text.
+#
+# Website : https://flyingion.com
+# Support : contact@flyingion.com
+
+import logging
+
+logger = logging.getLogger(__name__)
+

txhttputil/site/RootResource.py

@@ -0,0 +1,29 @@
+# Copyright (c) 2016-2026 Flying Ion Pty Ltd
+#
+# This software is open source; the MIT license applies.
+# See LICENSE for the full license text.
+#
+# Website : https://flyingion.com
+# Support : contact@flyingion.com
+
+from txhttputil.site.AuthUserDetails import AuthUserDetails
+from txhttputil.site.ResourceUtil import RESOURCES
+from txhttputil.site.StaticFileMultiPath import StaticFileMultiPath
+from txhttputil.site.StaticFileResource import StaticFileResource
+
+
+class RootResource(StaticFileResource):
+    """Root Resource
+
+    This resource is the root or subroot of a resource tree.
+    It first looks for resources created with "createRoot
+    """
+
+
+def createRootResource(
+    userAccess: AuthUserDetails, staticFileRoot: StaticFileMultiPath
+):
+    rootResource = RootResource(userAccess, staticFileRoot=staticFileRoot)
+    callResourceCreators(RESOURCES, rootResource, userAccess)
+
+    return rootResource

txhttputil/site/RootSiteTest.py

@@ -0,0 +1,8 @@
+# Copyright (c) 2016-2026 Flying Ion Pty Ltd
+#
+# This software is open source; the MIT license applies.
+# See LICENSE for the full license text.
+#
+# Website : https://flyingion.com
+# Support : contact@flyingion.com
+