plain 0.74.0__py3-none-any.whl → 0.76.0__py3-none-any.whl

plain/server/workers/base.py

@@ -0,0 +1,305 @@
+from __future__ import annotations
+
+#
+#
+# This file is part of gunicorn released under the MIT license.
+# See the LICENSE for more information.
+#
+# Vendored and modified for Plain.
+import io
+import os
+import signal
+import sys
+import time
+import traceback
+from datetime import datetime
+from random import randint
+from ssl import SSLError
+from typing import TYPE_CHECKING, Any
+
+from .. import util
+from ..http.errors import (
+    ConfigurationProblem,
+    InvalidHeader,
+    InvalidHeaderName,
+    InvalidHTTPVersion,
+    InvalidRequestLine,
+    InvalidRequestMethod,
+    InvalidSchemeHeaders,
+    LimitRequestHeaders,
+    LimitRequestLine,
+    ObsoleteFolding,
+    UnsupportedTransferCoding,
+)
+from ..http.wsgi import Response, default_environ
+from ..reloader import reloader_engines
+from .workertmp import WorkerTmp
+
+if TYPE_CHECKING:
+    import socket
+
+    from ..app import ServerApplication
+    from ..config import Config
+    from ..glogging import Logger
+    from ..http.message import Request
+
+# Maximum jitter to add to max_requests to stagger worker restarts
+MAX_REQUESTS_JITTER = 50
+
+
+class Worker:
+    SIGNALS = [
+        getattr(signal, f"SIG{x}")
+        for x in ("ABRT HUP QUIT INT TERM USR1 USR2 WINCH CHLD".split())
+    ]
+
+    PIPE = []
+
+    def __init__(
+        self,
+        age: int,
+        ppid: int,
+        sockets: list[socket.socket],
+        app: ServerApplication,
+        timeout: int,
+        cfg: Config,
+        log: Logger,
+    ):
+        """\
+        This is called pre-fork so it shouldn't do anything to the
+        current process. If there's a need to make process wide
+        changes you'll want to do that in ``self.init_process()``.
+        """
+        self.age = age
+        self.pid: str | int = "[booting]"
+        self.ppid = ppid
+        self.sockets = sockets
+        self.app = app
+        self.timeout = timeout
+        self.cfg = cfg
+        self.booted = False
+        self.aborted = False
+        self.reloader: Any = None
+
+        self.nr = 0
+
+        if cfg.max_requests > 0:
+            jitter = randint(0, MAX_REQUESTS_JITTER)
+            self.max_requests = cfg.max_requests + jitter
+        else:
+            self.max_requests = sys.maxsize
+
+        self.alive = True
+        self.log = log
+        self.tmp = WorkerTmp(cfg)
+
+    def __str__(self) -> str:
+        return f"<Worker {self.pid}>"
+
+    def notify(self) -> None:
+        """\
+        Your worker subclass must arrange to have this method called
+        once every ``self.timeout`` seconds. If you fail in accomplishing
+        this task, the master process will murder your workers.
+        """
+        self.tmp.notify()
+
+    def run(self) -> None:
+        """\
+        This is the mainloop of a worker process. You should override
+        this method in a subclass to provide the intended behaviour
+        for your particular evil schemes.
+        """
+        raise NotImplementedError()
+
+    def init_process(self) -> None:
+        """\
+        If you override this method in a subclass, the last statement
+        in the function should be to call this method with
+        super().init_process() so that the ``run()`` loop is initiated.
+        """
+
+        # Reseed the random number generator
+        util.seed()
+
+        # For waking ourselves up
+        self.PIPE = os.pipe()
+        for p in self.PIPE:
+            util.set_non_blocking(p)
+            util.close_on_exec(p)
+
+        # Prevent fd inheritance
+        for s in self.sockets:
+            util.close_on_exec(s)
+        util.close_on_exec(self.tmp.fileno())
+
+        self.wait_fds = self.sockets + [self.PIPE[0]]
+
+        self.log.close_on_exec()
+
+        self.init_signals()
+
+        # start the reloader
+        if self.cfg.reload:
+
+            def changed(fname: str) -> None:
+                self.log.info("Worker reloading: %s modified", fname)
+                self.alive = False
+                os.write(self.PIPE[1], b"1")
+                time.sleep(0.1)
+                sys.exit(0)
+
+            reloader_cls = reloader_engines["auto"]
+            self.reloader = reloader_cls(
+                extra_files=self.cfg.reload_extra_files, callback=changed
+            )
+
+        self.load_wsgi()
+        if self.reloader:
+            self.reloader.start()
+
+        # Enter main run loop
+        self.booted = True
+        self.run()
+
+    def load_wsgi(self) -> None:
+        try:
+            self.wsgi = self.app.wsgi()
+        except SyntaxError as e:
+            if not self.cfg.reload:
+                raise
+
+            self.log.exception(e)
+
+            # fix from PR #1228
+            # storing the traceback into exc_tb will create a circular reference.
+            # per https://docs.python.org/2/library/sys.html#sys.exc_info warning,
+            # delete the traceback after use.
+            try:
+                _, exc_val, exc_tb = sys.exc_info()
+                self.reloader.add_extra_file(exc_val.filename)
+
+                tb_string = io.StringIO()
+                traceback.print_tb(exc_tb, file=tb_string)
+                self.wsgi = util.make_fail_app(tb_string.getvalue())
+            finally:
+                del exc_tb
+
+    def init_signals(self) -> None:
+        # reset signaling
+        for s in self.SIGNALS:
+            signal.signal(s, signal.SIG_DFL)
+        # init new signaling
+        signal.signal(signal.SIGQUIT, self.handle_quit)
+        signal.signal(signal.SIGTERM, self.handle_exit)
+        signal.signal(signal.SIGINT, self.handle_quit)
+        signal.signal(signal.SIGWINCH, self.handle_winch)
+        signal.signal(signal.SIGUSR1, self.handle_usr1)
+        signal.signal(signal.SIGABRT, self.handle_abort)
+
+        # Don't let SIGTERM and SIGUSR1 disturb active requests
+        # by interrupting system calls
+        signal.siginterrupt(signal.SIGTERM, False)
+        signal.siginterrupt(signal.SIGUSR1, False)
+
+        if hasattr(signal, "set_wakeup_fd"):
+            signal.set_wakeup_fd(self.PIPE[1])
+
+    def handle_usr1(self, sig: int, frame: Any) -> None:
+        self.log.reopen_files()
+
+    def handle_exit(self, sig: int, frame: Any) -> None:
+        self.alive = False
+
+    def handle_quit(self, sig: int, frame: Any) -> None:
+        self.alive = False
+        time.sleep(0.1)
+        sys.exit(0)
+
+    def handle_abort(self, sig: int, frame: Any) -> None:
+        self.alive = False
+        sys.exit(1)
+
+    def handle_error(
+        self, req: Request | None, client: socket.socket, addr: Any, exc: Exception
+    ) -> None:
+        request_start = datetime.now()
+        addr = addr or ("", -1)  # unix socket case
+        if isinstance(
+            exc,
+            InvalidRequestLine
+            | InvalidRequestMethod
+            | InvalidHTTPVersion
+            | InvalidHeader
+            | InvalidHeaderName
+            | LimitRequestLine
+            | LimitRequestHeaders
+            | InvalidSchemeHeaders
+            | UnsupportedTransferCoding
+            | ConfigurationProblem
+            | ObsoleteFolding
+            | SSLError,
+        ):
+            status_int = 400
+            reason = "Bad Request"
+
+            if isinstance(exc, InvalidRequestLine):
+                mesg = f"Invalid Request Line '{str(exc)}'"
+            elif isinstance(exc, InvalidRequestMethod):
+                mesg = f"Invalid Method '{str(exc)}'"
+            elif isinstance(exc, InvalidHTTPVersion):
+                mesg = f"Invalid HTTP Version '{str(exc)}'"
+            elif isinstance(exc, UnsupportedTransferCoding):
+                mesg = f"{str(exc)}"
+                status_int = 501
+            elif isinstance(exc, ConfigurationProblem):
+                mesg = f"{str(exc)}"
+                status_int = 500
+            elif isinstance(exc, ObsoleteFolding):
+                mesg = f"{str(exc)}"
+            elif isinstance(exc, InvalidHeaderName | InvalidHeader):
+                mesg = f"{str(exc)}"
+                if not req and hasattr(exc, "req"):
+                    req = exc.req  # type: ignore[attr-defined]  # for access log
+            elif isinstance(exc, LimitRequestLine):
+                mesg = f"{str(exc)}"
+            elif isinstance(exc, LimitRequestHeaders):
+                reason = "Request Header Fields Too Large"
+                mesg = f"Error parsing headers: '{str(exc)}'"
+                status_int = 431
+            elif isinstance(exc, InvalidSchemeHeaders):
+                mesg = f"{str(exc)}"
+            elif isinstance(exc, SSLError):
+                reason = "Forbidden"
+                mesg = f"'{str(exc)}'"
+                status_int = 403
+
+            msg = "Invalid request from ip={ip}: {error}"
+            self.log.warning(msg.format(ip=addr[0], error=str(exc)))
+        else:
+            if hasattr(req, "uri"):
+                self.log.exception("Error handling request %s", req.uri)
+            else:
+                self.log.exception("Error handling request (no URI read)")
+            status_int = 500
+            reason = "Internal Server Error"
+            mesg = ""
+
+        if req is not None:
+            request_time = datetime.now() - request_start
+            environ = default_environ(req, client, self.cfg)
+            environ["REMOTE_ADDR"] = addr[0]
+            environ["REMOTE_PORT"] = str(addr[1])
+            resp = Response(req, client, self.cfg)
+            resp.status = f"{status_int} {reason}"
+            resp.response_length = len(mesg)
+            self.log.access(resp, req, environ, request_time)
+
+        try:
+            util.write_error(client, status_int, reason, mesg)
+        except Exception:
+            self.log.debug("Failed to send error message.")
+
+    def handle_winch(self, sig: int, fname: Any) -> None:
+        # Ignore SIGWINCH in worker. Fixes a crash on OpenBSD.
+        self.log.debug("worker: SIGWINCH ignored.")

plain/server/workers/gthread.py

@@ -0,0 +1,393 @@
+from __future__ import annotations
+
+#
+#
+# This file is part of gunicorn released under the MIT license.
+# See the LICENSE for more information.
+#
+# Vendored and modified for Plain.
+# design:
+# A threaded worker accepts connections in the main loop, accepted
+# connections are added to the thread pool as a connection job.
+# Keepalive connections are put back in the loop waiting for an event.
+# If no event happen after the keep alive timeout, the connection is
+# closed.
+# pylint: disable=no-else-break
+import errno
+import os
+import selectors
+import socket
+import ssl
+import sys
+import time
+from collections import deque
+from concurrent import futures
+from datetime import datetime
+from functools import partial
+from threading import RLock
+from types import FrameType
+from typing import TYPE_CHECKING, Any
+
+from .. import http, sock, util
+from ..http import wsgi
+from . import base
+
+if TYPE_CHECKING:
+    from ..config import Config
+    from ..glogging import Logger
+
+# Keep-alive connection timeout in seconds
+KEEPALIVE = 2
+
+# Maximum number of simultaneous client connections
+WORKER_CONNECTIONS = 1000
+
+
+class TConn:
+    def __init__(
+        self,
+        cfg: Config,
+        sock: socket.socket,
+        client: tuple[str, int],
+        server: tuple[str, int],
+    ) -> None:
+        self.cfg = cfg
+        self.sock = sock
+        self.client = client
+        self.server = server
+
+        self.timeout: float | None = None
+        self.parser: http.RequestParser | None = None
+        self.initialized: bool = False
+
+        # set the socket to non blocking
+        self.sock.setblocking(False)
+
+    def init(self) -> None:
+        self.initialized = True
+        self.sock.setblocking(True)
+
+        if self.parser is None:
+            # wrap the socket if needed
+            if self.cfg.is_ssl:
+                self.sock = sock.ssl_wrap_socket(self.sock, self.cfg)
+
+            # initialize the parser
+            self.parser = http.RequestParser(self.cfg, self.sock, self.client)
+
+    def set_timeout(self) -> None:
+        # set the timeout
+        self.timeout = time.time() + KEEPALIVE
+
+    def close(self) -> None:
+        util.close(self.sock)
+
+
+class ThreadWorker(base.Worker):
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        super().__init__(*args, **kwargs)
+        self.worker_connections: int = WORKER_CONNECTIONS
+        self.max_keepalived: int = WORKER_CONNECTIONS - self.cfg.threads
+        # initialise the pool
+        self.tpool: futures.ThreadPoolExecutor | None = None
+        self.poller: selectors.DefaultSelector | None = None
+        self._lock: RLock | None = None
+        self.futures: deque[futures.Future[tuple[bool, TConn]]] = deque()
+        self._keep: deque[TConn] = deque()
+        self.nr_conns: int = 0
+
+    @classmethod
+    def check_config(cls, cfg: Config, log: Logger) -> None:
+        max_keepalived = WORKER_CONNECTIONS - cfg.threads
+
+        if max_keepalived <= 0:
+            log.warning(
+                "No keepalived connections can be handled. "
+                "Check the number of worker connections and threads."
+            )
+
+    def init_process(self) -> None:
+        self.tpool = self.get_thread_pool()
+        self.poller = selectors.DefaultSelector()
+        self._lock = RLock()
+        super().init_process()
+
+    def get_thread_pool(self) -> futures.ThreadPoolExecutor:
+        """Override this method to customize how the thread pool is created"""
+        return futures.ThreadPoolExecutor(max_workers=self.cfg.threads)
+
+    def handle_quit(self, sig: int, frame: FrameType | None) -> None:
+        self.alive = False
+        self.tpool.shutdown(False)
+        time.sleep(0.1)
+        sys.exit(0)
+
+    def _wrap_future(self, fs: futures.Future[tuple[bool, TConn]], conn: TConn) -> None:
+        fs.conn = conn  # type: ignore[attr-defined]
+        self.futures.append(fs)
+        fs.add_done_callback(self.finish_request)
+
+    def enqueue_req(self, conn: TConn) -> None:
+        conn.init()
+        # submit the connection to a worker
+        fs = self.tpool.submit(self.handle, conn)
+        self._wrap_future(fs, conn)
+
+    def accept(self, server: tuple[str, int], listener: socket.socket) -> None:
+        try:
+            sock, client = listener.accept()
+            # initialize the connection object
+            conn = TConn(self.cfg, sock, client, server)
+
+            self.nr_conns += 1
+            # wait until socket is readable
+            with self._lock:
+                self.poller.register(
+                    conn.sock,
+                    selectors.EVENT_READ,
+                    partial(self.on_client_socket_readable, conn),
+                )
+        except OSError as e:
+            if e.errno not in (errno.EAGAIN, errno.ECONNABORTED, errno.EWOULDBLOCK):
+                raise
+
+    def on_client_socket_readable(self, conn: TConn, client: socket.socket) -> None:
+        with self._lock:
+            # unregister the client from the poller
+            self.poller.unregister(client)
+
+            if conn.initialized:
+                # remove the connection from keepalive
+                try:
+                    self._keep.remove(conn)
+                except ValueError:
+                    # race condition
+                    return
+
+        # submit the connection to a worker
+        self.enqueue_req(conn)
+
+    def murder_keepalived(self) -> None:
+        now = time.time()
+        while True:
+            with self._lock:
+                try:
+                    # remove the connection from the queue
+                    conn = self._keep.popleft()
+                except IndexError:
+                    break
+
+            delta = conn.timeout - now
+            if delta > 0:
+                # add the connection back to the queue
+                with self._lock:
+                    self._keep.appendleft(conn)
+                break
+            else:
+                self.nr_conns -= 1
+                # remove the socket from the poller
+                with self._lock:
+                    try:
+                        self.poller.unregister(conn.sock)
+                    except OSError as e:
+                        if e.errno != errno.EBADF:
+                            raise
+                    except KeyError:
+                        # already removed by the system, continue
+                        pass
+                    except ValueError:
+                        # already removed by the system continue
+                        pass
+
+                # close the socket
+                conn.close()
+
+    def is_parent_alive(self) -> bool:
+        # If our parent changed then we shut down.
+        if self.ppid != os.getppid():
+            self.log.info("Parent changed, shutting down: %s", self)
+            return False
+        return True
+
+    def run(self) -> None:
+        # init listeners, add them to the event loop
+        for listener in self.sockets:
+            listener.setblocking(False)
+            # a race condition during graceful shutdown may make the listener
+            # name unavailable in the request handler so capture it once here
+            server = listener.getsockname()
+            acceptor = partial(self.accept, server)
+            self.poller.register(listener, selectors.EVENT_READ, acceptor)
+
+        while self.alive:
+            # notify the arbiter we are alive
+            self.notify()
+
+            # can we accept more connections?
+            if self.nr_conns < self.worker_connections:
+                # wait for an event
+                events = self.poller.select(1.0)
+                for key, _ in events:
+                    callback = key.data
+                    callback(key.fileobj)
+
+                # check (but do not wait) for finished requests
+                result = futures.wait(
+                    self.futures, timeout=0, return_when=futures.FIRST_COMPLETED
+                )
+            else:
+                # wait for a request to finish
+                result = futures.wait(
+                    self.futures, timeout=1.0, return_when=futures.FIRST_COMPLETED
+                )
+
+            # clean up finished requests
+            for fut in result.done:
+                self.futures.remove(fut)
+
+            if not self.is_parent_alive():
+                break
+
+            # handle keepalive timeouts
+            self.murder_keepalived()
+
+        self.tpool.shutdown(False)
+        self.poller.close()
+
+        for s in self.sockets:
+            s.close()
+
+        futures.wait(self.futures, timeout=self.cfg.graceful_timeout)
+
+    def finish_request(self, fs: futures.Future[tuple[bool, TConn]]) -> None:
+        if fs.cancelled():
+            self.nr_conns -= 1
+            fs.conn.close()  # type: ignore[attr-defined]
+            return
+
+        try:
+            (keepalive, conn) = fs.result()
+            # if the connection should be kept alived add it
+            # to the eventloop and record it
+            if keepalive and self.alive:
+                # flag the socket as non blocked
+                conn.sock.setblocking(False)
+
+                # register the connection
+                conn.set_timeout()
+                with self._lock:
+                    self._keep.append(conn)
+
+                    # add the socket to the event loop
+                    self.poller.register(
+                        conn.sock,
+                        selectors.EVENT_READ,
+                        partial(self.on_client_socket_readable, conn),
+                    )
+            else:
+                self.nr_conns -= 1
+                conn.close()
+        except Exception:
+            # an exception happened, make sure to close the
+            # socket.
+            self.nr_conns -= 1
+            fs.conn.close()  # type: ignore[attr-defined]
+
+    def handle(self, conn: TConn) -> tuple[bool, TConn]:
+        keepalive = False
+        req = None
+        try:
+            # conn.parser is guaranteed to be initialized by enqueue_req -> conn.init()
+            assert conn.parser is not None
+            req = next(conn.parser)
+            if not req:
+                return (False, conn)
+
+            # handle the request
+            keepalive = self.handle_request(req, conn)
+            if keepalive:
+                return (keepalive, conn)
+        except http.errors.NoMoreData as e:
+            self.log.debug("Ignored premature client disconnection. %s", e)
+
+        except StopIteration as e:
+            self.log.debug("Closing connection. %s", e)
+        except ssl.SSLError as e:
+            if e.args[0] == ssl.SSL_ERROR_EOF:
+                self.log.debug("ssl connection closed")
+                conn.sock.close()
+            else:
+                self.log.debug("Error processing SSL request.")
+                self.handle_error(req, conn.sock, conn.client, e)
+
+        except OSError as e:
+            if e.errno not in (errno.EPIPE, errno.ECONNRESET, errno.ENOTCONN):
+                self.log.exception("Socket error processing request.")
+            else:
+                if e.errno == errno.ECONNRESET:
+                    self.log.debug("Ignoring connection reset")
+                elif e.errno == errno.ENOTCONN:
+                    self.log.debug("Ignoring socket not connected")
+                else:
+                    self.log.debug("Ignoring connection epipe")
+        except Exception as e:
+            self.handle_error(req, conn.sock, conn.client, e)
+
+        return (False, conn)
+
+    def handle_request(self, req: Any, conn: TConn) -> bool:
+        environ: dict[str, Any] = {}
+        resp: wsgi.Response | None = None
+        try:
+            request_start = datetime.now()
+            resp, environ = wsgi.create(
+                req, conn.sock, conn.client, conn.server, self.cfg
+            )
+            environ["wsgi.multithread"] = True
+            self.nr += 1
+            if self.nr >= self.max_requests:
+                if self.alive:
+                    self.log.info("Autorestarting worker after current request.")
+                    self.alive = False
+                resp.force_close()
+
+            if not self.alive:
+                resp.force_close()
+            elif len(self._keep) >= self.max_keepalived:
+                resp.force_close()
+
+            respiter = self.wsgi(environ, resp.start_response)
+            try:
+                if isinstance(respiter, environ["wsgi.file_wrapper"]):
+                    resp.write_file(respiter)
+                else:
+                    for item in respiter:
+                        resp.write(item)
+
+                resp.close()
+            finally:
+                request_time = datetime.now() - request_start
+                self.log.access(resp, req, environ, request_time)
+                if hasattr(respiter, "close"):
+                    respiter.close()
+
+            if resp.should_close():
+                self.log.debug("Closing connection.")
+                return False
+        except OSError:
+            # pass to next try-except level
+            util.reraise(*sys.exc_info())
+        except Exception:
+            if resp and resp.headers_sent:
+                # If the requests have already been sent, we should close the
+                # connection to indicate the error.
+                self.log.exception("Error handling request")
+                try:
+                    conn.sock.shutdown(socket.SHUT_RDWR)
+                    conn.sock.close()
+                except OSError:
+                    pass
+                raise StopIteration()
+            raise
+
+        return True