Pyjoyment 0.1.0__py3-none-any.whl

Pyjo/Server/Prethread.py

@@ -0,0 +1,394 @@
+"""
+Pyjo.Server.Prethread - Pre-threading non-blocking I/O HTTP and WebSocket server
+================================================================================
+::
+
+    from Pyjo.Server.Prethread import Prethread
+
+    prethread = Prethread(listen=['http://*:8080'])
+
+
+    @prethread.unsubscribe('request').on('request')
+    def request(prethread, tx):
+        # Request
+        method = tx.req.method
+        path = tx.req.url.path
+
+        # Response
+        tx.res.code = 200
+        tx.res.headers.content_type = 'text/plain'
+        tx.res.body = f'{method} request for {path}!'.encode()
+
+        # Resume transaction
+        tx.resume()
+
+
+    prethread.run()
+
+    # Or import the module, as in Google Python Style Guide
+
+    import Pyjo.Server.Prethread
+
+    prethread = Pyjo.Server.Prethread.new(listen=['http://*:8080'])
+
+
+    @prethread.unsubscribe('request').on('request')
+    def request(prethread, tx):
+        # Request
+        method = tx.req.method
+        path = tx.req.url.path
+
+        # Response
+        tx.res.code = 200
+        tx.res.headers.content_type = 'text/plain'
+        tx.res.body = f'{method} request for {path}!'.encode()
+
+        # Resume transaction
+        tx.resume()
+
+
+    prethread.run()
+
+:mod:`Pyjo.Server.Prethread` is a full featured pre-threading non-blocking
+I/O HTTP and WebSocket server, which works as :mod:`Pyjo.Server.Prefork`
+does, with threads of one process for the worker processes. Each worker is a
+copy of the server in a thread of its own, with the :data:`Pyjo.IOLoop.singleton`
+of the thread and the listen sockets of the manager. Note that the server uses
+signals for process management, so you should avoid modifying signal handlers
+in your applications.
+
+The workers share the memory of the process and the application that the
+manager has loaded, so the application has to be safe to use in several
+threads at the same time. Python runs one thread at a time where it has the
+global interpreter lock, so that the workers run at the same time only while
+they wait, for the network or a blocking call, and on all processor cores on
+a free-threaded Python. A worker that does not stop when it is told to, as
+one in a blocking call, cannot be killed, as a process can, so the manager
+leaves it, and it ends with the process.
+
+The workers need no :func:`os.fork`, so the server runs on Windows too, where
+:mod:`Pyjo.Server.Prefork` does not.
+
+Manager signals
+---------------
+
+The :mod:`Pyjo.Server.Prethread` manager process can be controlled at runtime
+with the following signals.
+
+INT, TERM
+~~~~~~~~~
+
+Shut down server gracefully. The manager and the workers close the listen
+sockets, so that a new connection is refused, and the workers finish the
+accepted ones, after :attr:`~Pyjo.Server.Daemon.Pyjo_Server_Daemon.shutdown_delay`,
+while the workers serve as before; on Windows, where the workers share the
+sockets of the manager, a new connection waits until the server ends. A
+second ``INT``, as of Ctrl-C pressed twice, shuts it down immediately, also
+during the delay, a second ``TERM`` does nothing.
+
+QUIT
+~~~~
+
+Shut down server immediately. On Windows, which has no ``QUIT``, ``BREAK``
+does it, as of Ctrl-Break.
+
+HUP
+~~~
+
+Restart the workers gracefully, or emit the event ``reload`` when it has
+subscribers.
+
+TTIN
+~~~~
+
+Increase worker pool by one.
+
+TTOU
+~~~~
+
+Decrease worker pool by one.
+
+The manager does not handle ``CHLD``, so that the application gets the exit
+status of its child processes.
+
+Windows has no ``HUP``, ``TTIN`` and ``TTOU``, and a ``TERM`` from another
+process terminates the process there without its handler, so Ctrl-C and
+Ctrl-Break of the console are what shut the server down.
+
+Events
+------
+
+:mod:`Pyjo.Server.Prethread` inherits all events from
+:mod:`Pyjo.Server.Prefork`, which pass the id of the thread of a worker, as
+:func:`threading.get_native_id` gives it, where :mod:`Pyjo.Server.Prefork`
+passes its process id. ::
+
+    prethread.on('spawn', lambda prethread, id: print(f'Worker {id} started'))
+
+See also
+--------
+
+:mod:`Pyjoyment.Command.prethread`, :mod:`Pyjo.Server.Prefork`,
+:mod:`Pyjoyment`, https://mojolicious.org.
+
+It has no counterpart in Mojolicious; it is inspired by
+:mod:`Pyjo.Server.Prefork`, the counterpart of `Mojo::Server::Prefork
+<https://github.com/mojolicious/mojo/blob/v9.49/lib/Mojo/Server/Prefork.pm>`_,
+as `Thrall <https://metacpan.org/pod/Thrall>`_ is by Starlight.
+
+Classes
+-------
+"""
+
+import Pyjo.IOLoop
+import Pyjo.Server.Prefork
+import Pyjo.URL
+
+import copy
+import os
+import signal
+import socket
+import sys
+import tempfile
+import threading
+import time
+import traceback
+import weakref
+
+from typing import Any, NotRequired, TypedDict
+
+
+class _Thread(TypedDict):
+    control: socket.socket
+    ended: threading.Event
+    killed: NotRequired[float]
+    thread: threading.Thread
+
+
+class Pyjo_Server_Prethread(Pyjo.Server.Prefork.object):
+    """
+    :mod:`Pyjo.Server.Prethread` inherits all attributes and methods from
+    :mod:`Pyjo.Server.Prefork`, with ``prethread.pid`` in a temporary
+    directory for the default of :attr:`~Pyjo.Server.Prefork.Pyjo_Server_Prefork.pid_file`.
+    """
+
+    # The signals of the manager, without CHLD, which is the one of the children of the application; Windows has INT,
+    # TERM and BREAK of Ctrl-Break, which takes the place of QUIT
+    if sys.platform == 'win32':
+        _MANAGED = (signal.SIGINT, signal.SIGTERM, signal.SIGBREAK)
+    else:
+        _MANAGED = (signal.SIGHUP, signal.SIGINT, signal.SIGTERM, signal.SIGQUIT, signal.SIGTTIN, signal.SIGTTOU)
+
+    # The seconds that a worker has to stop once it is told to stop immediately, before the manager leaves it
+    _LEAVE = 1
+
+    # The workers are threads, which need no fork
+    _FORKS = False
+
+    def __init__(self, *, pid_file: str | os.PathLike[str] | None = None, **kwargs: Any) -> None:
+        """::
+
+            prethread = Pyjo.Server.Prethread.new()
+            prethread = Pyjo.Server.Prethread.new(listen=['http://*:8080'], workers=10)
+
+        Construct a new :mod:`Pyjo.Server.Prethread` object.
+        """
+        super().__init__(
+            pid_file=pid_file if pid_file is not None else os.path.join(tempfile.gettempdir(), 'prethread.pid'),
+            **kwargs,
+        )
+        self._threads: dict[int, _Thread] = {}
+        self._pair: tuple[socket.socket, socket.socket] | None = None
+
+    # The heartbeats go through a socket pair in the place of a pipe, as select of Windows watches sockets only
+    def _pipe(self) -> tuple[int, int]:
+        reader, writer = self._pair = socket.socketpair()
+        return reader.fileno(), writer.fileno()
+
+    def _read_pipe(self, reader: int) -> bytes:
+        assert self._pair is not None
+        return self._pair[0].recv(4194304)
+
+    def _close_pipe(self, reader: int, writer: int) -> None:
+        if (pair := self._pair) is not None:
+            self._pair = None
+            pair[0].close()
+            pair[1].close()
+
+    def _send(self, data: bytes) -> None:
+        if (pair := self._pair) is None:
+            raise OSError('No pipe')
+        pair[1].sendall(data)
+
+    def _heartbeat(self, finished: bool) -> None:
+        try:
+            self._send(f'{threading.get_native_id()}:{int(finished)}\n'.encode())
+        except Exception:
+            Pyjo.IOLoop.singleton.stop()
+
+    def _kill_worker(self, pid: int, graceful: bool) -> bool:
+        if (t := self._threads.get(pid)) is None:
+            return False
+        if not graceful:
+            now = time.monotonic()
+            if (killed := t.get('killed')) is not None:
+                if killed + self._LEAVE > now:
+                    return True
+                self.app.log.error(f'Worker {pid} does not stop, leaving it')
+                self._threads.pop(pid)['control'].close()
+                return False
+            t['killed'] = now
+        try:
+            t['control'].send(b'T' if graceful else b'Q')
+        except OSError:
+            pass
+        return True
+
+    def _delay_worker(self, pid: int) -> None:
+        if (t := self._threads.get(pid)) is None:
+            return
+        try:
+            t['control'].send(b'D')
+        except OSError:
+            pass
+
+    # The listen sockets are left open on Windows, where the workers share the sockets of the manager, which their
+    # reactors watch until they stop, and a new connection waits for the end of the server, as in Mojolicious
+    def _close_listen(self) -> None:
+        if sys.platform != 'win32':
+            super()._close_listen()
+
+    def _spawn(self) -> None:
+        # The listen sockets of the manager, which the worker duplicates, or shares on Windows
+        loop = self.ioloop
+        listen = [
+            (location, loop.acceptor(id).handle) for location, id in zip(self.listen, self.acceptors, strict=True)
+        ]
+        control, wakee = socket.socketpair()
+        ended = threading.Event()
+        thread = threading.Thread(
+            target=self._serve, args=(listen, wakee, ended), name='Pyjo.Server.Prethread', daemon=True
+        )
+        thread.start()
+        assert (id := thread.native_id) is not None
+        self._threads[id] = {'control': control, 'ended': ended, 'thread': thread}
+        self.emit('spawn', id)._pool[id] = {'time': time.monotonic()}
+
+    def _wait(self) -> None:
+        super()._wait()
+
+        # The workers that have stopped, which the last line of a worker tells, with the moment that its thread takes to
+        # end after it
+        for id, t in list(self._threads.items()):
+            if t['ended'].is_set():
+                t['thread'].join(1)
+            if t['thread'].is_alive():
+                continue
+            t['control'].close()
+            del self._threads[id]
+            self.emit('reap', id)._stopped(id)
+
+    # The worker, a copy of the server in a thread, which runs the event loop of the thread until it stops
+    def _serve(
+        self, listen: list[tuple[Any, socket.socket | None]], control: socket.socket, ended: threading.Event
+    ) -> None:
+        worker = copy.copy(self)
+        loop = worker.ioloop = Pyjo.IOLoop.singleton
+        worker.acceptors, worker._connections, worker._servers = [], {}, {}
+        worker.cleanup, worker.silent = False, True
+        try:
+            # The listen sockets of the manager, duplicated, as the worker closes them. On Windows, the workers share the
+            # sockets of the manager and leave them open, as accept of a duplicate, of WSADuplicateSocket or of
+            # DuplicateHandle, can block, also on a non-blocking socket, when the duplicate of another worker accepts the
+            # connection that select reported to both
+            loop.max_accepts = self.accepts
+            if max := worker.max_clients:
+                loop.max_connections = max
+            for location, handle in listen:
+                assert handle is not None
+                worker._listen(
+                    _location(location, handle.fileno() if sys.platform == 'win32' else handle.dup().detach())
+                )
+
+            # Heartbeat messages
+            finished = False
+            ref = weakref.ref(worker)
+
+            def finish(loop: Any) -> None:
+                nonlocal finished
+                finished = True
+                if (worker := ref()) is not None:
+                    worker._close_listen()
+                    worker.max_requests = 1
+
+            loop.on('finish', finish)
+
+            def heartbeat(loop: Any) -> None:
+                if (worker := ref()) is not None:
+                    worker._heartbeat(finished)
+
+            loop.next_tick(heartbeat)
+            loop.recurring(self.heartbeat_interval, heartbeat)
+
+            # The manager stops the worker with a byte, gracefully, after the delay of the graceful shutdown, or
+            # immediately, also when it is gone
+            immediate = False
+
+            def command(reactor: Any, writable: bool) -> None:
+                nonlocal immediate
+                try:
+                    chunk = control.recv(4096)
+                except OSError:
+                    return
+                if not chunk or b'Q' in chunk:
+                    immediate = True
+                    loop.stop()
+                elif b'T' in chunk:
+                    loop.stop_gracefully()
+                elif (worker := ref()) is not None:
+                    loop.stop_gracefully(worker.shutdown_delay)
+
+            loop.reactor.io(control, command).watch(control, True, False)
+
+            self.app.log.info(f'Worker {threading.get_native_id()} started')
+            loop.start()
+
+            # Not in Mojolicious: the end of the server in the thread of the worker, unless it has stopped immediately
+            if not immediate:
+                worker._server_stopped()
+        except BaseException:
+            traceback.print_exc()
+        finally:
+            loop.reactor.remove(control)
+            control.close()
+            if sys.platform == 'win32':
+                for acceptor in [*(loop.acceptor(id) for id in worker.acceptors), *worker._servers.values()]:
+                    if acceptor is not None and (handle := acceptor.handle) is not None:
+                        acceptor.stop()
+                        handle.detach()
+            worker._cleanup()
+            loop.reset()
+
+            # The manager is woken up, to reap the worker at once
+            ended.set()
+            try:
+                self._send(b'\n')
+            except OSError:
+                pass
+
+
+# A listen location for the duplicate of a listen socket, or the socket on Windows, without the port and the path that
+# would find the socket of the manager in PYJO_REUSE
+def _location(location: Any, fd: int) -> str:
+    url = Pyjo.URL.new(str(location))
+    if url.protocol.endswith('+unix'):
+        url.host = None
+    else:
+        url.port = None
+    url.query.append(fd=fd)
+    return str(url)
+
+
+new = Pyjo_Server_Prethread.new
+object = Pyjo_Server_Prethread
+Prethread = Pyjo_Server_Prethread

Pyjo/Server/Reload/Backend/Poll.py

@@ -0,0 +1,114 @@
+"""
+Pyjo.Server.Reload.Backend.Poll - Auto-restarting development server backend using stat
+=======================================================================================
+::
+
+    from Pyjo.Server.Reload.Backend.Poll import Poll
+
+    backend = Poll()
+    if files := backend.modified_files():
+        print('Files have been changed')
+
+    # Or import the module, as in Google Python Style Guide
+
+    import Pyjo.Server.Reload.Backend.Poll
+
+    backend = Pyjo.Server.Reload.Backend.Poll.new()
+    if files := backend.modified_files():
+        print('Files have been changed')
+
+:mod:`Pyjo.Server.Reload.Backend.Poll` is a backend for
+:mod:`Pyjo.Server.Reload` that checks the size and the modification time
+of the files with :func:`os.stat`, and waits for
+:attr:`~Pyjo.Server.Reload.Backend.Pyjo_Server_Reload_Backend.watch_timeout`
+when none has changed.
+
+See also
+--------
+
+:mod:`Pyjoyment`, https://mojolicious.org.
+
+Ported from `Mojo::Server::Morbo::Backend::Poll <https://github.com/mojolicious/mojo/blob/v9.49/lib/Mojo/Server/Morbo/Backend/Poll.pm>`_
+of Mojolicious 9.49.
+
+Classes
+-------
+"""
+
+import Pyjo.File
+import Pyjo.Server.Reload.Backend
+
+import os
+import time
+
+from typing import Any, cast
+
+
+# The time the program started, as Perl's $^T
+_START = time.time()
+
+
+# Renamed from Mojolicious: Mojo::Server::Morbo::Backend::Poll, as the names of Futurama are not ours
+class Pyjo_Server_Reload_Backend_Poll(Pyjo.Server.Reload.Backend.object):
+    """
+    :mod:`Pyjo.Server.Reload.Backend.Poll` inherits all attributes and methods
+    from :mod:`Pyjo.Server.Reload.Backend` and implements the following new
+    ones.
+    """
+
+    def __init__(self, **kwargs: Any) -> None:
+        """::
+
+            backend = Pyjo.Server.Reload.Backend.Poll.new()
+
+        Construct a new :mod:`Pyjo.Server.Reload.Backend.Poll` object.
+        """
+        super().__init__(**kwargs)
+        self._cache: dict[str, list[float]] = {}
+
+    def modified_files(self) -> list[str]:
+        """::
+
+            files = backend.modified_files()
+
+        Check file size and mtime to determine which files have changed, this
+        is not particularly efficient, but very robust.
+        """
+        cache = self._cache
+        known = set(cache)
+        files: list[str] = []
+        for file in (f for watch in self.watch for f in _files(os.fspath(cast(str, watch)))):
+            known.discard(file)
+            try:
+                stat = os.stat(file)
+            except OSError:
+                continue
+            stats = cache.setdefault(file, [_START, stat.st_size])
+            if stat.st_mtime <= stats[0] and stat.st_size == stats[1]:
+                continue
+            stats[:] = [stat.st_mtime, stat.st_size]
+            files.append(file)
+
+        # Any leftover file means they were deleted
+        for file in sorted(known):
+            files.append(file)
+            del cache[file]
+
+        if not len(files):
+            time.sleep(self.watch_timeout)
+
+        return files
+
+
+# A readable file, or the files of a directory
+def _files(path: str) -> list[str]:
+    if os.path.isfile(path) and os.access(path, os.R_OK):
+        return [path]
+    # Not in Mojolicious: the bytecode that Python writes to __pycache__ when the server imports a module that it watches,
+    # which is not a change of a source, and would restart the server once after each start
+    return [file.to_str() for file in Pyjo.File.new(path).list_tree() if '__pycache__' not in file.to_list()]
+
+
+new = Pyjo_Server_Reload_Backend_Poll.new
+object = Pyjo_Server_Reload_Backend_Poll
+Poll = Pyjo_Server_Reload_Backend_Poll

Pyjo/Server/Reload/Backend/__init__.py

@@ -0,0 +1,126 @@
+"""
+Pyjo.Server.Reload.Backend - Auto-restarting development server backend
+=======================================================================
+::
+
+    from Pyjo.Server.Reload.Backend import Backend
+
+
+    class Pyjo_Server_Reload_Backend_Inotify(Backend):
+        def modified_files(self):
+            ...
+
+
+    new = Pyjo_Server_Reload_Backend_Inotify.new
+    object = Pyjo_Server_Reload_Backend_Inotify
+
+    # Or import the module, as in Google Python Style Guide
+
+    import Pyjo.Server.Reload.Backend
+
+
+    class Pyjo_Server_Reload_Backend_Inotify(Pyjo.Server.Reload.Backend.object):
+        def modified_files(self):
+            ...
+
+
+    new = Pyjo_Server_Reload_Backend_Inotify.new
+    object = Pyjo_Server_Reload_Backend_Inotify
+
+:mod:`Pyjo.Server.Reload.Backend` is an abstract base class for
+auto-restarting development server backends used by
+:mod:`Pyjo.Server.Reload`.
+
+A backend is a module named ``Pyjo.Server.Reload.Backend.<Name>``, with a
+function ``new`` that constructs it and the class as ``object``, which
+another distribution can add, as :func:`pkgutil.extend_path` keeps the
+namespace open, and the ``PYJO_RELOAD_BACKEND`` environment variable selects
+it.
+
+See also
+--------
+
+:mod:`Pyjoyment`, https://mojolicious.org.
+
+Ported from `Mojo::Server::Morbo::Backend <https://github.com/mojolicious/mojo/blob/v9.49/lib/Mojo/Server/Morbo/Backend.pm>`_
+of Mojolicious 9.49.
+
+Classes
+-------
+"""
+
+import Pyjo.Base
+
+import os
+
+from pkgutil import extend_path
+from typing import Any
+
+
+# Not in Mojolicious: a namespace that other distributions can add backends to
+__path__ = extend_path(__path__, __name__)
+
+
+# Renamed from Mojolicious: Mojo::Server::Morbo::Backend, as the names of Futurama are not ours
+class Pyjo_Server_Reload_Backend(Pyjo.Base.object):
+    """
+    :mod:`Pyjo.Server.Reload.Backend` inherits all methods from
+    :mod:`Pyjo.Base` and implements the following new ones.
+    """
+
+    def __init__(self, *, watch: list[Any] | None = None, watch_timeout: float | None = None) -> None:
+        """::
+
+            backend = Pyjo.Server.Reload.Backend.new()
+            backend = Pyjo.Server.Reload.Backend.new(watch=['lib'], watch_timeout=2)
+
+        Construct a new :mod:`Pyjo.Server.Reload.Backend` object.
+        """
+        super().__init__()
+
+        self.watch: list[Any] = watch if watch is not None else ['lib', 'templates']
+        """::
+
+            watch = backend.watch
+            backend.watch = ['/home/sri/my_app']
+
+        Files and directories to watch for changes, defaults to the application
+        script as well as the ``lib`` and ``templates`` directories in the
+        current working directory.
+        """
+
+        self._watch_timeout: float | None = watch_timeout
+
+    @property
+    def watch_timeout(self) -> float:
+        """::
+
+            timeout = backend.watch_timeout
+            backend.watch_timeout = 10
+
+        Maximum amount of time in seconds a backend may block when waiting for
+        files to change, defaults to the value of the ``PYJO_RELOAD_TIMEOUT``
+        environment variable or ``1``.
+        """
+        if self._watch_timeout is None:
+            self._watch_timeout = float(os.environ.get('PYJO_RELOAD_TIMEOUT') or 1)
+        return self._watch_timeout
+
+    @watch_timeout.setter
+    def watch_timeout(self, value: float) -> None:
+        self._watch_timeout = value
+
+    def modified_files(self) -> list[str]:
+        """::
+
+            files = backend.modified_files()
+
+        Check if files from :attr:`watch` have been modified since the last
+        check and return their names. Meant to be overloaded in a subclass.
+        """
+        raise NotImplementedError('Method "modified_files" not implemented by subclass')
+
+
+new = Pyjo_Server_Reload_Backend.new
+object = Pyjo_Server_Reload_Backend
+Backend = Pyjo_Server_Reload_Backend