beanqueue 0.2.0__tar.gz → 0.2.2__tar.gz

{beanqueue-0.2.0 → beanqueue-0.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: beanqueue
-Version: 0.2.0
+Version: 0.2.2
 Summary: BeanQueue or BQ for short, PostgreSQL SKIP LOCK based worker queue library
 License: MIT
 Author: Fang-Pen Lin
@@ -151,7 +151,7 @@ config = bq.Config(
 app = bq.BeanQueue(config=config)
 ```
 
-Then you can pass `--app` argument pointing to the app object to the process command like this:
+Then you can pass `--app` argument (or `-a` for short) pointing to the app object to the process command like this:
 
 ```bash
 python -m bq.cmds.process -a my_pkgs.bq.app images

{beanqueue-0.2.0 → beanqueue-0.2.2}/README.md

@@ -131,7 +131,7 @@ config = bq.Config(
 app = bq.BeanQueue(config=config)
 ```
 
-Then you can pass `--app` argument pointing to the app object to the process command like this:
+Then you can pass `--app` argument (or `-a` for short) pointing to the app object to the process command like this:
 
 ```bash
 python -m bq.cmds.process -a my_pkgs.bq.app images

{beanqueue-0.2.0 → beanqueue-0.2.2}/bq/app.py

@@ -1,11 +1,14 @@
 import functools
 import importlib
+import json
 import logging
 import platform
 import sys
 import threading
 import time
 import typing
+from wsgiref.simple_server import make_server
+from wsgiref.simple_server import WSGIRequestHandler
 
 import venusian
 from sqlalchemy import func
@@ -29,6 +32,21 @@ from .utils import load_module_var
 logger = logging.getLogger(__name__)
 
 
+class WSGIRequestHandlerWithLogger(WSGIRequestHandler):
+    logger = logging.getLogger("metrics_server")
+
+    def log_message(self, format, *args):
+        message = format % args
+        self.logger.info(
+            "%s - - [%s] %s\n"
+            % (
+                self.address_string(),
+                self.log_date_time_string(),
+                message.translate(self._control_char_table),
+            )
+        )
+
+
 class BeanQueue:
     def __init__(
         self,
@@ -150,7 +168,9 @@ class BeanQueue:
                 # is set too short. It could also be the administrator update the worker state to something else than
                 # RUNNING. Regardless the reason, let's stop processing.
                 logger.warning(
-                    "Current worker %s state is %s instead of running, quit processing"
+                    "Current worker %s state is %s instead of running, quit processing",
+                    current_worker.id,
+                    current_worker.state,
                 )
                 sys.exit(0)
 
@@ -159,6 +179,64 @@ class BeanQueue:
             db.add(current_worker)
             db.commit()
 
+    def _serve_http_request(
+        self, worker_id: typing.Any, environ: dict, start_response: typing.Callable
+    ) -> list[bytes]:
+        path = environ["PATH_INFO"]
+        if path == "/healthz":
+            db = self.make_session()
+            worker_service = self._make_worker_service(db)
+            worker = worker_service.get_worker(worker_id)
+            if worker is not None and worker.state == models.WorkerState.RUNNING:
+                start_response(
+                    "200 OK",
+                    [
+                        ("Content-Type", "application/json"),
+                    ],
+                )
+                return [
+                    json.dumps(dict(status="ok", worker_id=str(worker_id))).encode(
+                        "utf8"
+                    )
+                ]
+            else:
+                logger.warning("Bad worker %s state %s", worker_id, worker.state)
+                start_response(
+                    "500 Internal Server Error",
+                    [
+                        ("Content-Type", "application/json"),
+                    ],
+                )
+                return [
+                    json.dumps(
+                        dict(
+                            status="internal error",
+                            worker_id=str(worker_id),
+                            state=str(worker.state),
+                        )
+                    ).encode("utf8")
+                ]
+        # TODO: add other metrics endpoints
+        start_response(
+            "404 NOT FOUND",
+            [
+                ("Content-Type", "application/json"),
+            ],
+        )
+        return [json.dumps(dict(status="not found")).encode("utf8")]
+
+    def run_metrics_http_server(self, worker_id: typing.Any):
+        host = self.config.METRICS_HTTP_SERVER_INTERFACE
+        port = self.config.METRICS_HTTP_SERVER_PORT
+        with make_server(
+            host,
+            port,
+            functools.partial(self._serve_http_request, worker_id),
+            handler_class=WSGIRequestHandlerWithLogger,
+        ) as httpd:
+            logger.info("Run metrics HTTP server on %s:%s", host, port)
+            httpd.serve_forever()
+
     def process_tasks(
         self,
         channels: tuple[str, ...],
@@ -194,6 +272,18 @@ class BeanQueue:
         dispatch_service.listen(channels)
         db.commit()
 
+        metrics_server_thread = None
+        if self.config.METRICS_HTTP_SERVER_ENABLED:
+            WSGIRequestHandlerWithLogger.logger.setLevel(
+                self.config.METRICS_HTTP_SERVER_LOG_LEVEL
+            )
+            metrics_server_thread = threading.Thread(
+                target=self.run_metrics_http_server,
+                args=(worker.id,),
+            )
+            metrics_server_thread.daemon = True
+            metrics_server_thread.start()
+
         logger.info("Created worker %s, name=%s", worker.id, worker.name)
         events.worker_init.send(self, worker=worker)
 
@@ -249,6 +339,8 @@ class BeanQueue:
             db.rollback()
             logger.info("Shutting down ...")
             worker_update_thread.join(5)
+            if metrics_server_thread is not None:
+                metrics_server_thread.join(5)
 
         worker.state = models.WorkerState.SHUTDOWN
         db.add(worker)

{beanqueue-0.2.0 → beanqueue-0.2.2}/bq/config.py

@@ -30,6 +30,18 @@ class Config(BaseSettings):
     # which worker model to use
     WORKER_MODEL: str = "bq.Worker"
 
+    # Enable metrics HTTP server
+    METRICS_HTTP_SERVER_ENABLED: bool = True
+
+    # the metrics http server interface to listen
+    METRICS_HTTP_SERVER_INTERFACE: str = ""
+
+    # the metrics http server port to listen
+    METRICS_HTTP_SERVER_PORT: int = 8000
+
+    # default log level for metrics http server
+    METRICS_HTTP_SERVER_LOG_LEVEL: int = 30
+
     POSTGRES_SERVER: str = "localhost"
     POSTGRES_USER: str = "bq"
     POSTGRES_PASSWORD: str = ""

{beanqueue-0.2.0 → beanqueue-0.2.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "beanqueue"
-version = "0.2.0"
+version = "0.2.2"
 description = "BeanQueue or BQ for short, PostgreSQL SKIP LOCK based worker queue library"
 authors = ["Fang-Pen Lin <fangpen@launchplatform.com>"]
 license = "MIT"