queueflow 0.2.2__tar.gz → 0.3.0__tar.gz

{queueflow-0.2.2/queueflow.egg-info → queueflow-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: queueflow
-Version: 0.2.2
+Version: 0.3.0
 Summary: Python client for QueueFlow, a PostgreSQL-native distributed job queue and workflow engine.
 Home-page: https://github.com/elision-labs/queueflow-sdk-python
 Author: QueueFlow
@@ -93,9 +93,12 @@ qf = QueueFlow(
     base_url,             # required, e.g. "http://localhost:8000"
     token,                # required: tenant bearer token (any non-empty token on the dev server)
     worker_token=None,    # credential for qf.worker and run_worker (defaults to token; dev mode only)
+    timeout=30.0,         # default per-request timeout in seconds; a (connect, read) pair or None also work
 )
 ```
 
+Any generated api method accepts `_request_timeout=` to override the default for one call.
+
 The generated per-tag clients hang off the facade for anything the helpers do not cover:
 `qf.jobs`, `qf.workflows`, `qf.worker`, `qf.cron`, `qf.dlq`, `qf.system`, `qf.health_api`.
 
@@ -168,16 +171,50 @@ def transcode(job, ctx):
     return {"ok": True}
 
 stop = threading.Event()                          # set it (e.g. from a signal handler) to drain
-qf.run_worker("orders", {"send_email": send_email, "transcode": transcode}, stop=stop)
+result = qf.run_worker(
+    "orders",
+    {"send_email": send_email, "transcode": transcode},
+    stop=stop,
+    concurrency=4,
+)
+for job in result.unfinished:                     # handlers still running when the drain gave up
+    print("abandoned", job.id)
+```
+
+```python
+run_worker(
+    queue, handlers,
+    lease_secs=30,        # lease per job; each in-flight job is heartbeated at half this
+    wait_secs=20,         # long-poll wait when the queue is empty
+    stop=None,            # threading.Event: set it to stop
+    on_error=None,        # called on transient lease errors (default: throttled log)
+    concurrency=1,        # handlers running at once, on a ThreadPoolExecutor
+    poll_wait_secs=5,     # cap on wait_secs while `stop` is provided (bounds stop latency)
+    drain_timeout=30.0,   # seconds to wait for in-flight handlers after stop
+) -> WorkerRunResult      # .unfinished: jobs still running when the drain gave up
 ```
 
-`run_worker(queue, handlers, lease_secs=30, wait_secs=20, stop=None, on_error=None)` leases one
-job at a time, heartbeats each in-flight job at half the lease interval, stops reporting when the
+`run_worker` heartbeats each in-flight job at half the lease interval, stops reporting when the
 lease is lost, and applies the server's retry policy on errors. Handlers take `(job)` or
 `(job, ctx)` and return the result dict (or `None` for `{}`). Delivery is at-least-once, so make
 handlers idempotent. The low-level calls (`qf.worker.lease_jobs`, `heartbeat_job`, `complete_job`,
 `fail_job`) are also exposed.
 
+Concurrency: up to `concurrency` handlers run at once. Each lease asks for as many jobs as there
+are free slots, and no lease is issued while every slot is busy.
+
+Stop and drain: set `stop`. The generated urllib3 client cannot cancel an in-flight long poll, so
+while `stop` is provided each lease waits at most `poll_wait_secs` (never more than `wait_secs`),
+which bounds stop latency to about that long; without `stop`, the full `wait_secs` is used. After
+the loop exits, handlers already running get up to `drain_timeout` seconds to finish and report.
+Any handler still running after that is abandoned: its `ctx.cancelled` flips, nothing is reported
+(the lease expires and the server redelivers), and the job is listed in the returned
+`WorkerRunResult.unfinished`. Python cannot kill a thread, so an abandoned handler keeps its pool
+thread until it returns; long handlers should check `ctx.cancelled`.
+
+Timeouts: every request uses the client-wide `timeout` (30s by default, see Client), except the
+lease long poll, which gets its own timeout of the effective wait plus 35 seconds.
+
 Rules the runtime enforces, and that any hand-rolled loop over `qf.worker` must honour too:
 
 1. Worker routes authenticate with the **worker token**, not a tenant token. Pass it as
@@ -209,10 +246,15 @@ Rules the runtime enforces, and that any hand-rolled loop over `qf.worker` must
 ```python
 qf.system.get_stats()    # engine counters
 qf.system.list_tasks()   # registered task handler names
+qf.queues()              # live per-queue backlog: pending, scheduled, running, oldest pending age
 qf.health_api.get_health()
 qf.health_api.get_ready()
 ```
 
+`qf.queues()` (`GET /api/v1/queues`) returns a `QueueStats` per queue that currently has
+non-terminal jobs, so you can decide whether to add workers; idle queues are omitted.
+`qf.system.list_queues()` is the raw call.
+
 ## Authentication
 
 Every request carries `Authorization: Bearer <token>`. Two credentials exist:

{queueflow-0.2.2 → queueflow-0.3.0}/README.md

@@ -52,9 +52,12 @@ qf = QueueFlow(
     base_url,             # required, e.g. "http://localhost:8000"
     token,                # required: tenant bearer token (any non-empty token on the dev server)
     worker_token=None,    # credential for qf.worker and run_worker (defaults to token; dev mode only)
+    timeout=30.0,         # default per-request timeout in seconds; a (connect, read) pair or None also work
 )
 ```
 
+Any generated api method accepts `_request_timeout=` to override the default for one call.
+
 The generated per-tag clients hang off the facade for anything the helpers do not cover:
 `qf.jobs`, `qf.workflows`, `qf.worker`, `qf.cron`, `qf.dlq`, `qf.system`, `qf.health_api`.
 
@@ -127,16 +130,50 @@ def transcode(job, ctx):
     return {"ok": True}
 
 stop = threading.Event()                          # set it (e.g. from a signal handler) to drain
-qf.run_worker("orders", {"send_email": send_email, "transcode": transcode}, stop=stop)
+result = qf.run_worker(
+    "orders",
+    {"send_email": send_email, "transcode": transcode},
+    stop=stop,
+    concurrency=4,
+)
+for job in result.unfinished:                     # handlers still running when the drain gave up
+    print("abandoned", job.id)
+```
+
+```python
+run_worker(
+    queue, handlers,
+    lease_secs=30,        # lease per job; each in-flight job is heartbeated at half this
+    wait_secs=20,         # long-poll wait when the queue is empty
+    stop=None,            # threading.Event: set it to stop
+    on_error=None,        # called on transient lease errors (default: throttled log)
+    concurrency=1,        # handlers running at once, on a ThreadPoolExecutor
+    poll_wait_secs=5,     # cap on wait_secs while `stop` is provided (bounds stop latency)
+    drain_timeout=30.0,   # seconds to wait for in-flight handlers after stop
+) -> WorkerRunResult      # .unfinished: jobs still running when the drain gave up
 ```
 
-`run_worker(queue, handlers, lease_secs=30, wait_secs=20, stop=None, on_error=None)` leases one
-job at a time, heartbeats each in-flight job at half the lease interval, stops reporting when the
+`run_worker` heartbeats each in-flight job at half the lease interval, stops reporting when the
 lease is lost, and applies the server's retry policy on errors. Handlers take `(job)` or
 `(job, ctx)` and return the result dict (or `None` for `{}`). Delivery is at-least-once, so make
 handlers idempotent. The low-level calls (`qf.worker.lease_jobs`, `heartbeat_job`, `complete_job`,
 `fail_job`) are also exposed.
 
+Concurrency: up to `concurrency` handlers run at once. Each lease asks for as many jobs as there
+are free slots, and no lease is issued while every slot is busy.
+
+Stop and drain: set `stop`. The generated urllib3 client cannot cancel an in-flight long poll, so
+while `stop` is provided each lease waits at most `poll_wait_secs` (never more than `wait_secs`),
+which bounds stop latency to about that long; without `stop`, the full `wait_secs` is used. After
+the loop exits, handlers already running get up to `drain_timeout` seconds to finish and report.
+Any handler still running after that is abandoned: its `ctx.cancelled` flips, nothing is reported
+(the lease expires and the server redelivers), and the job is listed in the returned
+`WorkerRunResult.unfinished`. Python cannot kill a thread, so an abandoned handler keeps its pool
+thread until it returns; long handlers should check `ctx.cancelled`.
+
+Timeouts: every request uses the client-wide `timeout` (30s by default, see Client), except the
+lease long poll, which gets its own timeout of the effective wait plus 35 seconds.
+
 Rules the runtime enforces, and that any hand-rolled loop over `qf.worker` must honour too:
 
 1. Worker routes authenticate with the **worker token**, not a tenant token. Pass it as
@@ -168,10 +205,15 @@ Rules the runtime enforces, and that any hand-rolled loop over `qf.worker` must
 ```python
 qf.system.get_stats()    # engine counters
 qf.system.list_tasks()   # registered task handler names
+qf.queues()              # live per-queue backlog: pending, scheduled, running, oldest pending age
 qf.health_api.get_health()
 qf.health_api.get_ready()
 ```
 
+`qf.queues()` (`GET /api/v1/queues`) returns a `QueueStats` per queue that currently has
+non-terminal jobs, so you can decide whether to add workers; idle queues are omitted.
+`qf.system.list_queues()` is the raw call.
+
 ## Authentication
 
 Every request carries `Authorization: Bearer <token>`. Two credentials exist:

{queueflow-0.2.2 → queueflow-0.3.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "queueflow"
-version = "0.2.2"
+version = "0.3.0"
 description = "Python client for QueueFlow, a PostgreSQL-native distributed job queue and workflow engine."
 readme = "README.md"
 license = "MIT"

{queueflow-0.2.2 → queueflow-0.3.0}/queueflow/__init__.py

@@ -7,14 +7,14 @@
 
     REST API for QueueFlow, a PostgreSQL-native distributed job queue and workflow engine.
 
-    The version of the OpenAPI document: 1.0.0
+    The version of the OpenAPI document: 0.2.0
     Generated by OpenAPI Generator (https://openapi-generator.tech)
 
     Do not edit the class manually.
 """  # noqa: E501
 
 
-__version__ = "0.2.2"
+__version__ = "0.3.0"
 
 # import apis into sdk package
 from queueflow.api.cron_api import CronApi
@@ -67,6 +67,8 @@ from queueflow.models.list_jobs_response import ListJobsResponse
 from queueflow.models.list_workflows_response import ListWorkflowsResponse
 from queueflow.models.on_failure import OnFailure
 from queueflow.models.on_success import OnSuccess
+from queueflow.models.queue_stats import QueueStats
+from queueflow.models.queues_response import QueuesResponse
 from queueflow.models.ready_status import ReadyStatus
 from queueflow.models.replay_dead_letter_response import ReplayDeadLetterResponse
 from queueflow.models.stats_snapshot import StatsSnapshot

{queueflow-0.2.2 → queueflow-0.3.0}/queueflow/api/cron_api.py

@@ -5,7 +5,7 @@
 
     REST API for QueueFlow, a PostgreSQL-native distributed job queue and workflow engine.
 
-    The version of the OpenAPI document: 1.0.0
+    The version of the OpenAPI document: 0.2.0
     Generated by OpenAPI Generator (https://openapi-generator.tech)
 
     Do not edit the class manually.

{queueflow-0.2.2 → queueflow-0.3.0}/queueflow/api/dlq_api.py

@@ -5,7 +5,7 @@
 
     REST API for QueueFlow, a PostgreSQL-native distributed job queue and workflow engine.
 
-    The version of the OpenAPI document: 1.0.0
+    The version of the OpenAPI document: 0.2.0
     Generated by OpenAPI Generator (https://openapi-generator.tech)
 
     Do not edit the class manually.

{queueflow-0.2.2 → queueflow-0.3.0}/queueflow/api/health_api.py

@@ -5,7 +5,7 @@
 
     REST API for QueueFlow, a PostgreSQL-native distributed job queue and workflow engine.
 
-    The version of the OpenAPI document: 1.0.0
+    The version of the OpenAPI document: 0.2.0
     Generated by OpenAPI Generator (https://openapi-generator.tech)
 
     Do not edit the class manually.

{queueflow-0.2.2 → queueflow-0.3.0}/queueflow/api/jobs_api.py

@@ -5,7 +5,7 @@
 
     REST API for QueueFlow, a PostgreSQL-native distributed job queue and workflow engine.
 
-    The version of the OpenAPI document: 1.0.0
+    The version of the OpenAPI document: 0.2.0
     Generated by OpenAPI Generator (https://openapi-generator.tech)
 
     Do not edit the class manually.

{queueflow-0.2.2 → queueflow-0.3.0}/queueflow/api/system_api.py

@@ -5,7 +5,7 @@
 
     REST API for QueueFlow, a PostgreSQL-native distributed job queue and workflow engine.
 
-    The version of the OpenAPI document: 1.0.0
+    The version of the OpenAPI document: 0.2.0
     Generated by OpenAPI Generator (https://openapi-generator.tech)
 
     Do not edit the class manually.
@@ -16,6 +16,7 @@ from pydantic import validate_call, Field, StrictFloat, StrictStr, StrictInt
 from typing import Any, Dict, List, Optional, Tuple, Union
 from typing_extensions import Annotated
 
+from queueflow.models.queues_response import QueuesResponse
 from queueflow.models.stats_snapshot import StatsSnapshot
 from queueflow.models.tasks_response import TasksResponse
 
@@ -280,6 +281,249 @@ class SystemApi:
 
 
 
+    @validate_call
+    def list_queues(
+        self,
+        _request_timeout: Union[
+            None,
+            Annotated[StrictFloat, Field(gt=0)],
+            Tuple[
+                Annotated[StrictFloat, Field(gt=0)],
+                Annotated[StrictFloat, Field(gt=0)]
+            ]
+        ] = None,
+        _request_auth: Optional[Dict[StrictStr, Any]] = None,
+        _content_type: Optional[StrictStr] = None,
+        _headers: Optional[Dict[StrictStr, Any]] = None,
+        _host_index: Annotated[StrictInt, Field(ge=0, le=0)] = 0,
+    ) -> QueuesResponse:
+        """list_queues
+
+
+        :param _request_timeout: timeout setting for this request. If one
+                                 number provided, it will be total request
+                                 timeout. It can also be a pair (tuple) of
+                                 (connection, read) timeouts.
+        :type _request_timeout: int, tuple(int, int), optional
+        :param _request_auth: set to override the auth_settings for an a single
+                              request; this effectively ignores the
+                              authentication in the spec for a single request.
+        :type _request_auth: dict, optional
+        :param _content_type: force content-type for the request.
+        :type _content_type: str, Optional
+        :param _headers: set to override the headers for a single
+                         request; this effectively ignores the headers
+                         in the spec for a single request.
+        :type _headers: dict, optional
+        :param _host_index: set to override the host_index for a single
+                            request; this effectively ignores the host_index
+                            in the spec for a single request.
+        :type _host_index: int, optional
+        :return: Returns the result object.
+        """ # noqa: E501
+
+        _param = self._list_queues_serialize(
+            _request_auth=_request_auth,
+            _content_type=_content_type,
+            _headers=_headers,
+            _host_index=_host_index
+        )
+
+        _response_types_map: Dict[str, Optional[str]] = {
+            '200': "QueuesResponse",
+        }
+        response_data = self.api_client.call_api(
+            *_param,
+            _request_timeout=_request_timeout
+        )
+        response_data.read()
+        return self.api_client.response_deserialize(
+            response_data=response_data,
+            response_types_map=_response_types_map,
+        ).data
+
+
+    @validate_call
+    def list_queues_with_http_info(
+        self,
+        _request_timeout: Union[
+            None,
+            Annotated[StrictFloat, Field(gt=0)],
+            Tuple[
+                Annotated[StrictFloat, Field(gt=0)],
+                Annotated[StrictFloat, Field(gt=0)]
+            ]
+        ] = None,
+        _request_auth: Optional[Dict[StrictStr, Any]] = None,
+        _content_type: Optional[StrictStr] = None,
+        _headers: Optional[Dict[StrictStr, Any]] = None,
+        _host_index: Annotated[StrictInt, Field(ge=0, le=0)] = 0,
+    ) -> ApiResponse[QueuesResponse]:
+        """list_queues
+
+
+        :param _request_timeout: timeout setting for this request. If one
+                                 number provided, it will be total request
+                                 timeout. It can also be a pair (tuple) of
+                                 (connection, read) timeouts.
+        :type _request_timeout: int, tuple(int, int), optional
+        :param _request_auth: set to override the auth_settings for an a single
+                              request; this effectively ignores the
+                              authentication in the spec for a single request.
+        :type _request_auth: dict, optional
+        :param _content_type: force content-type for the request.
+        :type _content_type: str, Optional
+        :param _headers: set to override the headers for a single
+                         request; this effectively ignores the headers
+                         in the spec for a single request.
+        :type _headers: dict, optional
+        :param _host_index: set to override the host_index for a single
+                            request; this effectively ignores the host_index
+                            in the spec for a single request.
+        :type _host_index: int, optional
+        :return: Returns the result object.
+        """ # noqa: E501
+
+        _param = self._list_queues_serialize(
+            _request_auth=_request_auth,
+            _content_type=_content_type,
+            _headers=_headers,
+            _host_index=_host_index
+        )
+
+        _response_types_map: Dict[str, Optional[str]] = {
+            '200': "QueuesResponse",
+        }
+        response_data = self.api_client.call_api(
+            *_param,
+            _request_timeout=_request_timeout
+        )
+        response_data.read()
+        return self.api_client.response_deserialize(
+            response_data=response_data,
+            response_types_map=_response_types_map,
+        )
+
+
+    @validate_call
+    def list_queues_without_preload_content(
+        self,
+        _request_timeout: Union[
+            None,
+            Annotated[StrictFloat, Field(gt=0)],
+            Tuple[
+                Annotated[StrictFloat, Field(gt=0)],
+                Annotated[StrictFloat, Field(gt=0)]
+            ]
+        ] = None,
+        _request_auth: Optional[Dict[StrictStr, Any]] = None,
+        _content_type: Optional[StrictStr] = None,
+        _headers: Optional[Dict[StrictStr, Any]] = None,
+        _host_index: Annotated[StrictInt, Field(ge=0, le=0)] = 0,
+    ) -> RESTResponseType:
+        """list_queues
+
+
+        :param _request_timeout: timeout setting for this request. If one
+                                 number provided, it will be total request
+                                 timeout. It can also be a pair (tuple) of
+                                 (connection, read) timeouts.
+        :type _request_timeout: int, tuple(int, int), optional
+        :param _request_auth: set to override the auth_settings for an a single
+                              request; this effectively ignores the
+                              authentication in the spec for a single request.
+        :type _request_auth: dict, optional
+        :param _content_type: force content-type for the request.
+        :type _content_type: str, Optional
+        :param _headers: set to override the headers for a single
+                         request; this effectively ignores the headers
+                         in the spec for a single request.
+        :type _headers: dict, optional
+        :param _host_index: set to override the host_index for a single
+                            request; this effectively ignores the host_index
+                            in the spec for a single request.
+        :type _host_index: int, optional
+        :return: Returns the result object.
+        """ # noqa: E501
+
+        _param = self._list_queues_serialize(
+            _request_auth=_request_auth,
+            _content_type=_content_type,
+            _headers=_headers,
+            _host_index=_host_index
+        )
+
+        _response_types_map: Dict[str, Optional[str]] = {
+            '200': "QueuesResponse",
+        }
+        response_data = self.api_client.call_api(
+            *_param,
+            _request_timeout=_request_timeout
+        )
+        return response_data.response
+
+
+    def _list_queues_serialize(
+        self,
+        _request_auth,
+        _content_type,
+        _headers,
+        _host_index,
+    ) -> RequestSerialized:
+
+        _host = None
+
+        _collection_formats: Dict[str, str] = {
+        }
+
+        _path_params: Dict[str, str] = {}
+        _query_params: List[Tuple[str, str]] = []
+        _header_params: Dict[str, Optional[str]] = _headers or {}
+        _form_params: List[Tuple[str, str]] = []
+        _files: Dict[
+            str, Union[str, bytes, List[str], List[bytes], List[Tuple[str, bytes]]]
+        ] = {}
+        _body_params: Optional[bytes] = None
+
+        # process the path parameters
+        # process the query parameters
+        # process the header parameters
+        # process the form parameters
+        # process the body parameter
+
+
+        # set the HTTP header `Accept`
+        if 'Accept' not in _header_params:
+            _header_params['Accept'] = self.api_client.select_header_accept(
+                [
+                    'application/json'
+                ]
+            )
+
+
+        # authentication setting
+        _auth_settings: List[str] = [
+            'bearerAuth'
+        ]
+
+        return self.api_client.param_serialize(
+            method='GET',
+            resource_path='/api/v1/queues',
+            path_params=_path_params,
+            query_params=_query_params,
+            header_params=_header_params,
+            body=_body_params,
+            post_params=_form_params,
+            files=_files,
+            auth_settings=_auth_settings,
+            collection_formats=_collection_formats,
+            _host=_host,
+            _request_auth=_request_auth
+        )
+
+
+
+
     @validate_call
     def list_tasks(
         self,

{queueflow-0.2.2 → queueflow-0.3.0}/queueflow/api/worker_api.py

@@ -5,7 +5,7 @@
 
     REST API for QueueFlow, a PostgreSQL-native distributed job queue and workflow engine.
 
-    The version of the OpenAPI document: 1.0.0
+    The version of the OpenAPI document: 0.2.0
     Generated by OpenAPI Generator (https://openapi-generator.tech)
 
     Do not edit the class manually.

{queueflow-0.2.2 → queueflow-0.3.0}/queueflow/api/workflows_api.py

@@ -5,7 +5,7 @@
 
     REST API for QueueFlow, a PostgreSQL-native distributed job queue and workflow engine.
 
-    The version of the OpenAPI document: 1.0.0
+    The version of the OpenAPI document: 0.2.0
     Generated by OpenAPI Generator (https://openapi-generator.tech)
 
     Do not edit the class manually.

{queueflow-0.2.2 → queueflow-0.3.0}/queueflow/api_client.py

@@ -5,7 +5,7 @@
 
     REST API for QueueFlow, a PostgreSQL-native distributed job queue and workflow engine.
 
-    The version of the OpenAPI document: 1.0.0
+    The version of the OpenAPI document: 0.2.0
     Generated by OpenAPI Generator (https://openapi-generator.tech)
 
     Do not edit the class manually.
@@ -90,7 +90,7 @@ class ApiClient:
             self.default_headers[header_name] = header_value
         self.cookie = cookie
         # Set default User-Agent.
-        self.user_agent = 'OpenAPI-Generator/0.2.0/python'
+        self.user_agent = 'OpenAPI-Generator/0.3.0/python'
         self.client_side_validation = configuration.client_side_validation
 
     def __enter__(self):

{queueflow-0.2.2 → queueflow-0.3.0}/queueflow/configuration.py

@@ -5,7 +5,7 @@
 
     REST API for QueueFlow, a PostgreSQL-native distributed job queue and workflow engine.
 
-    The version of the OpenAPI document: 1.0.0
+    The version of the OpenAPI document: 0.2.0
     Generated by OpenAPI Generator (https://openapi-generator.tech)
 
     Do not edit the class manually.
@@ -503,8 +503,8 @@ class Configuration:
         return "Python SDK Debug Report:\n"\
                "OS: {env}\n"\
                "Python Version: {pyversion}\n"\
-               "Version of the API: 1.0.0\n"\
-               "SDK Package Version: 0.2.0".\
+               "Version of the API: 0.2.0\n"\
+               "SDK Package Version: 0.3.0".\
                format(env=sys.platform, pyversion=sys.version)
 
     def get_host_settings(self) -> List[HostSetting]:

{queueflow-0.2.2 → queueflow-0.3.0}/queueflow/exceptions.py

@@ -5,7 +5,7 @@
 
     REST API for QueueFlow, a PostgreSQL-native distributed job queue and workflow engine.
 
-    The version of the OpenAPI document: 1.0.0
+    The version of the OpenAPI document: 0.2.0
     Generated by OpenAPI Generator (https://openapi-generator.tech)
 
     Do not edit the class manually.