flowstash-runtime 0.9.2__tar.gz → 0.10.0__tar.gz

{flowstash_runtime-0.9.2 → flowstash_runtime-0.10.0}/PKG-INFO

@@ -1,10 +1,12 @@
 Metadata-Version: 2.3
 Name: flowstash-runtime
-Version: 0.9.2
+Version: 0.10.0
 Summary: Actual runtime engine and bootstrap layer for the flowstash platform.
+License: MIT
 Author: juraj.bezdek@gmail.com
 Author-email: juraj.bezdek@gmail.com
 Requires-Python: >=3.11
+Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
@@ -12,8 +14,8 @@ Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: apscheduler (>=3.11.0)
 Requires-Dist: dramatiq (>=1.16.0)
 Requires-Dist: fastapi (>=0.110.0)
-Requires-Dist: flowstash-clients (>=0.9.2,<0.10.0)
-Requires-Dist: flowstash-lib (>=0.9.2,<0.10.0)
+Requires-Dist: flowstash-clients (>=0.10.0,<0.11.0)
+Requires-Dist: flowstash-lib (>=0.10.0,<0.11.0)
 Requires-Dist: pyyaml (>=6.0.1)
 Requires-Dist: uvicorn (>=0.29.0)
 Requires-Dist: websockets (>=14.0)

{flowstash_runtime-0.9.2 → flowstash_runtime-0.10.0}/pyproject.toml

@@ -1,12 +1,13 @@
 [project]
 name = "flowstash-runtime"
-version = "0.9.2"
+version = "0.10.0"
 description = "Actual runtime engine and bootstrap layer for the flowstash platform."
 authors = [{name = "juraj.bezdek@gmail.com", email = "juraj.bezdek@gmail.com"}]
+license = "MIT"
 requires-python = ">=3.11"
 dependencies = [
-    "flowstash-clients>=0.9.2,<0.10.0",
-    "flowstash-lib>=0.9.2,<0.10.0",
+    "flowstash-clients>=0.10.0,<0.11.0",
+    "flowstash-lib>=0.10.0,<0.11.0",
     "fastapi>=0.110.0",
     "uvicorn>=0.29.0",
     "dramatiq>=1.16.0",

{flowstash_runtime-0.9.2 → flowstash_runtime-0.10.0}/src/flowstash/runtime/worker/backends/dramatiq/dramatiq_backend.py

@@ -8,6 +8,7 @@ import dramatiq
 import logging
 import asyncio
 import threading
+from datetime import datetime, UTC
 from typing import Any, Callable, Optional, Mapping, List, Union
 from flowstash.queue.backend import JobHandle, TaskBackend, Schedule
 from flowstash.context import IntegrationContext, integration_context, current_context
@@ -244,8 +245,17 @@ class DramatiqBackend(TaskBackend):
         if pipeline:
             headers["fw.pipeline"] = pipeline
 
-        delay = eta_or_delay if isinstance(eta_or_delay, (int, float)) else None
-        # Handle datetime eta if needed...
+        # Framework API expresses numeric delays in seconds; dramatiq wants ms.
+        delay: Optional[int] = None
+        if isinstance(eta_or_delay, (int, float)):
+            delay = max(0, int(eta_or_delay * 1000))
+        elif isinstance(eta_or_delay, datetime):
+            eta = (
+                eta_or_delay
+                if eta_or_delay.tzinfo
+                else eta_or_delay.replace(tzinfo=UTC)
+            )
+            delay = max(0, int((eta - datetime.now(UTC)).total_seconds() * 1000))
 
         message = func.send_with_options(
             args=args, kwargs=kwargs, delay=delay, headers=headers

{flowstash_runtime-0.9.2 → flowstash_runtime-0.10.0}/src/flowstash/runtime/worker/backends/managed/http_entrypoint.py

@@ -170,7 +170,7 @@ def _get_drain_controller(request: Request) -> ManagedTaskDrainController:
     return controller
 
 
-async def _flush_observability(timeout_s: float = 15.0) -> None:
+async def _flush_observability(timeout_s: float = 30.0) -> None:
     import asyncio
     from flowstash.observability.ingestion import AsyncManager
 

{flowstash_runtime-0.9.2 → flowstash_runtime-0.10.0}/src/flowstash/runtime/worker/backends/managed/lease_client.py

@@ -20,6 +20,8 @@ import asyncio
 import json
 import logging
 import os
+import random
+import uuid
 from dataclasses import dataclass
 from typing import Dict, List, Optional, Set
 
@@ -83,6 +85,9 @@ class LeaseBrokerClient:
         self._acquire_timeout = acquire_timeout
         self._reconnect_min = reconnect_min
         self._reconnect_max = reconnect_max
+        # Stable per-process identity so a reconnecting worker reclaims (transfers)
+        # its own leases instead of conflicting with its not-yet-reaped old conn.
+        self._worker_id = os.getenv("CLOUD_RUN_EXECUTION") or uuid.uuid4().hex
 
         self._held: Set[str] = set()
         self._waiters: Dict[str, List[asyncio.Future]] = {}
@@ -110,13 +115,17 @@ class LeaseBrokerClient:
             except Exception:
                 pass
 
+    def _connect_url(self) -> str:
+        sep = "&" if "?" in self._url else "?"
+        return f"{self._url}{sep}worker_id={self._worker_id}"
+
     async def _run(self) -> None:
         backoff = self._reconnect_min
         headers = {"Authorization": f"Bearer {self._token}"}
         while not self._closing:
             try:
                 async with websockets.connect(
-                    self._url, additional_headers=headers, open_timeout=10
+                    self._connect_url(), additional_headers=headers, open_timeout=10
                 ) as ws:
                     self._ws = ws
                     backoff = self._reconnect_min
@@ -138,7 +147,9 @@ class LeaseBrokerClient:
                 self._ws = None
             if self._closing:
                 break
-            await asyncio.sleep(backoff)
+            # Jittered backoff so a broker restart doesn't trigger a synchronized
+            # reconnect storm from every worker at once.
+            await asyncio.sleep(backoff * (0.5 + 0.5 * random.random()))
             backoff = min(backoff * 2, self._reconnect_max)
 
     # ── messaging ──────────────────────────────────────────────────────────
@@ -250,8 +261,11 @@ def get_lease_client() -> Optional[LeaseBrokerClient]:
         return _client
     _resolved = True
 
-    if os.getenv("LEASE_BROKER_ENABLED", "true").lower() in ("0", "false", "no"):
-        logger.info("lease broker disabled via LEASE_BROKER_ENABLED")
+    # Default OFF: deploying the worker code is inert until ops explicitly enables
+    # the guard (after the broker is deployed and validated). Otherwise an
+    # unreachable broker would fail every task closed (503).
+    if os.getenv("LEASE_BROKER_ENABLED", "false").lower() not in ("1", "true", "yes"):
+        logger.info("lease broker disabled (set LEASE_BROKER_ENABLED=true to enable)")
         return None
     if websockets is None:
         logger.warning("lease broker disabled: 'websockets' not installed")

{flowstash_runtime-0.9.2 → flowstash_runtime-0.10.0}/src/flowstash/runtime/worker/backends/managed/managed_consumer.py

@@ -411,7 +411,7 @@ class ManagedConsumer(TaskConsumer):
         except ValueError as e:
             logger.error(f"[run-task] {e}")
             await _record_missing_task(task_name)
-            await asyncio.to_thread(AsyncManager.get_instance().flush, 15.0)
+            await asyncio.to_thread(AsyncManager.get_instance().flush, 30.0)
             sys.exit(1)
 
         # Idempotency guard via the lease broker. Unlike HTTP tasks, a job that
@@ -428,7 +428,7 @@ class ManagedConsumer(TaskConsumer):
                     run_id,
                     res.outcome,
                 )
-                await asyncio.to_thread(AsyncManager.get_instance().flush, 15.0)
+                await asyncio.to_thread(AsyncManager.get_instance().flush, 30.0)
                 sys.exit(0)
             elif res.outcome == ACQUIRED:
                 lease_held = True
@@ -450,7 +450,7 @@ class ManagedConsumer(TaskConsumer):
         if lease_held:
             await lease.release(run_id, SUCCEEDED if success else FAILED)
 
-        await asyncio.to_thread(AsyncManager.get_instance().flush, 15.0)
+        await asyncio.to_thread(AsyncManager.get_instance().flush, 30.0)
         sys.exit(0 if success else 1)
 
     async def stop(self) -> None: