ergon-framework-python 0.1.7__tar.gz → 0.1.9__tar.gz

{ergon_framework_python-0.1.7 → ergon_framework_python-0.1.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ergon-framework-python
-Version: 0.1.7
+Version: 0.1.9
 Summary: Ergon internal task-oriented project bootstrapper
 Author-email: Ergondata Technologies <anza.vossos@protonmail.com>
 License-Expression: MIT

{ergon_framework_python-0.1.7 → ergon_framework_python-0.1.9}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ergon-framework-python"
-version = "0.1.7"
+version = "0.1.9"
 description = "Ergon internal task-oriented project bootstrapper"
 readme = "README.md"
 license = "MIT"
@@ -47,7 +47,7 @@ dev = [
     "pyright>=1.1.390",
     "pytest>=8.0.0",
     "pytest-asyncio>=0.25.0",
-    "ruff>=0.6.0",
+    "ruff>=0.6.0,<0.16",
 ]
 
 [build-system]

{ergon_framework_python-0.1.7 → ergon_framework_python-0.1.9}/src/ergon/__init__.py

@@ -10,4 +10,4 @@ __all__ = [
     "manager",
 ]
 
-__version__ = "0.1.7"
+__version__ = "0.1.9"

{ergon_framework_python-0.1.7 → ergon_framework_python-0.1.9}/src/ergon/connector/rabbitmq/async_service.py

@@ -155,11 +155,33 @@ class AsyncRabbitMQService:
 
         connection = self._connection
         self._connection = None
-        if connection is not None and not connection.is_closed:
-            try:
+        await self._close_connection_safely(connection, reason)
+
+    async def _close_connection_safely(
+        self,
+        connection: Optional[AbstractRobustConnection],
+        reason: str,
+    ) -> None:
+        """Stop transport and robust-reconnect ownership within a fixed deadline.
+
+        ``RobustConnection.is_closed`` only describes its current transport. Its
+        background reconnection task can still be alive after the broker closes
+        that transport, so ``close()`` must always run for an owned connection.
+        """
+        if connection is None:
+            return
+        close_timeout = min(5.0, self.client.connect_timeout)
+        try:
+            async with timeout_after(close_timeout):
                 await connection.close()
-            except Exception as exc:  # noqa: BLE001 - best-effort reset
-                logger.warning("Error closing RabbitMQ connection during reset (%s): %r", reason, exc)
+        except TimeoutError:
+            logger.error(
+                "Timed out closing RabbitMQ connection after %.1fs (%s)",
+                close_timeout,
+                reason,
+            )
+        except Exception as exc:  # noqa: BLE001 - best-effort reset
+            logger.warning("Error closing RabbitMQ connection (%s): %r", reason, exc)
 
     def _invalidate_consume_channel(self, reason: str = "explicit invalidation") -> None:
         """Drop cached consume channel + queue/exchange handles.
@@ -752,9 +774,9 @@ class AsyncRabbitMQService:
                     logger.warning("Error closing %s: %r", attr_name, exc)
             setattr(self, attr_name, None)
 
-        if self._connection is not None and not self._connection.is_closed:
-            await self._connection.close()
-            self._connection = None
+        connection = self._connection
+        self._connection = None
+        await self._close_connection_safely(connection, "service close")
 
         self._active_consumer_tag = None
         self._active_consumer_tags.clear()

{ergon_framework_python-0.1.7 → ergon_framework_python-0.1.9}/src/ergon/task/mixins/consumer.py

@@ -601,22 +601,28 @@ class AsyncConsumerMixin(ABC):
     # =====================================================================
     #   FETCH HANDLER (ASYNC)
     # =====================================================================
-    async def _handle_fetch(self, conn, policy: policies.FetchPolicy) -> tuple[bool, List[connector.Transaction]]:
+    async def _handle_fetch(
+        self,
+        conn,
+        policy: policies.FetchPolicy,
+        batch_size: int | None = None,
+    ) -> tuple[bool, List[connector.Transaction]]:
+        fetch_size = policy.batch.size if batch_size is None else batch_size
         self._mark_consumer_progress("fetching")
-        logger.info(f"Fetch handler started for batch size {policy.batch.size}", extra=policy.extra)
+        logger.info(f"Fetch handler started for batch size {fetch_size}", extra=policy.extra)
         fetch_start = time.perf_counter()
         success, result = await helpers.run_fn_async(
-            fn=lambda: conn.fetch_transactions_async(policy.batch.size, **policy.extra),
+            fn=lambda: conn.fetch_transactions_async(fetch_size, **policy.extra),
             retry=policy.retry,
             trace_name=f"{self.__class__.__name__}.fetch_transactions",
-            trace_attrs={"batch_size": policy.batch.size},
+            trace_attrs={"batch_size": fetch_size},
         )
         # Record fetch metrics
         fetched_count = len(result) if success and result else 0
         mixin_metrics.record_consumer_fetch(
             task_name=getattr(self, "name", self.__class__.__name__),
             connector_name=conn.__class__.__name__,
-            batch_size=policy.batch.size,
+            batch_size=fetch_size,
             fetched_count=fetched_count,
             duration=time.perf_counter() - fetch_start,
             success=success,
@@ -877,13 +883,146 @@ class AsyncConsumerMixin(ABC):
                     )
                     return await self._handle_exception(tr, timeout_error, pol.exception.retry)
 
+            async def _consume_continuous():
+                nonlocal batch_number, empty_count, processed
+
+                concurrency = policy.loop.concurrency.value
+                submitted = 0
+                in_flight: set[asyncio.Task] = set()
+
+                async def reap(done: set[asyncio.Task]) -> None:
+                    nonlocal processed
+                    for task in done:
+                        try:
+                            await task
+                        except Exception as exc:
+                            logger.error("[async] Execution error: %s", exc)
+                        finally:
+                            processed += 1
+
+                async def wait_for_completion() -> None:
+                    done, _ = await asyncio.wait(
+                        in_flight,
+                        return_when=asyncio.FIRST_COMPLETED,
+                    )
+                    in_flight.difference_update(done)
+                    await reap(done)
+                    self._mark_consumer_progress("polling")
+
+                try:
+                    while True:
+                        if policy.loop.limit is not None and submitted >= policy.loop.limit:
+                            if in_flight:
+                                await wait_for_completion()
+                                continue
+                            break
+
+                        free_slots = concurrency - len(in_flight)
+                        if free_slots == 0:
+                            await wait_for_completion()
+                            continue
+
+                        if policy.loop.limit is not None:
+                            free_slots = min(free_slots, policy.loop.limit - submitted)
+
+                        logger.debug(
+                            "Fetching transactions for %d free continuous-consumer slot(s) with fetch policy: %s",
+                            free_slots,
+                            policy.fetch.model_dump_json(indent=2),
+                        )
+                        success, result = await self._handle_fetch(
+                            conn,
+                            policy.fetch,
+                            batch_size=free_slots,
+                        )
+
+                        if not success:
+                            logger.error("Fetch failed → %s", result)
+                            # Unlike shutdown/cancellation, a fetch failure must not
+                            # cancel work already in progress: let in-flight
+                            # transactions finish (bounded by the transaction
+                            # runtime timeout) before propagating the failure.
+                            if in_flight:
+                                logger.warning(
+                                    "Fetch failed with %d transaction(s) in flight; draining before raising",
+                                    len(in_flight),
+                                )
+                                await asyncio.gather(*in_flight, return_exceptions=True)
+                                in_flight.clear()
+                            if isinstance(result, (asyncio.TimeoutError, futures.TimeoutError)):
+                                raise exceptions.FetchTimeoutException(str(result))
+                            raise exceptions.FetchException(str(result))
+
+                        transactions = result
+                        if transactions:
+                            empty_count = 0
+                            batch_number += 1
+                            self._mark_consumer_progress("processing")
+                            logger.info(
+                                "%d transaction(s) fetched for continuous processing",
+                                len(transactions),
+                            )
+                            mixin_metrics.record_consumer_batch(
+                                task_name=getattr(self, "name", self.__class__.__name__),
+                                batch_number=batch_number,
+                                batch_size=len(transactions),
+                                streaming=policy.loop.streaming,
+                            )
+                            for transaction in transactions:
+                                in_flight.add(
+                                    asyncio.create_task(
+                                        submit_start_processing(transaction, policy),
+                                    )
+                                )
+                                submitted += 1
+                            continue
+
+                        if in_flight:
+                            await wait_for_completion()
+                            continue
+
+                        self._mark_consumer_progress("idle")
+                        logger.info("Empty fetch detected at %s", datetime.now().isoformat())
+                        if not policy.loop.streaming:
+                            logger.info("Non-streaming mode detected, breaking loop")
+                            break
+
+                        mixin_metrics.record_consumer_empty_queue_wait(
+                            task_name=getattr(self, "name", self.__class__.__name__),
+                            wait_count=empty_count,
+                        )
+                        await utils.backoff_async(
+                            backoff=policy.fetch.empty.backoff,
+                            multiplier=policy.fetch.empty.backoff_multiplier,
+                            cap=policy.fetch.empty.backoff_cap,
+                            attempt=empty_count,
+                        )
+                        empty_count += 1
+                except BaseException:
+                    for task in in_flight:
+                        task.cancel()
+                    if in_flight:
+                        await asyncio.gather(*in_flight, return_exceptions=True)
+                    raise
+
+                elapsed_time = time.perf_counter() - start_time
+                logger.info(
+                    "[Consume continuous] Finished. Processed=%d in %.2f seconds",
+                    processed,
+                    elapsed_time,
+                )
+                return processed
+
+            if policy.loop.mode == "continuous":
+                return await _consume_continuous()
+
             while True:
                 batch_number += 1
 
                 # ============================================================
                 #  FETCH
                 # ============================================================
-                logger.info(f"Fetching transactions batch with fetch policy: {policy.fetch.model_dump_json(indent=2)}")
+                logger.debug(f"Fetching transactions batch with fetch policy: {policy.fetch.model_dump_json(indent=2)}")
                 success, result = await self._handle_fetch(conn, policy.fetch)
 
                 if not success:
@@ -941,7 +1080,7 @@ class AsyncConsumerMixin(ABC):
                 else:
                     batch_context = None  # Use current context
 
-                logger.info(
+                logger.debug(
                     f"Starting batch processing of "
                     f"{len(transactions)} transaction(s) "
                     f"from fetch handler with "

{ergon_framework_python-0.1.7 → ergon_framework_python-0.1.9}/src/ergon/task/policies.py

@@ -1,4 +1,4 @@
-from typing import Optional, Union
+from typing import Literal, Optional, Union
 
 from pydantic import BaseModel, Field, field_validator
 
@@ -161,11 +161,20 @@ class ExceptionPolicy(BaseModel):
 
 
 class ConsumerLoopPolicy(BaseModel):
+    mode: Literal["batch", "continuous"] = Field(default="batch")
     concurrency: ConcurrencyPolicy = Field(default_factory=ConcurrencyPolicy)
     timeout: Optional[float] = Field(default=None, ge=0)
     limit: Optional[int] = Field(default=None, ge=0)
     streaming: bool = Field(default=False)
 
+    @field_validator("mode", mode="before")
+    @classmethod
+    def _normalize_mode(cls, v):
+        v = _normalize_optional(v)
+        if v is None:
+            return "batch"
+        return v.strip().lower() if isinstance(v, str) else v
+
     @field_validator("timeout", "limit", mode="before")
     @classmethod
     def _normalize_optional_numbers(cls, v):

{ergon_framework_python-0.1.7 → ergon_framework_python-0.1.9}/src/ergon/task/runner.py

@@ -106,15 +106,18 @@ class _TaskLivenessSupervisor:
         self,
         provider: LivenessProvider,
         policy: TaskSupervisionPolicy,
-        request_shutdown,
-        hard_exit=os._exit,
+        request_shutdown: Callable[[str], None],
+        hard_exit: Callable[[int], None] | None = None,
     ) -> None:
         self.provider = provider
         self.policy = policy
         self.request_shutdown = request_shutdown
-        self.hard_exit = hard_exit
+        self.hard_exit = hard_exit or os._exit
         self._stop = threading.Event()
         self._thread: threading.Thread | None = None
+        self._shutdown_lock = threading.Lock()
+        self._shutdown_requested_at: float | None = None
+        self._shutdown_reason: str | None = None
 
     def start(self) -> None:
         self._thread = threading.Thread(
@@ -129,13 +132,39 @@ class _TaskLivenessSupervisor:
         if self._thread is not None and self._thread is not threading.current_thread():
             self._thread.join(timeout=max(1.0, self.policy.check_interval * 2))
 
+    def arm_shutdown_deadline(self, reason: str) -> None:
+        """Force process exit if any cooperative shutdown or cleanup wedges."""
+        with self._shutdown_lock:
+            if self._shutdown_requested_at is not None:
+                return
+            self._shutdown_requested_at = time.monotonic()
+            self._shutdown_reason = reason
+        _supervisor_logger.warning(
+            "Task shutdown deadline armed for %.1fs (%s)",
+            self.policy.shutdown_grace,
+            reason,
+        )
+
     def _run(self) -> None:
         started_at = time.monotonic()
         unhealthy_since: float | None = None
-        shutdown_requested_at: float | None = None
 
         while not self._stop.wait(self.policy.check_interval):
             now = time.monotonic()
+            with self._shutdown_lock:
+                shutdown_requested_at = self._shutdown_requested_at
+                shutdown_reason = self._shutdown_reason
+            if shutdown_requested_at is not None:
+                if now - shutdown_requested_at >= self.policy.shutdown_grace:
+                    _supervisor_logger.critical(
+                        "Task did not stop within %.1fs after shutdown began (%s); forcing exit",
+                        self.policy.shutdown_grace,
+                        shutdown_reason,
+                    )
+                    self._stop.set()
+                    self.hard_exit(int(ExitCode.ERROR))
+                continue
+
             if now - started_at < self.policy.startup_grace:
                 continue
 
@@ -157,22 +186,14 @@ class _TaskLivenessSupervisor:
                 _supervisor_logger.error("Task liveness check failed: %s", reason)
                 continue
 
-            if shutdown_requested_at is None and now - unhealthy_since >= self.policy.unhealthy_grace:
-                shutdown_requested_at = now
+            if now - unhealthy_since >= self.policy.unhealthy_grace:
                 _supervisor_logger.critical(
                     "Task remained unhealthy for %.1fs (%s); requesting process shutdown",
                     now - unhealthy_since,
                     reason,
                 )
+                self.arm_shutdown_deadline(reason)
                 self.request_shutdown(reason)
-                continue
-
-            if shutdown_requested_at is not None and now - shutdown_requested_at >= self.policy.shutdown_grace:
-                _supervisor_logger.critical(
-                    "Task did not stop within %.1fs after liveness shutdown request; forcing exit",
-                    self.policy.shutdown_grace,
-                )
-                self.hard_exit(int(ExitCode.ERROR))
 
 
 # =============================================================
@@ -313,8 +334,10 @@ async def __run_task_async(
             raise
     finally:
         _unregister_shutdown_callback(request_signal_shutdown)
-        if supervisor is not None and not liveness_shutdown_requested.is_set():
-            supervisor.stop()
+        if supervisor is not None:
+            supervisor.arm_shutdown_deadline(
+                "liveness shutdown cleanup" if liveness_shutdown_requested.is_set() else "task cleanup"
+            )
         try:
             if instance is not None:
                 await instance.exit()

{ergon_framework_python-0.1.7 → ergon_framework_python-0.1.9}/src/ergon_framework_python.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ergon-framework-python
-Version: 0.1.7
+Version: 0.1.9
 Summary: Ergon internal task-oriented project bootstrapper
 Author-email: Ergondata Technologies <anza.vossos@protonmail.com>
 License-Expression: MIT

{ergon_framework_python-0.1.7 → ergon_framework_python-0.1.9}/tests/test_smoke.py

@@ -4,4 +4,4 @@ import ergon
 
 
 def test_ergon_package_is_importable() -> None:
-    assert ergon.__version__ == "0.1.7"
+    assert ergon.__version__ == "0.1.9"