by-framework 0.2.2.dev15__tar.gz → 0.2.2.dev16__tar.gz

{by_framework-0.2.2.dev15 → by_framework-0.2.2.dev16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: by-framework
-Version: 0.2.2.dev15
+Version: 0.2.2.dev16
 Summary: 分布式 Agent 调度框架
 License-File: LICENSE
 Requires-Python: >=3.12

{by_framework-0.2.2.dev15 → by_framework-0.2.2.dev16}/docs/architecture/KEY_FILES.md

@@ -314,6 +314,22 @@ Entry anatomy:
   best-effort — a background component that fails to construct must never
   prevent the worker from consuming — and every one of them must be
   cancelled and awaited in `_shutdown()`, or shutdown hangs on it.
+  `_run_once()` acquires a concurrency slot via `_acquire_slot()`, never a
+  bare `await self.semaphore.acquire()` — a saturated pool (all
+  `max_concurrency` slots held by legitimately long-running tasks) must not
+  be indistinguishable from a genuinely stuck consume loop. `_acquire_slot()`
+  polls `len(self._running_tasks) >= self.max_concurrency` and calls
+  `_mark_consumer_tick()` on every poll while waiting, so `_is_consumer_healthy()`
+  (whose own comparison logic is untouched) keeps seeing fresh ticks for as
+  long as the loop is merely busy, not stalled. `self._slot_wait_poll_seconds`
+  is derived from `_consumer_health_timeout_seconds` (not hardcoded) so the
+  two stay proportional under any lease-TTL/stream-block-ms configuration.
+  Trade-off this does *not* solve, by design: it cannot distinguish a
+  legitimately long task from a deadlocked one that will never return — both
+  present as `_running_tasks` stuck at `max_concurrency`; task-level SLA
+  enforcement (detecting and cancelling an individual stuck task) is a
+  separate concern from consumer-loop liveness and is not implemented here —
+  see `CancelTaskCommand` for the existing per-task cancel path.
 
 - `src/by_framework/worker/health_server.py` — `WorkerHealthServer`: the
   `/readyz` readiness HTTP endpoint, on its own daemon thread (mirrors

{by_framework-0.2.2.dev15 → by_framework-0.2.2.dev16}/docs/architecture/worker-readiness-endpoint.md

@@ -216,7 +216,15 @@ regardless.
 2. `draining` — `SIGTERM`/`SIGINT` received, shutdown in progress
 3. `evicted` — `admin_lifecycle == "evicted"`
 4. `suspended` — `admin_lifecycle == "suspended"`
-5. `consumer_stalled` — consume loop stale past `_consumer_health_timeout_seconds`
+5. `consumer_stalled` — consume loop stale past `_consumer_health_timeout_seconds`.
+   "Stale" means the loop itself has stopped making progress (crashed task,
+   or an unexpected hang outside slot-waiting) — it deliberately excludes a
+   consume loop that is merely waiting for every `max_concurrency` slot to
+   free up while legitimately long-running tasks are in flight. A saturated-
+   but-alive loop keeps refreshing its liveness signal while it waits (see
+   `runner.py`'s `_acquire_slot()` in `docs/architecture/KEY_FILES.md`), so
+   it never surfaces here; only a loop that has actually stopped responding
+   does.
 6. `serving` — none of the above; `ready=true`
 
 `ready` is `true` if and only if `reason == "serving"`.

{by_framework-0.2.2.dev15 → by_framework-0.2.2.dev16}/libs/by-framework-langgraph/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "by-framework-langgraph"
-version = "0.0.3.dev4"
+version = "0.0.3.dev5"
 description = "LangGraph integration for by-framework"
 readme = "README.md"
 requires-python = ">=3.12"

{by_framework-0.2.2.dev15 → by_framework-0.2.2.dev16}/libs/by-framework-trace-langfuse/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "by-framework-trace-langfuse"
-version = "0.0.3.dev4"
+version = "0.0.3.dev5"
 description = "Langfuse trace provider for by-framework"
 readme = "README.md"
 requires-python = ">=3.12"

{by_framework-0.2.2.dev15 → by_framework-0.2.2.dev16}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "by-framework"
-version = "0.2.2.dev15"
+version = "0.2.2.dev16"
 description = "分布式 Agent 调度框架"
 readme = "README.md"
 requires-python = ">=3.12"

{by_framework-0.2.2.dev15 → by_framework-0.2.2.dev16}/src/by_framework/worker/runner.py

@@ -83,6 +83,7 @@ class WorkerRunner:
         self.worker = worker
         self.group_name = group_name or self._auto_group_name()
         self.consumer_name = worker.worker_id
+        self.max_concurrency = max_concurrency
         self.semaphore = asyncio.Semaphore(max_concurrency)
         self.fetch_count = fetch_count
         # OTel for worker.execute is emitted via a live wrapping span (see
@@ -121,6 +122,14 @@ class WorkerRunner:
             float(self.worker.heartbeat_lease_ttl_seconds) * 2.0,
             stream_block_seconds * 3.0,
         )
+        # Poll interval used by _acquire_slot() while waiting for a saturated
+        # semaphore. Derived from _consumer_health_timeout_seconds (not a
+        # hardcoded constant) so it stays well under the health-check
+        # deadline under any configuration, with a floor to avoid busy-
+        # polling if the timeout is configured very small.
+        self._slot_wait_poll_seconds = max(
+            self._consumer_health_timeout_seconds / 4.0, 1.0
+        )
         # Set as the first step of _shutdown() - see
         # docs/architecture/worker-readiness-endpoint.md Decision 6.
         self._draining: bool = False
@@ -386,9 +395,23 @@ class WorkerRunner:
         finally:
             await self.redis.xack(stream_name, self.group_name, msg_id)
 
+    async def _acquire_slot(self) -> None:
+        """Wait for a concurrency slot, marking the consumer tick on every
+        poll so a saturated-but-alive loop isn't judged stalled.
+
+        Polling on `len(self._running_tasks)` instead of wrapping
+        `semaphore.acquire()` in `asyncio.wait_for()` avoids relying on
+        Semaphore cancellation semantics for a lock this runner is the only
+        waiter on — see design.md's "why not asyncio.wait_for" section.
+        """
+        while len(self._running_tasks) >= self.max_concurrency:
+            self._mark_consumer_tick()
+            await asyncio.sleep(self._slot_wait_poll_seconds)
+        await self.semaphore.acquire()
+
     async def _run_once(self) -> bool:
         """Fetch and start processing messages."""
-        await self.semaphore.acquire()
+        await self._acquire_slot()
 
         try:
             messages = await self.fetch_messages(
@@ -401,7 +424,7 @@ class WorkerRunner:
 
             for i, (stream_name, msg_id, data_dict) in enumerate(messages):
                 if i > 0:
-                    await self.semaphore.acquire()
+                    await self._acquire_slot()
 
                 task = asyncio.create_task(
                     self._process_and_release(stream_name, msg_id, data_dict)

{by_framework-0.2.2.dev15 → by_framework-0.2.2.dev16}/tests/worker/test_runner.py

@@ -193,6 +193,81 @@ class StartupOrderStop(RuntimeError):
     """Sentinel used to stop the infinite runner loop in startup-order tests."""
 
 
+class _ContinuousMockRedis(MockRedisRunner):
+    """Returns a fresh AskAgentCommand message on every xreadgroup call.
+
+    Used by the semaphore-saturation tests to simulate a control stream
+    that always has more work waiting, so the consume loop is genuinely
+    busy (not idle) while a concurrency slot is unavailable.
+    """
+
+    def __init__(self, agent_type: str = "dummy_agent"):
+        super().__init__(message_to_return=None)
+        self.agent_type = agent_type
+        self._counter = 0
+
+    async def xreadgroup(self, groupname, consumername, streams, count=1, block=0):
+        self.called_xreadgroup = True
+        self._counter += 1
+        msg = AskAgentCommand(
+            header=MessageHeader(
+                message_id=f"msg-{self._counter}",
+                session_id="sess-1",
+                trace_id="trace-1",
+                target_agent_type=self.agent_type,
+            ),
+            content="test",
+        )
+        return [
+            [
+                RedisKeys.ctrl_stream(self.agent_type).encode(),
+                [
+                    (
+                        f"{self._counter}-0".encode(),
+                        {b"data": json.dumps(msg.to_dict()).encode()},
+                    )
+                ],
+            ]
+        ]
+
+
+class _SlowWorker(DummyWorker):
+    """Every task takes `delay` seconds - stands in for a long-running
+    LLM/tool-call agent task that legitimately occupies a concurrency slot."""
+
+    def __init__(self, delay: float):
+        super().__init__()
+        self.delay = delay
+        self.completed = 0
+
+    async def _handle_message(self, command, **kwargs):
+        await asyncio.sleep(self.delay)
+        self.completed += 1
+        return AgentTaskResult(status=AgentState.COMPLETED.value)
+
+
+class _MixedDurationWorker(DummyWorker):
+    """The first message processed takes `long_delay`; every subsequent one
+    takes `short_delay` - simulates one slot pinned by a long task while
+    the rest of the pool cycles normally."""
+
+    def __init__(self, long_delay: float, short_delay: float):
+        super().__init__()
+        self.long_delay = long_delay
+        self.short_delay = short_delay
+        self.completed = 0
+        self._long_dispatched = False
+        self._lock = asyncio.Lock()
+
+    async def _handle_message(self, command, **kwargs):
+        async with self._lock:
+            is_long = not self._long_dispatched
+            self._long_dispatched = True
+        await asyncio.sleep(self.long_delay if is_long else self.short_delay)
+        self.completed += 1
+        return AgentTaskResult(status=AgentState.COMPLETED.value)
+
+
 class TestWorkerRunner(unittest.IsolatedAsyncioTestCase):
 
     async def test_runner_pull_and_dispatch(self):
@@ -1398,6 +1473,139 @@ class TestWorkerRunner(unittest.IsolatedAsyncioTestCase):
             with self.assertRaises(asyncio.CancelledError):
                 await runner._consumer_task
 
+    async def test_runner_healthy_during_single_slot_saturation_live(self):
+        """max_concurrency=1: a single task running well past the (shrunk)
+        health timeout, with more messages continuously available, must
+        never flip _is_consumer_healthy() to False - this is the exact
+        scenario from the PRD's Background (real asyncio scheduling, no
+        mocked clock): the consume loop is busy, not stuck."""
+        worker = _SlowWorker(delay=0.3)
+        redis_mock = _ContinuousMockRedis()
+        runner = WorkerRunner(
+            redis_client=redis_mock,
+            worker=worker,
+            group_name="test_group",
+            max_concurrency=1,
+            fetch_count=1,
+        )
+        # Shrink well below the task duration so a stale tick would be
+        # caught, and re-derive the poll interval the same way __init__
+        # does (a smaller floor than production's 1.0s so the test runs
+        # fast, without changing the ratio the design specifies).
+        runner._consumer_health_timeout_seconds = 0.1
+        runner._slot_wait_poll_seconds = max(
+            runner._consumer_health_timeout_seconds / 4.0, 0.02
+        )
+        runner._mark_consumer_tick()
+        runner._consumer_task = asyncio.ensure_future(runner._consume_loop())
+        try:
+            loop = asyncio.get_running_loop()
+            deadline = loop.time() + 1.0
+            saw_unhealthy = False
+            while loop.time() < deadline:
+                if not runner._is_consumer_healthy():
+                    saw_unhealthy = True
+                    break
+                await asyncio.sleep(0.01)
+            self.assertFalse(
+                saw_unhealthy,
+                "consumer was judged unhealthy while merely saturated, not stuck",
+            )
+            # Sanity: the loop was actually doing work throughout, not
+            # idling because fetch never returned anything.
+            self.assertGreater(worker.completed, 0)
+        finally:
+            runner._consumer_task.cancel()
+            with self.assertRaises(asyncio.CancelledError):
+                await runner._consumer_task
+            await runner.wait_for_tasks()
+
+    async def test_runner_healthy_during_partial_saturation_live(self):
+        """max_concurrency>1: one slot pinned by a long task while the rest
+        of the pool keeps cycling normally must never flip
+        _is_consumer_healthy() to False."""
+        worker = _MixedDurationWorker(long_delay=0.4, short_delay=0.05)
+        redis_mock = _ContinuousMockRedis()
+        runner = WorkerRunner(
+            redis_client=redis_mock,
+            worker=worker,
+            group_name="test_group",
+            max_concurrency=3,
+            fetch_count=1,
+        )
+        runner._consumer_health_timeout_seconds = 0.1
+        runner._slot_wait_poll_seconds = max(
+            runner._consumer_health_timeout_seconds / 4.0, 0.02
+        )
+        runner._mark_consumer_tick()
+        runner._consumer_task = asyncio.ensure_future(runner._consume_loop())
+        try:
+            loop = asyncio.get_running_loop()
+            deadline = loop.time() + 1.0
+            saw_unhealthy = False
+            while loop.time() < deadline:
+                if not runner._is_consumer_healthy():
+                    saw_unhealthy = True
+                    break
+                await asyncio.sleep(0.01)
+            self.assertFalse(
+                saw_unhealthy,
+                "consumer was judged unhealthy while one slot was merely "
+                "pinned by a long task, not stuck",
+            )
+            # The short-task slots must have cycled multiple times while
+            # the long task was still occupying its own slot.
+            self.assertGreater(worker.completed, 1)
+        finally:
+            runner._consumer_task.cancel()
+            with self.assertRaises(asyncio.CancelledError):
+                await runner._consumer_task
+            await runner.wait_for_tasks()
+
+    async def test_runner_consumer_stalled_detected_despite_free_slot_live(self):
+        """Regression guard: a free concurrency slot must not mask a
+        genuinely stuck consume loop. If fetch/dispatch itself stops
+        making progress - not because every slot is occupied -
+        _is_consumer_healthy() must still flip to False. Uses real
+        asyncio scheduling (wall-clock sleep past the shrunk timeout, no
+        mocked clock) so this fix cannot degrade into "saturated (or
+        merely idle) == always healthy"."""
+        worker = DummyWorker()
+        redis_mock = MockRedisRunner(message_to_return=[])
+        runner = WorkerRunner(
+            redis_client=redis_mock,
+            worker=worker,
+            group_name="test_group",
+            max_concurrency=5,
+        )
+        runner._consumer_health_timeout_seconds = 0.2
+        runner._slot_wait_poll_seconds = 0.05
+        runner._mark_consumer_tick()
+
+        hang_forever = asyncio.Event()
+
+        async def _hanging_fetch(*args, **kwargs):
+            await hang_forever.wait()
+            return []
+
+        # Simulate the consume loop getting stuck somewhere other than the
+        # slot wait (e.g. an unexpected hang inside fetch/dispatch) while a
+        # slot is freely available - _acquire_slot()'s polling branch must
+        # never even be entered here (len(_running_tasks) stays 0, well
+        # under max_concurrency), so it cannot be the thing masking this.
+        runner.fetch_messages = _hanging_fetch
+
+        runner._consumer_task = asyncio.ensure_future(runner._consume_loop())
+        try:
+            await asyncio.sleep(runner._consumer_health_timeout_seconds + 0.3)
+            self.assertFalse(runner._is_consumer_healthy())
+            self.assertEqual(len(runner._running_tasks), 0)
+        finally:
+            hang_forever.set()
+            runner._consumer_task.cancel()
+            with self.assertRaises(asyncio.CancelledError):
+                await runner._consumer_task
+
 
 if __name__ == "__main__":
     unittest.main()

{by_framework-0.2.2.dev15 → by_framework-0.2.2.dev16}/uv.lock

@@ -315,7 +315,7 @@ wheels = [
 
 [[package]]
 name = "by-framework"
-version = "0.2.2.dev15"
+version = "0.2.2.dev16"
 source = { editable = "." }
 dependencies = [
     { name = "dill" },
@@ -531,7 +531,7 @@ provides-extras = ["dev"]
 
 [[package]]
 name = "by-framework-langgraph"
-version = "0.0.3.dev4"
+version = "0.0.3.dev5"
 source = { editable = "libs/by-framework-langgraph" }
 dependencies = [
     { name = "by-framework" },
@@ -577,7 +577,7 @@ provides-extras = ["dev", "langfuse", "phoenix"]
 
 [[package]]
 name = "by-framework-trace-langfuse"
-version = "0.0.3.dev4"
+version = "0.0.3.dev5"
 source = { editable = "libs/by-framework-trace-langfuse" }
 dependencies = [
     { name = "by-framework" },