by-framework 0.2.2.dev10__py3-none-any.whl → 0.2.2.dev12__py3-none-any.whl

by_framework/__main__.py

@@ -12,6 +12,17 @@ import importlib
 from .worker.app import run_worker
 
 
+def _parse_cluster_nodes(value: str):
+    """Parse a comma-separated "host:port,host:port" string, mirroring
+    RedisConfig.from_env()'s REDIS_CLUSTER_HOST/REDIS_CLUSTER_NODES parsing
+    so the CLI and env-var paths accept the same format."""
+    nodes = []
+    for node in value.split(","):
+        node_host, node_port = node.rsplit(":", 1)
+        nodes.append((node_host, int(node_port)))
+    return nodes
+
+
 def parse_args():
     """Parse command line arguments for the CLI runner."""
     parser = argparse.ArgumentParser(description="By-Framework CLI Runner")
@@ -25,12 +36,55 @@ def parse_args():
         ),
     )
     parser.add_argument(
-        "--redis-host", type=str, default="localhost", help="Redis server hostname"
+        "--redis-host",
+        type=str,
+        default=None,
+        help="Redis server hostname (default: REDIS_HOST env var, then 'localhost')",
+    )
+    parser.add_argument(
+        "--redis-port",
+        type=int,
+        default=None,
+        help="Redis server port (default: REDIS_PORT env var, then 6379)",
+    )
+    parser.add_argument(
+        "--redis-db",
+        type=int,
+        default=None,
+        help="Redis database number (default: REDIS_DATABASE env var, then 0)",
     )
     parser.add_argument(
-        "--redis-port", type=int, default=6379, help="Redis server port"
+        "--redis-password",
+        type=str,
+        default=None,
+        help="Redis password (default: REDIS_PASSWORD env var)",
+    )
+    parser.add_argument(
+        "--redis-username",
+        type=str,
+        default=None,
+        help="Redis username, for ACL-enabled Redis (default: REDIS_USERNAME env var)",
+    )
+    parser.add_argument(
+        "--redis-mode",
+        type=str,
+        choices=["standalone", "cluster"],
+        default=None,
+        help=(
+            "Redis deployment mode (default: REDIS_MODE env var; implied by "
+            "--redis-cluster-nodes/REDIS_CLUSTER_HOST alone if neither is set)"
+        ),
+    )
+    parser.add_argument(
+        "--redis-cluster-nodes",
+        type=str,
+        default=None,
+        help=(
+            "Comma-separated Redis Cluster seed nodes, 'host:port,host:port' "
+            "(default: REDIS_CLUSTER_HOST/REDIS_CLUSTER_NODES env var). "
+            "Passing this alone implies --redis-mode cluster."
+        ),
     )
-    parser.add_argument("--redis-db", type=int, default=0, help="Redis database number")
     parser.add_argument(
         "--worker-id", type=str, default="worker-1", help="Unique worker identifier"
     )
@@ -47,6 +101,17 @@ def parse_args():
     parser.add_argument(
         "--redis-max-connections", type=int, help="Max Redis connections allowed"
     )
+    parser.add_argument(
+        "--health-port",
+        type=int,
+        default=None,
+        help=(
+            "Port for the local /readyz readiness endpoint (Docker/Kubernetes "
+            "health checks). Opt-in only - omit to leave it disabled, no "
+            "default port is provided. Also settable via BYAI_WORKER_HEALTH_PORT. "
+            "See docs/architecture/worker-readiness-endpoint.md."
+        ),
+    )
 
     return parser.parse_args()
 
@@ -68,17 +133,31 @@ def main():
 
     print(f"Starting worker class: {worker_class.__name__} (ID: {args.worker_id})")
 
-    # Delegate to the common launcher
+    redis_cluster_nodes = (
+        _parse_cluster_nodes(args.redis_cluster_nodes)
+        if args.redis_cluster_nodes
+        else None
+    )
+
+    # Delegate to the common launcher. Every redis_* arg defaults to None
+    # here (not a literal like "localhost"), same as run_worker()'s own
+    # defaults, so an unset CLI flag actually falls through to the
+    # corresponding REDIS_* env var instead of silently shadowing it.
     run_worker(
         worker_class=worker_class,
         worker_id=args.worker_id,
         redis_host=args.redis_host,
         redis_port=args.redis_port,
         redis_db=args.redis_db,
+        redis_password=args.redis_password,
+        redis_username=args.redis_username,
+        redis_mode=args.redis_mode,
+        redis_cluster_nodes=redis_cluster_nodes,
         workspace_dir=args.workspace,
         max_concurrency=args.max_concurrency,
         fetch_count=args.fetch_count,
         redis_max_connections=args.redis_max_connections,
+        health_port=args.health_port,
     )
 
 

by_framework/admin/cli.py

@@ -50,7 +50,6 @@ app.add_typer(metrics_app, name="metrics", help="Metrics and observability")
 
 console = Console()
 err_console = Console(stderr=True)
-_DEFAULT_REDIS_URL = "redis://localhost:6379/0"
 _redis_url: Optional[str] = None
 
 
@@ -85,10 +84,12 @@ def _get_redis(redis_url: Optional[str] = None):
     configured_url = redis_url if redis_url is not None else _redis_url
     if configured_url:
         return init_redis_from_url(configured_url)
-    config = RedisConfig.from_env()
-    if config.mode == "cluster":
-        return init_redis(config=config)
-    return init_redis_from_url(_DEFAULT_REDIS_URL)
+    # Always resolve through RedisConfig.from_env() - for both standalone and
+    # cluster mode - so REDIS_HOST/PORT/PASSWORD/USERNAME/DATABASE are honored
+    # the same way run_worker()'s config resolution honors them. Previously
+    # only cluster mode read env_config; standalone mode silently ignored it
+    # and connected to a hardcoded localhost:6379/0.
+    return init_redis(config=RedisConfig.from_env())
 
 
 # --------------------------------------------------------------------------- #

by_framework/client/client.py

@@ -14,6 +14,7 @@ from typing import (TYPE_CHECKING, Any, AsyncIterator, Dict, List, Optional, Pro
 
 from by_framework.common.constants import (
     CANCEL_MESSAGE_ID_PREFIX,
+    CLIENT_SOURCE_AGENT_TYPE,
     EXECUTION_ID_PREFIX,
     MESSAGE_ID_PREFIX,
     RedisKeys,
@@ -864,7 +865,7 @@ class GatewayClient:
                             trace_id=trace_id,
                             target_agent_type=params["target_agent_type"],
                             parent_message_id=params["parent_message_id"] or "",
-                            source_agent_type="client",
+                            source_agent_type=CLIENT_SOURCE_AGENT_TYPE,
                             route_policy=route_policy,
                             route_status=availability.status,
                             stream_name=availability.stream_name or "",
@@ -968,7 +969,7 @@ class GatewayClient:
                         "session_id": params["session_id"],
                         "trace_id": trace_id,
                         "parent_message_id": params["parent_message_id"] or "",
-                        "source_agent_type": "client",
+                        "source_agent_type": CLIENT_SOURCE_AGENT_TYPE,
                         "target_agent_type": params["target_agent_type"],
                         "stream_name": route.stream_name,
                         "status": "QUEUED",
@@ -1117,7 +1118,7 @@ class GatewayClient:
                     message_id=message_id,
                     parent_message_id=parent_message_id,
                     worker_id=target_worker_id,
-                    source_agent_type="client",
+                    source_agent_type=CLIENT_SOURCE_AGENT_TYPE,
                     target_agent_type=target_agent_type,
                     route_policy=route_policy,
                     route_status=route_status,

by_framework/common/config.py

@@ -43,6 +43,8 @@ class RedisConfig:
         """
         password = os.environ.get("REDIS_PASSWORD", "")
         username = os.environ.get("REDIS_USERNAME") or None
+        host = os.environ.get("REDIS_HOST") or "localhost"
+        port_str = os.environ.get("REDIS_PORT")
         max_connections = os.environ.get("REDIS_MAX_CONNECTIONS")
         cluster_host_str = os.environ.get("REDIS_CLUSTER_HOST")
         cluster_nodes_str = cluster_host_str or os.environ.get("REDIS_CLUSTER_NODES")
@@ -61,8 +63,8 @@ class RedisConfig:
             if db_str is not None:
                 logger.warning("REDIS_DB is deprecated, use REDIS_DATABASE instead")
         return cls(
-            host=os.environ.get("REDIS_HOST", "localhost"),
-            port=int(os.environ.get("REDIS_PORT", "6379")),
+            host=host,
+            port=int(port_str) if port_str else 6379,
             db=int(db_str) if db_str is not None else 0,
             password=password,
             username=username,

by_framework/common/constants.py

@@ -297,6 +297,100 @@ class RedisKeys:
             v2_suffix=f"task_group:{{{group_id}}}:results",
         )
 
+    @classmethod
+    def wait_index(cls, shard: int) -> str:
+        """ZSET index of suspended callers waiting for a sub-task reply.
+
+        member = encoded wait-index member (see core/wait_index.py),
+        score = deadline in epoch milliseconds. Sharded so sweepers can
+        claim disjoint slices without a global lock; the shard is derived
+        from session_id (see wait_index_shard()).
+
+        Cross-entity index (spans every session), so deliberately left
+        untagged relative to the per-session keys it points at — same rule
+        as trace_index_session/admin_workers.
+        """
+        return cls._versioned(
+            v1_key=f"byai_gateway:wait:index:{shard}",
+            v2_suffix=f"wait:index:{shard}",
+        )
+
+    @classmethod
+    def wait_sweep_lock(cls, shard: int) -> str:
+        """Short-lived claim on one wait_index() shard, held while sweeping it.
+
+        Ownership is advisory: it only keeps two sweepers from doing the same
+        triage at the same moment. Losing it (expiry, a partitioned worker)
+        cannot corrupt anything, because every action a sweep takes is
+        idempotent — a duplicate synthesized reply is caught by the same
+        wait-index gate that catches a duplicate real one. That is why the
+        shards need no leader election: a dead worker's claim simply expires
+        and another worker picks the shard up on its next cycle.
+
+        Cross-entity like the shard it guards, so deliberately untagged.
+        """
+        return cls._versioned(
+            v1_key=f"byai_gateway:wait:sweep_lock:{shard}",
+            v2_suffix=f"wait:sweep_lock:{shard}",
+        )
+
+    @classmethod
+    def wait_consumed(cls, session_id: str, member_digest: str) -> str:
+        """Short-lived marker: "this wait-index entry was already resolved".
+
+        Written by the idempotency gate right after it wins the ZREM for a
+        member, and read when a later ZREM for the same member returns 0.
+        It is the *only* thing that distinguishes the two meanings of that
+        0 — "someone already consumed this wait" (drop the duplicate) from
+        "this wait was never registered" (a pre-upgrade or expired entry,
+        which must be let through). Without it, every rolling upgrade would
+        silently drop in-flight replies.
+
+        Per-session entity, so hash-tagged with the session in v2.
+        """
+        return cls._versioned(
+            v1_key=f"byai_gateway:wait:consumed:{session_id}:{member_digest}",
+            v2_suffix=f"wait:consumed:{{{session_id}}}:{member_digest}",
+        )
+
+    @classmethod
+    def wait_renew_origin(cls, session_id: str, member_digest: str) -> str:
+        """The deadline a wait's renewal budget is measured from.
+
+        Written once (SET NX) by the first sweep that finds the entry due, so
+        it holds the wait's *original* deadline even after renewals have
+        overwritten the ZSET score. Without it a renewal budget cannot exist
+        at all: every sweep would re-measure from the score it just pushed
+        out, and a callee whose worker is alive but making no progress would
+        be renewed forever.
+
+        Sweeper-private: nothing on the reply path reads or writes it, so it
+        is deliberately NOT part of the wait-index member (which must stay
+        rebuildable from a reply alone — see core/wait_index.py). Expiring is
+        safe by design: losing it only restarts the budget from the current
+        deadline, so the TTL is sized well above any plausible budget.
+
+        Per-session entity, so hash-tagged with the session in v2.
+        """
+        return cls._versioned(
+            v1_key=f"byai_gateway:wait:renew_origin:{session_id}:{member_digest}",
+            v2_suffix=f"wait:renew_origin:{{{session_id}}}:{member_digest}",
+        )
+
+    @classmethod
+    def harness_state(cls, execution_id: str) -> str:
+        """Serialized in-flight native-agent-harness loop state.
+
+        Keyed purely by execution_id — the same identity the registry
+        already reattaches on RESUME — so any worker instance that picks up
+        the eventual ResumeCommand can rehydrate the loop, not only the one
+        that started it.
+        """
+        return cls._versioned(
+            v1_key=f"byai_gateway:harness_state:{execution_id}",
+            v2_suffix=f"harness_state:{{{execution_id}}}",
+        )
+
     # --- Registry ---
     @classmethod
     def known_workers(cls) -> str:
@@ -517,8 +611,32 @@ class RedisKeys:
 MESSAGE_ID_PREFIX = "msg-"
 EXECUTION_ID_PREFIX = "exec-"
 TASK_GROUP_ID_PREFIX = "tg-"
+# A single call_agent (non-group) dispatch stores its result in the same
+# task_group_results Hash a real group uses, under a group id derived from
+# the sub-task's own message_id — i.e. a group of size 1. Keeps one result
+# storage/recovery path instead of two.
+TASK_GROUP_SINGLE_ID_PREFIX = "tg-single-"
 CANCEL_MESSAGE_ID_PREFIX = "msg-cancel-"
 
+# Sentinel GatewayClient writes as an execution record's source_agent_type for
+# a dispatch it made itself (client/client.py's initialize_execution and
+# record_failed_route_decision). It is NOT an agent type: nothing declares it,
+# so nothing consumes RedisKeys.ctrl_stream(CLIENT_SOURCE_AGENT_TYPE).
+#
+# Load-bearing wherever a resumed execution recovers its caller from its own
+# record instead of from the resume header (GatewayWorker._resolve_reply_command
+# / GatewayProcessor._resolve_reply_header): a root execution's record carries
+# this, and treating it as a caller both posts the result to a stream no one
+# reads and suppresses the end-of-stream event the session data plane owes the
+# user — the visible half of the bug being prevented.
+CLIENT_SOURCE_AGENT_TYPE = "client"
+
+
+def single_call_task_group_id(child_message_id: str) -> str:
+    """Group id under which a single (non-group) call_agent result is stored."""
+    return f"{TASK_GROUP_SINGLE_ID_PREFIX}{child_message_id}"
+
+
 # --- Redis Hash Field Prefixes ---
 # Field prefixes in Session Registry Hash
 EXEC_FIELD_PREFIX = "exec:"
@@ -529,6 +647,10 @@ MSG_MAP_PREFIX = "msg_map:"
 TASK_GROUP_FIELD_TOTAL = "total"
 TASK_GROUP_FIELD_COMPLETED = "completed"
 TASK_GROUP_FIELD_SOURCE_AGENT = "source_agent_type"
+# Set once dispatch fails partway through a batch; any reply that arrives
+# for an aborted group is discarded instead of resuming the (already
+# terminated) caller execution.
+TASK_GROUP_FIELD_ABORTED = "aborted"
 
 
 # --- Timing and Sleep Constants ---
@@ -538,12 +660,133 @@ CONTROL_LOOP_SLEEP_SECONDS = 0.01
 WAIT_FOR_TASKS_TIMEOUT_SECONDS = 5.0
 # Task group Key TTL (seconds), default 1 day
 TASK_GROUP_TTL_SECONDS = 86400
+# Native agent harness loop-state Key TTL (seconds), default 1 day
+HARNESS_STATE_TTL_SECONDS = 86400
 # First retry wait time (seconds)
 FIRST_RETRY_WAIT_SECONDS = 1.0
 # Maximum retry count
 MAX_RETRY_COUNT = 3
 
 
+# --- Suspended-caller liveness (wait index) ---
+# Number of RedisKeys.wait_index() shards. Fixed: changing it re-maps every
+# session to a different shard, so in-flight entries would be swept by no
+# one. Treat as a cross-SDK protocol constant, not a tunable.
+WAIT_INDEX_SHARDS = 16
+# Default deadline for a call_agent(wait_for_reply=True) reply (1 hour).
+# Machine waiting on machine.
+DEFAULT_REPLY_TIMEOUT_MS = 3_600_000
+# Default deadline for an ask_user reply. Machine waiting on a human, so it
+# is deliberately decoupled from DEFAULT_REPLY_TIMEOUT_MS and aligned with
+# the session TTL (which is in seconds) instead.
+DEFAULT_ASK_USER_TIMEOUT_MS = RedisKeys.DEFAULT_SESSION_TTL * 1000
+# How often a worker's sweeper scans the shards it owns (seconds).
+WAIT_SWEEP_INTERVAL_SECONDS = 30
+# TTL of a RedisKeys.wait_sweep_lock() claim. Must comfortably exceed one
+# shard's sweep so the owner doesn't lose the shard mid-pass, and stay short
+# enough that a crashed sweeper's shards are picked up again quickly.
+WAIT_SWEEP_LOCK_TTL_SECONDS = 60
+# Most due entries one sweep resolves per shard per cycle. Bounds the work
+# (and the Redis traffic) of a single pass after an outage leaves a large
+# backlog; the remainder is simply picked up next cycle, since entries stay
+# in the index until a reply clears them.
+WAIT_SWEEP_BATCH_LIMIT = 200
+# Fixed extension applied when a sweep finds the callee still making
+# progress. Deliberately a constant rather than the original timeout: the
+# wait-index member must stay reconstructible from a reply alone, so it
+# cannot carry the caller's original timeout.
+WAIT_RENEW_INCREMENT_MS = 300_000
+# Hard ceiling on renewals, as a multiple of the caller's own timeout: a wait
+# may be renewed until `registered_at + N * timeout`, after which the callee
+# is declared CHILD_TIMEOUT even though its worker is still alive.
+#
+# Without a ceiling the "worker lease alive -> renew" rule is unconditional,
+# so a callee that is running but making no progress (a hung LLM call, a
+# deadlock) suspends its caller forever — the one failure mode the deadline
+# was supposed to bound. N is deliberately expressed against the caller's
+# timeout rather than a renewal count, so a caller that asked for 10 minutes
+# is not held to the same absolute budget as one that asked for four hours,
+# and so retuning WAIT_RENEW_INCREMENT_MS cannot silently change the bound.
+#
+# Why 3: N must exceed 1 (N == 1 is "never renew", which kills every callee
+# that is merely slow); N == 2 leaves a single extra window, so one
+# under-estimated timeout is enough to kill healthy work; N == 3 means the
+# caller's own estimate has to be off by 200% before that happens, while
+# still bounding the default case at 3 hours — two orders of magnitude below
+# DEFAULT_SESSION_TTL, which matters because once the session data expires
+# there is no execution record left to compensate against and the wait is
+# simply dropped. Override per deployment via
+# BY_FRAMEWORK_WAIT_RENEW_MAX_MULTIPLE.
+WAIT_RENEW_MAX_MULTIPLE = 3
+# TTL of RedisKeys.wait_renew_origin(). Must comfortably exceed the largest
+# budget in use (N * timeout), or the budget silently restarts mid-wait.
+WAIT_RENEW_ORIGIN_TTL_SECONDS = TASK_GROUP_TTL_SECONDS
+# How long RedisKeys.wait_consumed() remembers that a wait was already
+# resolved, i.e. how far apart two copies of the same reply may be and still
+# be recognized as duplicates.
+#
+# Sized off DEFAULT_SESSION_TTL, which is the lifetime of the session
+# registry — and the session registry is what keeps a *wait entry* relevant.
+# A marker that expires while entries of that session are still live leaves
+# two holes, and the second is the dangerous one:
+#
+#   1. A repeated ask_user answer (a human may take days; the ask_user
+#      deadline is DEFAULT_SESSION_TTL itself) is no longer recognized as a
+#      duplicate and wakes the caller a second time.
+#   2. Worse: a stale duplicate sub-agent reply, having lost the marker that
+#      would stop it at its own candidate, falls through to the ask_user
+#      candidate for the same caller and claims a wait that is still live —
+#      after which the real answer is dropped as "already consumed".
+#
+# Both close once the marker outlives every wait it may have to arbitrate,
+# i.e. the session TTL. Erring long costs a handful of idle 1-byte keys with
+# the same lifetime as the session registry they belong to; erring short
+# costs a lost user answer.
+WAIT_CONSUMED_TTL_SECONDS = RedisKeys.DEFAULT_SESSION_TTL
+# How often a sweeper prunes entries that are provably beyond use (see
+# WAIT_PRUNE_AFTER_SECONDS). Deliberately far coarser than
+# WAIT_SWEEP_INTERVAL_SECONDS: this is garbage collection on a multi-day
+# horizon, and it is the only work a sweeper does when compensation is off.
+WAIT_PRUNE_INTERVAL_SECONDS = 3600
+# How far in the past a wait entry's score must lie before pruning it.
+#
+# Every writer of an entry sets its score to its own `now` plus a
+# non-negative offset (registration adds the caller's timeout, a renewal adds
+# WAIT_RENEW_INCREMENT_MS), and only ever does so while the caller's
+# execution record exists. So `now - score > this` proves the entry was last
+# touched more than a session TTL ago, hence that the session registry the
+# sweep would interrogate has expired and no triage is possible any more:
+# the entry can only ever produce "caller missing". Pruning it is therefore
+# not a decision, which is why it needs no opt-in.
+#
+# The margin over DEFAULT_SESSION_TTL is what makes that strict rather than
+# coincident: DEFAULT_ASK_USER_TIMEOUT_MS *equals* the session TTL, so a
+# threshold trimmed to the session TTL exactly would land on the boundary of
+# a live ask_user wait and lose to any clock skew between the worker that
+# registered the entry and the one sweeping it. A day is far beyond plausible
+# skew, and being late costs one ZSET member per unresolved call for one
+# extra day — the asymmetry says err long.
+WAIT_PRUNE_AFTER_SECONDS = RedisKeys.DEFAULT_SESSION_TTL + 86400
+
+
+class LivenessErrorCode:
+    """error_code values carried by synthesized/recovered resume replies.
+
+    Cross-SDK wire contract — Python/TS/Java must emit the same strings;
+    callers match on them. Append only, never rename.
+    """
+
+    # The callee's worker lease expired while its execution was non-terminal.
+    CHILD_WORKER_LOST = "CHILD_WORKER_LOST"
+    # The callee was alive but produced no reply before the deadline.
+    CHILD_TIMEOUT = "CHILD_TIMEOUT"
+    # The dispatch was never picked up by any worker.
+    CHILD_NEVER_STARTED = "CHILD_NEVER_STARTED"
+    # The callee finished and its result was persisted, but the reply
+    # message was lost; the result was recovered from storage.
+    REPLY_LOST_RECOVERED = "REPLY_LOST_RECOVERED"
+
+
 # --- Filesystem Constants ---
 DEFAULT_WORKSPACE_DIR = "/workspace"
 

by_framework/core/protocol/event_type.py

@@ -21,6 +21,10 @@ class EventType(str, Enum):
         TASK_CREATE: Task creation event
         STEP_COMPLETE: Step completion event
         TASK_STOP: Task stop event
+        ORPHANED_REPLY: A reply that arrived for an already-resolved wait and
+            was therefore dropped by the idempotency gate. Diagnostic only —
+            the sub-agent did real work whose result nobody will consume, so
+            it must not vanish silently.
     """
 
     ANSWER_DELTA = "answerDelta"
@@ -32,3 +36,4 @@ class EventType(str, Enum):
     TASK_CREATE = "taskCreate"
     STEP_COMPLETE = "stepComplete"
     TASK_STOP = "taskStop"
+    ORPHANED_REPLY = "orphanedReply"

by_framework/core/registry.py

@@ -239,6 +239,33 @@ async def check_worker_online(
     return is_legacy or last_seen > 0
 
 
+async def acquire_scoped_lock(
+    redis: Redis,
+    key: str,
+    token: str,
+    ttl_seconds: int,
+) -> bool:
+    """Claim `key` for `token` if nobody holds it (Redlock acquire half).
+
+    The stored value must stay a cjson-decodable object carrying a "token"
+    field: `_REFRESH_LOCK_SCRIPT` / `_RELEASE_LOCK_SCRIPT` parse it that way,
+    and a bare token string would decode as unparseable legacy data, making
+    the holder unable to release its own lock.
+    """
+    stored = await redis.set(
+        key,
+        json.dumps({"token": token}, separators=(",", ":")),
+        nx=True,
+        ex=ttl_seconds,
+    )
+    return bool(stored)
+
+
+async def release_scoped_lock(redis: Redis, key: str, token: str) -> bool:
+    """Release a lock taken with acquire_scoped_lock(), if still owned."""
+    return bool(await redis.eval(_RELEASE_LOCK_SCRIPT, 1, key, token or ""))
+
+
 async def check_agent_type_online(
     redis: Redis,
     agent_type: str,
@@ -1225,6 +1252,10 @@ class WorkerRegistry:
             "active": self._get_int_hash_value(raw_counts, "active_count"),
             "queued": self._get_int_hash_value(raw_counts, "queued_count"),
             "running": self._get_int_hash_value(raw_counts, "running_count"),
+            "waiting_agent": self._get_int_hash_value(
+                raw_counts, "waiting_agent_count"
+            ),
+            "waiting_user": self._get_int_hash_value(raw_counts, "waiting_user_count"),
             "cancelling": self._get_int_hash_value(raw_counts, "cancelling_count"),
             "completed": self._get_int_hash_value(raw_counts, "completed_count"),
             "failed": self._get_int_hash_value(raw_counts, "failed_count"),
@@ -1245,6 +1276,10 @@ class WorkerRegistry:
         status_counts = {
             "QUEUED": counts["queued"],
             "RUNNING": counts["running"],
+            # Suspended callers persist as WAITING_* rather than QUEUED; these
+            # rows keep them visible (zero-valued entries are filtered below).
+            "WAITING_AGENT": counts["waiting_agent"],
+            "WAITING_USER": counts["waiting_user"],
             "CANCELLING": counts["cancelling"],
             "COMPLETED": counts["completed"],
             "FAILED": counts["failed"],