devops-bot-sdk 1.4.8__tar.gz → 1.4.11__tar.gz

{devops_bot_sdk-1.4.8 → devops_bot_sdk-1.4.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: devops-bot-sdk
-Version: 1.4.8
+Version: 1.4.11
 Summary: DevOps Bot Desktop SDK — thin client for the AgentOS Electron desktop app
 Author: noumanaziz2128
 License-Expression: LicenseRef-Proprietary

{devops_bot_sdk-1.4.8 → devops_bot_sdk-1.4.11}/devops_bot_sdk.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: devops-bot-sdk
-Version: 1.4.8
+Version: 1.4.11
 Summary: DevOps Bot Desktop SDK — thin client for the AgentOS Electron desktop app
 Author: noumanaziz2128
 License-Expression: LicenseRef-Proprietary

{devops_bot_sdk-1.4.8 → devops_bot_sdk-1.4.11}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "devops-bot-sdk"
-version = "1.4.8"
+version = "1.4.11"
 description = "DevOps Bot Desktop SDK — thin client for the AgentOS Electron desktop app"
 readme = "README.md"
 license = "LicenseRef-Proprietary"

{devops_bot_sdk-1.4.8 → devops_bot_sdk-1.4.11}/sdk/__init__.py

@@ -1,6 +1,6 @@
 """AgentOS Desktop SDK — thin HTTPS/SSE client for the Electron app.
 
-Version: 1.4.8
+Version: 1.4.11
 
 Public surface:
   BackendClient.from_config()        — create client from ~/.agentos/config.toml
@@ -30,7 +30,7 @@ Rules:
   - All data egress through submit_webhook only
 """
 
-__version__ = "1.4.8"
+__version__ = "1.4.11"
 __author__  = "AgentOS"
 
 from sdk.client import BackendClient

{devops_bot_sdk-1.4.8 → devops_bot_sdk-1.4.11}/sdk/client.py

@@ -36,7 +36,7 @@ from sdk.sse import _check_status, stream_with_reconnect
 
 logger = logging.getLogger(__name__)
 
-SDK_VERSION    = "1.4.8"
+SDK_VERSION    = "1.4.11"
 _POLL_INTERVAL = 3.0
 _POLL_TIMEOUT  = 600.0
 _ORCHESTRATE_TIMEOUT      = 2700.0  # 45 min — covers approval wait + VPS execution time
@@ -661,6 +661,35 @@ class BackendClient:
         except Exception:
             return None
 
+    async def move_to_backlog(self, task_id: str) -> bool:
+        """PUT {BE}/tasks-ml/{id} status=backlog (best-effort).
+
+        Used when a local run hits the Claude usage limit: the ticket goes BACK to
+        Backlog (not Done) and the loop retries it after the limit resets. Returns
+        True on success; logs and returns False otherwise (e.g. the ML backend
+        rejects the transition for a task with no running timer).
+        """
+        from sdk.config import BE_DEFAULT_URL
+
+        url = f"{BE_DEFAULT_URL.rstrip('/')}/tasks-ml/{task_id}"
+        # multipart form field `status=backlog` — drop any json Content-Type so
+        # httpx sets the multipart boundary itself.
+        headers = {
+            k: v for k, v in self._headers.items() if k.lower() != "content-type"
+        }
+        headers.update({"accept": "*/*", "developer-name": "null", "product": ""})
+        try:
+            async with httpx.AsyncClient(timeout=20.0) as client:
+                resp = await client.put(url, files={"status": (None, "backlog")}, headers=headers)
+            if resp.status_code >= 400:
+                logger.warning("client.move_to_backlog failed task=%s status=%s body=%s",
+                               task_id, resp.status_code, resp.text[:200])
+                return False
+            return True
+        except Exception as exc:  # noqa: BLE001
+            logger.warning("client.move_to_backlog error task=%s err=%s", task_id, exc)
+            return False
+
     async def notify_whatsapp(
         self,
         task_id: str,

{devops_bot_sdk-1.4.8 → devops_bot_sdk-1.4.11}/sdk/ipc/handlers.py

@@ -29,6 +29,7 @@ from __future__ import annotations
 
 import asyncio
 import logging
+import os
 import re
 from typing import Any, AsyncGenerator, Callable
 
@@ -711,6 +712,29 @@ async def handle_pipeline_approve(
     if pr_urls:
         agg_result += "\n\nPRs:\n" + "\n".join(pr_urls)
 
+    # ── Claude usage-limit hit: DON'T mark Done. Move the ticket back to Backlog
+    #    and signal the auto-loop to retry this task after the window resets. ──
+    if any(r["result"].get("limit_exceeded") for r in runs):
+        reset_epoch = max((r["result"].get("limit_reset_epoch") or 0) for r in runs) or None
+        moved = await client.move_to_backlog(task_id)
+        await send(Envelope(
+            type="step_update", thread_id=thread_id,
+            data={"task_id": task_id,
+                  "current_step": (("Claude usage limit reached — ticket moved to Backlog"
+                                    if moved else "Claude usage limit reached (Backlog move failed)")
+                                   + "; will retry after the limit resets")},
+        ).model_dump())
+        try:
+            await client.notify_whatsapp(
+                task_id,
+                f"⚠ Task {task_id}: Claude usage limit reached — moved to Backlog; "
+                "will auto-retry after the window resets.",
+                summary="usage limit",
+            )
+        except Exception:  # noqa: BLE001
+            pass
+        return {"status": "limit_exceeded", "reset_epoch": reset_epoch, "task_id": task_id}
+
     # ── Notify on WhatsApp if nothing succeeded ──────────────────────────
     if not ok_any:
         err = "; ".join(str(r["result"].get("error") or "unknown") for r in runs) or "no repos processed"
@@ -1429,6 +1453,40 @@ _APPROVAL_POLL_TIMEOUT_S = 1200.0
 _MAX_CONTEXT_RESUMES = 3
 # Max mid-run crucial-decision approval gates per ticket (safety bound).
 _MAX_CRUCIAL_GATES = 10
+# Max times a ticket is retried after hitting the Claude usage limit.
+_MAX_LIMIT_RETRIES = 3
+# Cap on how long we'll sleep waiting for a usage-limit window to reset.
+_LIMIT_MAX_WAIT_S = 6 * 60 * 60.0
+# Fallback wait when the limit message carries no reset time.
+_LIMIT_FALLBACK_WAIT_S = float(os.getenv("AGENTOS_LIMIT_DEFAULT_WAIT_S", "") or 3600.0)
+
+
+def _limit_wait_seconds(reset_epoch: int | None) -> float:
+    """Seconds to wait before retrying after a usage-limit hit.
+
+    Uses the reset timestamp the limit message carried (+60s buffer); otherwise a
+    fallback. Always capped at _LIMIT_MAX_WAIT_S so a bad timestamp can't park the
+    loop for days.
+    """
+    import time as _t
+    if reset_epoch:
+        wait = max(0.0, float(reset_epoch) - _t.time()) + 60.0
+    else:
+        wait = _LIMIT_FALLBACK_WAIT_S
+    return min(wait, _LIMIT_MAX_WAIT_S)
+
+
+async def _cooperative_sleep(seconds: float, should_stop: Callable[[], bool] | None) -> bool:
+    """Sleep `seconds`, checking should_stop periodically. Returns False if a stop
+    was requested mid-sleep (so the caller can pause cleanly), else True."""
+    waited = 0.0
+    while waited < seconds:
+        if should_stop is not None and should_stop():
+            return False
+        chunk = min(30.0, seconds - waited)
+        await asyncio.sleep(chunk)
+        waited += chunk
+    return True
 
 
 async def _poll_approval(
@@ -1558,52 +1616,86 @@ async def handle_orchestrate_auto_local_run(
             # Per-ticket isolation: a failure here (network, git, a raising
             # repo) must NOT abort the remaining tickets — log it and move on.
             try:
-                # Phase 1 — server plans + gates. NO local code/bash runs yet.
-                await _run_local_pipeline(client, task_payload, send)
-                session_id = next(
-                    (sid for sid, p in _PENDING_LOCAL.items() if p.get("task_id") == tid),
-                    None,
-                )
-                if not session_id:
-                    # Planning never reached the approval gate for this task — skip it.
-                    await send(Envelope(
-                        type="step_update", thread_id=thread_id,
-                        data={"current_step": f"{jira_key}: skipped (no approval gate reached)"},
-                    ).model_dump())
-                    continue
+                # Retry the ticket if a run hits the Claude usage limit:
+                # handle_pipeline_approve moves it to Backlog and signals here; we
+                # wait for the window to reset and re-run — up to _MAX_LIMIT_RETRIES.
+                attempt = 0
+                while True:
+                    attempt += 1
+                    # Phase 1 — server plans + gates. NO local code/bash runs yet.
+                    await _run_local_pipeline(client, task_payload, send)
+                    session_id = next(
+                        (sid for sid, p in _PENDING_LOCAL.items() if p.get("task_id") == tid),
+                        None,
+                    )
+                    if not session_id:
+                        # Planning never reached the approval gate — skip this ticket.
+                        await send(Envelope(
+                            type="step_update", thread_id=thread_id,
+                            data={"current_step": f"{jira_key}: skipped (no approval gate reached)"},
+                        ).model_dump())
+                        break
 
-                # First-approval gate: when agentMode is ON, WAIT for the human's
-                # WhatsApp approval before ANY local code/bash. When OFF, proceed.
-                if agent_mode:
-                    await send(Envelope(
-                        type=Envelope.TYPE_AWAITING_APPROVAL, thread_id=thread_id,
-                        data={"jira_key": jira_key, "session_id": session_id, "task_id": tid,
-                              "current_step": f"{jira_key}: awaiting approval"},
-                    ).model_dump())
-                    decision = await _poll_approval(client, tid, should_stop)
-                    if decision == "stopped":
-                        _PENDING_LOCAL.pop(session_id, None)
+                    # First-approval gate: when agentMode is ON, WAIT for the human's
+                    # WhatsApp approval before ANY local code/bash. When OFF, proceed.
+                    if agent_mode:
                         await send(Envelope(
-                            type=Envelope.TYPE_DONE, thread_id=thread_id,
-                            data={"mode": "orchestrate-auto-local", "status": "paused",
-                                  "processed": processed, "task_count": total},
+                            type=Envelope.TYPE_AWAITING_APPROVAL, thread_id=thread_id,
+                            data={"jira_key": jira_key, "session_id": session_id, "task_id": tid,
+                                  "current_step": f"{jira_key}: awaiting approval"},
                         ).model_dump())
-                        return
-                    if decision != "approved":
-                        _PENDING_LOCAL.pop(session_id, None)
+                        decision = await _poll_approval(client, tid, should_stop)
+                        if decision == "stopped":
+                            _PENDING_LOCAL.pop(session_id, None)
+                            await send(Envelope(
+                                type=Envelope.TYPE_DONE, thread_id=thread_id,
+                                data={"mode": "orchestrate-auto-local", "status": "paused",
+                                      "processed": processed, "task_count": total},
+                            ).model_dump())
+                            return
+                        if decision != "approved":
+                            _PENDING_LOCAL.pop(session_id, None)
+                            await send(Envelope(
+                                type="step_update", thread_id=thread_id,
+                                data={"current_step": f"{jira_key}: {decision} — skipped"},
+                            ).model_dump())
+                            break
+
+                    # Phase 2 — run the user's local `claude`, report back.
+                    outcome = await handle_pipeline_approve(
+                        client,
+                        {"session_id": session_id, "decision": "approved", "thread_id": thread_id},
+                        send,
+                    )
+
+                    # Usage limit → ticket is now in Backlog. Wait for reset + retry.
+                    if isinstance(outcome, dict) and outcome.get("status") == "limit_exceeded":
+                        if attempt <= _MAX_LIMIT_RETRIES:
+                            wait_s = _limit_wait_seconds(outcome.get("reset_epoch"))
+                            await send(Envelope(
+                                type="step_update", thread_id=thread_id,
+                                data={"current_step": f"{jira_key}: Claude usage limit — waiting "
+                                                      f"{round(wait_s / 60)}m for reset, then retry "
+                                                      f"({attempt}/{_MAX_LIMIT_RETRIES})"},
+                            ).model_dump())
+                            if not await _cooperative_sleep(wait_s, should_stop):
+                                await send(Envelope(
+                                    type=Envelope.TYPE_DONE, thread_id=thread_id,
+                                    data={"mode": "orchestrate-auto-local", "status": "paused",
+                                          "processed": processed, "task_count": total},
+                                ).model_dump())
+                                return
+                            continue  # re-run this ticket from phase 1
+                        # Out of retries — leave it in Backlog for a later run.
                         await send(Envelope(
                             type="step_update", thread_id=thread_id,
-                            data={"current_step": f"{jira_key}: {decision} — skipped"},
+                            data={"current_step": f"{jira_key}: usage limit — max retries reached, "
+                                                  f"left in Backlog"},
                         ).model_dump())
-                        continue
+                        break
 
-                # Phase 2 — run the user's local `claude`, report back.
-                await handle_pipeline_approve(
-                    client,
-                    {"session_id": session_id, "decision": "approved", "thread_id": thread_id},
-                    send,
-                )
-                processed += 1
+                    processed += 1
+                    break
             except Exception as exc:  # noqa: BLE001 — isolate per-ticket failures
                 logger.warning("orchestrate_auto.task_failed", extra={"task_id": tid, "error": str(exc)})
                 await send(Envelope(

{devops_bot_sdk-1.4.8 → devops_bot_sdk-1.4.11}/sdk/local_exec.py

@@ -20,6 +20,7 @@ from __future__ import annotations
 import asyncio
 import json
 import os
+import re
 import shutil
 from pathlib import Path
 from typing import Any, Awaitable, Callable
@@ -29,8 +30,21 @@ from typing import Any, Awaitable, Callable
 # user's machine, not the server.
 DEFAULT_ALLOWED_TOOLS = ["Read", "Write", "Edit", "Glob", "Grep", "Bash"]
 
-# Hard ceiling so a runaway agent can't hang the sidecar forever.
-_DEFAULT_TIMEOUT_S = 1800.0
+# Run limits. The OLD design capped TOTAL runtime at 30 min, which killed long
+# but healthy runs (big builds, many steps) mid-work. Instead we use an IDLE
+# timeout — abort only if claude emits NOTHING for this long (truly stuck) — plus
+# a generous absolute backstop so a runaway can't hang forever. Both env-tunable.
+def _f(env: str, default: float) -> float:
+    try:
+        return float(os.getenv(env, "") or default)
+    except ValueError:
+        return default
+
+
+# No output for this long → the run is stuck → abort.
+_IDLE_TIMEOUT_S = _f("AGENTOS_LOCAL_IDLE_TIMEOUT_S", 900.0)      # 15 min of silence
+# Absolute backstop on total wall-clock regardless of activity.
+_DEFAULT_TIMEOUT_S = _f("AGENTOS_LOCAL_MAX_S", 4 * 60 * 60.0)    # 4 h
 
 # StreamReader buffer for the claude CLI's stdout. `--output-format stream-json`
 # emits one JSON object per line, and a single event can embed a whole file's
@@ -46,6 +60,37 @@ _CONTEXT_MARKERS = (
     "token limit", "prompt is too long", "exceeds the maximum", "context_length_exceeded",
 )
 
+# Markers that indicate the Claude USAGE/RATE limit (the N-hour window) was hit —
+# distinct from a context overflow. The run didn't fail on the task; the account
+# is throttled until the window resets.
+_LIMIT_MARKERS = (
+    "usage limit reached", "usage limit", "rate limit", "rate_limit",
+    "too many requests", "limit will reset", "limit reached", "overloaded_error",
+    "resets at", "429",
+)
+# A unix-seconds reset timestamp claude sometimes appends (e.g. "...reached|1719500400").
+_RESET_EPOCH_RE = re.compile(r"\b(1[6-9]\d{8}|20\d{8})\b")
+# Fallback wait when the limit message carries no reset time (env-tunable).
+_LIMIT_DEFAULT_WAIT_S = _f("AGENTOS_LIMIT_DEFAULT_WAIT_S", 60 * 60.0)   # 1 h
+
+
+def _is_limit(text: str | None) -> bool:
+    low = (text or "").lower()
+    return any(m in low for m in _LIMIT_MARKERS)
+
+
+def _parse_reset_epoch(text: str | None) -> int | None:
+    """A unix-seconds reset time embedded in a limit message, if present."""
+    if not text:
+        return None
+    m = _RESET_EPOCH_RE.search(text)
+    if m:
+        try:
+            return int(m.group(1))
+        except ValueError:
+            return None
+    return None
+
 
 def _ctx_exceeded(text: str | None) -> bool:
     low = (text or "").lower()
@@ -74,6 +119,7 @@ async def run_claude_local(
     github_token: str | None = None,
     resume_session_id: str | None = None,
     approval_dir: str | None = None,
+    idle_timeout: float = _IDLE_TIMEOUT_S,
 ) -> dict[str, Any]:
     """Run the local `claude` CLI non-interactively against ``project_path``.
 
@@ -168,13 +214,22 @@ async def run_claude_local(
     result_event: dict | None = None
     files_changed: list[str] = []
     session_id: str | None = resume_session_id
+    import time as _time
+    started = _time.monotonic()
+    stall = {"timed_out": False, "reason": ""}
 
     async def _drain_stdout() -> None:
         nonlocal result_event, session_id
         assert proc.stdout is not None
         while True:
             try:
-                raw = await proc.stdout.readline()
+                # IDLE timeout: abort only if claude produces NO output for this
+                # long (genuinely stuck), so long-but-active runs aren't killed.
+                raw = await asyncio.wait_for(proc.stdout.readline(), timeout=idle_timeout)
+            except asyncio.TimeoutError:
+                stall["timed_out"] = True
+                stall["reason"] = f"no output for {idle_timeout:.0f}s (stuck)"
+                return
             except ValueError:
                 # Line exceeded even _STREAM_LIMIT_BYTES. readline() has already
                 # advanced past the oversized chunk, so we can safely skip this
@@ -182,6 +237,11 @@ async def run_claude_local(
                 continue
             if not raw:
                 break
+            # Absolute backstop on total wall-clock (checked as events arrive).
+            if _time.monotonic() - started > timeout:
+                stall["timed_out"] = True
+                stall["reason"] = f"exceeded absolute cap {timeout:.0f}s"
+                return
             line = raw.decode("utf-8", errors="replace").strip()
             if not line:
                 continue
@@ -212,16 +272,20 @@ async def run_claude_local(
                 except Exception:
                     pass  # progress forwarding is best-effort
 
-    try:
-        await asyncio.wait_for(_drain_stdout(), timeout=timeout)
-        await asyncio.wait_for(proc.wait(), timeout=30.0)
-    except asyncio.TimeoutError:
+    await _drain_stdout()
+    if not stall["timed_out"]:
+        try:
+            await asyncio.wait_for(proc.wait(), timeout=30.0)
+        except asyncio.TimeoutError:
+            stall["timed_out"] = True
+            stall["reason"] = "process did not exit after the stream closed"
+    if stall["timed_out"]:
         try:
             proc.kill()
         except ProcessLookupError:
             pass
         return {
-            "ok": False, "result": "", "error": f"claude CLI timed out after {timeout:.0f}s",
+            "ok": False, "result": "", "error": f"claude CLI timed out — {stall['reason']}",
             "input_tokens": 0, "output_tokens": 0, "total_tokens": 0,
             "cost_usd": 0.0, "files_changed": files_changed,
             "session_id": session_id, "context_exceeded": False,
@@ -235,17 +299,31 @@ async def run_claude_local(
             "input_tokens": 0, "output_tokens": 0, "total_tokens": 0,
             "cost_usd": 0.0, "files_changed": files_changed,
             "session_id": session_id, "context_exceeded": _ctx_exceeded(err),
+            "limit_exceeded": _is_limit(err), "limit_reset_epoch": _parse_reset_epoch(err),
         }
 
     usage = result_event.get("usage") or {}
     in_tok = int(usage.get("input_tokens", 0) or 0)
     out_tok = int(usage.get("output_tokens", 0) or 0)
     is_error = bool(result_event.get("is_error", False))
+    result_text = str(result_event.get("result", ""))
+    subtype = str(result_event.get("subtype", ""))
+
+    # Claude USAGE/RATE limit: detected from the result/subtype even when the CLI
+    # didn't flag is_error — so a limit-truncated run is NOT treated as success
+    # (which would wrongly mark the ticket Done).
+    limit_hit = _is_limit(result_text) or "limit" in subtype.lower()
+    reset_epoch = _parse_reset_epoch(result_text)
+
+    err_text = None
+    if is_error:
+        err_text = result_text or "claude reported an error"
+    elif limit_hit:
+        err_text = result_text or "claude usage limit reached"
 
-    err_text = None if not is_error else str(result_event.get("result", "claude reported an error"))
     return {
-        "ok": not is_error,
-        "result": str(result_event.get("result", "")),
+        "ok": (not is_error) and (not limit_hit),
+        "result": result_text,
         "input_tokens": in_tok,
         "output_tokens": out_tok,
         "total_tokens": in_tok + out_tok,
@@ -254,4 +332,6 @@ async def run_claude_local(
         "error": err_text,
         "session_id": session_id,
         "context_exceeded": is_error and _ctx_exceeded(err_text),
+        "limit_exceeded": limit_hit,
+        "limit_reset_epoch": reset_epoch,
     }