devops-bot-sdk 1.6.64__tar.gz → 1.6.66__tar.gz

{devops_bot_sdk-1.6.64 → devops_bot_sdk-1.6.66}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: devops-bot-sdk
-Version: 1.6.64
+Version: 1.6.66
 Summary: DevOps Bot Desktop SDK — thin client for the AgentOS Electron desktop app
 Author: noumanaziz2128
 License-Expression: LicenseRef-Proprietary

{devops_bot_sdk-1.6.64 → devops_bot_sdk-1.6.66}/devops_bot_sdk.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: devops-bot-sdk
-Version: 1.6.64
+Version: 1.6.66
 Summary: DevOps Bot Desktop SDK — thin client for the AgentOS Electron desktop app
 Author: noumanaziz2128
 License-Expression: LicenseRef-Proprietary

{devops_bot_sdk-1.6.64 → devops_bot_sdk-1.6.66}/sdk/__init__.py

@@ -1,6 +1,6 @@
 """AgentOS Desktop SDK — thin HTTPS/SSE client for the Electron app.
 
-Version: 1.6.64
+Version: 1.6.66
 
 Public surface:
   BackendClient.from_config()        — create client from ~/.agentos/config.toml
@@ -34,7 +34,7 @@ Rules:
   - All data egress through submit_webhook only
 """
 
-__version__ = "1.6.64"   # SINGLE SOURCE OF TRUTH — bump on every change; pyproject,
+__version__ = "1.6.66"   # SINGLE SOURCE OF TRUTH — bump on every change; pyproject,
 __author__  = "AgentOS"   # sdk.client.SDK_VERSION and the UI server version all read this.
 
 from sdk.client import BackendClient

{devops_bot_sdk-1.6.64 → devops_bot_sdk-1.6.66}/sdk/activity/models.py

@@ -16,6 +16,7 @@ retried at any hop is stored once.
 """
 from __future__ import annotations
 
+import json
 import os
 import sys
 import time
@@ -140,6 +141,57 @@ def _wire(obj: Any) -> dict[str, Any]:
     return out
 
 
+#: String caps the relay validates (devops-bot-ai ``modules/vps_activity/schemas.py``).
+#: A value over its cap is a 422, and the shipper drops a row the relay refuses — so
+#: one over-long path or command line used to cost the whole event. Clamp instead.
+WIRE_MAX_CHARS: dict[str, dict[str, int]] = {
+    "event": {
+        "action": 100,
+        "osUser": 255,
+        "processName": 255,
+        "command": 20_000,
+        "target": 4096,
+        "agentRunId": 128,
+        "sessionKey": 128,
+    },
+    "session": {
+        "osUser": 255,
+        "remoteIp": 45,
+        "agentRunId": 128,
+        "initiatedByKey": 128,
+    },
+}
+_CLAMP_MARK = "…"
+#: Free-form dicts the relay does not cap. Bounded here so that, with the string caps
+#: above, no single row can outgrow a request (2 MB gzipped / 10 MB raw) — a row that
+#: could would be 413'd on its own and dropped.
+WIRE_DICT_FIELDS = ("metadata", "actorReasons")
+MAX_DICT_FIELD_BYTES = 64 * 1024
+
+
+def fit_wire_limits(kind: str, wire: dict[str, Any]) -> dict[str, Any]:
+    """``wire`` with every over-long field shortened to fit the relay's limits."""
+    out = wire
+
+    def _set(name: str, value: Any) -> None:
+        nonlocal out
+        if out is wire:
+            out = dict(wire)
+        out[name] = value
+
+    for name, cap in WIRE_MAX_CHARS.get(kind, {}).items():
+        value = wire.get(name)
+        if isinstance(value, str) and len(value) > cap:
+            _set(name, value[: cap - len(_CLAMP_MARK)] + _CLAMP_MARK)
+    for name in WIRE_DICT_FIELDS:
+        value = wire.get(name)
+        if isinstance(value, dict):
+            size = len(json.dumps(value, separators=(",", ":"), default=str))
+            if size > MAX_DICT_FIELD_BYTES:
+                _set(name, {"truncated": True, "originalBytes": size})
+    return out
+
+
 # ── models ───────────────────────────────────────────────────────────────────
 
 

{devops_bot_sdk-1.6.64 → devops_bot_sdk-1.6.66}/sdk/activity/redact.py

@@ -72,6 +72,7 @@ _PATTERNS: list[tuple[re.Pattern[str], str]] = [
 
 #: Hard cap so a pathological command line can't bloat the spool.
 MAX_COMMAND_CHARS = 20_000
+TRUNCATED = "…[truncated]"
 
 
 def redact_command(command: str | None) -> str | None:
@@ -82,5 +83,7 @@ def redact_command(command: str | None) -> str | None:
     for pattern, repl in _PATTERNS:
         out = pattern.sub(repl, out)
     if len(out) > MAX_COMMAND_CHARS:
-        out = out[:MAX_COMMAND_CHARS] + "…[truncated]"
+        # The marker counts toward the cap: the relay 422s anything longer, and the
+        # shipper drops a refused event rather than shipping it truncated.
+        out = out[: MAX_COMMAND_CHARS - len(TRUNCATED)] + TRUNCATED
     return out

{devops_bot_sdk-1.6.64 → devops_bot_sdk-1.6.66}/sdk/activity/shipper.py

@@ -7,8 +7,9 @@ Delivery rules — a row leaves the spool ONLY when the relay answered 202 (or i
 provably undeliverable):
 
   2xx                          delivered → delete the chunk
-  network error, 5xx, 408, 429 retry (short backoff); if still failing, keep
+  network error, 5xx, 408      retry (short backoff); if still failing, keep
                                everything and try again at the next flush
+  429                          wait out the relay's 60 s window, then retry
   401, 426                     token / SDK version needs an operator → keep, stop
                                this flush (retrying in a tight loop helps nobody)
   any other 4xx                the relay READ the chunk and refused it. Dropping all
@@ -16,8 +17,9 @@ provably undeliverable):
                                so the chunk is BISECTED until the offending row is
                                isolated; only that single row is dropped.
 
-A flush drains the whole spool, chunk by chunk (≤500 events / ≤1 MB each), because
-five minutes of full OS auditing is routinely thousands of events.
+A flush drains the whole spool, chunk by chunk (≤500 events / ≤4 MB each), paced under
+the relay's 60-requests-a-minute budget, because five minutes of full OS auditing is
+routinely thousands of events.
 """
 from __future__ import annotations
 
@@ -36,9 +38,18 @@ logger = logging.getLogger("sdk.activity.shipper")
 RETRY_DELAYS_S = (2.0, 8.0)
 #: Safety cap on chunks per flush so one flush can't run away.
 MAX_CHUNKS_PER_FLUSH = 200
+#: The relay allows 60 batches per minute per token. Firing chunks back-to-back blew
+#: through that after ~60 chunks; the 2 s / 8 s retries cannot outlast a 60 s window,
+#: so the flush gave up and a busy host never caught up. Pace below the budget...
+MIN_POST_GAP_S = 1.1
+#: ...and if a 429 still lands (another process on the same token), wait the window
+#: out rather than abandoning the flush.
+RATE_LIMIT_WAIT_S = 61.0
+MAX_RATE_LIMIT_WAITS = 3
 
 _NEEDS_OPERATOR = {401, 426}
 _TRANSIENT_4XX = {408, 429}
+_RATE_LIMITED = 429
 
 Sleep = Callable[[float], Awaitable[Any]]
 
@@ -73,20 +84,27 @@ def _verdict(status: int | None) -> str:
 
 async def _attempt(sender: Sender, chunk: Chunk, sleep: Sleep) -> str:
     """One chunk with short in-flush retries → ``ok`` / ``drop`` / ``stop`` / ``later``."""
-    for delay in (0.0, *RETRY_DELAYS_S):
-        if delay:
-            await sleep(delay)
+    retry_delays = list(RETRY_DELAYS_S)
+    rate_limit_waits = 0
+    while True:
         try:
             status: int | None = await sender.post_vps_activity_batch(chunk.to_wire())
         except Exception as exc:  # noqa: BLE001 — httpx errors, DNS, TLS, ...
             logger.warning("activity: ship failed (%s); will retry", exc)
             status = None
+        if status == _RATE_LIMITED and rate_limit_waits < MAX_RATE_LIMIT_WAITS:
+            rate_limit_waits += 1
+            logger.info("activity: relay rate-limited; waiting %.0fs", RATE_LIMIT_WAIT_S)
+            await sleep(RATE_LIMIT_WAIT_S)
+            continue
         verdict = _verdict(status)
         if verdict != "retry":
             if verdict == "stop":
                 logger.warning("activity: relay answered %s — needs operator attention", status)
             return verdict
-    return "later"
+        if not retry_delays:
+            return "later"
+        await sleep(retry_delays.pop(0))
 
 
 async def _deliver(
@@ -135,10 +153,12 @@ async def flush_once(
             from sdk.client import BackendClient
 
             sender = BackendClient.from_config()
-        for _ in range(max_chunks):
+        for i in range(max_chunks):
             chunk = spool.next_chunk()
             if not chunk:
                 break
+            if i:
+                await sleep(MIN_POST_GAP_S)
             if not await _deliver(spool, sender, chunk, result, sleep):
                 break
         else:

{devops_bot_sdk-1.6.64 → devops_bot_sdk-1.6.66}/sdk/activity/spool.py

@@ -23,14 +23,21 @@ from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Any
 
+from sdk.activity.models import fit_wire_limits
+
 logger = logging.getLogger("sdk.activity.spool")
 
 #: Rows kept at most. ~200k events is days of a busy host; beyond it, shed oldest.
 DEFAULT_MAX_ROWS = 200_000
-#: Wire limits per request — the relay and the gateway enforce the same cap.
+#: Wire limits per request — the relay and the gateway enforce the same row cap.
 MAX_EVENTS_PER_CHUNK = 500
 MAX_SESSIONS_PER_CHUNK = 500
-MAX_CHUNK_BYTES = 1_000_000
+#: Uncompressed JSON per request. The relay takes 10 MB decompressed / 2 MB gzipped;
+#: audit JSON gzips well past 2:1, so 4 MB stays under both. A 1 MB cap meant a host
+#: with long command lines sent small chunks, ran into the relay's per-minute request
+#: budget and fell behind until the spool shed its oldest rows. A chunk that still
+#: comes back 413 is bisected by the shipper, so this is a target, not a cliff.
+MAX_CHUNK_BYTES = 4_000_000
 
 _SCHEMA = """
 CREATE TABLE IF NOT EXISTS spool (
@@ -110,7 +117,10 @@ class Spool:
         with self._lock:
             self._db.execute(
                 "INSERT OR IGNORE INTO spool (kind, key, payload) VALUES ('event', ?, ?)",
-                (wire["eventId"], json.dumps(wire, separators=(",", ":"))),
+                (
+                    wire["eventId"],
+                    json.dumps(fit_wire_limits("event", wire), separators=(",", ":")),
+                ),
             )
             self._trim()
 
@@ -127,7 +137,10 @@ class Spool:
                 "INSERT INTO spool (kind, key, payload) VALUES ('session', ?, ?) "
                 "ON CONFLICT (kind, key) DO UPDATE SET "
                 "payload = excluded.payload, rev = rev + 1",
-                (wire["externalKey"], json.dumps(wire, separators=(",", ":"))),
+                (
+                    wire["externalKey"],
+                    json.dumps(fit_wire_limits("session", wire), separators=(",", ":")),
+                ),
             )
             self._trim()
 
@@ -176,7 +189,8 @@ class Spool:
                 break
             if chunk.rows and size + len(payload) > max_bytes:
                 break
-            chunk.rows.append(Row(seq, rev, kind, json.loads(payload)))
+            # Re-fit on the way out too: rows queued by an older SDK predate the clamp.
+            chunk.rows.append(Row(seq, rev, kind, fit_wire_limits(kind, json.loads(payload))))
             n_events += kind == "event"
             n_sessions += kind == "session"
             size += len(payload)

{devops_bot_sdk-1.6.64 → devops_bot_sdk-1.6.66}/tests/test_activity_core.py

@@ -176,7 +176,7 @@ class TestRedaction:
     def test_is_length_capped(self):
         out = redact_command("a" * (MAX_COMMAND_CHARS + 500))
         assert out.endswith("…[truncated]")
-        assert len(out) < MAX_COMMAND_CHARS + 50
+        assert len(out) == MAX_COMMAND_CHARS  # the relay 422s anything longer
 
 
 class TestClassifier:

{devops_bot_sdk-1.6.64 → devops_bot_sdk-1.6.66}/tests/test_activity_pipeline.py

@@ -66,6 +66,45 @@ class TestSpool:
             _sess("S1", actorType="HUMAN", endedAt="2026-10-09T12:00:00.000Z")
         ]
 
+    def test_over_long_fields_are_clamped_to_the_relay_caps_not_left_to_be_refused(self, spool):
+        from sdk.activity.models import WIRE_MAX_CHARS
+
+        spool.add_event(_ev("E1", command="c" * 30_000, target="/t" * 5000, processName="p" * 999))
+        spool.add_session(_sess("S1", osUser="u" * 999))
+        chunk = spool.next_chunk()
+        ev, sess = chunk.events[0], chunk.sessions[0]
+        for name in ("command", "target", "processName"):
+            assert len(ev[name]) == WIRE_MAX_CHARS["event"][name]
+        assert len(sess["osUser"]) == WIRE_MAX_CHARS["session"]["osUser"]
+        assert ev["action"] == "exec"  # short fields untouched
+
+    def test_an_oversized_metadata_dict_is_replaced_so_the_row_still_ships(self, spool):
+        from sdk.activity.models import MAX_DICT_FIELD_BYTES
+
+        spool.add_event(_ev("E1", metadata={"blob": "x" * (MAX_DICT_FIELD_BYTES * 2)}))
+        spool.add_event(_ev("E2", metadata={"small": 1}))
+        events = spool.next_chunk().events
+        assert events[0]["metadata"]["truncated"] is True
+        assert events[0]["metadata"]["originalBytes"] > MAX_DICT_FIELD_BYTES
+        assert events[1]["metadata"] == {"small": 1}
+
+    def test_a_single_worst_case_row_fits_comfortably_in_one_request(self, spool):
+        from sdk.activity.models import MAX_DICT_FIELD_BYTES
+
+        spool.add_event(_ev(
+            "E1", command="c" * 99_999, target="t" * 99_999, processName="p" * 9_999,
+            metadata={"blob": "x" * (MAX_DICT_FIELD_BYTES * 10)},
+        ))
+        (row,) = spool.next_chunk().rows
+        import json as _json
+
+        assert len(_json.dumps(row.wire)) < 200_000  # vs 2 MB gzipped / 10 MB raw at the relay
+
+    def test_default_chunk_byte_cap_stays_under_the_relay_body_limit(self):
+        from sdk.activity import spool as spool_mod
+
+        assert 1_000_000 < spool_mod.MAX_CHUNK_BYTES < 10 * 1024 * 1024
+
     def test_rows_come_back_in_arrival_order_session_first(self, spool):
         spool.add_session(_sess("S1"))
         spool.add_event(_ev("E1"))
@@ -231,6 +270,28 @@ class TestShipper:
         result = _run(shipper.flush_once(spool, sender, sleep=Sleeps()))
         assert result.shipped_rows == 1
 
+    def test_a_429_waits_out_the_relay_window_instead_of_giving_up(self, spool):
+        spool.add_event(_ev("E1"))
+        sender = FakeSender(script=[429, 429, 429, 202])
+        sleeps = Sleeps()
+        result = _run(shipper.flush_once(spool, sender, sleep=sleeps))
+        assert result.shipped_rows == 1 and result.stopped is None
+        assert sleeps.delays == [shipper.RATE_LIMIT_WAIT_S] * 3
+
+    def test_endless_429s_keep_the_data_for_the_next_flush(self, spool):
+        spool.add_event(_ev("E1"))
+        sender = FakeSender(default=429)
+        result = _run(shipper.flush_once(spool, sender, sleep=Sleeps()))
+        assert result.stopped == "later" and spool.count() == 1
+
+    def test_chunks_are_paced_under_the_relay_request_budget(self, spool):
+        for i in range(1200):
+            spool.add_event(_ev(f"E{i:05d}"))
+        sleeps = Sleeps()
+        _run(shipper.flush_once(spool, FakeSender(), sleep=sleeps))
+        assert sleeps.delays == [shipper.MIN_POST_GAP_S] * 2  # 3 chunks, 2 gaps
+        assert shipper.MIN_POST_GAP_S * 60 > 60  # stays under 60 requests a minute
+
     @pytest.mark.parametrize("status", [401, 426])
     def test_token_or_version_problems_stop_without_retrying_or_dropping(self, spool, status):
         spool.add_event(_ev("E1"))