devops-bot-sdk 1.6.70__tar.gz → 1.6.71__tar.gz

{devops_bot_sdk-1.6.70 → devops_bot_sdk-1.6.71}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: devops-bot-sdk
-Version: 1.6.70
+Version: 1.6.71
 Summary: DevOps Bot Desktop SDK — thin client for the AgentOS Electron desktop app
 Author: noumanaziz2128
 License-Expression: LicenseRef-Proprietary

{devops_bot_sdk-1.6.70 → devops_bot_sdk-1.6.71}/devops_bot_sdk.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: devops-bot-sdk
-Version: 1.6.70
+Version: 1.6.71
 Summary: DevOps Bot Desktop SDK — thin client for the AgentOS Electron desktop app
 Author: noumanaziz2128
 License-Expression: LicenseRef-Proprietary

{devops_bot_sdk-1.6.70 → devops_bot_sdk-1.6.71}/devops_bot_sdk.egg-info/SOURCES.txt

@@ -144,6 +144,7 @@ sdk/ui/static/tickets.html
 sdk/vendor/aidraftsman-cli-0.1.0.tgz
 tests/test_activity_collectors.py
 tests/test_activity_core.py
+tests/test_activity_live_tracking.py
 tests/test_activity_macos.py
 tests/test_activity_os_setup.py
 tests/test_activity_pipeline.py

{devops_bot_sdk-1.6.70 → devops_bot_sdk-1.6.71}/sdk/__init__.py

@@ -1,6 +1,6 @@
 """AgentOS Desktop SDK — thin HTTPS/SSE client for the Electron app.
 
-Version: 1.6.70
+Version: 1.6.71
 
 Public surface:
   BackendClient.from_config()        — create client from ~/.agentos/config.toml
@@ -34,7 +34,7 @@ Rules:
   - All data egress through submit_webhook only
 """
 
-__version__ = "1.6.70"   # SINGLE SOURCE OF TRUTH — bump on every change; pyproject,
+__version__ = "1.6.71"   # SINGLE SOURCE OF TRUTH — bump on every change; pyproject,
 __author__  = "AgentOS"   # sdk.client.SDK_VERSION and the UI server version all read this.
 
 from sdk.client import BackendClient

{devops_bot_sdk-1.6.70 → devops_bot_sdk-1.6.71}/sdk/activity/settings.py

@@ -23,8 +23,12 @@ from sdk.activity.spool import Spool
 
 logger = logging.getLogger("sdk.activity.settings")
 
-DEFAULT_SHIP_INTERVAL_S = 300.0
+# Live tracking: one POST a minute is far under the relay's 60/min budget, and the
+# five-minute default meant the newest activity was always up to five minutes stale.
+DEFAULT_SHIP_INTERVAL_S = 60.0
 MIN_SHIP_INTERVAL_S = 30.0
+#: Events older than this are discarded instead of shipped (``0`` keeps everything).
+DEFAULT_MAX_AGE_S = 3600.0
 _FALSEY = {"0", "false", "no", "off"}
 _TRUTHY = {"1", "true", "yes", "on"}
 
@@ -63,6 +67,22 @@ def ship_interval_seconds() -> float:
         return DEFAULT_SHIP_INTERVAL_S
 
 
+def max_age_seconds() -> float:
+    """``AGENTOS_ACTIVITY_MAX_AGE_S`` — how old an event may be and still ship.
+
+    Default one hour: the trail is for live tracking, and a backlog shipped
+    oldest-first kept busy hosts days behind. ``0`` disables the cut-off.
+    """
+    raw = setting("AGENTOS_ACTIVITY_MAX_AGE_S")
+    if not raw:
+        return DEFAULT_MAX_AGE_S
+    try:
+        return max(0.0, float(raw))
+    except ValueError:
+        logger.warning("AGENTOS_ACTIVITY_MAX_AGE_S=%r is not a number; using default", raw)
+        return DEFAULT_MAX_AGE_S
+
+
 def spool_path() -> Path:
     """``~/.agentos/activity.db`` — resolved at call time so tests can redirect HOME."""
     return Path.home() / ".agentos" / "activity.db"

{devops_bot_sdk-1.6.70 → devops_bot_sdk-1.6.71}/sdk/activity/shipper.py

@@ -1,4 +1,4 @@
-"""Ship the activity spool to devops-bot-ai every 5 minutes.
+"""Ship the activity spool to devops-bot-ai every minute, newest first.
 
 Same shape as ``sdk.vps_health``: an asyncio loop with a ``start()`` the UI
 server's lifespan calls, cancelled by the host on shutdown.
@@ -25,8 +25,10 @@ from __future__ import annotations
 
 import asyncio
 import logging
+import time
 from collections.abc import Awaitable, Callable
 from dataclasses import dataclass
+from datetime import datetime, timezone
 from typing import Any, Protocol
 
 from sdk.activity import settings
@@ -188,17 +190,42 @@ def discard_spool(spool: Spool | None = None) -> int:
         spool.ack(chunk)
 
 
+def purge_stale_rows(spool: Spool | None = None, now: float | None = None) -> int:
+    """Discard what is older than ``settings.max_age_seconds()``. Returns rows removed."""
+    max_age = settings.max_age_seconds()
+    if max_age <= 0:
+        return 0
+    spool = spool or settings.get_spool()
+    cutoff = (now if now is not None else time.time()) - max_age
+    cutoff_iso = (datetime.fromtimestamp(cutoff, timezone.utc)
+                  .isoformat(timespec="milliseconds").replace("+00:00", "Z"))
+    dropped = spool.purge_stale(cutoff_iso)
+    if dropped:
+        logger.info("activity.stale_discarded rows=%d older_than_s=%.0f "
+                    "(live tracking: only recent activity is shipped)", dropped, max_age)
+    return dropped
+
+
 async def ship_tick(
     is_vps: Callable[[], Awaitable[bool | None]] | None = None,
 ) -> FlushResult | None:
-    """One cron tick: ship only from the caller's assigned VPS.
+    """One cron tick: ship only from the caller's assigned VPS, newest first.
 
     The relay files every batch under the TOKEN's VPS, so a laptop running the SDK
     with the same token put its own events in the VPS's audit trail. On a machine
     that is definitely not the VPS the queue is discarded — kept, it would only grow
     forever, since that machine will never be allowed to send it. When the answer
-    is unknown (profile unreachable) everything is kept for the next tick.
+    is unknown (profile unreachable) everything recent is kept for the next tick.
+
+    Every tick first discards events older than ``AGENTOS_ACTIVITY_MAX_AGE_S``
+    (default one hour): the trail is for live tracking, and a backlog — a host
+    that was offline, or flooded by a brute-force attack — must never hold the
+    current activity back.
     """
+    try:
+        purge_stale_rows()
+    except Exception as exc:  # noqa: BLE001 — a purge failure must not stop shipping
+        logger.warning("activity: stale purge failed: %s", exc)
     if is_vps is None:
         from sdk import vps_identity
         is_vps = vps_identity.is_assigned_vps
@@ -222,7 +249,7 @@ async def ship_tick(
 
 
 async def run_forever(interval_s: float | None = None, *, initial_delay_s: float = 30.0) -> None:
-    """Flush every ``interval_s`` (default 5 minutes) until cancelled."""
+    """Flush every ``interval_s`` (default 60 s) until cancelled."""
     await asyncio.sleep(initial_delay_s)
     while True:
         try:

{devops_bot_sdk-1.6.70 → devops_bot_sdk-1.6.71}/sdk/activity/spool.py

@@ -169,33 +169,68 @@ class Spool:
         max_sessions: int = MAX_SESSIONS_PER_CHUNK,
         max_bytes: int = MAX_CHUNK_BYTES,
     ) -> Chunk:
-        """The oldest rows that fit the per-request limits, in arrival order.
-
-        Arrival order matters: a session is queued before its first event, so
-        taking rows by ``seq`` ships a session no later than the events that
-        reference it. Does NOT remove anything — call :meth:`ack` after delivery.
+        """Queued sessions, then the NEWEST events, within the per-request limits.
+
+        Live tracking needs the latest activity first. Shipping oldest-first left
+        busy hosts (a brute-forced RDP port alone is thousands of events an hour)
+        days behind, so the dashboard showed Tuesday's events on Thursday. Sessions
+        still go first, in arrival order: they are few, and an event must never
+        reach the relay ahead of the session it references. Does NOT remove
+        anything — call :meth:`ack` after delivery.
         """
         chunk = Chunk()
-        n_events = n_sessions = size = 0
+        size = 0
         with self._lock:
-            rows = self._db.execute(
-                "SELECT seq, rev, kind, payload FROM spool ORDER BY seq LIMIT ?",
-                (max_events + max_sessions,),
+            sessions = self._db.execute(
+                "SELECT seq, rev, kind, payload FROM spool WHERE kind = 'session' "
+                "ORDER BY seq LIMIT ?",
+                (max_sessions,),
             ).fetchall()
-        for seq, rev, kind, payload in rows:
-            if kind == "event" and n_events >= max_events:
-                break  # keep strict order: stop rather than skip ahead
-            if kind == "session" and n_sessions >= max_sessions:
-                break
+            events = self._db.execute(
+                "SELECT seq, rev, kind, payload FROM spool WHERE kind = 'event' "
+                "ORDER BY seq DESC LIMIT ?",
+                (max_events,),
+            ).fetchall()
+        for seq, rev, kind, payload in [*sessions, *events]:
             if chunk.rows and size + len(payload) > max_bytes:
                 break
             # Re-fit on the way out too: rows queued by an older SDK predate the clamp.
             chunk.rows.append(Row(seq, rev, kind, fit_wire_limits(kind, json.loads(payload))))
-            n_events += kind == "event"
-            n_sessions += kind == "session"
             size += len(payload)
         return chunk
 
+    def purge_stale(self, cutoff_iso: str) -> int:
+        """Delete events that happened before ``cutoff_iso``, and sessions that ENDED
+        before it. Returns the rows removed.
+
+        For live tracking an event an hour old is history, and shipping it only
+        delays the current ones. Open sessions are kept whatever their age — a
+        login that is still going on is live state. ``cutoff_iso`` uses the wire
+        format (``2026-10-22T07:06:01.971Z``), so string order is time order.
+        """
+        with self._lock:
+            try:
+                cur = self._db.execute(
+                    "DELETE FROM spool WHERE "
+                    "(kind = 'event' AND json_extract(payload, '$.occurredAt') < ?) OR "
+                    "(kind = 'session' AND json_extract(payload, '$.endedAt') IS NOT NULL "
+                    " AND json_extract(payload, '$.endedAt') < ?)",
+                    (cutoff_iso, cutoff_iso),
+                )
+                return int(cur.rowcount or 0)
+            except sqlite3.OperationalError:
+                # SQLite built without JSON1: filter in Python instead.
+                stale = []
+                for seq, kind, payload in self._db.execute(
+                    "SELECT seq, kind, payload FROM spool"
+                ).fetchall():
+                    wire = json.loads(payload)
+                    when = wire.get("occurredAt") if kind == "event" else wire.get("endedAt")
+                    if when and str(when) < cutoff_iso:
+                        stale.append((seq,))
+                self._db.executemany("DELETE FROM spool WHERE seq = ?", stale)
+                return len(stale)
+
     # ── ack ──────────────────────────────────────────────────────────────────
 
     def ack(self, chunk: Chunk) -> None:

{devops_bot_sdk-1.6.70 → devops_bot_sdk-1.6.71}/tests/test_activity_collectors.py

@@ -31,8 +31,15 @@ def _isolated(tmp_path, monkeypatch):
 
 
 def _queued():
+    """Queued sessions and events in the order the collector RECORDED them.
+
+    The shipper sends newest-first (live tracking); these tests are about what a
+    collector recorded and in which order, so read the rows back by ``seq``.
+    """
     chunk = settings.get_spool().next_chunk()
-    return chunk.sessions, chunk.events
+    rows = sorted(chunk.rows, key=lambda r: r.seq)
+    return ([r.wire for r in rows if r.kind == "session"],
+            [r.wire for r in rows if r.kind == "event"])
 
 
 # ─────────────────────────────── Linux / auditd ───────────────────────────────

devops_bot_sdk-1.6.71/tests/test_activity_live_tracking.py

@@ -0,0 +1,145 @@
+"""Live tracking: the newest activity ships first, and stale backlog is discarded.
+
+The incident: hosts that had been offline, or flooded by a brute-forced RDP port,
+shipped their spool oldest-first. At 10:00 several VPS were delivering events from
+two and three days earlier, so the dashboard was never live.
+"""
+from __future__ import annotations
+
+import asyncio
+import time
+from datetime import datetime, timezone
+
+import pytest
+
+from sdk.activity import settings, shipper
+from sdk.activity.spool import Spool
+
+NOW = datetime(2026, 10, 22, 10, 0, 0, tzinfo=timezone.utc).timestamp()
+
+
+def _iso(minutes_ago: float) -> str:
+    t = datetime.fromtimestamp(NOW - minutes_ago * 60, timezone.utc)
+    return t.isoformat(timespec="milliseconds").replace("+00:00", "Z")
+
+
+def _ev(eid: str, minutes_ago: float) -> dict:
+    return {"eventId": eid, "occurredAt": _iso(minutes_ago), "os": "LINUX", "source": "AUDITD",
+            "category": "COMMAND", "action": "exec", "actorType": "HUMAN", "actorConfidence": 90}
+
+
+def _sess(key: str, started_ago: float, ended_ago: float | None = None) -> dict:
+    s = {"externalKey": key, "kind": "SSH", "actorType": "HUMAN", "actorConfidence": 90,
+         "startedAt": _iso(started_ago)}
+    if ended_ago is not None:
+        s["endedAt"] = _iso(ended_ago)
+    return s
+
+
+@pytest.fixture
+def spool(tmp_path):
+    s = Spool(tmp_path / "live.db")
+    yield s
+    s.close()
+
+
+@pytest.fixture(autouse=True)
+def hermetic(monkeypatch):
+    import sdk.config as cfg
+    monkeypatch.setattr(cfg, "_dotenv_value", lambda key: "")
+    monkeypatch.delenv("AGENTOS_ACTIVITY_MAX_AGE_S", raising=False)
+
+
+class TestStaleBacklogIsDiscarded:
+    def test_events_older_than_an_hour_are_dropped(self, spool):
+        spool.add_event(_ev("days-old", 3 * 24 * 60))
+        spool.add_event(_ev("61-min", 61))
+        spool.add_event(_ev("59-min", 59))
+        spool.add_event(_ev("now", 0))
+        assert shipper.purge_stale_rows(spool, now=NOW) == 2
+        assert sorted(e["eventId"] for e in spool.next_chunk().events) == ["59-min", "now"]
+
+    def test_an_open_session_is_kept_however_old(self, spool):
+        spool.add_session(_sess("still-logged-in", started_ago=600))
+        spool.add_session(_sess("ended-long-ago", started_ago=600, ended_ago=300))
+        spool.add_session(_sess("ended-just-now", started_ago=600, ended_ago=5))
+        assert shipper.purge_stale_rows(spool, now=NOW) == 1
+        assert sorted(s["externalKey"] for s in spool.next_chunk().sessions) == [
+            "ended-just-now", "still-logged-in"]
+
+    def test_the_cut_off_is_configurable(self, spool, monkeypatch):
+        monkeypatch.setenv("AGENTOS_ACTIVITY_MAX_AGE_S", "86400")
+        spool.add_event(_ev("3h", 180))
+        spool.add_event(_ev("2d", 2 * 24 * 60))
+        assert shipper.purge_stale_rows(spool, now=NOW) == 1
+        assert [e["eventId"] for e in spool.next_chunk().events] == ["3h"]
+
+    def test_zero_keeps_everything(self, spool, monkeypatch):
+        monkeypatch.setenv("AGENTOS_ACTIVITY_MAX_AGE_S", "0")
+        spool.add_event(_ev("ancient", 10 * 24 * 60))
+        assert shipper.purge_stale_rows(spool, now=NOW) == 0
+        assert spool.count() == 1
+
+    def test_python_fallback_matches_json1(self, spool, monkeypatch):
+        """SQLite without JSON1 filters in Python, to the same result."""
+        import sqlite3
+
+        spool.add_event(_ev("old", 120))
+        spool.add_event(_ev("new", 1))
+        real = spool._db
+
+        class NoJson:
+            def execute(self, sql, *a):
+                if "json_extract" in sql:
+                    raise sqlite3.OperationalError("no such function: json_extract")
+                return real.execute(sql, *a)
+
+            def executemany(self, sql, *a):
+                return real.executemany(sql, *a)
+        monkeypatch.setattr(spool, "_db", NoJson())
+        assert shipper.purge_stale_rows(spool, now=NOW) == 1
+        monkeypatch.setattr(spool, "_db", real)
+        assert [e["eventId"] for e in spool.next_chunk().events] == ["new"]
+
+
+class TestNewestFirst:
+    def test_a_backlog_never_holds_back_the_current_events(self, spool):
+        for i in range(1200):                       # a flood, oldest first
+            spool.add_event(_ev(f"old{i}", 50 - i * 0.01))
+        spool.add_event(_ev("current", 0))
+        first = spool.next_chunk(max_events=500)
+        assert first.events[0]["eventId"] == "current"
+
+
+class TestShipTickPurgesFirst:
+    def test_stale_rows_are_gone_before_anything_ships(self, tmp_path, monkeypatch):
+        s = Spool(tmp_path / "tick.db")
+        s.add_event(_ev("stale", 120))
+        s.add_event(_ev("fresh", 1))
+        monkeypatch.setattr(settings, "get_spool", lambda: s)
+        monkeypatch.setattr(time, "time", lambda: NOW)
+        seen = []
+
+        async def fake_flush():
+            seen.extend(e["eventId"] for e in s.next_chunk().events)
+            return shipper.FlushResult()
+        monkeypatch.setattr(shipper, "flush_once", fake_flush)
+
+        async def yes():
+            return True
+        asyncio.run(shipper.ship_tick(yes))
+        s.close()
+        assert seen == ["fresh"]
+
+    def test_purge_also_runs_when_vps_identity_is_unknown(self, tmp_path, monkeypatch):
+        s = Spool(tmp_path / "tick2.db")
+        s.add_event(_ev("stale", 120))
+        monkeypatch.setattr(settings, "get_spool", lambda: s)
+        monkeypatch.setattr(time, "time", lambda: NOW)
+
+        async def unknown():
+            return None
+        asyncio.run(shipper.ship_tick(unknown))
+        n = s.count()
+        s.close()
+        assert n == 0

{devops_bot_sdk-1.6.70 → devops_bot_sdk-1.6.71}/tests/test_activity_macos.py

@@ -31,8 +31,15 @@ def _isolated(tmp_path, monkeypatch):
 
 
 def _queued():
+    """Queued sessions and events in the order the collector RECORDED them.
+
+    The shipper sends newest-first (live tracking); these tests are about what a
+    collector recorded and in which order, so read the rows back by ``seq``.
+    """
     chunk = settings.get_spool().next_chunk()
-    return chunk.sessions, chunk.events
+    rows = sorted(chunk.rows, key=lambda r: r.seq)
+    return ([r.wire for r in rows if r.kind == "session"],
+            [r.wire for r in rows if r.kind == "event"])
 
 
 USERS = {501: "alice", 502: "agentos", 0: "root"}

{devops_bot_sdk-1.6.70 → devops_bot_sdk-1.6.71}/tests/test_activity_pipeline.py

@@ -83,10 +83,10 @@ class TestSpool:
 
         spool.add_event(_ev("E1", metadata={"blob": "x" * (MAX_DICT_FIELD_BYTES * 2)}))
         spool.add_event(_ev("E2", metadata={"small": 1}))
-        events = spool.next_chunk().events
-        assert events[0]["metadata"]["truncated"] is True
-        assert events[0]["metadata"]["originalBytes"] > MAX_DICT_FIELD_BYTES
-        assert events[1]["metadata"] == {"small": 1}
+        events = {e["eventId"]: e for e in spool.next_chunk().events}
+        assert events["E1"]["metadata"]["truncated"] is True
+        assert events["E1"]["metadata"]["originalBytes"] > MAX_DICT_FIELD_BYTES
+        assert events["E2"]["metadata"] == {"small": 1}
 
     def test_a_single_worst_case_row_fits_comfortably_in_one_request(self, spool):
         from sdk.activity.models import MAX_DICT_FIELD_BYTES
@@ -105,21 +105,23 @@ class TestSpool:
 
         assert 1_000_000 < spool_mod.MAX_CHUNK_BYTES < 10 * 1024 * 1024
 
-    def test_rows_come_back_in_arrival_order_session_first(self, spool):
+    def test_sessions_first_then_the_newest_events(self, spool):
+        """Live tracking: the latest activity ships first. Sessions still lead, so an
+        event never reaches the relay ahead of the session it references."""
         spool.add_session(_sess("S1"))
         spool.add_event(_ev("E1"))
         spool.add_event(_ev("E2"))
         chunk = spool.next_chunk()
         assert [r.kind for r in chunk.rows] == ["session", "event", "event"]
-        assert [e["eventId"] for e in chunk.events] == ["E1", "E2"]
+        assert [e["eventId"] for e in chunk.events] == ["E2", "E1"]
 
     def test_next_chunk_respects_the_event_limit_and_leaves_the_rest(self, spool):
         for i in range(5):
             spool.add_event(_ev(f"E{i}"))
         chunk = spool.next_chunk(max_events=2)
-        assert [e["eventId"] for e in chunk.events] == ["E0", "E1"]
+        assert [e["eventId"] for e in chunk.events] == ["E4", "E3"]
         spool.ack(chunk)
-        assert [e["eventId"] for e in spool.next_chunk(max_events=10).events] == ["E2", "E3", "E4"]
+        assert [e["eventId"] for e in spool.next_chunk(max_events=10).events] == ["E2", "E1", "E0"]
 
     def test_next_chunk_respects_the_byte_limit(self, spool):
         for i in range(10):
@@ -169,7 +171,7 @@ class TestSpool:
             for i in range(5):
                 s.add_event(_ev(f"E{i}"))
             assert s.count() == 3
-            assert [e["eventId"] for e in s.next_chunk().events] == ["E2", "E3", "E4"]
+            assert [e["eventId"] for e in s.next_chunk().events] == ["E4", "E3", "E2"]
         finally:
             s.close()
 
@@ -179,7 +181,7 @@ class TestSpool:
             spool.add_event(_ev(f"E{i}"))
         left, right = spool.next_chunk().split()
         assert [r.kind for r in left.rows] == ["session", "event"]
-        assert [e["eventId"] for e in right.events] == ["E1", "E2", "E3"]
+        assert [e["eventId"] for e in right.events] == ["E2", "E1", "E0"]
 
 
 class FakeSender:
@@ -355,7 +357,8 @@ class TestSettings:
     def test_defaults(self):
         assert settings.enabled() is True
         assert settings.audit_enabled() is False
-        assert settings.ship_interval_seconds() == 300.0
+        assert settings.ship_interval_seconds() == 60.0
+        assert settings.max_age_seconds() == 3600.0
 
     def test_can_be_disabled(self, monkeypatch):
         monkeypatch.setenv("AGENTOS_ACTIVITY", "off")
@@ -373,7 +376,7 @@ class TestSettings:
 
     def test_garbage_interval_falls_back_to_default(self, monkeypatch):
         monkeypatch.setenv("AGENTOS_ACTIVITY_SHIP_INTERVAL_S", "often")
-        assert settings.ship_interval_seconds() == 300.0
+        assert settings.ship_interval_seconds() == settings.DEFAULT_SHIP_INTERVAL_S
 
     def test_start_is_a_noop_when_disabled(self, monkeypatch):
         monkeypatch.setenv("AGENTOS_ACTIVITY", "0")
@@ -438,8 +441,9 @@ class TestRecorder:
         recorder.agent_run_finished(RUN, ok=False, detail="boom")
         sessions, events = _queued()
         assert len(sessions) == 1 and "endedAt" in sessions[0]
-        assert [e["action"] for e in events] == ["run.start", "run.end"]
-        assert events[1]["exitCode"] == 1 and events[1]["severity"] == "WARNING"
+        by_action = {e["action"]: e for e in events}   # shipped newest-first
+        assert sorted(by_action) == ["run.end", "run.start"]
+        assert by_action["run.end"]["exitCode"] == 1 and by_action["run.end"]["severity"] == "WARNING"
 
     def test_a_non_uuid_agent_id_is_omitted_not_sent(self):
         recorder.agent_run_started(RUN, agent_id="nightly-report")