flightdeck-sensor 0.7.1__tar.gz → 0.8.0__tar.gz

{flightdeck_sensor-0.7.1 → flightdeck_sensor-0.8.0}/CLAUDE.md

@@ -16,7 +16,7 @@
 
 29. **Token counting carries over from tokencap.** Do not rewrite the counting logic.
     Extend it. Pre-call estimation, post-call reconciliation, delta correction -- these
-    are proven and must not be changed without Supervisor approval.
+    are proven and must not be changed without the maintainer's approval.
 
 30. **capture_prompts defaults to False.** The default init() call never captures
     content. Always verify this default has not been accidentally changed.

{flightdeck_sensor-0.7.1 → flightdeck_sensor-0.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: flightdeck-sensor
-Version: 0.7.1
+Version: 0.8.0
 Summary: In-process agent observability sensor for Flightdeck
 Project-URL: Repository, https://github.com/flightdeckhq/flightdeck
 Author-email: Omri Arnon <omri.arnon@gmail.com>
@@ -222,8 +222,8 @@ the single-count assertion and the capture round-trip).
   if framework reports surface calls that slip past the current
   coverage, file a new issue.
 - **`litellm.text_completion` / `atext_completion` and
-  `litellm.responses` / `aresponses`.** Neither is wrapped (root
-  `README.md` Roadmap), but some routes are still reported. The legacy
+  `litellm.responses` / `aresponses`.** Neither is wrapped (planned in
+  [#170](https://github.com/flightdeckhq/flightdeck/issues/170)), but some routes are still reported. The legacy
   completion API calls `completion` through the name bound in
   `litellm.main`, which the module-attribute swap does not reach; on
   an OpenAI chat model the OpenAI interceptor reports the SDK call it

{flightdeck_sensor-0.7.1 → flightdeck_sensor-0.8.0}/README.md

@@ -162,8 +162,8 @@ the single-count assertion and the capture round-trip).
   if framework reports surface calls that slip past the current
   coverage, file a new issue.
 - **`litellm.text_completion` / `atext_completion` and
-  `litellm.responses` / `aresponses`.** Neither is wrapped (root
-  `README.md` Roadmap), but some routes are still reported. The legacy
+  `litellm.responses` / `aresponses`.** Neither is wrapped (planned in
+  [#170](https://github.com/flightdeckhq/flightdeck/issues/170)), but some routes are still reported. The legacy
   completion API calls `completion` through the name bound in
   `litellm.main`, which the module-attribute swap does not reach; on
   an OpenAI chat model the OpenAI interceptor reports the SDK call it

{flightdeck_sensor-0.7.1 → flightdeck_sensor-0.8.0}/flightdeck_sensor/compat/__init__.py

@@ -11,6 +11,6 @@ Stability: these helpers exist as workarounds for upstream bugs.
 Each module's docstring names the upstream package + version where
 the bug was observed and points at the README "Known framework
 constraints" section. The helpers ship behind their own modules so
-removing one as upstream fixes land is a clean delete + Roadmap
-checkbox tick, not a sweep through the sensor proper.
+removing one as upstream fixes land is a clean delete that closes
+its roadmap issue, not a sweep through the sensor proper.
 """

{flightdeck_sensor-0.7.1 → flightdeck_sensor-0.8.0}/flightdeck_sensor/compat/crewai_mcp.py

@@ -30,7 +30,7 @@ path, raw ``model_json_schema()`` calls) sees the cleaned schema.
 
 Remove this module once mcpadapt emits valid schemas. See README
 "Known framework constraints" for the operator-facing explanation
-and the README Roadmap for the removal checkbox.
+and roadmap issue #193 for the removal.
 """
 
 from __future__ import annotations

flightdeck_sensor-0.8.0/flightdeck_sensor/transport/body_limit.py

@@ -0,0 +1,154 @@
+"""Keep every event body the sensor POSTs within the server's size limit.
+
+Ingestion reads at most 1 MiB of an event body, and NATS carries the
+event on with the same 1 MiB ``max_payload``; a larger body is rejected
+whole (413 from a proxy with nginx's default limit, 400 from ingestion
+when the cut body fails to parse), and the call's token counts with it.
+A captured prompt, a tool result or a sub-agent message can cross that.
+An event over the limit has the strings of its captured ``content`` cut,
+each to the same largest length that fits, ending in a marker; when no
+cut fits, it is sent without content. The Claude Code plugin does the
+same (``plugin/hooks/scripts/core/poster.mjs``).
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from typing import Any
+
+_log = logging.getLogger("flightdeck_sensor.transport.body_limit")
+
+#: Largest event body the sensor sends, as measured by
+#: :func:`encoded_size`; the remaining quarter of the 1 MiB limit covers
+#: the token id and name ingestion adds to a ``session_start``.
+MAX_EVENT_BODY_BYTES = 768 * 1024
+
+# Characters ingestion's Go encoder writes as six-byte unicode escapes
+# when it re-encodes a session_start (one byte each here).
+_GO_ESCAPED = (b"<", b">", b"&")
+
+
+def encoded_size(body: bytes) -> int:
+    """The size of a JSON body once ingestion has re-encoded it.
+
+    The sensor's ``json.dumps`` already escapes every non-ASCII
+    character, which Go writes as shorter UTF-8, so text grows only by
+    ``<``, ``>`` and ``&``. Go also writes some floats longer than
+    Python does (``1e16`` as ``10000000000000000``), which this does not
+    count; a ``session_start``, the one event ingestion re-encodes,
+    carries text content. Every event is measured this way; for the
+    others it only over-counts.
+    """
+    return len(body) + 5 * sum(body.count(ch) for ch in _GO_ESCAPED)
+
+
+def truncation_marker(kept_bytes: int, total_bytes: int) -> str:
+    """The text that replaces the cut end of a string in oversized content."""
+    return f"… [flightdeck: kept {kept_bytes} of {total_bytes} bytes to fit the event size limit]"
+
+
+def _cap_strings(value: Any, cap_bytes: int) -> Any:
+    """Every string in ``value`` longer than ``cap_bytes`` of UTF-8, cut and marked."""
+    if isinstance(value, str):
+        # ``surrogatepass`` keeps a lone surrogate (from a
+        # ``surrogateescape`` decode) from raising; the cut drops it.
+        raw = value.encode("utf-8", "surrogatepass")
+        if len(raw) <= cap_bytes:
+            return value
+        # ``errors="ignore"`` drops a code point the cut split.
+        kept = raw[:cap_bytes].decode("utf-8", errors="ignore")
+        marked = kept + truncation_marker(len(kept.encode("utf-8", "surrogatepass")), len(raw))
+        # A string barely over the cap would grow by the marker; keep it.
+        return marked if len(marked.encode("utf-8", "surrogatepass")) < len(raw) else value
+    if isinstance(value, list):
+        return [_cap_strings(item, cap_bytes) for item in value]
+    if isinstance(value, dict):
+        return {key: _cap_strings(item, cap_bytes) for key, item in value.items()}
+    return value
+
+
+def _longest_string_bytes(value: Any) -> int:
+    if isinstance(value, str):
+        return len(value.encode("utf-8", "surrogatepass"))
+    items = value if isinstance(value, list) else value.values() if isinstance(value, dict) else ()
+    return max((_longest_string_bytes(item) for item in items), default=0)
+
+
+def _serialize_cut(payload: dict[str, Any], content: Any, cap: int) -> bytes:
+    return json.dumps({**payload, "content": _cap_strings(content, cap)}).encode()
+
+
+# Payload fields holding a sub-agent message stub that points at the
+# event's content (``{has_content: true, content_bytes, captured_at}``).
+_MESSAGE_STUB_FIELDS = ("incoming_message", "outgoing_message")
+
+
+def _without_content(payload: dict[str, Any]) -> dict[str, Any]:
+    """``payload`` sent without its content: no stub still claims it."""
+    out = {**payload, "has_content": False, "content": None}
+    for field in _MESSAGE_STUB_FIELDS:
+        stub = out.get(field)
+        if isinstance(stub, dict) and stub.get("has_content"):
+            out[field] = {**stub, "has_content": False}
+    return out
+
+
+def serialize_event(payload: dict[str, Any]) -> bytes:
+    """The JSON body for one event, kept within :data:`MAX_EVENT_BODY_BYTES`.
+
+    An event without content, or one over the limit even without it, is
+    sent as it is.
+    """
+    body = json.dumps(payload).encode()
+    size = encoded_size(body)
+    content = payload.get("content")
+    if size <= MAX_EVENT_BODY_BYTES or content is None:
+        return body
+    event_type = payload.get("event_type")
+    bare = json.dumps(_without_content(payload)).encode()
+    if encoded_size(bare) > MAX_EVENT_BODY_BYTES:
+        _log.warning(
+            "%s event is %d bytes and over the event size limit without its content; sent as it is",
+            event_type,
+            size,
+        )
+        return body
+    # The body only grows with the cap, so when even an empty cap does
+    # not fit, no cap does; that skips the search (each step serialises
+    # the whole event).
+    if encoded_size(_serialize_cut(payload, content, 0)) > MAX_EVENT_BODY_BYTES:
+        _log.warning(
+            "%s event is %d bytes and its content cannot be cut to fit the event size limit; "
+            "sent without it",
+            event_type,
+            size,
+        )
+        return bare
+    lo, hi = 0, min(_longest_string_bytes(content), MAX_EVENT_BODY_BYTES)
+    best: bytes | None = None
+    best_cap = 0
+    # Binary search for the largest per-string length that fits.
+    while lo <= hi:
+        cap = (lo + hi) // 2
+        candidate = _serialize_cut(payload, content, cap)
+        if encoded_size(candidate) <= MAX_EVENT_BODY_BYTES:
+            best, best_cap, lo = candidate, cap, cap + 1
+        else:
+            hi = cap - 1
+    if best is not None:
+        _log.warning(
+            "%s event is %d bytes; content strings over %d bytes were cut "
+            "to fit the event size limit",
+            event_type,
+            size,
+            best_cap,
+        )
+        return best
+    _log.warning(
+        "%s event is %d bytes and its content cannot be cut to fit the event size limit; "
+        "sent without it",
+        event_type,
+        size,
+    )
+    return bare

{flightdeck_sensor-0.7.1 → flightdeck_sensor-0.8.0}/flightdeck_sensor/transport/client.py

@@ -19,6 +19,7 @@ from pydantic import ValidationError
 
 from flightdeck_sensor.core.exceptions import DirectiveError
 from flightdeck_sensor.core.types import Directive, DirectiveAction
+from flightdeck_sensor.transport.body_limit import serialize_event
 from flightdeck_sensor.transport.retry import with_retry
 
 if TYPE_CHECKING:
@@ -78,7 +79,7 @@ class ControlPlaneClient:
           field defaults to ``False`` for backward compatibility with
           older ingestion deployments.
         """
-        return self._post("/v1/events", payload)
+        return self._post("/v1/events", payload, data=serialize_event(payload))
 
     def sync_directives(
         self,
@@ -150,10 +151,20 @@ class ControlPlaneClient:
     # Internals
     # ------------------------------------------------------------------
 
-    def _post(self, path: str, body: dict[str, Any]) -> tuple[Directive | None, bool]:
-        """POST JSON and parse the response envelope for directive + attached."""
+    def _post(
+        self,
+        path: str,
+        body: dict[str, Any],
+        data: bytes | None = None,
+    ) -> tuple[Directive | None, bool]:
+        """POST JSON and parse the response envelope for directive + attached.
+
+        ``data`` is the already-encoded body when the caller has one
+        (an event, kept within the size limit by ``serialize_event``).
+        """
         url = f"{self._base_url}{path}"
-        data = json.dumps(body).encode()
+        if data is None:
+            data = json.dumps(body).encode()
         req = urllib.request.Request(
             url,
             data=data,

{flightdeck_sensor-0.7.1 → flightdeck_sensor-0.8.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "flightdeck-sensor"
-version = "0.7.1"
+version = "0.8.0"
 description = "In-process agent observability sensor for Flightdeck"
 readme = "README.md"
 license = "Apache-2.0"

flightdeck_sensor-0.8.0/tests/unit/test_body_limit.py

@@ -0,0 +1,206 @@
+"""transport/body_limit.py: every event body stays within the server's
+1 MiB limit, less headroom, by cutting the strings of captured content."""
+
+from __future__ import annotations
+
+import json
+import logging
+import re
+from typing import TYPE_CHECKING, Any
+
+from flightdeck_sensor.transport import body_limit
+from flightdeck_sensor.transport.body_limit import (
+    MAX_EVENT_BODY_BYTES,
+    _cap_strings,
+    encoded_size,
+    serialize_event,
+    truncation_marker,
+)
+
+if TYPE_CHECKING:
+    import pytest
+
+_MARKER_RE = re.compile(r"… \[flightdeck: kept (\d+) of (\d+) bytes to fit the event size limit\]$")
+
+
+def _post_call(content: Any) -> dict[str, Any]:
+    return {
+        "event_type": "post_call",
+        "session_id": "s1",
+        "tokens_input": 42,
+        "has_content": True,
+        "content": content,
+    }
+
+
+def test_an_event_within_the_limit_is_sent_unchanged(caplog: pytest.LogCaptureFixture) -> None:
+    payload = _post_call({"messages": [{"role": "user", "content": "hi"}]})
+    with caplog.at_level(logging.WARNING):
+        assert serialize_event(payload) == json.dumps(payload).encode()
+    assert caplog.records == []
+
+
+def test_oversized_content_is_cut_and_the_tokens_kept(caplog: pytest.LogCaptureFixture) -> None:
+    big = "a" * (3 * 1024 * 1024)
+    payload = _post_call(
+        {
+            "provider": "anthropic",
+            "messages": [{"role": "user", "content": big}],
+            "response": {"content": [{"type": "text", "text": "short"}]},
+        }
+    )
+    with caplog.at_level(logging.WARNING):
+        body = serialize_event(payload)
+    assert encoded_size(body) <= MAX_EVENT_BODY_BYTES
+    sent = json.loads(body)
+    assert sent["tokens_input"] == 42
+    assert sent["has_content"] is True
+    assert sent["content"]["provider"] == "anthropic"
+    assert sent["content"]["response"] == payload["content"]["response"]
+    cut = sent["content"]["messages"][0]["content"]
+    match = _MARKER_RE.search(cut)
+    assert match, cut[-120:]
+    kept, total = int(match.group(1)), int(match.group(2))
+    assert total == len(big)
+    assert cut == "a" * kept + truncation_marker(kept, total)
+    assert kept > MAX_EVENT_BODY_BYTES * 0.9
+    assert len(caplog.records) == 1
+    assert "content strings over" in caplog.records[0].getMessage()
+    assert "aaaa" not in caplog.records[0].getMessage()
+
+
+def test_every_long_string_gets_the_same_cap() -> None:
+    payload = _post_call(
+        {"system": "b" * 600_000, "messages": [{"role": "user", "content": "c" * 600_000}]}
+    )
+    sent = json.loads(serialize_event(payload))
+
+    def kept(s: str) -> int:
+        m = _MARKER_RE.search(s)
+        assert m
+        return int(m.group(1))
+
+    assert kept(sent["content"]["system"]) == kept(sent["content"]["messages"][0]["content"])
+
+
+def test_a_multibyte_character_is_never_split() -> None:
+    payload = _post_call({"messages": [{"role": "user", "content": "\U0001f600" * 400_000}]})
+    body = serialize_event(payload)
+    assert encoded_size(body) <= MAX_EVENT_BODY_BYTES
+    text = json.loads(body)["content"]["messages"][0]["content"]
+    assert "�" not in text
+    assert re.match(r"^(\U0001f600)+… \[flightdeck: kept \d+ of 1600000 bytes", text)
+
+
+def test_markup_heavy_content_fits_once_go_re_encodes_it() -> None:
+    # Ingestion's Go encoder writes <, > and & as six-byte escapes when
+    # it re-encodes a session_start.
+    payload = {
+        "event_type": "session_start",
+        "session_id": "s1",
+        "has_content": True,
+        "content": {"response": {"body": "<a>" * 240_000}},
+    }
+    body = serialize_event(payload)
+    assert encoded_size(body) <= MAX_EVENT_BODY_BYTES
+    assert len(body) + 5 * (body.count(b"<") + body.count(b">")) <= MAX_EVENT_BODY_BYTES
+
+
+def test_encoded_size_counts_go_escapes() -> None:
+    assert encoded_size(b'{"a":"x"}') == 9
+    assert encoded_size(b'{"a":"<&>"}') == 11 + 3 * 5
+
+
+def test_without_any_fitting_cut_the_event_goes_without_content(
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    blocks = [{"type": "text", "text": ""} for _ in range(60_000)]
+    payload = _post_call({"messages": [], "response": {"content": blocks}})
+    with caplog.at_level(logging.WARNING):
+        sent = json.loads(serialize_event(payload))
+    assert sent["content"] is None
+    assert sent["has_content"] is False
+    assert sent["tokens_input"] == 42
+    assert "cannot be cut" in caplog.records[0].getMessage()
+
+
+def test_an_event_over_the_limit_without_content_keeps_it(caplog: pytest.LogCaptureFixture) -> None:
+    payload = {
+        **_post_call({"messages": [{"role": "user", "content": "hi"}]}),
+        "note": "q" * MAX_EVENT_BODY_BYTES,
+    }
+    with caplog.at_level(logging.WARNING):
+        assert serialize_event(payload) == json.dumps(payload).encode()
+    assert "without its content" in caplog.records[0].getMessage()
+
+
+def test_an_oversized_event_with_no_content_is_sent_as_it_is() -> None:
+    payload = {"event_type": "tool_call", "content": None, "note": "q" * MAX_EVENT_BODY_BYTES}
+    assert serialize_event(payload) == json.dumps(payload).encode()
+
+
+def test_a_string_the_marker_would_lengthen_is_kept() -> None:
+    short = "s" * 70
+    assert _cap_strings(short, 10) == short
+    long = "l" * 200
+    assert _cap_strings(long, 10) == "l" * 10 + truncation_marker(10, 200)
+    assert _cap_strings({"a": [short, long], "n": 3, "b": None}, 10) == {
+        "a": [short, "l" * 10 + truncation_marker(10, 200)],
+        "n": 3,
+        "b": None,
+    }
+
+
+def test_a_cut_never_splits_a_multibyte_character() -> None:
+    # Each emoji is four bytes; a six-byte cap lands inside the second.
+    cut = _cap_strings("\U0001f600" * 40, 6)
+    assert "\ufffd" not in cut
+    assert cut == "\U0001f600" + truncation_marker(4, 160)
+
+
+def test_a_lone_surrogate_in_oversized_content_is_cut_not_raised() -> None:
+    # A tool result decoded with errors="surrogateescape" carries \udcXX.
+    payload = {
+        "event_type": "session_start",
+        "has_content": True,
+        "content": {"response": {"body": "\udc80" * 300_000}},
+    }
+    body = serialize_event(payload)
+    assert encoded_size(body) <= MAX_EVENT_BODY_BYTES
+    assert json.loads(body)["has_content"] is True
+
+
+def test_dropping_the_content_clears_a_message_stub_that_points_at_it() -> None:
+    payload = {
+        "event_type": "session_start",
+        "has_content": True,
+        "incoming_message": {"has_content": True, "content_bytes": 3_000_000, "captured_at": "t"},
+        "content": {"response": {"body": [{"k": i} for i in range(200_000)]}},
+    }
+    sent = json.loads(serialize_event(payload))
+    assert sent["content"] is None
+    assert sent["has_content"] is False
+    assert sent["incoming_message"] == {
+        "has_content": False,
+        "content_bytes": 3_000_000,
+        "captured_at": "t",
+    }
+
+
+def test_content_with_nothing_to_cut_skips_the_search(monkeypatch: pytest.MonkeyPatch) -> None:
+    calls = []
+    real = body_limit._serialize_cut
+
+    def counting(payload: dict[str, Any], content: Any, cap: int) -> bytes:
+        calls.append(cap)
+        return real(payload, content, cap)
+
+    monkeypatch.setattr(body_limit, "_serialize_cut", counting)
+    vectors = [[0.123456789] * 3072 for _ in range(50)]
+    sent = json.loads(serialize_event(_post_call({"embedding_output": vectors})))
+    assert sent["content"] is None
+    assert calls == [0]
+
+
+def test_the_budget_leaves_room_under_the_one_mib_limit() -> None:
+    assert MAX_EVENT_BODY_BYTES <= 0.8 * 1024 * 1024

{flightdeck_sensor-0.7.1 → flightdeck_sensor-0.8.0}/tests/unit/test_provider_enum.py

@@ -66,8 +66,8 @@ def test_provider_has_exactly_six_members() -> None:
     value-side drift; this one catches the count-side).
 
     Sub-agent observability added two targets (CREWAI, LANGGRAPH). AutoGen
-    support is on the Roadmap and ships in a future PR alongside
-    LLM-call interception for it.
+    is archived upstream and not planned; its successor, Microsoft Agent
+    Framework, is roadmap issue #185.
     """
     assert set(Provider) == {
         Provider.ANTHROPIC,

{flightdeck_sensor-0.7.1 → flightdeck_sensor-0.8.0}/tests/unit/test_transport.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import json
 import logging
 import threading
 import time
@@ -12,6 +13,7 @@ import pytest
 
 from flightdeck_sensor.core.exceptions import DirectiveError
 from flightdeck_sensor.core.types import Directive, DirectiveAction
+from flightdeck_sensor.transport.body_limit import MAX_EVENT_BODY_BYTES
 from flightdeck_sensor.transport.client import (
     _MAX_QUEUE_SIZE,
     ControlPlaneClient,
@@ -64,6 +66,25 @@ def test_post_event_returns_attached_true_when_backend_attaches(
     assert attached is True
 
 
+def test_post_event_keeps_an_oversized_event_within_the_body_limit(
+    mock_control_plane: Any,
+) -> None:
+    client = ControlPlaneClient(mock_control_plane["url"], "tok")
+    payload = {
+        "session_id": "123",
+        "event_type": "post_call",
+        "tokens_input": 900_000,
+        "has_content": True,
+        "content": {"messages": [{"role": "user", "content": "x" * (2 * 1024 * 1024)}]},
+    }
+    client.post_event(payload)
+    [request] = mock_control_plane["requests"]
+    sent = request["body"]
+    assert sent["tokens_input"] == 900_000
+    assert len(json.dumps(sent)) <= MAX_EVENT_BODY_BYTES
+    assert sent["content"]["messages"][0]["content"].endswith("bytes to fit the event size limit]")
+
+
 def test_connectivity_failure_continue_returns_none() -> None:
     client = ControlPlaneClient("http://127.0.0.1:1", "tok", unavailable_policy="continue")
     directive, attached = client.post_event({"session_id": "x", "event_type": "post_call"})

{flightdeck_sensor-0.7.1 → flightdeck_sensor-0.8.0}/uv.lock

@@ -1263,7 +1263,7 @@ wheels = [
 
 [[package]]
 name = "flightdeck-sensor"
-version = "0.7.1"
+version = "0.8.0"
 source = { editable = "." }
 dependencies = [
     { name = "pydantic", version = "2.12.5", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.14' and extra == 'extra-17-flightdeck-sensor-dev') or (extra == 'extra-17-flightdeck-sensor-dev' and extra == 'group-17-flightdeck-sensor-sdk-majors')" },