agent-tokenops 2.0.0__tar.gz → 2.1.0__tar.gz

{agent_tokenops-2.0.0/src/agent_tokenops.egg-info → agent_tokenops-2.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agent-tokenops
-Version: 2.0.0
+Version: 2.1.0
 Summary: Run-aware token governance for agents: budgets, policies and halts on Chronicle and the AgentPlane control plane
 Author: Susheem Koul, Tisha Chawla
 License-Expression: MIT
@@ -31,7 +31,7 @@ Requires-Dist: python-dotenv>=1.0
 Requires-Dist: httpx>=0.27
 Requires-Dist: fastapi>=0.115
 Provides-Extra: contract
-Requires-Dist: agentplane-control-plane>=1.0.1; extra == "contract"
+Requires-Dist: agentplane-control-plane>=1.0.2; extra == "contract"
 Provides-Extra: examples
 Requires-Dist: ddgs>=9.0; extra == "examples"
 Requires-Dist: langchain-core>=0.3; extra == "examples"

{agent_tokenops-2.0.0 → agent_tokenops-2.1.0}/pyproject.toml

@@ -66,7 +66,7 @@ dev = [
 # The tests that need it `pytest.importorskip("control_plane")`, so a plain `dev` install
 # stays green. To run them: `pip install -e ".[contract]"`.
 contract = [
-  "agentplane-control-plane>=1.0.1",
+  "agentplane-control-plane>=1.0.2",
 ]
 examples = [
   "ddgs>=9.0",

{agent_tokenops-2.0.0 → agent_tokenops-2.1.0/src/agent_tokenops.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agent-tokenops
-Version: 2.0.0
+Version: 2.1.0
 Summary: Run-aware token governance for agents: budgets, policies and halts on Chronicle and the AgentPlane control plane
 Author: Susheem Koul, Tisha Chawla
 License-Expression: MIT
@@ -31,7 +31,7 @@ Requires-Dist: python-dotenv>=1.0
 Requires-Dist: httpx>=0.27
 Requires-Dist: fastapi>=0.115
 Provides-Extra: contract
-Requires-Dist: agentplane-control-plane>=1.0.1; extra == "contract"
+Requires-Dist: agentplane-control-plane>=1.0.2; extra == "contract"
 Provides-Extra: examples
 Requires-Dist: ddgs>=9.0; extra == "examples"
 Requires-Dist: langchain-core>=0.3; extra == "examples"

{agent_tokenops-2.0.0 → agent_tokenops-2.1.0}/src/agent_tokenops.egg-info/SOURCES.txt

@@ -41,6 +41,22 @@ src/tokenops/framework/models.py
 src/tokenops/framework/observation_utils.py
 src/tokenops/framework/policy_labels.py
 src/tokenops/framework/pricing.py
+src/tokenops/framework/pruning.py
+src/tokenops/framework/pruning_c.py
+src/tokenops/framework/pruning_clex.py
+src/tokenops/framework/pruning_clike.py
+src/tokenops/framework/pruning_go.py
+src/tokenops/framework/pruning_java.py
+src/tokenops/framework/pruning_js.py
+src/tokenops/framework/pruning_jslex.py
+src/tokenops/framework/pruning_languages.py
+src/tokenops/framework/pruning_python.py
+src/tokenops/framework/pruning_rust.py
+src/tokenops/framework/pruning_shell.py
+src/tokenops/framework/pruning_shlex.py
+src/tokenops/framework/pruning_sql.py
+src/tokenops/framework/pruning_sqllex.py
+src/tokenops/framework/pruning_types.py
 src/tokenops/framework/request_context.py
 src/tokenops/framework/usage.py
 src/tokenops/framework/policies/__init__.py
@@ -56,6 +72,7 @@ src/tokenops/framework/policies/step_cap.py
 src/tokenops/framework/policies/time_budget.py
 src/tokenops/framework/policies/tool_fix.py
 src/tokenops/framework/policies/tool_output_cap.py
+src/tokenops/framework/policies/tool_output_prune.py
 src/tokenops/interfaces/__init__.py
 src/tokenops/interfaces/http.py
 src/tokenops/interfaces/instrument.py
@@ -111,5 +128,8 @@ tests/test_time_budget.py
 tests/test_tokenops_run.py
 tests/test_tool_fix.py
 tests/test_tool_output_cap.py
+tests/test_tool_output_prune.py
+tests/test_tool_output_prune_languages.py
+tests/test_tool_output_prune_scripts.py
 tests/test_wrap_adapters.py
 tests/test_wrap_complete_chronicle.py

{agent_tokenops-2.0.0 → agent_tokenops-2.1.0}/src/agent_tokenops.egg-info/requires.txt

@@ -8,7 +8,7 @@ httpx>=0.27
 fastapi>=0.115
 
 [contract]
-agentplane-control-plane>=1.0.1
+agentplane-control-plane>=1.0.2
 
 [examples]
 ddgs>=9.0

{agent_tokenops-2.0.0 → agent_tokenops-2.1.0}/src/tokenops/_version.py

@@ -1,3 +1,3 @@
 """Single source of the package version (read by the build backend and at runtime)."""
 
-__version__ = "2.0.0"
+__version__ = "2.1.0"

{agent_tokenops-2.0.0 → agent_tokenops-2.1.0}/src/tokenops/capabilities/crossing.py

@@ -124,6 +124,40 @@ def on_leave(info: CallInfo) -> None:
         _inflight_seg.set(None)
 
 
+_TEXT_KEYS = ("output", "stdout", "text", "content")
+
+
+def _tool_text(result: Any) -> str | None:
+    """The text of a tool result: the string itself, or the first string under a conventional key of a dict."""
+    if isinstance(result, str):
+        return result
+    if isinstance(result, dict):
+        for key in _TEXT_KEYS:
+            value = result.get(key)
+            if isinstance(value, str):
+                return value
+    return None
+
+
+def _shape_tool_result(governor: Any, controls: Any, name: str, result: Any) -> Any:
+    """Let the pruner policies rewrite the tool result text *before* it is observed, so the cap measures what the agent
+    will receive. The rewrite is left on ``controls.shaped_result`` for the host to apply; the wrapped function's own
+    return value is never changed here."""
+    text = _tool_text(result)
+    shape = getattr(governor, "shape_tool_result", None)
+    if text is None or shape is None:
+        return result
+    shaped = shape(name, text)
+    if shaped is None:
+        return result
+    if controls is not None and hasattr(controls, "shaped_result"):
+        controls.shaped_result = shaped
+    if isinstance(result, str):
+        return shaped
+    key = next(k for k in _TEXT_KEYS if isinstance(result.get(k), str))
+    return {**result, key: shaped}
+
+
 def on_crossing(info: CallInfo, result: Any) -> None:
     """Project crossing → Observation when a hop attribution + governance are bound."""
     from tokenops.framework.boundary import emit_observation, observation_from_crossing
@@ -146,6 +180,8 @@ def on_crossing(info: CallInfo, result: Any) -> None:
         comp = getattr(call, "compaction", None) if call is not None else None
         if comp is not None:
             compaction = dict(comp)
+    if _kind(info) == "tool" and governor is not None:
+        result = _shape_tool_result(governor, controls, info.name, result)
     obs = observation_from_crossing(
         boundary_id=info.name,
         kind=_kind(info),

{agent_tokenops-2.0.0 → agent_tokenops-2.1.0}/src/tokenops/framework/config.py

@@ -51,6 +51,7 @@ from tokenops.framework.policies import (
     time_budget,
     tool_fix,
     tool_output_cap,
+    tool_output_prune,
 )
 
 
@@ -139,6 +140,23 @@ POLICY_TEMPLATES: Mapping[str, PolicyTemplate] = MappingProxyType(
             factory=lambda p, c: tool_output_cap.build(p.get("cap_tokens", 8000)),
             default_params={"cap_tokens": 8000},
         ),
+        "tool_output_prune": PolicyTemplate(
+            display_name="Tool output prune",
+            factory=lambda p, c: tool_output_prune.build(
+                comments=p.get("comments", True),
+                docstrings=p.get("docstrings", True),
+                min_saved_chars=p.get("min_saved_chars", 200),
+                languages=p.get("languages", ["python"]),
+                keep_jsdoc_types=p.get("keep_jsdoc_types", True),
+            ),
+            default_params={
+                "comments": True,
+                "docstrings": True,
+                "min_saved_chars": 200,
+                "languages": ["python"],
+                "keep_jsdoc_types": True,
+            },
+        ),
         "progress_guard": PolicyTemplate(
             display_name="Progress guard",
             factory=lambda p, c: progress_guard.build(

{agent_tokenops-2.0.0 → agent_tokenops-2.1.0}/src/tokenops/framework/core.py

@@ -327,6 +327,10 @@ class Detector(ABC):
         """Inspect on a clock, independent of events (timeouts, stalls)."""
         return None
 
+    def shape_tool_result(self, boundary_id: str, text: str) -> str | None:
+        """Rewrite a tool result's text before it is observed (pruning). Return the new text, or None to leave it."""
+        return None
+
     def reset(self, trace_id: str) -> None:
         """Drop any per-run state held for ``trace_id``."""
         return None

{agent_tokenops-2.0.0 → agent_tokenops-2.1.0}/src/tokenops/framework/engine.py

@@ -137,6 +137,9 @@ class ApplyControls:
     call: _ResolvedCall = field(default_factory=_ResolvedCall)
     retry: bool = False
     tool_result_override: str | None = None  # deep INJECT: substitute the last tool result
+    shaped_result: str | None = (
+        None  # a pruner's rewrite of the last tool result (applied before the cap measures)
+    )
     max_carry: int = 3  # steer messages carried onto one dispatch; the rest are dropped
     dropped_carry: int = 0  # how many were dropped, for the dashboard
 
@@ -154,6 +157,12 @@ class ApplyControls:
         self.tool_result_override = None
         return val
 
+    def take_shaped_result(self) -> str | None:
+        """Consume the pruned text of the last tool result, if a pruner rewrote it. One-shot — cleared on read."""
+        val = self.shaped_result
+        self.shaped_result = None
+        return val
+
     def apply(self, action: Action) -> None:
         kind = action.kind
         if kind is ActionKind.ALLOW:
@@ -319,6 +328,15 @@ class Governor:
         self._enforce(signals)
         return step
 
+    def shape_tool_result(self, boundary_id: str, text: str) -> str | None:
+        """Run each detector's ``shape_tool_result`` in registration order; the new text, or None if none changed it."""
+        shaped = text
+        for d in self._detectors:
+            new = d.shape_tool_result(boundary_id, shaped)
+            if new is not None:
+                shaped = new
+        return None if shaped == text else shaped
+
     def tick(self, now: float) -> None:
         signals = [s for d in self._detectors if (s := d.tick(now, self.ledger)) is not None]
         self._enforce(signals)

{agent_tokenops-2.0.0 → agent_tokenops-2.1.0}/src/tokenops/framework/policies/__init__.py

@@ -22,6 +22,7 @@ context_compaction     in process: the recent step window
 output_runaway         the current call's output only
 tool_fix               the current call's tool request only
 tool_output_cap        the current call's tool output only
+tool_output_prune      the current call's tool output only (rewrites it before the cap measures it)
 time_budget            the wall clock since the hop started
 =====================  ==========================================================
 
@@ -40,6 +41,7 @@ from tokenops.framework.policies import (
     time_budget,
     tool_fix,
     tool_output_cap,
+    tool_output_prune,
 )
 
 __all__ = [
@@ -54,4 +56,5 @@ __all__ = [
     "time_budget",
     "tool_fix",
     "tool_output_cap",
+    "tool_output_prune",
 ]

agent_tokenops-2.1.0/src/tokenops/framework/policies/tool_output_prune.py

@@ -0,0 +1,80 @@
+"""tool_output_prune — hide what a coding agent rarely needs from a tool result, before the cap measures it.
+
+LLD row:
+    Detect: the tool result text is a block of source code in an enabled language (Python by default; Go, Java, Rust,
+            C/C++ on request): clean lexing, that language's structure, not a diff, grep hit list or test report.
+    Fix:    SHAPE — replace the result the agent sees by the same text with comments and docstrings hidden. Code
+            lines stay verbatim and the line count is unchanged (hidden blocks become blank lines), so exact-text
+            edits and cited line numbers still hold. Never HALT, never drops code, no model, no tokenizer.
+
+Unlike ``tool_output_cap`` this does not wait for a result to be large: it shapes a result of any size, and it runs
+*before* the cap so the cap measures what the agent will actually receive. The shaped text is handed to the host
+through ``controls.take_shaped_result()``; a host that does not call it is unaffected. It works from the result text
+alone (the command is not needed), so it only prunes text it can prove is source code in an enabled language, and refuses when two languages score close.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+from tokenops.framework.core import Action, ActionKind, Detector, LedgerView, Policy, Signal
+from tokenops.framework.pruning import prune_any
+from tokenops.framework.pruning_languages import resolve_languages
+
+
+class ToolOutputPruneDetector(Detector):
+    """Shapes a tool result; raises no signals (the saving is reported through the controls)."""
+
+    name = "tool_output_prune"
+
+    def __init__(
+        self,
+        *,
+        comments: bool = True,
+        docstrings: bool = True,
+        min_saved_chars: int = 200,
+        languages: str | Sequence[str] = ("python",),
+        keep_jsdoc_types: bool = True,
+    ) -> None:
+        self.handlers = resolve_languages(languages, keep_jsdoc_types=keep_jsdoc_types)
+        self.comments = comments
+        self.docstrings = docstrings
+        self.min_saved_chars = min_saved_chars
+
+    def shape_tool_result(self, boundary_id: str, text: str) -> str | None:
+        return prune_any(
+            text,
+            self.handlers,
+            comments=self.comments,
+            docstrings=self.docstrings,
+            min_saved_chars=self.min_saved_chars,
+        )
+
+
+class ToolOutputPrunePolicy(Policy):
+    """Nothing to decide: the detector shapes the result itself. Kept so the template registers like the others."""
+
+    name = "tool_output_prune"
+
+    def decide(self, signal: Signal, view: LedgerView) -> Action:
+        return Action(kind=ActionKind.ALLOW, trace_id=signal.trace_id, reason=signal.reason)
+
+
+def build(
+    *,
+    comments: bool = True,
+    docstrings: bool = True,
+    min_saved_chars: int = 200,
+    languages: str | Sequence[str] = ("python",),
+    keep_jsdoc_types: bool = True,
+) -> tuple[Detector, Policy]:
+    return (
+        ToolOutputPruneDetector(
+            comments=comments,
+            docstrings=docstrings,
+            min_saved_chars=min_saved_chars,
+            languages=languages,
+            keep_jsdoc_types=keep_jsdoc_types,
+        ),
+        ToolOutputPrunePolicy(),
+    )

agent_tokenops-2.1.0/src/tokenops/framework/pruning.py

@@ -0,0 +1,185 @@
+"""Hide comments and docstrings in a tool result that is source code, from the result text alone.
+
+The pruner does not know the command that produced the text, and the text may be a fragment (a ``sed`` range can start or
+end inside a docstring, mid-block, or mid-statement). So it only acts when a language handler can show the text is that
+language's source, and it only hides what the handler can prove is a comment or docstring. When in doubt it returns
+``None`` and the result goes through untouched. It never changes a code line, and it keeps the number of lines the same
+(a hidden block becomes blank lines), so line numbers the agent has seen or will cite still hold.
+
+This module is the shared driver: the detection exclusions (diffs, ``grep -n`` hits, numbered listings, test reports),
+the line-count-preserving rewrite, the note, the minimum-saving rule and the safety contract. A ``LanguageHandler``
+(see ``pruning_types``) only finds the spans. Pure, model-free and dependency-free.
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Sequence
+
+from tokenops.framework.pruning_python import HIDDEN_DOCSTRING, PythonHandler
+from tokenops.framework.pruning_types import LanguageHandler, Span
+
+__all__ = [
+    "HIDDEN_DOCSTRING",
+    "NOTE",
+    "looks_like_python_source",
+    "prune",
+    "prune_any",
+    "select_handler",
+    "prune_python",
+]
+
+NOTE = (
+    "[TokenOps: comments and docstrings are hidden in this output; line numbers are unchanged. "
+    "`grep -n` shows a specific comment or docstring line.]"
+)
+
+# Output that merely contains code lines: diffs, grep -n hits, test reports. Never pruned.
+_NOT_SOURCE = re.compile(
+    r"^(diff --git |@@ |\+\+\+ |--- [ab]/|={3,} \S.* ={3,}$|E {3}|[\w./\\-]+:\d+:)", re.M
+)
+_NUMBERED = re.compile(
+    r"^\s*\d+[\t:|]\s", re.M
+)  # cat -n / nl listings: most lines start with a number
+MIN_LINES = 8
+
+_PYTHON = PythonHandler()
+
+
+def _passes_exclusions(text: str) -> bool:
+    """False for an output that merely contains code lines (a diff, grep hits, a numbered listing, a test report)."""
+    lines = [ln for ln in text.split("\n") if ln.strip()]
+    if len(lines) < MIN_LINES or _NOT_SOURCE.search(text):
+        return False
+    return len(_NUMBERED.findall(text)) <= 0.4 * len(lines)
+
+
+def _is_source_of(text: str, handler: LanguageHandler) -> bool:
+    """True when `text` is a block of `handler`'s language, not an output that merely contains some code."""
+    return _passes_exclusions(text) and handler.score(text) >= handler.min_score
+
+
+def looks_like_python_source(text: str) -> bool:
+    """True when `text` is a block of Python source, not an output that merely contains some code."""
+    return _is_source_of(text, _PYTHON)
+
+
+def select_handler(text: str, handlers: Sequence[LanguageHandler]) -> LanguageHandler | None:
+    """The handler whose language `text` is, or None when none is clearly it.
+
+    One enabled handler needs no comparison. With several, the best score must reach its minimum and clearly beat the
+    runner-up (at least 1.5 times and 2 more); two close scores mean the language is ambiguous, so nothing is pruned."""
+    if len(handlers) == 1:
+        return handlers[0]
+    ranked = sorted(
+        ((h.score(text), i, h) for i, h in enumerate(handlers)), key=lambda t: (-t[0], t[1])
+    )
+    if not ranked or ranked[0][0] < ranked[0][2].min_score:
+        return None
+    top = ranked[0][0]
+    second = ranked[1][0]
+    if second and (top < 1.5 * second or top - second < 2):
+        return None
+    return ranked[0][2]
+
+
+def _position(span: Span) -> tuple[int, int]:
+    return span.first_row, span.col
+
+
+def _hide_comment(lines: list[str], s: Span) -> None:
+    """Blank or trim the comment `s` in `lines`, keeping every code character and the number of lines."""
+    first = lines[s.first_row - 1]
+    before = first[: s.col]
+    if s.first_row == s.last_row:
+        after = first[s.end_col :] if s.end_col >= 0 else ""
+        if not after.strip():
+            lines[s.first_row - 1] = before.rstrip() if before.strip() else ""
+        elif not before.strip():
+            lines[s.first_row - 1] = before + after.lstrip()
+        else:
+            lines[s.first_row - 1] = before.rstrip() + " " + after.lstrip()
+        return
+    lines[s.first_row - 1] = before.rstrip() if before.strip() else ""
+    for i in range(s.first_row, s.last_row - 1):
+        lines[i] = ""
+    last = lines[s.last_row - 1]
+    after = last[s.end_col :] if s.end_col >= 0 else ""
+    lines[s.last_row - 1] = after.lstrip() if after.strip() else ""
+
+
+def prune(
+    text: str,
+    handler: LanguageHandler,
+    *,
+    comments: bool = True,
+    docstrings: bool = True,
+    min_saved_chars: int = 200,
+) -> str | None:
+    """`text` with comments and/or docstrings hidden and the same number of lines, or None to leave it as it is.
+
+    None when: it is not `handler`'s source, the handler refuses it, it would save fewer than `min_saved_chars` (after
+    the note), or the handler says the code would differ."""
+    if not (comments or docstrings) or not _is_source_of(text, handler):
+        return None
+    found = handler.spans(text)
+    if found is None:
+        return None
+    original = text.split("\n")
+    lines = list(original)
+    n = len(lines)
+    wanted = [
+        s
+        for s in found
+        if (comments if s.kind == "comment" else docstrings) and not handler.keep(s)
+    ]
+
+    # Right to left, so that editing a comment never moves the column of one before it on the same line.
+    for s in sorted((s for s in wanted if s.kind == "comment"), key=_position, reverse=True):
+        if s.last_row <= n:
+            _hide_comment(lines, s)
+    for s in wanted:
+        if s.kind == "docstring" and s.last_row <= n:
+            first = original[s.first_row - 1]
+            indent = first[: len(first) - len(first.lstrip(" \t"))]
+            lines[s.first_row - 1] = indent + handler.hidden_doc
+            for i in range(s.first_row, s.last_row):
+                lines[i] = ""
+
+    out = "\n".join(lines)
+    if len(text) - len(out) - len(NOTE) - 1 < min_saved_chars:
+        return None
+    if not handler.same_code(text, out):
+        return None
+    # Never strip trailing blank lines: a hidden last comment leaves some, and the line count must not change.
+    if text.endswith("\n"):
+        return out + NOTE + "\n"
+    return out + "\n" + NOTE
+
+
+def prune_any(
+    text: str,
+    handlers: Sequence[LanguageHandler],
+    *,
+    comments: bool = True,
+    docstrings: bool = True,
+    min_saved_chars: int = 200,
+) -> str | None:
+    """`prune` with the handler that `select_handler` picks for `text` among `handlers`."""
+    if not handlers or not (comments or docstrings) or not _passes_exclusions(text):
+        return None
+    handler = select_handler(text, handlers)
+    if handler is None:
+        return None
+    return prune(
+        text, handler, comments=comments, docstrings=docstrings, min_saved_chars=min_saved_chars
+    )
+
+
+def prune_python(
+    text: str, *, comments: bool = True, docstrings: bool = True, min_saved_chars: int = 200
+) -> str | None:
+    """`prune` with the Python handler."""
+    return prune(
+        text, _PYTHON, comments=comments, docstrings=docstrings, min_saved_chars=min_saved_chars
+    )

agent_tokenops-2.1.0/src/tokenops/framework/pruning_c.py

@@ -0,0 +1,28 @@
+"""C and C++ handler for tool-output pruning.
+
+``#pragma``, ``#if 0`` blocks and preprocessor lines are code, so they are never touched. A ``//`` comment that ends in a
+backslash continues on the next line. Raw strings (``R"x(...)x"``) and digit separators (``1'000``) are handled by the lexer.
+"""
+
+from __future__ import annotations
+
+import re
+
+from tokenops.framework.pruning_clike import CStyleHandler
+from tokenops.framework.pruning_types import CLexConfig
+
+_NOT_OTHER = r"(?!func\b|fn\b|def\b|pub\b|public\b|private\b|protected\b|static\s+void\s+main)"
+_STRUCTURE = re.compile(
+    r"^\s*#\s*(?:include|define|undef|ifdef|ifndef|if|elif|else|endif|pragma|error)\b"
+    r"|^\s*(?:typedef\b|namespace\b|template\s*<|extern\s+\"C\")"
+    r"|^\s*(?:struct|union|enum|class)\s+\w+\s*(?::[^{]*)?\{?$"
+    rf"|^{_NOT_OTHER}[A-Za-z_][\w\s\*&:<>,~]*\s[\*&]*\w+(?:::\w+)*\([^;{{}}]*\)\s*(?:const\s*)?(?:\{{|$)"
+    r"|\b(?:nullptr|NULL)\b|\bstd::\w+|\w->\w|\b(?:printf|fprintf|malloc|calloc|free|sizeof)\s*\(",
+    re.M,
+)
+
+
+class CHandler(CStyleHandler):
+    name = "c"
+    cfg = CLexConfig(line_continuation=True, cpp_raw_strings=True, digit_separators=True)
+    structure = _STRUCTURE