induscode 0.1.0__py3-none-any.whl

induscode/window_budget/__init__.py

@@ -0,0 +1,76 @@
+"""Window-budget — public barrel (port of TS ``src/window-budget/index.ts``).
+
+The context-window budgeting subsystem: token measurement, slice planning,
+and transcript condensing (with branch-archival collapsed in behind a scope
+flag). Re-exports the frozen contract types plus the live functions: the
+budget math (``estimate_tokens`` / ``is_over_budget`` / ``plan_slice``), the
+summarization core (``summarize`` / ``condense_scope``), and the
+conductor-consumable condenser factory (``create_condenser`` / ``condense``).
+
+This is the app's OWN compaction engine over :mod:`indusagi.ai` messages and
+``complete_simple`` — it feeds the conductor's ``CondenseFn`` seam and is
+deliberately NOT merged with the framework's :mod:`indusagi.runtime.memory`
+compactor (different message types, policy semantics, prompts, and digest
+headers; see :mod:`induscode.window_budget.condenser`).
+"""
+
+from .budget import (
+    budget_limit,
+    estimate_message_tokens,
+    estimate_tokens,
+    is_over_budget,
+    plan_slice,
+    prefix_tokens,
+)
+from .condenser import DEFAULT_POLICY, condense, condense_scope, create_condenser
+from .contract import (
+    AgentMessage,
+    Api,
+    BudgetPolicy,
+    CompleteFn,
+    CondensePlan,
+    Condenser,
+    CondenserDeps,
+    CondenseScope,
+    Model,
+    Summary,
+    TokenEstimate,
+    Usage,
+)
+from .summarize import (
+    CONDENSER_BRIEF,
+    SummarizeDeps,
+    build_summary_prompt,
+    flatten_transcript,
+    summarize,
+)
+
+__all__ = [
+    "AgentMessage",
+    "Api",
+    "BudgetPolicy",
+    "CONDENSER_BRIEF",
+    "CompleteFn",
+    "CondensePlan",
+    "CondenseScope",
+    "Condenser",
+    "CondenserDeps",
+    "DEFAULT_POLICY",
+    "Model",
+    "SummarizeDeps",
+    "Summary",
+    "TokenEstimate",
+    "Usage",
+    "budget_limit",
+    "build_summary_prompt",
+    "condense",
+    "condense_scope",
+    "create_condenser",
+    "estimate_message_tokens",
+    "estimate_tokens",
+    "flatten_transcript",
+    "is_over_budget",
+    "plan_slice",
+    "prefix_tokens",
+    "summarize",
+]

induscode/window_budget/budget/__init__.py

@@ -0,0 +1,26 @@
+"""Window-budget / budget — pure budget math (port of TS
+``src/window-budget/budget/index.ts``).
+
+The arithmetic core of the window-budget subsystem, free of I/O, prompts, and
+model calls:
+
+- :func:`estimate_tokens` / :func:`estimate_message_tokens` /
+  :func:`prefix_tokens` — heuristic token measurement and the forward
+  cumulative-token array.
+- :func:`is_over_budget` / :func:`budget_limit` — the window-relative trigger.
+- :func:`plan_slice` — forward prefix-sum + binary-search cut planning that
+  never splits a tool_call from its tool_result.
+"""
+
+from .estimate import estimate_message_tokens, estimate_tokens, prefix_tokens
+from .gate import budget_limit, is_over_budget
+from .slice import plan_slice
+
+__all__ = [
+    "budget_limit",
+    "estimate_message_tokens",
+    "estimate_tokens",
+    "is_over_budget",
+    "plan_slice",
+    "prefix_tokens",
+]

induscode/window_budget/budget/estimate.py

@@ -0,0 +1,273 @@
+"""Token estimation — the heuristic "meter stick" for the window-budget
+subsystem (port of TS ``src/window-budget/budget/estimate.ts``).
+
+Before we can decide whether the transcript is over budget (``gate.py``) or
+where to slice it (``slice.py``), we need a cheap, deterministic estimate of
+how many tokens each :data:`AgentMessage` occupies. We do *not* call a
+tokenizer; we walk the serialized content and apply a small, transparent
+weighting model.
+
+Design notes (clean-room, deliberately distinct from any chars/4 + flat-image
+baseline):
+
+- The estimate is computed over a *serialization* of each message's payload,
+  not over a JSON blob of the whole object — structural framing (braces,
+  quotes, field names) is accounted for with small per-block adds rather than
+  being measured character-for-character.
+- Constants below are this module's OWN tunables (a fractional
+  bytes-per-token divisor plus per-block framing adds and a flat image cost).
+  They are intentionally not the legacy fingerprint values — and they are
+  NOT the framework's: ``indusagi``'s own estimator uses 4.0 chars/token;
+  this app-level meter keeps its independent 3.6. Do not conflate the two.
+- The estimator is conservative (rounds up, adds framing) so the gate fires a
+  little early rather than a little late — overflowing the window is worse
+  than condensing one turn sooner than strictly necessary.
+
+Port note: TS probed messages structurally (``isRecord`` + ``.role`` reads
+over plain objects). Python transcripts are frozen dataclasses
+(:mod:`indusagi.ai` messages plus the :mod:`indusagi.agent` custom kinds), so
+the probing here is ``getattr`` duck typing over a ``role`` discriminator —
+and the agent's custom messages (``bashExecution`` / ``custom`` /
+``branchSummary`` / ``compactionSummary``) get explicit branches because the
+Python ``indusagi.ai.AgentMessage`` alias does not include them (analysis 05
+risk-2): falling through to generic serialization for known roles would
+silently mis-weigh them.
+"""
+
+from __future__ import annotations
+
+import dataclasses
+import json
+import math
+
+from ..contract import AgentMessage
+
+__all__ = [
+    "estimate_message_tokens",
+    "estimate_tokens",
+    "prefix_tokens",
+]
+
+
+# ---------------------------------------------------------------------------
+# Tunable constants (this module's own heuristic values)
+# ---------------------------------------------------------------------------
+
+# Average serialized characters per token. English-ish text and JSON tend to
+# pack a little under four characters per token for most tokenizers; we use a
+# slightly conservative 3.6 so the estimate skews high. (App-OWN constant —
+# the framework's estimator uses 4.0; keep them distinct.)
+CHARS_PER_TOKEN = 3.6
+
+# Flat token cost charged for one inline image block (resolution-agnostic).
+IMAGE_TOKENS = 1024
+
+# Tokens added per message to cover role/turn framing in the wire format.
+MESSAGE_FRAMING_TOKENS = 4
+
+# Tokens added per structured block (text/thinking/toolCall/toolResult part).
+BLOCK_FRAMING_TOKENS = 2
+
+# Tokens added per tool call to cover the call envelope (id + name framing).
+TOOLCALL_ENVELOPE_TOKENS = 6
+
+
+# ---------------------------------------------------------------------------
+# Per-message serialization → character weight
+# ---------------------------------------------------------------------------
+
+# Sentinel distinguishing "attribute absent" from "attribute is None" — the
+# Python analogue of the TS `=== undefined` probe.
+_MISSING = object()
+
+
+def _jsonable(value: object) -> object:
+    """``json.dumps`` fallback for non-JSON values: dataclasses flatten to
+    their field dict; anything else degrades to ``str()`` (we only ever
+    measure the serialized *length*, fidelity is irrelevant)."""
+    if dataclasses.is_dataclass(value) and not isinstance(value, type):
+        return dataclasses.asdict(value)
+    return str(value)
+
+
+def _safe_json(value: object) -> str:
+    """JSON-serialize that never raises (cycles/exotic values → empty string).
+    Compact separators mirror ``JSON.stringify``'s no-spaces output so the
+    character weights stay comparable to the TS originals."""
+    try:
+        return json.dumps(value, default=_jsonable, separators=(",", ":"))
+    except Exception:
+        return ""
+
+
+def _weigh_block(block: object) -> tuple[int, int]:
+    """Sum the character weight of a single content block and report whether
+    it was an image (images are billed as a flat token cost, not by character
+    length).
+
+    Returns ``(chars, images)``: ``chars`` feeds the chars-per-token divisor;
+    ``images`` is a count multiplied by :data:`IMAGE_TOKENS` later.
+    """
+    if isinstance(block, str):
+        return (len(block), 0)
+
+    block_type = getattr(block, "type", None)
+    if block_type == "text":
+        text = getattr(block, "text", None)
+        return (len(text) if isinstance(text, str) else 0, 0)
+    if block_type == "thinking":
+        thinking = getattr(block, "thinking", None)
+        return (len(thinking) if isinstance(thinking, str) else 0, 0)
+    if block_type == "image":
+        # Flat cost; do NOT measure the (often base64) `data` length.
+        return (0, 1)
+    if block_type == "toolCall":
+        name = getattr(block, "name", None)
+        chars = len(name) if isinstance(name, str) else 0
+        arguments = getattr(block, "arguments", _MISSING)
+        if arguments is not _MISSING:
+            chars += len(_safe_json(arguments))
+        return (chars, 0)
+
+    # Unknown/custom block: fall back to measuring its serialized form.
+    return (len(_safe_json(block)), 0)
+
+
+def _weigh_content(content: object) -> tuple[int, int, int]:
+    """Weigh a string-or-blocks ``content`` field: ``(chars, images,
+    framing)`` where framing counts one :data:`BLOCK_FRAMING_TOKENS` per
+    block (a bare string counts as one block)."""
+    chars = 0
+    images = 0
+    framing = 0
+    if isinstance(content, str):
+        chars += len(content)
+        framing += BLOCK_FRAMING_TOKENS
+    elif isinstance(content, (list, tuple)):
+        for block in content:
+            block_chars, block_images = _weigh_block(block)
+            chars += block_chars
+            images += block_images
+            framing += BLOCK_FRAMING_TOKENS
+    return (chars, images, framing)
+
+
+def _weigh_message(message: AgentMessage) -> tuple[int, int, int]:
+    """Total raw character weight + image count + framing-token total for one
+    message, dispatched by ``role``. Truly unknown roles fall through to a
+    generic serialization so they still contribute to the estimate.
+    """
+    chars = 0
+    images = 0
+    framing = MESSAGE_FRAMING_TOKENS
+
+    role = getattr(message, "role", None)
+
+    if role == "user":
+        content_chars, content_images, content_framing = _weigh_content(
+            getattr(message, "content", None)
+        )
+        return (chars + content_chars, images + content_images, framing + content_framing)
+
+    if role == "assistant":
+        content = getattr(message, "content", None)
+        if isinstance(content, (list, tuple)):
+            for block in content:
+                block_chars, block_images = _weigh_block(block)
+                chars += block_chars
+                images += block_images
+                framing += BLOCK_FRAMING_TOKENS
+                if getattr(block, "type", None) == "toolCall":
+                    framing += TOOLCALL_ENVELOPE_TOKENS
+        error_message = getattr(message, "errorMessage", None)
+        if isinstance(error_message, str):
+            chars += len(error_message)
+        return (chars, images, framing)
+
+    if role == "toolResult":
+        tool_name = getattr(message, "toolName", None)
+        if isinstance(tool_name, str):
+            chars += len(tool_name)
+        content_chars, content_images, content_framing = _weigh_content(
+            getattr(message, "content", None)
+        )
+        chars += content_chars
+        images += content_images
+        framing += content_framing + TOOLCALL_ENVELOPE_TOKENS
+        return (chars, images, framing)
+
+    # --- agent custom session messages (EXPLICIT branches — the Python
+    # `indusagi.ai.AgentMessage` alias excludes these; see contract.py) -----
+
+    if role == "bashExecution":
+        command = getattr(message, "command", None)
+        output = getattr(message, "output", None)
+        if isinstance(command, str):
+            chars += len(command)
+        if isinstance(output, str):
+            chars += len(output)
+        return (chars, images, framing + BLOCK_FRAMING_TOKENS)
+
+    if role == "custom":
+        content_chars, content_images, content_framing = _weigh_content(
+            getattr(message, "content", None)
+        )
+        return (chars + content_chars, images + content_images, framing + content_framing)
+
+    if role in ("branchSummary", "compactionSummary"):
+        summary = getattr(message, "summary", None)
+        if isinstance(summary, str):
+            chars += len(summary)
+        return (chars, images, framing + BLOCK_FRAMING_TOKENS)
+
+    # Unrecognized role: serialize generically (matches the TS default arm).
+    chars += len(_safe_json(message))
+    return (chars, images, framing + BLOCK_FRAMING_TOKENS)
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+
+def estimate_message_tokens(message: AgentMessage) -> int:
+    """Estimate the token cost of one message in isolation. Exposed so the
+    slice planner can build a prefix-sum without re-deriving the per-message
+    weight.
+
+    Cost = ``ceil(serialized_chars / CHARS_PER_TOKEN)`` + framing tokens +
+    ``images * IMAGE_TOKENS``. Always ``>= 0`` and biased high (conservative).
+    """
+    chars, images, framing = _weigh_message(message)
+    text_tokens = math.ceil(chars / CHARS_PER_TOKEN)
+    return text_tokens + framing + images * IMAGE_TOKENS
+
+
+def estimate_tokens(messages: list[AgentMessage]) -> int:
+    """Estimate the total token cost of a transcript: the sum of every
+    message's :func:`estimate_message_tokens`. This is a pure heuristic — the
+    gate prefers a provider-anchored figure when one exists, and only falls
+    back to this.
+
+    Note: an empty transcript estimates to exactly ``0`` (no per-message
+    framing is charged when there are no messages).
+    """
+    total = 0
+    for message in messages:
+        total += estimate_message_tokens(message)
+    return total
+
+
+def prefix_tokens(messages: list[AgentMessage]) -> list[int]:
+    """Build a FORWARD prefix-sum array of per-message token estimates.
+
+    Returns a list of length ``len(messages) + 1`` where ``result[i]`` is the
+    total estimated tokens of ``messages[0:i]`` (so ``result[0] == 0`` and
+    ``result[len(messages)] == estimate_tokens(messages)``). This
+    monotonically non-decreasing array is exactly what ``plan_slice``
+    binary-searches to find the cut boundary in O(log n).
+    """
+    prefix: list[int] = [0] * (len(messages) + 1)
+    for index, message in enumerate(messages):
+        prefix[index + 1] = prefix[index] + estimate_message_tokens(message)
+    return prefix

induscode/window_budget/budget/gate.py

@@ -0,0 +1,60 @@
+"""Budget gate — the trigger predicate for the window-budget subsystem
+(port of TS ``src/window-budget/budget/gate.ts``).
+
+:func:`is_over_budget` answers the single question "is the transcript large
+enough that we should condense before the next turn?" by comparing a token
+estimate against a window-relative threshold derived from the model's
+``contextWindow`` and the caller-supplied
+:class:`~induscode.window_budget.contract.BudgetPolicy`.
+
+The threshold is computed as::
+
+    limit = (contextWindow - reserve_tokens?) * trigger_ratio
+
+i.e. optional fixed headroom is carved off the window first, then the
+remaining capacity is scaled by the window-relative trigger ratio. This keeps
+the trigger free of baked-in token counts — ``trigger_ratio`` lives in
+``(0, 1]`` and ``reserve_tokens`` is an optional config value, never a
+fingerprint constant.
+"""
+
+from __future__ import annotations
+
+from ..contract import AgentMessage, BudgetPolicy, Model
+from .estimate import estimate_tokens
+
+__all__ = [
+    "budget_limit",
+    "is_over_budget",
+]
+
+
+def budget_limit(model: Model, policy: BudgetPolicy) -> float:
+    """Compute the absolute token limit at which condensing should fire for a
+    given model and policy. Exposed for callers that want the number itself
+    (e.g. to display headroom) rather than just the boolean.
+
+    ``reserve_tokens`` (when present) is subtracted from the window before
+    the ratio is applied; the result is floored at ``0`` so a misconfigured
+    reserve larger than the window can never yield a negative limit.
+    """
+    reserve = policy.reserve_tokens if policy.reserve_tokens is not None else 0
+    usable = max(0, model.contextWindow - reserve)
+    return usable * policy.trigger_ratio
+
+
+def is_over_budget(
+    messages: list[AgentMessage],
+    model: Model,
+    policy: BudgetPolicy,
+) -> bool:
+    """Decide whether the transcript is over budget for the given model and
+    policy.
+
+    Uses the heuristic :func:`~.estimate.estimate_tokens` over the serialized
+    transcript and compares it to :func:`budget_limit`. Returns ``True`` once
+    the estimate strictly exceeds the limit — the signal the conductor uses
+    to kick off a condense.
+    """
+    estimate = estimate_tokens(messages)
+    return estimate > budget_limit(model, policy)

induscode/window_budget/budget/slice.py

@@ -0,0 +1,145 @@
+"""Slice planner — decides *where* to cut the transcript when it is over
+budget (port of TS ``src/window-budget/budget/slice.ts``).
+
+:func:`plan_slice` partitions a transcript into a ``dropped`` prefix (older
+messages destined for a summary) and a ``kept`` suffix (the verbatim recent
+tail), choosing the boundary so the tail holds roughly ``policy.keep_recent``
+tokens. It does this with a FORWARD prefix-sum + BINARY SEARCH (not a
+backward accumulate-and-snap):
+
+1. Build the cumulative-token array ``prefix`` (forward, monotonic).
+2. The total is ``prefix[n]``; we want the tail to be ~``keep_recent``
+   tokens, so the *ideal* cut sits at the first index ``i`` whose suffix
+   ``prefix[n] - prefix[i]`` has fallen to ``<= keep_recent``. Equivalently,
+   the first ``i`` with ``prefix[i] >= prefix[n] - keep_recent``. That
+   predicate is monotonic in ``i``, so we BINARY-SEARCH the lower bound of
+   the threshold ``prefix[n] - keep_recent`` over ``prefix``.
+3. Snap the resulting index FORWARD to the nearest *legal* boundary so we
+   never land between a tool_call and its matching tool_result. A boundary is
+   legal iff the message it lands on is not a ``toolResult`` (a tool result
+   always belongs with the assistant turn that issued the call, so the tail
+   may never begin with one).
+
+If the whole transcript already fits within ``keep_recent``, or snapping
+forward consumes everything, the cut is ``0`` (nothing condensable — keep it
+all).
+
+Port note: role probing is ``getattr`` duck typing over the message
+dataclasses (the TS structural ``isRecord``/``.role`` reads), so the agent's
+custom session messages — which carry roles like ``bashExecution`` /
+``branchSummary`` — are all legal tail starts; only ``toolResult`` is glued
+to its issuing assistant turn.
+"""
+
+from __future__ import annotations
+
+from ..contract import AgentMessage, BudgetPolicy, CondensePlan
+from .estimate import prefix_tokens
+
+__all__ = [
+    "plan_slice",
+]
+
+
+# ---------------------------------------------------------------------------
+# Boundary legality
+# ---------------------------------------------------------------------------
+
+
+def _is_legal_tail_start(message: AgentMessage) -> bool:
+    """A ``toolResult`` message must stay glued to the assistant turn that
+    issued its ``toolCall``. So the *kept tail* may never begin with a tool
+    result — that would orphan it from the (dropped) call. Every other role
+    is a legal tail-start.
+
+    :param message: the message that would become ``kept[0]`` for a candidate cut
+    :returns: ``True`` if a cut landing on this message is legal
+    """
+    return getattr(message, "role", None) != "toolResult"
+
+
+def _snap_forward(messages: list[AgentMessage], candidate: int) -> int:
+    """Snap a candidate cut index FORWARD to the nearest legal boundary:
+    advance past any leading ``toolResult`` messages so the kept tail starts
+    on a user/assistant/custom turn, never mid tool-call group.
+
+    :returns: the snapped index in ``[candidate, len(messages)]``
+    """
+    cut = candidate
+    while cut < len(messages) and not _is_legal_tail_start(messages[cut]):
+        cut += 1
+    return cut
+
+
+# ---------------------------------------------------------------------------
+# Forward prefix-sum + binary search
+# ---------------------------------------------------------------------------
+
+
+def _lower_bound(prefix: list[int], threshold: float) -> int:
+    """Binary-search the lower bound: the smallest index ``i`` in
+    ``[0, len(prefix))`` with ``prefix[i] >= threshold``. ``prefix`` is
+    monotonically non-decreasing, so the predicate flips exactly once and a
+    standard half-open bisection finds it in O(log n). Returns
+    ``len(prefix) - 1`` if no index satisfies it.
+    """
+    lo = 0
+    hi = len(prefix) - 1
+    while lo < hi:
+        mid = (lo + hi) // 2
+        if prefix[mid] >= threshold:
+            hi = mid
+        else:
+            lo = mid + 1
+    return lo
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+
+def plan_slice(messages: list[AgentMessage], policy: BudgetPolicy) -> CondensePlan:
+    """Plan the condense slice for a transcript under a
+    :class:`~induscode.window_budget.contract.BudgetPolicy`.
+
+    Chooses the cut via forward prefix-sum + binary search for the boundary
+    that keeps ~``policy.keep_recent`` recent tokens, then snaps that
+    boundary forward to a legal point (never between a tool_call and its
+    tool_result).
+
+    :returns: a :class:`~induscode.window_budget.contract.CondensePlan` with
+        the chosen ``cut``, the ``kept`` tail (``messages[cut:]``) and the
+        ``dropped`` head (``messages[:cut]``). ``cut == 0`` when nothing is
+        condensable.
+    """
+    n = len(messages)
+    if n == 0:
+        return CondensePlan(cut=0, kept=[], dropped=[])
+
+    # Forward cumulative-token array: prefix[i] = tokens of messages[0:i].
+    prefix = prefix_tokens(messages)
+    total = prefix[n]
+
+    # If the whole transcript already fits in the recent-tail budget, keep all.
+    if total <= policy.keep_recent:
+        return CondensePlan(cut=0, kept=list(messages), dropped=[])
+
+    # Ideal cut: first index whose suffix has shrunk to <= keep_recent, i.e.
+    # the lower bound of (total - keep_recent) over the forward prefix array.
+    threshold = total - policy.keep_recent
+    ideal = _lower_bound(prefix, threshold)
+
+    # Snap forward to a legal boundary (don't orphan a tool result from its call).
+    cut = _snap_forward(messages, ideal)
+
+    # Snapping forward can swallow everything (e.g. a long trailing
+    # tool-result run): then there is nothing to drop — keep all.
+    if cut >= n:
+        return CondensePlan(cut=0, kept=list(messages), dropped=[])
+
+    return CondensePlan(
+        cut=cut,
+        kept=list(messages[cut:]),
+        dropped=list(messages[:cut]),
+    )