leancontext 2.0.0__py3-none-any.whl

leancontext/messages.py

@@ -0,0 +1,152 @@
+"""Protocol-aware message reduction — the gateway/wire surface.
+
+This is how LeanContext plugs into gateways (LiteLLM), SDK client wrappers, and proxies
+*without* the structure-blindness that hurts wire-level compressors: the chat
+protocols already tag tool outputs (OpenAI ``role="tool"``; Anthropic
+``tool_result`` blocks), so we can find and reduce exactly those — and nothing
+else. We never touch system/user/assistant instruction text. Fail-open throughout.
+
+Cache-safety: reductions are deterministic and content-addressed, so the same tool
+output always serialises to the same bytes → the provider prompt-cache keeps hitting.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from .core import reduce_text
+
+
+def detect_format(messages: list) -> str:
+    """Best-effort detection of the message protocol."""
+    for m in messages:
+        if not isinstance(m, dict):
+            continue
+        if isinstance(m.get("parts"), list):
+            return "gemini"
+        if m.get("role") in ("tool", "function"):
+            return "openai"
+        content = m.get("content")
+        if isinstance(content, list):
+            for block in content:
+                if isinstance(block, dict) and block.get("type") == "tool_result":
+                    return "anthropic"
+    return "openai"
+
+
+def _reduce_str(text: Any, opts: dict) -> Any:
+    if not isinstance(text, str):
+        return text
+    return reduce_text(text, **opts).text
+
+
+# --- OpenAI / chat-completions format ----------------------------------------
+
+def _reduce_openai_message(m: Any, opts: dict) -> Any:
+    if not isinstance(m, dict) or m.get("role") not in ("tool", "function"):
+        return m
+    content = m.get("content")
+    if isinstance(content, str):
+        nm = dict(m)
+        nm["content"] = _reduce_str(content, opts)
+        return nm
+    if isinstance(content, list):
+        nm = dict(m)
+        nm["content"] = [_reduce_openai_part(p, opts) for p in content]
+        return nm
+    return m
+
+
+def _reduce_openai_part(part: Any, opts: dict) -> Any:
+    if (
+        isinstance(part, dict)
+        and part.get("type") in ("text", "output_text")
+        and isinstance(part.get("text"), str)
+    ):
+        np = dict(part)
+        np["text"] = _reduce_str(part["text"], opts)
+        return np
+    return part
+
+
+# --- Anthropic messages format -----------------------------------------------
+
+def _reduce_anthropic_message(m: Any, opts: dict) -> Any:
+    if not isinstance(m, dict):
+        return m
+    content = m.get("content")
+    if not isinstance(content, list):
+        return m
+    new_blocks, changed = [], False
+    for block in content:
+        if isinstance(block, dict) and block.get("type") == "tool_result":
+            bc = block.get("content")
+            if isinstance(bc, str):
+                nb = dict(block)
+                nb["content"] = _reduce_str(bc, opts)
+                new_blocks.append(nb)
+                changed = True
+                continue
+            if isinstance(bc, list):
+                nb = dict(block)
+                nb["content"] = [_reduce_anthropic_textblock(x, opts) for x in bc]
+                new_blocks.append(nb)
+                changed = True
+                continue
+        new_blocks.append(block)
+    if not changed:
+        return m
+    nm = dict(m)
+    nm["content"] = new_blocks
+    return nm
+
+
+def _reduce_anthropic_textblock(x: Any, opts: dict) -> Any:
+    if isinstance(x, dict) and x.get("type") == "text" and isinstance(x.get("text"), str):
+        nx = dict(x)
+        nx["text"] = _reduce_str(x["text"], opts)
+        return nx
+    return x
+
+
+# --- Gemini format -----------------------------------------------------------
+# Gemini uses `contents` -> `parts`, where a tool result is a `functionResponse`
+# part whose `response` is a dict. We reduce the large string values inside that
+# dict, keeping the dict shape Gemini requires. Typed SDK objects (non-dict)
+# pass through untouched.
+
+def _reduce_gemini_message(content: Any, opts: dict) -> Any:
+    if not isinstance(content, dict) or not isinstance(content.get("parts"), list):
+        return content
+    new_parts, changed = [], False
+    for part in content["parts"]:
+        fr = part.get("functionResponse") if isinstance(part, dict) else None
+        resp = fr.get("response") if isinstance(fr, dict) else None
+        if isinstance(fr, dict) and isinstance(resp, dict):
+            reduced = {k: (_reduce_str(v, opts) if isinstance(v, str) else v) for k, v in resp.items()}
+            new_parts.append({**part, "functionResponse": {**fr, "response": reduced}})
+            changed = True
+        else:
+            new_parts.append(part)
+    if not changed:
+        return content
+    return {**content, "parts": new_parts}
+
+
+# --- public ------------------------------------------------------------------
+
+def reduce_messages(messages: Any, *, fmt: str = "auto", **opts) -> Any:
+    """Return a new message list with tool outputs reduced. Input is not mutated.
+
+    Handles OpenAI (`role:"tool"`), Anthropic (`tool_result` blocks), and Gemini
+    (`functionResponse` parts). Only tool-result content is touched; instructions
+    are never altered. Anything unrecognised passes through unchanged (fail open).
+    """
+    if not isinstance(messages, list):
+        return messages
+    resolved = detect_format(messages) if fmt == "auto" else fmt
+    if resolved == "anthropic":
+        return [_reduce_anthropic_message(m, opts) for m in messages]
+    if resolved == "gemini":
+        return [_reduce_gemini_message(m, opts) for m in messages]
+    return [_reduce_openai_message(m, opts) for m in messages]

leancontext/paging.py

@@ -0,0 +1,104 @@
+"""Paging: drop aged tool outputs from the wire, keep them retrievable.
+
+Reducing shrinks each payload; paging goes further by removing old payloads from
+context once the agent has moved on. The output is replaced with a small reference
+(a few tens of tokens) and the original is stored, so the agent can fetch it back
+with the expand tool when it needs the detail again.
+
+Refs are content hashes, so they're deterministic. The store is in-memory by
+default, or disk-backed (set ``root``) for retrieval across processes.
+"""
+
+from __future__ import annotations
+
+import os
+import re
+
+from .tokens import content_ref, count_tokens
+
+REF_SCHEME = "lc"
+_REF_RE = re.compile(r"lc://([0-9a-f]{6,40})")
+
+
+class ContentStore:
+    """Maps a content hash → original content. In-memory, or disk-backed if ``root`` set."""
+
+    def __init__(self, root: str | None = None):
+        self.root = root
+        self._mem: dict[str, str] = {}
+        if self.root:
+            os.makedirs(self.root, exist_ok=True)
+
+    def _path(self, ref: str) -> str:
+        return os.path.join(self.root, f"{ref}.txt")  # type: ignore[arg-type]
+
+    def put(self, content: str) -> str:
+        ref = content_ref(content)
+        if self.root:
+            with open(self._path(ref), "w", encoding="utf-8") as fh:
+                fh.write(content)
+        else:
+            self._mem[ref] = content
+        return ref
+
+    def get(self, ref: str) -> str | None:
+        if self.root:
+            try:
+                with open(self._path(ref), encoding="utf-8") as fh:
+                    return fh.read()
+            except OSError:
+                return None
+        return self._mem.get(ref)
+
+
+_DEFAULT_STORE = ContentStore()
+
+
+def _normalize(ref: str) -> str:
+    m = _REF_RE.search(ref)
+    return m.group(1) if m else ref.strip()
+
+
+def store(content: str, using: ContentStore | None = None) -> str:
+    """Stash content and return its ref id."""
+    return (using or _DEFAULT_STORE).put(content)
+
+
+def expand(ref: str, using: ContentStore | None = None) -> str | None:
+    """Return the original content for a ref (accepts 'lc://<id>' or a bare id)."""
+    return (using or _DEFAULT_STORE).get(_normalize(ref))
+
+
+def reference_line(content: str, summary: str | None = None,
+                   using: ContentStore | None = None) -> str:
+    """Stash content and return a compact, expandable reference line."""
+    ref = store(content, using=using)
+    tokens = count_tokens(content)
+    tail = f" — {summary}" if summary else ""
+    return f"[{REF_SCHEME}://{ref} · {tokens} tokens · call leancontext_expand to view{tail}]"
+
+
+def page(content: str, *, summary: str | None = None,
+         using: ContentStore | None = None) -> str:
+    """Collapse aged content to an expandable reference (O(1) on the wire)."""
+    return reference_line(content, summary=summary, using=using)
+
+
+#: Tool spec to expose ``expand`` to an agent (OpenAI/Anthropic/MCP-compatible shape).
+EXPAND_TOOL_SPEC = {
+    "name": "leancontext_expand",
+    "description": (
+        "Retrieve the full original content for a LeanContext reference id "
+        "(format: lc://<id>) that was collapsed to save tokens."
+    ),
+    "input_schema": {
+        "type": "object",
+        "properties": {
+            "ref": {
+                "type": "string",
+                "description": "The reference id, e.g. 'lc://a1b2c3d4' or the bare id.",
+            },
+        },
+        "required": ["ref"],
+    },
+}

leancontext/reducers/__init__.py

@@ -0,0 +1,36 @@
+"""Typed reducers.
+
+Each reducer module exposes a ``REDUCER`` (kind, detector, reduce function,
+priority). ``REGISTRY`` is the ordered list the core iterates for detection and
+dispatch, so adding a reducer means adding one module and listing it here.
+"""
+
+from .base import Reducer
+from .diff import REDUCER as _diff
+from .diff import reduce_diff
+from .html import REDUCER as _html
+from .html import reduce_html
+from .json_data import REDUCER as _json
+from .json_data import reduce_json
+from .logs import REDUCER as _logs
+from .logs import reduce_logs
+from .stacktrace import REDUCER as _stacktrace
+from .stacktrace import reduce_stacktrace
+from .table import REDUCER as _table
+from .table import reduce_table
+
+# Detection runs in priority order (lowest first): json, stacktrace, diff, html, log.
+REGISTRY: list[Reducer] = sorted(
+    [_json, _stacktrace, _diff, _html, _logs, _table], key=lambda r: r.priority
+)
+
+__all__ = [
+    "Reducer",
+    "REGISTRY",
+    "reduce_logs",
+    "reduce_json",
+    "reduce_diff",
+    "reduce_stacktrace",
+    "reduce_html",
+    "reduce_table",
+]

leancontext/reducers/base.py

@@ -0,0 +1,19 @@
+"""The shape every reducer registers with.
+
+A reducer bundles three things: the kind name, a detector that says whether a
+payload is this kind, and the reduce function. Detection priority is explicit
+(lower runs first), so the order is clear and stable.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class Reducer:
+    kind: str
+    detect: Callable[[str], bool]
+    reduce: Callable[[str], tuple[str, list[str]]]
+    priority: int

leancontext/reducers/diff.py

@@ -0,0 +1,54 @@
+"""Diff reducer.
+
+Keeps every change line (``+``/``-``), hunk header (``@@``) and file header verbatim
+— those are the signal. Collapses long runs of unchanged context lines to the first
+and last line plus a count. Deterministic; value-preserving for all changes.
+"""
+
+from __future__ import annotations
+
+import re
+
+from .base import Reducer
+
+_DIFF_HUNK = re.compile(r"(?m)^@@ -\d+(?:,\d+)? \+\d+(?:,\d+)? @@")
+
+_KEEP_PREFIXES = (
+    "+", "-", "@@", "diff ", "index ", "new file", "deleted file",
+    "rename ", "similarity ", "copy ", "old mode", "new mode",
+)
+
+
+def reduce_diff(text: str) -> tuple[str, list[str]]:
+    lines = text.splitlines()
+    out: list[str] = []
+    ctx: list[str] = []
+
+    def flush() -> None:
+        if not ctx:
+            return
+        if len(ctx) <= 3:
+            out.extend(ctx)
+        else:
+            out.append(ctx[0])
+            out.append(f"  ⟪… {len(ctx) - 2} unchanged lines⟫")
+            out.append(ctx[-1])
+        ctx.clear()
+
+    for line in lines:
+        if line.startswith(_KEEP_PREFIXES):
+            flush()
+            out.append(line)
+        else:
+            ctx.append(line)
+    flush()
+
+    notes = [f"kept all change/header lines; collapsed unchanged context ({len(lines)}→{len(out)} lines)"]
+    return "\n".join(out), notes
+
+
+def _detect(text: str) -> bool:
+    return text.lstrip().startswith("diff --git") or bool(_DIFF_HUNK.search(text))
+
+
+REDUCER = Reducer("diff", _detect, reduce_diff, priority=30)

leancontext/reducers/html.py

@@ -0,0 +1,64 @@
+"""HTML reducer — for web-fetch / scraped tool outputs.
+
+Strips tags, scripts, styles and collapses whitespace, keeping the visible text and
+the links (URLs are signal, so they're preserved). Stdlib only, deterministic.
+"""
+
+from __future__ import annotations
+
+import re
+from html.parser import HTMLParser
+
+from .base import Reducer
+
+_SKIP = {"script", "style", "noscript", "svg", "head", "template"}
+
+
+class _Extract(HTMLParser):
+    def __init__(self) -> None:
+        super().__init__(convert_charrefs=True)
+        self.parts: list[str] = []
+        self.links: list[str] = []
+        self._skip = 0
+
+    def handle_starttag(self, tag, attrs):
+        if tag in _SKIP:
+            self._skip += 1
+        if tag == "a":
+            for key, val in attrs:
+                if key == "href" and val:
+                    self.links.append(val)
+
+    def handle_endtag(self, tag):
+        if tag in _SKIP and self._skip > 0:
+            self._skip -= 1
+
+    def handle_data(self, data):
+        if self._skip == 0:
+            text = data.strip()
+            if text:
+                self.parts.append(text)
+
+
+def reduce_html(text: str) -> tuple[str, list[str]]:
+    parser = _Extract()
+    parser.feed(text)
+    body = re.sub(r"[ \t]+", " ", "\n".join(parser.parts)).strip()
+    links = list(dict.fromkeys(parser.links))
+
+    out = body
+    if links:
+        out += "\n\nLinks: " + " ".join(links)
+    notes = [f"stripped HTML tags/scripts/styles; kept visible text + {len(links)} links"]
+    return out, notes
+
+
+def _detect(text: str) -> bool:
+    stripped = text.lstrip()
+    head = stripped[:512].lower()
+    if "<!doctype html" in head or "<html" in head:
+        return True
+    return stripped.startswith("<") and text.lower().count("</") >= 5
+
+
+REDUCER = Reducer("html", _detect, reduce_html, priority=40)

leancontext/reducers/json_data.py

@@ -0,0 +1,61 @@
+"""JSON / RAG reducer.
+
+The dominant waste in JSON tool output is *repeated keys*: a list of 200 records
+re-states every field name 200 times. We factor the schema out once and emit the
+values columnar. All values are preserved, so this is near-lossless.
+
+Falls back to whitespace-stripped (minified) JSON when the payload isn't a record
+list — still a real saving on pretty-printed output, with zero information loss.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from .base import Reducer
+
+
+def _find_records(data: Any) -> list[dict] | None:
+    """Locate a homogeneous-ish list of dicts at the top level or one level down."""
+    if isinstance(data, list) and data and all(isinstance(x, dict) for x in data):
+        return data
+    if isinstance(data, dict):
+        for value in data.values():
+            if isinstance(value, list) and value and all(isinstance(x, dict) for x in value):
+                return value
+    return None
+
+
+def _fmt(value: Any) -> str:
+    if isinstance(value, str):
+        return value
+    return json.dumps(value, separators=(",", ":"), ensure_ascii=False)
+
+
+def reduce_json(text: str) -> tuple[str, list[str]]:
+    data = json.loads(text)
+    records = _find_records(data)
+
+    if records is not None and len(records) >= 3:
+        keys = list(dict.fromkeys(k for row in records for k in row.keys()))
+        header = "fields: " + " | ".join(keys)
+        rows = [" | ".join(_fmt(row.get(k, "")) for k in keys) for row in records]
+        notes = [f"columnar: {len(records)} records × {len(keys)} fields, keys factored out once"]
+        return header + "\n" + "\n".join(rows), notes
+
+    compact = json.dumps(data, separators=(",", ":"), ensure_ascii=False)
+    return compact, ["minified json (indentation/whitespace removed, lossless)"]
+
+
+def _detect(text: str) -> bool:
+    if text.lstrip()[:1] not in "[{":
+        return False
+    try:
+        json.loads(text)
+        return True
+    except Exception:
+        return False
+
+
+REDUCER = Reducer("json", _detect, reduce_json, priority=10)

leancontext/reducers/logs.py

@@ -0,0 +1,91 @@
+"""Collapse repetitive log lines.
+
+Near-identical lines collapse to one representative plus a count, while every
+error/anomaly line and every one-off pattern is kept as-is.
+
+To decide "near-identical", we mask the volatile parts of a line (timestamps, ips,
+uuids, hex, numbers, quoted strings) into a template. Lines that share a template
+are the same event with different values, so we keep one and count the rest.
+Templates seen only once, or carrying a severity keyword, are kept verbatim, since
+the rare line is usually the one that matters.
+
+Deterministic: first-seen order is preserved, so the same input gives the same output.
+"""
+
+from __future__ import annotations
+
+import re
+
+from .base import Reducer
+
+_LOG_HINT = re.compile(
+    r"(?im)^\s*(?:\d{4}-\d{2}-\d{2}[T ]|\[?(?:INFO|DEBUG|WARN|WARNING|ERROR|FATAL|TRACE|CRITICAL)\b)"
+)
+_SEVERITY = re.compile(r"(?i)\b(ERROR|FATAL|CRITICAL|EXCEPTION|PANIC|TRACEBACK|WARN|WARNING)\b")
+
+# Order matters: more specific patterns first so they win before the generic
+# number mask consumes their digits.
+_MASKS = (
+    (re.compile(r"\d{4}-\d{2}-\d{2}[T ][\d:.,]+(?:Z|[+-]\d{2}:?\d{2})?"), "§ts"),
+    (re.compile(r"\b\d{1,3}(?:\.\d{1,3}){3}\b"), "§ip"),
+    (re.compile(r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b"), "§uuid"),
+    (re.compile(r"0x[0-9a-fA-F]+"), "§hex"),
+    (re.compile(r'"[^"]*"'), "§s"),
+    (re.compile(r"\b\d+(?:\.\d+)?\b"), "§n"),
+)
+
+
+def _template(line: str) -> str:
+    t = line
+    for rx, repl in _MASKS:
+        t = rx.sub(repl, t)
+    return t.strip()
+
+
+def reduce_logs(text: str) -> tuple[str, list[str]]:
+    lines = text.splitlines()
+    groups: dict[str, list] = {}  # template -> [representative, count, is_severity]
+    order: list[str] = []
+
+    for line in lines:
+        if not line.strip():
+            continue
+        key = _template(line)
+        sev = bool(_SEVERITY.search(line))
+        g = groups.get(key)
+        if g is None:
+            groups[key] = [line, 1, sev]
+            order.append(key)
+        else:
+            g[1] += 1
+            g[2] = g[2] or sev
+
+    out: list[str] = []
+    kept_verbatim = 0
+    for key in order:
+        line, count, is_sev = groups[key]
+        if is_sev:
+            kept_verbatim += 1
+            out.append(line if count == 1 else f"{line}    ⟪×{count}⟫")
+        elif count == 1:
+            kept_verbatim += 1
+            out.append(line)
+        else:
+            out.append(f"{line}    ⟪×{count} similar⟫")
+
+    notes = [
+        f"{len(order)} unique patterns from {len(lines)} lines; "
+        f"{kept_verbatim} anomaly/unique lines kept verbatim"
+    ]
+    return "\n".join(out), notes
+
+
+def _detect(text: str) -> bool:
+    lines = text.splitlines()
+    if len(lines) < 5:
+        return False
+    hits = sum(1 for ln in lines if _LOG_HINT.match(ln))
+    return hits >= max(3, len(lines) * 0.3)
+
+
+REDUCER = Reducer("log", _detect, reduce_logs, priority=50)

leancontext/reducers/stacktrace.py

@@ -0,0 +1,59 @@
+"""Stack-trace reducer.
+
+The exception (last line) and the boundary frames are the signal; the deep middle
+of the call stack is usually noise. Keeps the header, the first frame, the last two
+frames, and the full exception/tail verbatim; collapses the middle with a count.
+Raises on non-tracebacks → core falls back to passthrough (fail open).
+"""
+
+from __future__ import annotations
+
+from .base import Reducer
+
+_KEEP_HEAD = 1
+_KEEP_TAIL = 2
+
+
+def reduce_stacktrace(text: str) -> tuple[str, list[str]]:
+    lines = text.splitlines()
+    file_idx = [i for i, ln in enumerate(lines) if ln.lstrip().startswith('File "')]
+    if not file_idx:
+        raise ValueError("not a python traceback")
+
+    header = lines[: file_idx[0]]
+    frames = [
+        lines[start : (file_idx[k + 1] if k + 1 < len(file_idx) else len(lines))]
+        for k, start in enumerate(file_idx)
+    ]
+
+    # Peel the trailing non-indented lines off the last frame: that's the exception.
+    last = frames[-1]
+    split = len(last)
+    for m in range(1, len(last)):
+        if last[m].strip() and not last[m].startswith((" ", "\t")):
+            split = m
+            break
+    tail = last[split:]
+    frames[-1] = last[:split]
+
+    out = list(header)
+    if len(frames) <= _KEEP_HEAD + _KEEP_TAIL + 1:
+        for fr in frames:
+            out.extend(fr)
+    else:
+        for fr in frames[:_KEEP_HEAD]:
+            out.extend(fr)
+        out.append(f"  ⟪… {len(frames) - _KEEP_HEAD - _KEEP_TAIL} stack frames hidden⟫")
+        for fr in frames[-_KEEP_TAIL:]:
+            out.extend(fr)
+    out.extend(tail)
+
+    notes = [f"kept {min(len(frames), _KEEP_HEAD + _KEEP_TAIL)} of {len(frames)} frames + exception"]
+    return "\n".join(out), notes
+
+
+def _detect(text: str) -> bool:
+    return "Traceback (most recent call last)" in text
+
+
+REDUCER = Reducer("stacktrace", _detect, reduce_stacktrace, priority=20)

leancontext/reducers/table.py

@@ -0,0 +1,32 @@
+"""Whitespace-aligned table reducer.
+
+Command-line tools (kubectl, docker, ps, ls -l, df) pad columns with runs of
+spaces so they line up. That padding is pure tokens. We collapse each run of two
+or more spaces to a single space and trim line ends. Every value is kept; only
+the alignment is dropped, so this is lossless for the data.
+"""
+
+from __future__ import annotations
+
+import re
+
+from .base import Reducer
+
+_GAP = re.compile(r"[ \t]{2,}")
+
+
+def reduce_table(text: str) -> tuple[str, list[str]]:
+    out = [_GAP.sub(" ", line).rstrip() for line in text.splitlines()]
+    return "\n".join(out), ["collapsed column padding; values preserved"]
+
+
+def _detect(text: str) -> bool:
+    lines = [ln for ln in text.splitlines() if ln.strip()]
+    if len(lines) < 3:
+        return False
+    # A line looks columnar when it has at least two padded gaps (3+ columns).
+    columnar = sum(1 for ln in lines if len(_GAP.findall(ln)) >= 2)
+    return columnar >= max(3, len(lines) * 0.6)
+
+
+REDUCER = Reducer("table", _detect, reduce_table, priority=60)