slimctx 0.1.0__py3-none-any.whl

slimctx/__init__.py

@@ -0,0 +1,30 @@
+"""slimctx — a zero-dependency, fully-reversible context compression layer
+for AI agents.
+
+    from slimctx import Pipeline, Config
+
+    pipe = Pipeline(Config(target_tokens=32_000))
+    result = pipe.compress(messages)          # OpenAI/Anthropic style dicts
+    result.savings_ratio                      # e.g. 0.72
+    pipe.retrieve("a1b2c3...")                # byte-exact original back
+
+Guarantees:
+  * pure Python stdlib — no models, no downloads, no network, ever
+  * every lossy transform stores the original first (always reversible)
+  * deterministic + memoized -> stable prefixes -> provider caches hit
+  * error/warning content is never dropped by any compressor
+"""
+
+from .pipeline import Config, Pipeline, Result
+from .store import MemoryStore, SqliteStore
+from .tokens import estimate_tokens
+
+__version__ = "0.1.0"
+__all__ = [
+    "Pipeline",
+    "Config",
+    "Result",
+    "MemoryStore",
+    "SqliteStore",
+    "estimate_tokens",
+]

slimctx/code_compressor.py

@@ -0,0 +1,196 @@
+"""Code compression: structural skeletons instead of raw truncation.
+
+For Python we use the real AST: keep imports, class/function signatures,
+docstring first-lines, and module-level assignments; elide bodies with a
+line-count note. The model keeps the *map* of the file — usually what it
+actually needs — and can retrieve the full source via the store marker.
+
+Query-aware body retention: when the caller passes the user's query, the
+few function bodies most relevant to it are kept verbatim while the rest
+are elided — the model sees the whole map plus the exact code it is being
+asked about. (Blanket skeletonization forces a retrieval round-trip for
+the one function the user actually cares about.)
+
+For other languages we fall back to a brace/keyword skeleton: keep lines
+that open declarations, drop deep indentation blocks with elision notes.
+"""
+
+from __future__ import annotations
+
+import ast
+import re
+from typing import List, Optional
+
+from .relevance import Bm25, _stem, tokenize
+
+MAX_VERBATIM_BODIES = 3
+DUPLICATE_JACCARD = 0.8
+
+_PY_HINTS = re.compile(r"^\s*(def |class |import |from |@|if __name__)", re.MULTILINE)
+_DECL_RE = re.compile(
+    r"^\s*(?:export\s+)?(?:public|private|protected|static|async|final|abstract)?\s*"
+    r"(?:function|class|interface|struct|enum|impl|fn|func|def|type|const|var|let|"
+    r"void|int|string|bool|public|private)\b"
+)
+
+
+def looks_like_code(text: str) -> bool:
+    lines = text.splitlines()
+    if len(lines) < 8:
+        return False
+    if len(_PY_HINTS.findall(text)) >= 3:
+        return True
+    decl_hits = sum(1 for ln in lines[:80] if _DECL_RE.match(ln))
+    brace_lines = sum(1 for ln in lines[:80] if ln.rstrip().endswith(("{", "}", ");")))
+    return decl_hits + brace_lines >= len(lines[:80]) * 0.25
+
+
+def compress_code(text: str, query: str = "") -> Optional[str]:
+    result = _python_skeleton(text, query)
+    if result is not None:
+        return result
+    return _generic_skeleton(text)
+
+
+def _relevant_function_ids(tree: ast.AST, lines: List[str], query: str) -> set:
+    """ids of function nodes whose bodies should be kept verbatim."""
+    if not query:
+        return set()
+    funcs = [
+        n for n in ast.walk(tree)
+        if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef))
+    ]
+    if not funcs:
+        return set()
+    sources = [
+        "\n".join(lines[f.lineno - 1 : getattr(f, "end_lineno", f.lineno)])
+        for f in funcs
+    ]
+    bm25 = Bm25(sources)
+    scored = sorted(
+        ((bm25.score(i, query), i) for i in range(len(funcs))), reverse=True
+    )
+    # Relative gate: an absolute score threshold fails when the query term
+    # appears in every function (IDF -> 0). What matters is which bodies
+    # score *best*, as long as there is any signal at all. Diversity gate:
+    # a body near-identical to one already kept adds no information
+    # (common in codebases with many boilerplate-similar functions).
+    best = scored[0][0]
+    if best <= 1e-9:
+        return set()
+    keep: set = set()
+    kept_tokens: List[set] = []
+    for score, i in scored:
+        if len(keep) >= MAX_VERBATIM_BODIES or score < best * 0.5:
+            break
+        toks = {_stem(t) for t in tokenize(sources[i])}
+        if any(_jaccard(toks, prev) > DUPLICATE_JACCARD for prev in kept_tokens):
+            continue
+        keep.add(id(funcs[i]))
+        kept_tokens.append(toks)
+    return keep
+
+
+def _jaccard(a: set, b: set) -> float:
+    if not a or not b:
+        return 0.0
+    return len(a & b) / len(a | b)
+
+
+def _python_skeleton(text: str, query: str = "") -> Optional[str]:
+    try:
+        tree = ast.parse(text)
+    except (SyntaxError, ValueError, RecursionError, MemoryError):
+        return None
+
+    lines = text.splitlines()
+    out: List[str] = []
+    verbatim_ids = _relevant_function_ids(tree, lines, query)
+
+    def emit_docstring(node: ast.AST, indent: str) -> None:
+        doc = ast.get_docstring(node, clean=True)
+        if doc:
+            first = doc.splitlines()[0]
+            out.append(f'{indent}"""{first}"""')
+
+    def signature_lines(node: ast.AST) -> List[str]:
+        # decorators + the def/class line(s) up to the colon
+        start = min(
+            [d.lineno for d in getattr(node, "decorator_list", [])] + [node.lineno]
+        )
+        body_start = node.body[0].lineno if node.body else node.lineno + 1
+        sig = []
+        for ln in lines[start - 1 : body_start - 1]:
+            sig.append(ln)
+            if ln.rstrip().endswith(":"):
+                break
+        return sig
+
+    def body_size(node: ast.AST) -> int:
+        return (getattr(node, "end_lineno", None) or node.lineno) - node.lineno + 1
+
+    def visit(node: ast.AST, depth: int) -> None:
+        indent = "    " * depth
+        for child in ast.iter_child_nodes(node):
+            if isinstance(child, (ast.Import, ast.ImportFrom)) and depth == 0:
+                out.append(lines[child.lineno - 1])
+            elif isinstance(child, ast.ClassDef):
+                out.extend(signature_lines(child))
+                emit_docstring(child, indent + "    ")
+                visit(child, depth + 1)
+            elif isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef)):
+                if id(child) in verbatim_ids:
+                    start = min(
+                        [d.lineno for d in child.decorator_list] + [child.lineno]
+                    )
+                    end = getattr(child, "end_lineno", child.lineno)
+                    out.extend(lines[start - 1 : end])
+                else:
+                    out.extend(signature_lines(child))
+                    emit_docstring(child, indent + "    ")
+                    out.append(f"{indent}    ... # {body_size(child)} lines")
+            elif isinstance(child, (ast.Assign, ast.AnnAssign)) and depth == 0:
+                src_line = lines[child.lineno - 1]
+                out.append(src_line if len(src_line) < 120 else src_line[:117] + "...")
+
+    visit(tree, 0)
+    if not out:
+        return None
+    skeleton = "\n".join(out)
+    if len(skeleton) >= len(text) * 0.85:
+        return None  # not worth it
+    total = len(lines)
+    return f"# skeleton of {total}-line python file (bodies elided)\n{skeleton}"
+
+
+def _generic_skeleton(text: str) -> Optional[str]:
+    lines = text.splitlines()
+    out: List[str] = []
+    elided = 0
+
+    def flush() -> None:
+        nonlocal elided
+        if elided:
+            out.append(f"    ... // {elided} lines elided")
+            elided = 0
+
+    for ln in lines:
+        stripped = ln.strip()
+        indent = len(ln) - len(ln.lstrip())
+        is_structural = (
+            _DECL_RE.match(ln)
+            or stripped.startswith(("//", "/*", "*", "#", "import ", "package ", "using "))
+            or (indent == 0 and stripped in ("}", "};"))
+        )
+        if is_structural:
+            flush()
+            out.append(ln)
+        elif indent <= 2 and stripped.endswith("{"):
+            flush()
+            out.append(ln)
+        else:
+            elided += 1
+    flush()
+    if not out or len("\n".join(out)) >= len(text) * 0.85:
+        return None
+    return f"// skeleton of {len(lines)}-line file (bodies elided)\n" + "\n".join(out)

slimctx/json_compressor.py

@@ -0,0 +1,234 @@
+"""JSON compression: lossless tabularization first, relevance-ranked
+row selection second.
+
+Stage 1 (lossless): a homogeneous array of objects repeats every key name
+in every element. Re-emitting it as a header + rows table removes that
+redundancy without dropping a single value:
+
+    [{"id": 1, "name": "a", "ok": true}, ...]   ->
+    #table cols=id|name|ok rows=200
+    1|a|true
+    ...
+
+Nested objects are flattened to dotted columns (meta.region) when uniform.
+
+Stage 2 (lossy, only if a row budget is given and exceeded): rank rows and
+keep the best under budget. A row is kept when it (a) contains an error /
+salient value, (b) is a statistical outlier on any numeric column (>2
+sigma), (c) is in the head or tail (positional anchors), or (d) scores
+high BM25 relevance against the query. Kept rows keep original order.
+
+Everything else (heterogeneous arrays, scalars, deep trees) is minified
+(no whitespace) — also lossless.
+"""
+
+from __future__ import annotations
+
+import json
+import math
+from typing import Any, Dict, List, Optional, Sequence, Tuple
+
+from .relevance import Bm25, salience_score
+
+CORE_FIELD_THRESHOLD = 0.8  # key must appear in >=80% of rows to be a column
+HEAD_KEEP = 3
+TAIL_KEEP = 2
+
+
+def try_parse(text: str) -> Optional[Any]:
+    stripped = text.strip()
+    if not stripped or stripped[0] not in "[{":
+        return None
+    try:
+        return json.loads(stripped)
+    except (json.JSONDecodeError, ValueError, RecursionError):
+        # RecursionError: pathologically nested input (e.g. 100k open
+        # brackets) must degrade to "not JSON", not crash the pipeline.
+        return None
+
+
+def _flatten(obj: Dict[str, Any], prefix: str = "", max_depth: int = 2) -> Dict[str, Any]:
+    flat: Dict[str, Any] = {}
+    for k, v in obj.items():
+        key = f"{prefix}{k}"
+        if isinstance(v, dict) and max_depth > 0 and 0 < len(v) <= 6:
+            flat.update(_flatten(v, f"{key}.", max_depth - 1))
+        else:
+            flat[key] = v
+    return flat
+
+
+def _find_object_array(data: Any) -> Optional[Tuple[List[Dict[str, Any]], Optional[str]]]:
+    """Return (array, wrapper_key) if data is/contains an array of objects."""
+    if isinstance(data, list):
+        if len(data) >= 2 and all(isinstance(x, dict) for x in data):
+            return data, None
+        return None
+    if isinstance(data, dict):
+        # common API shapes: {"items": [...]}, {"results": [...]}, etc.
+        candidates = [
+            (k, v) for k, v in data.items()
+            if isinstance(v, list) and len(v) >= 2 and all(isinstance(x, dict) for x in v)
+        ]
+        if len(candidates) == 1:
+            k, v = candidates[0]
+            return v, k
+    return None
+
+
+def _cell(value: Any) -> str:
+    if value is None:
+        return ""
+    if isinstance(value, bool):
+        return "true" if value else "false"
+    if isinstance(value, (int, float)):
+        return json.dumps(value)
+    if isinstance(value, str):
+        return value.replace("|", "\\|").replace("\n", "\\n")
+    return json.dumps(value, separators=(",", ":"))
+
+
+def _numeric_outlier_rows(rows: List[Dict[str, Any]], cols: Sequence[str]) -> set:
+    """Indices of rows that are >2 sigma from the mean on any numeric column."""
+    outliers: set = set()
+    for col in cols:
+        values = []
+        for i, row in enumerate(rows):
+            v = row.get(col)
+            if isinstance(v, (int, float)) and not isinstance(v, bool):
+                values.append((i, float(v)))
+        if len(values) < 8:
+            continue
+        nums = [v for _, v in values]
+        mean = sum(nums) / len(nums)
+        var = sum((x - mean) ** 2 for x in nums) / len(nums)
+        std = math.sqrt(var)
+        if std == 0:
+            continue
+        for i, v in values:
+            if abs(v - mean) > 2 * std:
+                outliers.add(i)
+    return outliers
+
+
+def tabularize(
+    text: str,
+    query: str = "",
+    max_rows: Optional[int] = None,
+) -> Optional[Tuple[str, bool]]:
+    """Compress a JSON string. Returns (compressed, was_lossy) or None if
+    the input is not JSON we can improve."""
+    data = try_parse(text)
+    if data is None:
+        return None
+
+    found = _find_object_array(data)
+    if found is None:
+        minified = json.dumps(data, separators=(",", ":"), ensure_ascii=False)
+        if len(minified) < len(text) * 0.9:
+            return minified, False
+        return None
+
+    rows_raw, wrapper = found
+    rows = [_flatten(r) for r in rows_raw]
+
+    # Determine core columns (present in >= threshold of rows).
+    freq: Dict[str, int] = {}
+    for row in rows:
+        for k in row:
+            freq[k] = freq.get(k, 0) + 1
+    n = len(rows)
+    cols = [k for k, c in freq.items() if c >= n * CORE_FIELD_THRESHOLD]
+    if len(cols) < 2:
+        return None  # too heterogeneous to tabularize profitably
+    # Stable column order: order of first appearance.
+    seen_order: List[str] = []
+    for row in rows:
+        for k in row:
+            if k in cols and k not in seen_order:
+                seen_order.append(k)
+    cols = seen_order
+
+    # Constant-column extraction (lossless): a column whose value is
+    # identical in every row is stated once in a legend, not repeated
+    # per row. Common in API payloads (repo, branch, region, type...).
+    consts: Dict[str, Any] = {}
+    for col in list(cols):
+        if freq.get(col) == n:
+            first = rows[0].get(col)
+            if all(row.get(col) == first for row in rows):
+                consts[col] = first
+                cols.remove(col)
+    if len(cols) < 1:
+        return None
+
+    kept_indices = list(range(n))
+    was_lossy = False
+    if max_rows is not None and n > max_rows:
+        kept_indices = _select_rows(rows, cols, query, max_rows)
+        was_lossy = True
+
+    lines = []
+    label = f" key={wrapper}" if wrapper else ""
+    shown = len(kept_indices)
+    row_note = f"rows={n}" if not was_lossy else f"rows={shown}of{n}"
+    lines.append(f"#table{label} cols={'|'.join(cols)} {row_note}")
+    if consts:
+        legend = " | ".join(f"{k}={_cell(v)}" for k, v in consts.items())
+        lines.append(f"#every-row: {legend}")
+    prev = -1
+    for i in kept_indices:
+        if was_lossy and prev >= 0 and i != prev + 1:
+            lines.append(f"…({i - prev - 1} rows omitted)…")
+        row = rows[i]
+        cells = [_cell(row.get(c)) for c in cols]
+        extras = {k: v for k, v in row.items() if k not in cols and k not in consts}
+        line = "|".join(cells)
+        if extras:
+            line += " +" + json.dumps(extras, separators=(",", ":"), ensure_ascii=False)
+        lines.append(line)
+        prev = i
+    return "\n".join(lines), was_lossy
+
+
+def _select_rows(
+    rows: List[Dict[str, Any]], cols: Sequence[str], query: str, max_rows: int
+) -> List[int]:
+    n = len(rows)
+    keep: set = set(range(min(HEAD_KEEP, n))) | set(range(max(0, n - TAIL_KEEP), n))
+
+    row_texts = [json.dumps(r, ensure_ascii=False) for r in rows]
+    # Salience pins a row only when it is *distinctive*: a keyword that
+    # appears in most rows (e.g. `timeout=30` in every signature) carries
+    # no information about which rows matter.
+    salient = [i for i, txt in enumerate(row_texts) if salience_score(txt) >= 0.75]
+    if len(salient) <= max(3, n // 5):
+        keep.update(salient)
+    keep |= _numeric_outlier_rows(rows, cols)
+
+    if query:
+        bm25 = Bm25(row_texts)
+        scored = sorted(
+            ((bm25.score(i, query), i) for i in range(n) if i not in keep),
+            reverse=True,
+        )
+        for score, i in scored:
+            if score <= 0.15:
+                break
+            # Strong matches always earn a slot, even past the row budget —
+            # the row that answers the query is the last thing to sacrifice.
+            if score >= 0.5 or len(keep) < max_rows:
+                keep.add(i)
+            elif len(keep) >= max_rows:
+                break
+
+    # Fill remaining budget with evenly spaced samples for coverage.
+    if len(keep) < max_rows:
+        remaining = [i for i in range(n) if i not in keep]
+        want = max_rows - len(keep)
+        if remaining and want > 0:
+            step = max(1, len(remaining) // want)
+            for i in remaining[::step][:want]:
+                keep.add(i)
+
+    return sorted(keep)

slimctx/log_compressor.py

@@ -0,0 +1,132 @@
+"""Log compression via template mining (Drain-lite).
+
+Agent tool output is dominated by *repeated* log lines that differ only in
+volatile fields (timestamps, IDs, durations). Generic text compressors
+score and drop individual sentences; they can't exploit this structure.
+We can:
+
+    2026-07-19T10:00:01Z GET /api/users/8231 200 12ms
+    2026-07-19T10:00:02Z GET /api/users/9917 200 9ms
+    ... x 1430 more ...
+
+becomes one line:
+
+    <TS> GET /api/users/<NUM> 200 <NUM>ms   [x1432]
+
+Algorithm:
+1. Mask volatile tokens per line (timestamps, UUIDs, hex, numbers, IPs).
+2. Group lines by masked template.
+3. Emit each template once with its count and 1-2 verbatim examples.
+4. High-salience lines (errors, warnings, tracebacks) are NEVER collapsed —
+   they pass through verbatim, in original order, because they are the
+   signal the model is being asked to find.
+
+Lossy — so the original is always placed in the store first.
+"""
+
+from __future__ import annotations
+
+import re
+from collections import OrderedDict
+from dataclasses import dataclass, field
+from typing import List
+
+from .relevance import salience_score
+
+_VOLATILE = [
+    (re.compile(r"\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?"), "<TS>"),
+    (re.compile(r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b"), "<UUID>"),
+    (re.compile(r"\b(?:\d{1,3}\.){3}\d{1,3}(?::\d+)?\b"), "<IP>"),
+    (re.compile(r"\b[0-9a-fA-F]{12,}\b"), "<HEX>"),
+    # No trailing \b: numbers fused to units ("12ms", "512Mi") must mask
+    # too, or every line becomes its own template.
+    (re.compile(r"\d+(?:\.\d+)?"), "<NUM>"),
+]
+
+# A template must repeat at least this many times before we collapse it;
+# below that, collapsing costs more tokens (marker overhead) than it saves.
+MIN_REPEAT = 3
+
+
+def _template_of(line: str) -> str:
+    for pattern, token in _VOLATILE:
+        line = pattern.sub(token, line)
+    return line
+
+
+@dataclass
+class _Group:
+    template: str
+    count: int = 0
+    first_example: str = ""
+    last_example: str = ""
+    first_index: int = 0
+    salient_lines: List[str] = field(default_factory=list)
+
+
+def looks_like_log(text: str) -> bool:
+    """Heuristic: many short lines, high proportion with timestamps/levels."""
+    lines = [ln for ln in text.splitlines() if ln.strip()]
+    if len(lines) < 10:
+        return False
+    hits = 0
+    probe = lines[:50]
+    level_re = re.compile(r"\b(INFO|DEBUG|WARN|WARNING|ERROR|FATAL|TRACE)\b")
+    for ln in probe:
+        if _VOLATILE[0][0].search(ln) or level_re.search(ln):
+            hits += 1
+    return hits >= len(probe) * 0.4
+
+
+def compress_log(text: str, max_examples_per_template: int = 1) -> str:
+    """Collapse repeated log templates; keep salient lines verbatim."""
+    lines = text.splitlines()
+    groups: "OrderedDict[str, _Group]" = OrderedDict()
+    salient: List[tuple] = []  # (index, line) — kept verbatim, in order
+
+    for i, line in enumerate(lines):
+        if not line.strip():
+            continue
+        if salience_score(line) >= 0.6:
+            salient.append((i, line))
+            continue
+        tpl = _template_of(line)
+        grp = groups.get(tpl)
+        if grp is None:
+            grp = _Group(template=tpl, first_example=line, first_index=i)
+            groups[tpl] = grp
+        grp.count += 1
+        grp.last_example = line
+
+    out: List[str] = []
+    emitted_salient = 0
+
+    # Interleave: emit groups in first-appearance order, and salient lines
+    # at their original positions relative to the groups around them.
+    events: List[tuple] = []
+    for grp in groups.values():
+        events.append((grp.first_index, "group", grp))
+    for idx, line in salient:
+        events.append((idx, "salient", line))
+    events.sort(key=lambda e: e[0])
+
+    for _, kind, payload in events:
+        if kind == "salient":
+            out.append(payload)
+            emitted_salient += 1
+        else:
+            grp = payload
+            if grp.count < MIN_REPEAT:
+                out.append(grp.first_example)
+                if grp.count == 2:
+                    out.append(grp.last_example)
+            else:
+                out.append(f"{grp.template}   [x{grp.count}]")
+                if max_examples_per_template >= 1:
+                    out.append(f"  e.g. {grp.first_example}")
+
+    header = (
+        f"[log summary: {len(lines)} lines -> {len(out)} shown; "
+        f"{len(groups)} unique patterns; {emitted_salient} error/warn lines kept verbatim]"
+    )
+    return header + "\n" + "\n".join(out)