python-token-killer 0.1.0__py3-none-any.whl

ptk/minimizers/_dict.py

@@ -0,0 +1,167 @@
+"""Dict/JSON minimizer — biggest token wins live here."""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from ptk._base import Minimizer, strip_nullish
+
+
+class DictMinimizer(Minimizer):
+    """Compress dicts via null-stripping, key shortening, and compact serialization.
+
+    Strategies applied in order:
+    1. Strip nullish values (None, "", [], {})
+    2. Flatten single-child nesting  (aggressive)
+    3. Shorten keys to abbreviated forms (aggressive)
+    4. Serialize with minimal separators
+
+    The `format` kwarg controls output encoding:
+      - "json"    (default) — compact json.dumps
+      - "kv"      — key:value lines (great for flat dicts)
+      - "tabular" — header-once tabular format (auto for list-of-dicts values)
+    """
+
+    def _minimize(self, obj: Any, *, aggressive: bool = False, **kw: Any) -> str:
+        if not isinstance(obj, dict):
+            obj = {"_": obj}
+
+        d = strip_nullish(obj)
+
+        if aggressive:
+            d = _flatten_single_children(d)
+            d = _shorten_dotted_keys(d)
+            d = _shorten_keys(d)
+
+        fmt = kw.get("format", "json")
+        if fmt == "kv":
+            return _to_kv(d)
+        if fmt == "tabular":
+            return _to_tabular(d)
+        return json.dumps(d, separators=(",", ":"), default=str)
+
+
+# ── internal helpers ────────────────────────────────────────────────────
+
+
+def _flatten_single_children(d: dict[str, Any], _depth: int = 0) -> dict[str, Any]:
+    """Collapse {"a": {"b": val}} → {"a.b": val} up to 4 levels."""
+    if _depth > 4:
+        return d
+    out: dict[str, Any] = {}
+    for k, v in d.items():
+        if isinstance(v, dict) and len(v) == 1:
+            inner_k, inner_v = next(iter(v.items()))
+            flattened = _flatten_single_children({f"{k}.{inner_k}": inner_v}, _depth + 1)
+            out.update(flattened)
+        else:
+            out[k] = v
+    return out
+
+
+# common verbose key → short form (extensible)
+_KEY_MAP: dict[str, str] = {
+    "description": "desc",
+    "message": "msg",
+    "timestamp": "ts",
+    "created_at": "ts",
+    "updated_at": "upd",
+    "configuration": "cfg",
+    "config": "cfg",
+    "environment": "env",
+    "database": "db",
+    "information": "info",
+    "response": "resp",
+    "request": "req",
+    "function": "fn",
+    "parameters": "params",
+    "arguments": "args",
+    "exception": "exc",
+    "traceback": "tb",
+    "status_code": "code",
+    "content_type": "ctype",
+    "application": "app",
+    "transaction": "txn",
+    "identifier": "id",
+    "metadata": "meta",
+    "properties": "props",
+    "connection": "conn",
+    "password": "pw",
+    "username": "user",
+    "directory": "dir",
+    "reference": "ref",
+    "implementation": "impl",
+    "notifications": "notifs",
+    "repository": "repo",
+}
+
+
+def _shorten_keys(d: dict[str, Any]) -> dict[str, Any]:
+    """Recursively shorten known verbose keys.
+
+    If two keys map to the same short form (e.g. 'timestamp' and 'created_at'
+    both → 'ts'), the FIRST key wins and the second is kept unshortened to
+    prevent silent data loss.
+    """
+    out: dict[str, Any] = {}
+    used_shorts: set[str] = set()
+    for k, v in d.items():
+        short = _KEY_MAP.get(k, k)
+        # avoid collision: if short form already used, keep original key
+        if short in used_shorts and short != k:
+            short = k
+        used_shorts.add(short)
+        if isinstance(v, dict):
+            out[short] = _shorten_keys(v)
+        elif isinstance(v, list):
+            out[short] = [_shorten_keys(i) if isinstance(i, dict) else i for i in v]
+        else:
+            out[short] = v
+    return out
+
+
+def _shorten_dotted_keys(d: dict[str, Any]) -> dict[str, Any]:
+    """Shorten individual segments of dotted keys (from flattening)."""
+    out: dict[str, Any] = {}
+    for k, v in d.items():
+        if isinstance(k, str) and "." in k:
+            parts = [_KEY_MAP.get(p, p) for p in k.split(".")]
+            out[".".join(parts)] = v
+        else:
+            out[k] = v
+    return out
+
+
+def _to_kv(d: dict[str, Any], _prefix: str = "") -> str:
+    """Flat key:value format — one line per leaf."""
+    lines: list[str] = []
+    for k, v in d.items():
+        full = f"{_prefix}{k}"
+        if isinstance(v, dict):
+            lines.append(_to_kv(v, f"{full}."))
+        else:
+            lines.append(f"{full}:{v}")
+    return "\n".join(lines)
+
+
+def _to_tabular(d: dict[str, Any]) -> str:
+    """Render any list-of-dicts values as header-once tabular rows.
+
+    Non-list values render as kv pairs above the table.
+    """
+    kv_lines: list[str] = []
+    table_lines: list[str] = []
+
+    for k, v in d.items():
+        if isinstance(v, list) and v and isinstance(v[0], dict):
+            # collect union of all keys (preserving order)
+            fields: list[str] = list(dict.fromkeys(f for row in v for f in row))
+            table_lines.append(f"{k}[{len(v)}]{{{','.join(fields)}}}:")
+            for row in v:
+                vals = (str(row.get(f, "")) for f in fields)
+                table_lines.append(f"  {','.join(vals)}")
+        else:
+            kv_lines.append(f"{k}:{v}")
+
+    return "\n".join(kv_lines + table_lines)

ptk/minimizers/_diff.py

@@ -0,0 +1,83 @@
+"""Diff minimizer — fold unchanged context, keep only meaningful changes."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from ptk._base import Minimizer
+
+
+class DiffMinimizer(Minimizer):
+    """Compress git diffs by folding unchanged context lines.
+
+    Strategies:
+    1. Keep hunk headers (@@) and file headers (---, +++)
+    2. Keep added (+) and removed (-) lines
+    3. Collapse context (space-prefixed) lines to `... N lines ...`
+    4. Aggressive: strip file mode changes, index lines, trailing whitespace diffs
+    """
+
+    # max unchanged context lines to keep around changes
+    CONTEXT_LINES = 2
+
+    def _minimize(self, obj: Any, *, aggressive: bool = False, **kw: Any) -> str:
+        text = obj if isinstance(obj, str) else str(obj)
+        ctx = kw.get("context_lines", 0 if aggressive else self.CONTEXT_LINES)
+        return _fold_diff(text, context=ctx, aggressive=aggressive)
+
+
+def _fold_diff(text: str, *, context: int, aggressive: bool) -> str:
+    lines = text.split("\n")
+    result: list[str] = []
+    context_buf: list[str] = []
+
+    for line in lines:
+        # always skip noise in aggressive mode
+        if aggressive and _is_noise(line):
+            continue
+
+        if _is_significant(line):
+            # flush context buffer — keep only last `context` lines
+            if context_buf:
+                _flush_context(result, context_buf, keep=context)
+                context_buf.clear()
+            result.append(line)
+        else:
+            context_buf.append(line)
+
+    # flush remaining context
+    if context_buf:
+        _flush_context(result, context_buf, keep=context)
+
+    return "\n".join(result)
+
+
+def _is_significant(line: str) -> bool:
+    """Lines that carry meaningful diff information."""
+    return (
+        line.startswith(("+", "-", "@@", "diff ", "\\ ")) and not line.startswith(("+++", "---"))
+    ) or line.startswith(("+++", "---"))
+
+
+def _is_noise(line: str) -> bool:
+    """Lines that are almost never useful to an LLM."""
+    return (
+        line.startswith(("index ", "old mode", "new mode", "similarity"))
+        or line.startswith("rename ")
+        or (line.startswith("Binary files") and "differ" in line)
+    )
+
+
+def _flush_context(result: list[str], buf: list[str], *, keep: int) -> None:
+    """Collapse a block of context lines, keeping at most `keep` on each end."""
+    if len(buf) <= keep * 2 + 1:
+        result.extend(buf)
+        return
+
+    if keep > 0:
+        result.extend(buf[:keep])
+    folded = len(buf) - keep * 2
+    if folded > 0:
+        result.append(f" ... {folded} lines ...")
+    if keep > 0:
+        result.extend(buf[-keep:])

ptk/minimizers/_list.py

@@ -0,0 +1,87 @@
+"""List/array minimizer — dedup, sampling, schema-once tabular."""
+
+from __future__ import annotations
+
+import json
+from collections import Counter
+from typing import Any
+
+from ptk._base import Minimizer, strip_nullish
+
+
+class ListMinimizer(Minimizer):
+    """Compress lists via dedup, sampling, and tabular encoding.
+
+    Strategies:
+    1. Strip nullish items
+    2. Deduplicate exact-match items with counts
+    3. For uniform list-of-dicts: schema-once tabular output
+    4. For large arrays: statistical sampling (aggressive)
+    """
+
+    # max items before sampling kicks in (aggressive mode)
+    SAMPLE_THRESHOLD = 50
+
+    def _minimize(self, obj: Any, *, aggressive: bool = False, **kw: Any) -> str:
+        if not isinstance(obj, (list, tuple)):
+            return json.dumps(obj, separators=(",", ":"), default=str)
+
+        items = [strip_nullish(i) if isinstance(i, dict) else i for i in obj if i is not None]
+
+        if not items:
+            return "[]"
+
+        # ── uniform list-of-dicts → tabular ─────────────────────────
+        if isinstance(items[0], dict) and all(isinstance(i, dict) for i in items):
+            if aggressive and len(items) > self.SAMPLE_THRESHOLD:
+                items = _sample(items, self.SAMPLE_THRESHOLD)
+            return _tabular(items)
+
+        # ── primitive list → dedup with counts ──────────────────────
+        if aggressive and len(items) > self.SAMPLE_THRESHOLD:
+            items = items[: self.SAMPLE_THRESHOLD]
+
+        return _dedup_list(items)
+
+
+def _tabular(rows: list[dict[str, Any]]) -> str:
+    """Schema-once tabular: declare fields in header, one CSV-ish row per item."""
+    fields: list[str] = list(dict.fromkeys(f for row in rows for f in row))
+    header = f"[{len(rows)}]{{{','.join(fields)}}}:"
+    body = "\n".join(",".join(str(row.get(f, "")) for f in fields) for row in rows)
+    return f"{header}\n{body}"
+
+
+def _dedup_list(items: list[Any]) -> str:
+    """Collapse duplicate primitives: [a, a, a, b] → a (x3)\nb"""
+    # json-serialize each item for stable hashing
+    serialized = [json.dumps(i, separators=(",", ":"), default=str) for i in items]
+    counts = Counter(serialized)
+
+    # preserve first-seen order
+    seen: set[str] = set()
+    lines: list[str] = []
+    for s in serialized:
+        if s in seen:
+            continue
+        seen.add(s)
+        c = counts[s]
+        # strip quotes from simple strings for readability
+        display = s.strip('"') if s.startswith('"') and s.endswith('"') else s
+        lines.append(f"{display} (x{c})" if c > 1 else display)
+
+    return "\n".join(lines)
+
+
+def _sample(items: list[dict[str, Any]], n: int) -> list[dict[str, Any]]:
+    """Deterministic even-spaced sampling — keeps first and last."""
+    if n <= 0:
+        return []
+    if len(items) <= n:
+        return items
+    if n == 1:
+        return [items[0]]
+    step = (len(items) - 1) / (n - 1)
+    indices = {round(i * step) for i in range(n)}
+    sampled = [items[i] for i in sorted(indices)]
+    return sampled

ptk/minimizers/_log.py

@@ -0,0 +1,94 @@
+"""Log minimizer — dedup repeated lines, error-only filtering."""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+
+from ptk._base import Minimizer, dedup_lines
+
+# precompiled
+_LOG_LEVEL = re.compile(
+    r"\b(ERROR|WARN(?:ING)?|CRITICAL|FATAL|EXCEPTION|SEVERE|PANIC)\b",
+    re.IGNORECASE,
+)
+_TIMESTAMP = re.compile(
+    r"^\d{4}[-/]\d{2}[-/]\d{2}[T ]\d{2}:\d{2}:\d{2}[.,]?\d*\s*(?:[+-]\d{2}:?\d{2}|Z)?\s*",
+    re.MULTILINE,
+)
+# stack trace markers — lines matching these are always preserved
+_STACKTRACE_RE = re.compile(
+    r'^\s*(Traceback \(most recent|File "|\s+raise |'
+    r"\w+Error:|\w+Exception:|\w+Warning:|at \S+\.\S+\()",
+    re.MULTILINE,
+)
+# test-runner pass markers — never kept as context in errors_only mode
+_PASS_MARKERS: frozenset[str] = frozenset(
+    {
+        "PASSED",
+        " PASS",
+        "--- PASS",
+        "... ok",
+        " ok",
+        "test result: ok",
+        "✓",
+    }
+)
+
+
+class LogMinimizer(Minimizer):
+    """Compress log output via deduplication and error filtering.
+
+    Strategies:
+    1. Strip timestamps (aggressive) — timestamps are rarely useful to an LLM
+    2. Collapse consecutive duplicate lines with counts
+    3. Keep only error/warning lines (aggressive)
+    """
+
+    def _minimize(self, obj: Any, *, aggressive: bool = False, **kw: Any) -> str:
+        text = obj if isinstance(obj, str) else str(obj)
+
+        # bail early on blank input
+        if not text.strip():
+            return ""
+
+        if aggressive:
+            text = _TIMESTAMP.sub("", text)
+
+        text = dedup_lines(text)
+
+        if aggressive or kw.get("errors_only", False):
+            text = _errors_only(text)
+
+        return text.strip()
+
+
+def _errors_only(text: str) -> str:
+    """Keep error/warning lines, stack traces, 'failed' keyword lines, + context.
+
+    Pass-marker lines (PASSED, --- PASS, ... ok, etc.) are never kept even
+    as context around errors — they are pure noise in an error report.
+    """
+    lines = text.split("\n")
+    keep: set[int] = set()
+    for i, line in enumerate(lines):
+        ll = line.lower()
+        is_important = (
+            _LOG_LEVEL.search(line)
+            or _STACKTRACE_RE.match(line)
+            or "failed" in ll
+            or "fail:" in ll  # go test: --- FAIL: TestName
+            or " fail " in ll  # standalone FAIL line
+            or "error" in ll  # build errors, compiler output
+            or "panicked" in ll  # rust panic
+            or "assertion" in ll  # assertion errors
+        )
+        if is_important:
+            # keep the important line + 1 line of context before/after,
+            # but exclude pure pass-marker lines from the context window
+            for j in range(max(0, i - 1), min(len(lines), i + 2)):
+                if not any(marker in lines[j] for marker in _PASS_MARKERS):
+                    keep.add(j)
+    if not keep:
+        return text  # no errors found — return deduped version
+    return "\n".join(lines[i] for i in sorted(keep))

ptk/minimizers/_text.py

@@ -0,0 +1,182 @@
+"""Text minimizer — whitespace normalization, stopword removal, abbreviation."""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+
+from ptk._base import Minimizer
+
+# ── precompiled ─────────────────────────────────────────────────────────
+
+_MULTI_SPACE = re.compile(r"[ \t]+")
+_MULTI_NEWLINE = re.compile(r"\n{3,}")
+
+# high-frequency English stopwords that add tokens but rarely carry meaning
+# in LLM context (intentionally conservative — we never want to destroy meaning)
+_STOPWORDS: frozenset[str] = frozenset(
+    {
+        "the",
+        "a",
+        "an",
+        "is",
+        "are",
+        "was",
+        "were",
+        "be",
+        "been",
+        "being",
+        "have",
+        "has",
+        "had",
+        "do",
+        "does",
+        "did",
+        "will",
+        "would",
+        "shall",
+        "should",
+        "may",
+        "might",
+        "must",
+        "can",
+        "could",
+        "that",
+        "which",
+        "who",
+        "whom",
+        "this",
+        "these",
+        "those",
+        "am",
+        "its",
+        "very",
+        "just",
+        "also",
+        "really",
+        "quite",
+        "rather",
+    }
+)
+
+# common long phrases → shorter equivalents
+_PHRASE_ABBREVIATIONS: tuple[tuple[str, str], ...] = (
+    ("in order to", "to"),
+    ("as well as", "&"),
+    ("due to the fact that", "because"),
+    ("in the event that", "if"),
+    ("at this point in time", "now"),
+    ("for the purpose of", "to"),
+    ("in addition to", "also"),
+    ("with regard to", "re:"),
+    ("a large number of", "many"),
+    ("the majority of", "most"),
+    ("in the process of", "while"),
+    ("on the other hand", "however"),
+    ("take into account", "consider"),
+    ("it is important to note that", "note:"),
+    ("please note that", "note:"),
+    ("it should be noted that", "note:"),
+)
+
+# single-word abbreviations (applied via regex word boundaries)
+_WORD_ABBREVIATIONS: dict[str, str] = {
+    "implementation": "impl",
+    "implementations": "impls",
+    "configuration": "config",
+    "configurations": "configs",
+    "production": "prod",
+    "development": "dev",
+    "environment": "env",
+    "environments": "envs",
+    "application": "app",
+    "applications": "apps",
+    "infrastructure": "infra",
+    "authentication": "auth",
+    "authorization": "authz",
+    "repository": "repo",
+    "repositories": "repos",
+    "documentation": "docs",
+    "specification": "spec",
+    "specifications": "specs",
+    "requirements": "reqs",
+    "approximately": "~",
+    "notification": "notif",
+    "notifications": "notifs",
+}
+
+# filler phrases stripped entirely (claw-compactor Abbrev pattern)
+_FILLER_PHRASES: tuple[str, ...] = (
+    "Furthermore, ",
+    "Furthermore,",
+    "In addition, ",
+    "In addition,",
+    "Moreover, ",
+    "Moreover,",
+    "Additionally, ",
+    "Additionally,",
+    "Having said that, ",
+    "Having said that,",
+    "It is worth noting that ",
+    "As mentioned earlier, ",
+    "As previously stated, ",
+)
+
+_WORD_ABBREV_RE = re.compile(
+    r"\b(" + "|".join(re.escape(w) for w in _WORD_ABBREVIATIONS) + r")\b",
+    re.IGNORECASE,
+)
+
+
+class TextMinimizer(Minimizer):
+    """Compress natural language text for LLM consumption.
+
+    Strategies:
+    1. Normalize whitespace (always)
+    2. Apply phrase abbreviations (always)
+    3. Remove stopwords (aggressive) — safe for context/instructions, not prose
+    """
+
+    def _minimize(self, obj: Any, *, aggressive: bool = False, **kw: Any) -> str:
+        text = obj if isinstance(obj, str) else str(obj)
+
+        # normalize whitespace
+        text = _MULTI_SPACE.sub(" ", text)
+        text = _MULTI_NEWLINE.sub("\n\n", text)
+
+        # strip filler phrases
+        for filler in _FILLER_PHRASES:
+            text = text.replace(filler, "")
+
+        # abbreviate common phrases
+        for long, short in _PHRASE_ABBREVIATIONS:
+            text = text.replace(long, short)
+
+        # abbreviate common words (case-preserving)
+        text = _WORD_ABBREV_RE.sub(_word_abbrev_replace, text)
+
+        if aggressive:
+            text = _remove_stopwords(text)
+
+        return text.strip()
+
+
+def _word_abbrev_replace(m: re.Match[str]) -> str:
+    """Replace matched word with its abbreviation, preserving case style."""
+    word = m.group(0)
+    abbrev = _WORD_ABBREVIATIONS[word.lower()]
+    if word.isupper():
+        return abbrev.upper()
+    if word[0].isupper():
+        return abbrev.capitalize()
+    return abbrev
+
+
+def _remove_stopwords(text: str) -> str:
+    """Remove stopwords while preserving line structure."""
+    lines: list[str] = []
+    for line in text.split("\n"):
+        words = line.split()
+        filtered = [w for w in words if w.lower().strip(".,;:!?") not in _STOPWORDS]
+        lines.append(" ".join(filtered))
+    return "\n".join(lines)