kompactor 0.5.0__py3-none-any.whl

kompactor/config.py

@@ -0,0 +1,231 @@
+"""Configuration: a quality-vs-tokens preset, optional per-knob overrides,
+language profiles, and per-project overrides.
+
+Stored in ~/.kompactor/config.json and edited from the dashboard (`kompactor ui`):
+
+    {
+      "preset": 3,
+      "overrides": {"bash_max_lines": 250},
+      "languages": {"mode": "auto", "selected": ["python", "go", "typescript", "react"]},
+      "projects": {"/path/to/repo": {"preset": 4, "overrides": {}}}
+    }
+"""
+
+import json
+import os
+from pathlib import Path
+
+from . import store
+
+PRESETS = {
+    1: {"name": "Max quality", "blurb": "Only trims huge outputs and exact noise. Claude sees almost everything.",
+        "bash": True, "bash_wrap": True, "bash_plain": False, "bash_min_tokens": 1500, "bash_max_lines": 600, "max_line_chars": 2000, "grep_per_file": 50, "test_hide_passing": False, "test_boilerplate": False,
+        "trim_stack_frames": False, "diff_context": -1, "git_log_compact": False, "read_outline": False,
+        "read_outline_lines": 1500, "graph": True, "session_map_tokens": 0, "read_graph_context": False, "read_dedupe": True, "style": "off", "code_style": "normal", "subagent_rules": True, "never_compact": []},
+    2: {"name": "Quality", "blurb": "Compacts noisy commands; big files are still read whole.",
+        "bash": True, "bash_wrap": True, "bash_plain": True, "bash_min_tokens": 1000, "bash_max_lines": 400, "max_line_chars": 1000, "grep_per_file": 25, "test_hide_passing": True, "test_boilerplate": False,
+        "trim_stack_frames": False, "diff_context": -1, "git_log_compact": True, "read_outline": True,
+        "read_outline_lines": 800, "graph": True, "session_map_tokens": 0, "read_graph_context": True, "read_dedupe": True, "style": "lite", "code_style": "normal", "subagent_rules": True, "never_compact": []},
+    3: {"name": "Balanced", "blurb": "Recommended. Outline-first reads, compact tool output, terse replies.",
+        "bash": True, "bash_wrap": True, "bash_plain": True, "bash_min_tokens": 300, "bash_max_lines": 200, "max_line_chars": 500, "grep_per_file": 12, "test_hide_passing": True, "test_boilerplate": False,
+        "trim_stack_frames": True, "diff_context": -1, "git_log_compact": True, "read_outline": True,
+        "read_outline_lines": 400, "graph": True, "session_map_tokens": 500, "read_graph_context": True, "read_dedupe": True, "style": "lite", "code_style": "lean", "subagent_rules": True, "never_compact": []},
+    4: {"name": "Savings", "blurb": "Tighter caps, trimmed diff context, smaller file threshold.",
+        "bash": True, "bash_wrap": True, "bash_plain": True, "bash_min_tokens": 150, "bash_max_lines": 120, "max_line_chars": 300, "grep_per_file": 8, "test_hide_passing": True, "test_boilerplate": True,
+        "trim_stack_frames": True, "diff_context": 3, "git_log_compact": True, "read_outline": True,
+        "read_outline_lines": 250, "graph": True, "session_map_tokens": 700, "read_graph_context": True, "read_dedupe": True, "style": "full", "code_style": "lean", "subagent_rules": True, "never_compact": []},
+    5: {"name": "Max savings", "blurb": "Most aggressive. Claude may need `kompactor show` more often.",
+        "bash": True, "bash_wrap": True, "bash_plain": True, "bash_min_tokens": 80, "bash_max_lines": 80, "max_line_chars": 200, "grep_per_file": 5, "test_hide_passing": True, "test_boilerplate": True,
+        "trim_stack_frames": True, "diff_context": 1, "git_log_compact": True, "read_outline": True,
+        "read_outline_lines": 150, "graph": True, "session_map_tokens": 900, "read_graph_context": True, "read_dedupe": True, "style": "ultra", "code_style": "lean", "subagent_rules": True, "never_compact": []},
+}
+KNOBS = [k for k in PRESETS[3] if k not in ("name", "blurb")]
+
+# UI metadata for every knob: label, help, type and range.
+KNOB_INFO = {
+    "bash": ("Compact command output", "Shrink Bash/terminal output before Claude reads it.", "bool"),
+    "bash_wrap": ("Compact failing commands too", "Claude Code won't let hooks touch the output of a command that "
+                  "fails, and cuts long failure output down to its first and last few thousand characters, often "
+                  "losing the actual errors. With this on, tests, builds, linters and installs run through Kompactor "
+                  "(Claude Code's shell prefix setting) so failures arrive compacted and complete, with the same exit "
+                  "code. Servers and watch modes are never touched.", "bool"),
+    "bash_plain": ("Let Kompactor trim command output", "Tells Claude to run commands without `| head`, `| tail` or "
+                   "`| grep`. Kompactor's compaction keeps every failure, while Claude's own trimming can cut the one "
+                   "line that mattered and lead to a re-run. Only used while failing commands are compacted too "
+                   "(shell prefix active).", "bool"),
+    "bash_min_tokens": ("Leave outputs smaller than", "Outputs under this many tokens are passed through untouched.",
+                        "int", 0, 5000, 50),
+    "bash_max_lines": ("Max lines kept per output", "Head and tail are kept; error lines from the middle are always kept.",
+                       "int", 40, 1000, 10),
+    "max_line_chars": ("Longest line kept", "Longer lines (minified code, giant JSON) are clipped with a '+N chars' note.",
+                       "int", 120, 4000, 20),
+    "grep_per_file": ("Search matches per file", "grep/rg matches shown per file before '+N more'.", "int", 1, 100, 1),
+    "test_hide_passing": ("Hide passing tests", "Keep failures and the summary; drop the passing-test lines.", "bool"),
+    "test_boilerplate": ("Drop test-runner headers", "Remove session banners, separators and timing lines.", "bool"),
+    "trim_stack_frames": ("Trim library stack frames",
+                          "Collapse site-packages / node_modules / Go runtime frames in tracebacks.", "bool"),
+    "diff_context": ("Diff context lines", "Unchanged lines kept around each change in diffs (-1 = all).", "int", -1, 10, 1),
+    "git_log_compact": ("One-line git log", "Turn verbose git log into 'sha date author: subject'.", "bool"),
+    "read_outline": ("Outline-first file reads",
+                     "First full read of a big file returns its structure; reading again returns the whole file.", "bool"),
+    "read_outline_lines": ("Outline files longer than", "Line count from which a file is outlined first.",
+                           "int", 50, 3000, 50),
+    "graph": ("Code graph", "Index functions, classes, imports and calls so Claude can find code without reading "
+              "whole files (`kompactor graph query`).", "bool"),
+    "session_map_tokens": ("Code map", "Token budget for a map of the most depended-on modules, given to Claude the "
+                           "first time it searches or reads code in a session (0 = off). Sessions that only run "
+                           "commands don't pay for it.", "int", 0, 3000, 100),
+    "read_graph_context": ("Show usages in outlines", "Outlines also say which modules import the file and which "
+                           "tests cover it.", "bool"),
+    "read_dedupe": ("Skip unchanged re-reads", "Re-reading an unchanged file gets a short note (once).", "bool"),
+    "style": ("Prose style", "How Claude writes: off, lite (answer first, no recap), full (short sentences, no "
+              "narration), ultra (telegraphic). Code, commands and errors are never shortened.",
+              "choice", ["off", "lite", "full", "ultra"]),
+    "code_style": ("Code style", "How much code Claude writes: normal, lean (reuse first, smallest diff, check callers "
+                   "and tests), strict (also proposes a smaller alternative before building).",
+                   "choice", ["normal", "lean", "strict"]),
+    "subagent_rules": ("Apply rules to subagents", "Subagents get the same prose and code rules.", "bool"),
+    "never_compact": ("Never compact these commands", "Commands whose output is always passed through untouched, "
+                      "comma separated (e.g. terraform plan, kubectl describe).", "list"),
+}
+
+GENERAL = {"turn_summary": True, "statusline": True, "retention_days": 30}
+
+DEFAULT_FILE = {"preset": 3, "overrides": {}, "languages": {"mode": "auto", "selected": []}, "projects": {},
+                "general": dict(GENERAL)}
+
+
+def path() -> Path:
+    return store.home() / "config.json"
+
+
+def read_file() -> dict:
+    data = json.loads(json.dumps(DEFAULT_FILE))
+    p = path()
+    if p.exists():
+        try:
+            raw = json.loads(p.read_text())
+        except (OSError, ValueError):
+            raw = {}
+        if "preset" not in raw and any(k in raw for k in KNOBS):  # v0.1 flat format
+            raw = {"overrides": {k: v for k, v in raw.items() if k in KNOBS}}
+        for k, v in raw.items():
+            if isinstance(v, dict) and isinstance(data.get(k), dict):
+                data[k].update(v)
+            else:
+                data[k] = v
+    return data
+
+
+def write_file(data: dict) -> None:
+    p = path()
+    tmp = p.with_suffix(".tmp")
+    tmp.write_text(json.dumps(data, indent=2))
+    os.replace(tmp, p)
+
+
+def project_root(cwd: str) -> str:
+    """Nearest ancestor with a VCS or project marker; cwd itself otherwise."""
+    if not cwd:
+        return ""
+    p = Path(cwd).resolve()
+    for d in [p, *p.parents]:
+        if any((d / m).exists() for m in (".git", "go.mod", "pyproject.toml", "package.json", ".hg")):
+            return str(d)
+    return str(p)
+
+
+def coerce(key: str, value):
+    info = KNOB_INFO.get(key)
+    if not info:
+        return value
+    if info[2] == "bool":
+        return value if isinstance(value, bool) else str(value).lower() in ("1", "true", "yes", "on")
+    if info[2] == "int":
+        return max(info[3], min(info[4], int(value)))
+    if info[2] == "choice" and value not in info[3]:
+        raise ValueError(f"{key} must be one of {info[3]}")
+    if info[2] == "list":
+        items = value if isinstance(value, list) else str(value).split(",")
+        return [str(v).strip() for v in items if str(v).strip()][:50]
+    return value
+
+
+REPO_FILE = ".kompactor.json"  # committed to a repo: the level and settings for everyone working in it
+
+
+def repo_settings(root: str) -> dict:
+    """{"preset": N, "overrides": {...}} from the repo's .kompactor.json, validated; {} if none."""
+    try:
+        d = json.loads((Path(root) / REPO_FILE).read_text())
+    except (OSError, ValueError, TypeError):
+        return {}
+    if not isinstance(d, dict):
+        return {}
+    out = {}
+    if str(d.get("preset", "")).isdigit() and int(d["preset"]) in PRESETS:
+        out["preset"] = int(d["preset"])
+    ov = {}
+    for k, v in (d.get("overrides") or {}).items() if isinstance(d.get("overrides"), dict) else ():
+        if k in KNOBS:
+            try:
+                ov[k] = coerce(k, v)
+            except (ValueError, TypeError):
+                pass
+    if ov:
+        out["overrides"] = ov
+    return out
+
+
+def load(cwd: str = "") -> dict:
+    """Effective settings for a working directory, most specific last:
+    your level and overrides ⊕ the repo's .kompactor.json ⊕ your settings for this project ⊕ env."""
+    data = read_file()
+    root = project_root(cwd) if cwd else ""
+    proj = data["projects"].get(root, {}) if root else {}
+    repo = repo_settings(root) if root else {}
+    preset = int(proj.get("preset", repo.get("preset", data.get("preset", 3))))
+    preset = preset if preset in PRESETS else 3
+    cfg = {k: PRESETS[preset][k] for k in KNOBS}
+    cfg.update({k: v for k, v in data.get("overrides", {}).items() if k in KNOBS})
+    cfg.update(repo.get("overrides", {}))
+    cfg.update({k: v for k, v in proj.get("overrides", {}).items() if k in KNOBS})
+    for k in KNOBS:
+        env = os.environ.get(f"KOMPACTOR_{k.upper()}")
+        if env is not None:
+            try:
+                cfg[k] = coerce(k, env)
+            except ValueError:
+                pass
+    if os.environ.get("KOMPACTOR_PRESET", "").isdigit() and int(os.environ["KOMPACTOR_PRESET"]) in PRESETS:
+        preset = int(os.environ["KOMPACTOR_PRESET"])
+        cfg.update({k: PRESETS[preset][k] for k in KNOBS})
+    cfg["preset"] = preset
+    cfg["project"] = root
+    cfg.update({k: v for k, v in data.get("general", {}).items() if k in GENERAL})
+
+    from . import langs
+    lang_cfg = data.get("languages", {})
+    if lang_cfg.get("mode") == "manual" and lang_cfg.get("selected"):
+        cfg["languages"] = list(lang_cfg["selected"])
+    else:
+        cfg["languages"] = langs.detect(root) if root else []
+    return cfg
+
+
+def save(key: str, value) -> None:
+    """CLI helper: `kompactor config preset 4` or `kompactor config bash_max_lines 150`."""
+    data = read_file()
+    if key == "preset":
+        data["preset"] = int(value)
+    elif key in KNOBS:
+        data["overrides"][key] = coerce(key, value)
+    elif key in GENERAL:
+        data["general"][key] = int(value) if isinstance(GENERAL[key], int) and not isinstance(GENERAL[key], bool) \
+            else coerce("bash", value)
+    elif key == "languages":
+        data["languages"] = ({"mode": "auto", "selected": []} if value == "auto"
+                             else {"mode": "manual", "selected": [v.strip() for v in str(value).split(",") if v.strip()]})
+    else:
+        raise KeyError(key)
+    write_file(data)

kompactor/estimate.py

@@ -0,0 +1,145 @@
+"""Replay past Claude Code sessions through Kompactor to estimate savings.
+
+Reads ~/.claude/projects/**/*.jsonl, finds every Bash and Read tool result,
+and runs them through the same filters the hooks use. A token removed from a
+tool result is not just saved once: it would have been re-sent (as a cache
+read) on every later API call in that session until the next compaction. The
+estimate accounts for that, which is why context savings compound.
+"""
+
+import glob
+import json
+import os
+import re
+from collections import defaultdict
+from pathlib import Path
+
+from . import filters, outline, tokens
+
+# Relative price weights (input = 1). Same ratios across current Claude models.
+W_INPUT, W_CACHE_WRITE, W_CACHE_READ, W_OUTPUT = 1.0, 1.25, 0.1, 5.0
+LINE_NO = re.compile(r"^\s*\d+[\t→]", re.M)
+# After an outline, assume Claude still reads this share of the file by range.
+OUTLINE_FOLLOWUP = 0.4
+
+
+def _text(content) -> str:
+    if isinstance(content, str):
+        return content
+    if isinstance(content, list):
+        return "\n".join(b.get("text", "") for b in content if isinstance(b, dict) and b.get("type") == "text")
+    return ""
+
+
+def transcripts(root=None, project=None):
+    root = Path(root or Path.home() / ".claude" / "projects")
+    pattern = f"{glob.escape(str(root / project)) if project else glob.escape(str(root)) + '/*'}/**/*.jsonl"
+    return sorted(glob.glob(pattern, recursive=True), key=os.path.getmtime)
+
+
+def analyze_file(path: str, stats: dict, opts=None) -> None:
+    calls = []            # ordered unique API calls: (usage dict)
+    seen_ids = set()
+    pending = {}          # tool_use_id -> (name, input)
+    results = []          # (call_index_when_ingested, name, input, text, is_error)
+    boundaries = []       # call indexes where compaction happened
+    with open(path, errors="replace") as f:
+        for line in f:
+            try:
+                d = json.loads(line)
+            except ValueError:
+                continue
+            if d.get("type") == "system" and d.get("subtype") == "compact_boundary":
+                boundaries.append(len(calls))
+            msg = d.get("message")
+            if not isinstance(msg, dict):
+                continue
+            if d.get("type") == "assistant":
+                mid = msg.get("id") or d.get("requestId")
+                if mid and mid not in seen_ids and isinstance(msg.get("usage"), dict):
+                    seen_ids.add(mid)
+                    calls.append(msg["usage"])
+                for b in msg.get("content") or []:
+                    if isinstance(b, dict) and b.get("type") == "tool_use":
+                        pending[b.get("id")] = (b.get("name"), b.get("input") or {})
+            elif d.get("type") == "user" and isinstance(msg.get("content"), list):
+                for b in msg["content"]:
+                    if isinstance(b, dict) and b.get("type") == "tool_result" and b.get("tool_use_id") in pending:
+                        name, inp = pending.pop(b["tool_use_id"])
+                        results.append((len(calls), name, inp, _text(b.get("content")), bool(b.get("is_error"))))
+    if not calls:
+        return
+
+    stats["sessions"] += 1
+    for u in calls:
+        i, cw, cr, o = (u.get("input_tokens", 0) or 0, u.get("cache_creation_input_tokens", 0) or 0,
+                        u.get("cache_read_input_tokens", 0) or 0, u.get("output_tokens", 0) or 0)
+        stats["billed"] += i + cw + cr + o
+        stats["cost"] += i * W_INPUT + cw * W_CACHE_WRITE + cr * W_CACHE_READ + o * W_OUTPUT
+        stats["output"] += o
+        stats["calls"] += 1
+
+    def lifetime(k):
+        """Number of API calls that would re-send a result first ingested at call k."""
+        end = next((b for b in boundaries if b > k), len(calls))
+        return max(1, end - k)
+
+    reads = {}  # path -> (signature, text) of last read, for dedupe
+    for k, name, inp, text, is_err in results:
+        n = lifetime(k)
+        t = tokens.count(text)
+        stats["tool_tokens"] += t
+        stats["tool_by_name"][name] += t
+        saved, feature = 0, None
+        if name == "Bash" and t >= (opts or {}).get("bash_min_tokens", 150):
+            cmd = inp.get("command", "")
+            new = filters.compress(cmd, text, 1 if is_err else 0, stash_id=1, opts=opts)
+            kept = tokens.count(new) + 15  # + the "[kompactor: …]" footer
+            if kept < t * 0.9:
+                saved, feature = t - kept, "bash:" + filters.classify(cmd)
+                stats["top_cmds"].append((saved * n, cmd[:80], t, kept))
+        elif name == "Read":
+            p = inp.get("file_path", "")
+            key = (p, inp.get("offset"), inp.get("limit"))
+            if key in reads and reads[key] == text and (opts or {}).get("read_dedupe", True):
+                saved, feature = max(0, t - 60), "read:dedupe"
+            elif (opts or {}).get("read_outline", True) and inp.get("offset") is None and inp.get("limit") is None \
+                    and os.path.splitext(p)[1] in outline.SOURCE_EXT:
+                src = LINE_NO.sub("", text)
+                if src.count("\n") + 1 >= (opts or {}).get("read_outline_lines", 400):
+                    skel = tokens.count(outline.render_outline(p, src)) + 60
+                    saved = max(0, int(t * (1 - OUTLINE_FOLLOWUP)) - skel)
+                    feature = "read:outline"
+            reads[key] = text
+        elif name in ("Edit", "Write", "MultiEdit", "NotebookEdit"):
+            p = inp.get("file_path", "")
+            for key in [k_ for k_ in reads if k_[0] == p]:
+                del reads[key]
+        if saved:
+            stats["saved_tool"] += saved
+            stats["saved_billed"] += saved * n
+            stats["saved_cost"] += saved * W_CACHE_WRITE + saved * (n - 1) * W_CACHE_READ
+            stats["by_feature"][feature] += saved * n
+            stats["count_feature"][feature] += 1
+
+
+def run(root=None, project=None, limit=None, style_pct=None, opts=None):
+    """Replay transcripts. `opts` are effective settings (config.load()); defaults to the current global config."""
+    if opts is None:
+        from . import config
+        opts = config.load()
+    stats = defaultdict(float)
+    stats.update(tool_by_name=defaultdict(int), by_feature=defaultdict(int), count_feature=defaultdict(int), top_cmds=[])
+    files = transcripts(root, project)
+    if limit:
+        files = files[-limit:]
+    for f in files:
+        try:
+            analyze_file(f, stats, opts)
+        except OSError:
+            continue
+    stats["files"] = len(files)
+    if style_pct:
+        # Output-style rules are not replayable; apply an assumed prose reduction to output tokens.
+        stats["style_cost"] = stats["output"] * style_pct * W_OUTPUT
+    return stats