skimread 0.1.0__py3-none-any.whl

skimread/__init__.py

@@ -0,0 +1,8 @@
+"""skimread: skim before you read.
+
+A zero-dependency, local, lossless context guard for AI coding agents.
+It shrinks the two biggest token sinks in agentic coding, big file reads and
+noisy command output, while keeping every byte one call away.
+"""
+
+__version__ = "0.1.0"

skimread/__main__.py

@@ -0,0 +1,4 @@
+from .cli import main
+
+if __name__ == "__main__":
+    raise SystemExit(main())

skimread/ansi.py

@@ -0,0 +1,55 @@
+"""Terminal output normalisation: ANSI escapes, carriage returns, backspaces."""
+
+import re
+
+_CSI = re.compile(r"\x1b\[[0-9;?<>=!]*[ -/]*[@-~]")
+_OSC = re.compile(r"\x1b\][^\x07\x1b]*(?:\x07|\x1b\\)")
+_ESC_OTHER = re.compile(r"\x1b[@-Z\\-_]")
+_CTRL = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]")
+
+
+def strip_ansi(text: str) -> str:
+    text = _OSC.sub("", text)
+    text = _CSI.sub("", text)
+    text = _ESC_OTHER.sub("", text)
+    return text
+
+
+def _resolve_cr(line: str) -> str:
+    """Emulate a terminal: '\r' returns to column 0 and later text overwrites."""
+    if "\r" not in line:
+        return line
+    parts = line.split("\r")
+    # Keep the last non-empty segment; progress bars rewrite the same line.
+    out = ""
+    for seg in parts:
+        if not seg:
+            continue
+        if len(seg) >= len(out):
+            out = seg
+        else:
+            out = seg + out[len(seg):]
+    return out
+
+
+def normalize(raw: str) -> str:
+    """Return plain text with one logical line per terminal line."""
+    text = raw.replace("\r\n", "\n")
+    text = strip_ansi(text)
+    lines = []
+    for line in text.split("\n"):
+        line = _resolve_cr(line)
+        if "\x08" in line:
+            buf = []
+            for ch in line:
+                if ch == "\x08":
+                    if buf:
+                        buf.pop()
+                else:
+                    buf.append(ch)
+            line = "".join(buf)
+        line = _CTRL.sub("", line)
+        lines.append(line.rstrip())
+    while lines and not lines[-1]:
+        lines.pop()
+    return "\n".join(lines)

skimread/cli.py

@@ -0,0 +1,368 @@
+"""skimread command line."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import sys
+import time
+from pathlib import Path
+from typing import List, Optional
+
+from . import __version__, store
+from .digest import build_digest, run_command
+from .outline import analyze, render_outline
+from .tokens import estimate_tokens, fmt_bytes, fmt_tokens
+
+SUBCOMMANDS = {"run", "pipe", "show", "list", "outline", "read", "stats", "hook", "setup", "mcp", "config",
+               "prune", "version", "-h", "--help", "--version"}
+
+
+# --- helpers shared with the MCP server ------------------------------------------------
+
+def parse_range(spec: Optional[str], n: int) -> Optional[tuple]:
+    if not spec:
+        return None
+    m = re.match(r"^\s*(\d*)\s*[-:]\s*(\d*)\s*$", spec)
+    if m:
+        a = int(m.group(1)) if m.group(1) else 1
+        b = int(m.group(2)) if m.group(2) else n
+        return max(1, a), min(n, b)
+    if spec.strip().isdigit():
+        a = int(spec)
+        return max(1, a), min(n, a)
+    raise SystemExit(f"skimread: bad range {spec!r} (use A-B, A:, :B or A)")
+
+
+def _numbered(lines: List[str], start: int) -> str:
+    w = len(str(start + len(lines)))
+    return "\n".join(f"{start + i:>{w}}| {t}" for i, t in enumerate(lines))
+
+
+def format_show(rid: int, rng: Optional[str] = None, grep: Optional[str] = None, context: int = 2,
+                show_all: bool = False, cap: int = 400) -> str:
+    run = store.load_run(rid)
+    if run is None:
+        return f"skimread: no stored output with id {rid} (see `skimread list`)"
+    lines = run["text"].split("\n")
+    n = len(lines)
+    head = f"# skimread #{rid}: $ {run.get('cmd', '')}  ·  exit {run.get('exit')}  ·  {n} lines"
+    if grep:
+        try:
+            rx = re.compile(grep, re.I)
+        except re.error as e:
+            return f"skimread: bad regex: {e}"
+        hits = [i for i, t in enumerate(lines) if rx.search(t)]
+        if not hits:
+            return head + f"\n(no lines match /{grep}/)"
+        keep = set()
+        for i in hits:
+            keep.update(range(max(0, i - context), min(n, i + context + 1)))
+        out = [head + f"  ·  {len(hits)} matches for /{grep}/"]
+        prev = -1
+        shown = 0
+        for i in sorted(keep):
+            if prev >= 0 and i != prev + 1:
+                out.append("   …")
+            out.append(f"{i + 1:>5}| {lines[i]}")
+            prev = i
+            shown += 1
+            if shown >= cap * 2 and not show_all:
+                out.append(f"   … stopped after {shown} lines; narrow the pattern or pass --all")
+                break
+        return "\n".join(out)
+    r = parse_range(rng, n)
+    if r:
+        a, b = r
+        sel = lines[a - 1:b]
+        note = ""
+        if len(sel) > cap * 4 and not show_all:
+            sel = sel[: cap * 4]
+            note = f"\n   … range truncated to {cap * 4} lines; pass --all to print everything"
+        return head + "\n" + _numbered(sel, a) + note
+    if n > cap and not show_all:
+        return (head + f"\n" + _numbered(lines[:cap // 2], 1) +
+                f"\n   … {n - cap} lines omitted ({cap // 2 + 1}-{n - cap // 2}); use a range, --grep, or --all …\n" +
+                _numbered(lines[n - cap // 2:], n - cap // 2 + 1))
+    return head + "\n" + _numbered(lines, 1)
+
+
+def read_range(path: str, start: Optional[int] = None, end: Optional[int] = None) -> str:
+    try:
+        text = Path(path).read_text(encoding="utf-8", errors="replace")
+    except OSError as e:
+        return f"skimread: {e}"
+    lines = text.split("\n")
+    if text.endswith("\n"):
+        lines.pop()
+    n = len(lines)
+    a = max(1, int(start or 1))
+    b = min(n, int(end or n))
+    if a > b:
+        return f"skimread: empty range {a}-{b} (file has {n} lines)"
+    return f"# {path} lines {a}-{b} of {n}\n" + _numbered(lines[a - 1:b], a)
+
+
+def format_list(limit: int = 20) -> str:
+    runs = store.list_runs(limit)
+    if not runs:
+        return "skimread: no stored runs yet"
+    out = ["   id  exit   lines   tokens  when              command"]
+    for m in runs:
+        when = time.strftime("%Y-%m-%d %H:%M", time.localtime(m.get("ts", 0)))
+        cmd = (m.get("cmd") or "")[:70]
+        out.append(f"{m.get('id', 0):>5}  {str(m.get('exit')):>4}  {m.get('lines', 0):>6}  {fmt_tokens(m.get('tokens_in', 0)):>7}  {when}  {cmd}")
+    return "\n".join(out)
+
+
+def format_stats(price: Optional[float] = None, as_json: bool = False) -> str:
+    cfg = store.load_config()
+    price = price if price is not None else float(cfg["price_per_mtok"])
+    runs = 0
+    run_in = run_out = 0
+    guards = 0
+    guard_full = guard_outline = slices = full_reads = 0
+    by_fmt: dict = {}
+    for ev in store.events():
+        k = ev.get("kind")
+        if k == "run":
+            runs += 1
+            run_in += int(ev.get("tokens_in", 0))
+            run_out += int(ev.get("tokens_out", 0))
+            f = ev.get("fmt", "generic")
+            d = by_fmt.setdefault(f, [0, 0, 0])
+            d[0] += 1
+            d[1] += int(ev.get("tokens_in", 0))
+            d[2] += int(ev.get("tokens_out", 0))
+        elif k == "read_guard":
+            guards += 1
+            guard_full += int(ev.get("tokens", 0))
+            guard_outline += int(ev.get("outline_tokens", 0))
+        elif k == "read_slice":
+            slices += int(ev.get("tokens", 0))
+        elif k == "read_full":
+            full_reads += int(ev.get("tokens", 0))
+    run_saved = max(0, run_in - run_out)
+    read_spent = guard_outline + slices + full_reads
+    read_saved = max(0, guard_full - read_spent)
+    total = run_saved + read_saved
+    usd = total / 1e6 * price
+    if as_json:
+        return json.dumps({"runs": runs, "run_tokens_in": run_in, "run_tokens_out": run_out, "run_saved": run_saved,
+                           "read_guards": guards, "read_tokens_full": guard_full, "read_tokens_spent": read_spent,
+                           "read_saved": read_saved, "total_saved": total, "usd_saved": round(usd, 4),
+                           "price_per_mtok": price, "by_format": by_fmt}, indent=2)
+    lines = [
+        "skimread stats (all time)",
+        f"  commands digested : {runs:>7}   {fmt_tokens(run_in):>7} tok → {fmt_tokens(run_out):>7} tok   saved {fmt_tokens(run_saved)}"
+        + (f" ({100 * run_saved / run_in:.0f}%)" if run_in else ""),
+        f"  big reads guarded : {guards:>7}   {fmt_tokens(guard_full):>7} tok → {fmt_tokens(read_spent):>7} tok   saved {fmt_tokens(read_saved)}"
+        + (f" ({100 * read_saved / guard_full:.0f}%)" if guard_full else ""),
+        f"  total saved       : ~{fmt_tokens(total)} tokens ≈ ${usd:.2f} at ${price:g}/M input tokens",
+    ]
+    if by_fmt:
+        lines.append("  by format:")
+        for f, (c, i, o) in sorted(by_fmt.items(), key=lambda kv: -kv[1][1]):
+            pct = f"{100 * (i - o) / i:.0f}%" if i else "-"
+            lines.append(f"    {f:<10} {c:>5} runs  {fmt_tokens(i):>7} → {fmt_tokens(o):>7}  ({pct})")
+    return "\n".join(lines)
+
+
+# --- subcommands ---------------------------------------------------------------------------
+
+def cmd_run(args: argparse.Namespace) -> int:
+    if args.c:
+        cmd = args.c
+    elif args.argv:
+        argv = args.argv[1:] if args.argv and args.argv[0] == "--" else args.argv
+        import shlex
+        cmd = " ".join(shlex.quote(a) for a in argv)
+    else:
+        print("skimread run: give a command (skimread run -c 'pytest -q' or skimread run -- pytest -q)", file=sys.stderr)
+        return 2
+    res = run_command(cmd, cwd=args.cwd, timeout=args.timeout)
+    d = build_digest(res["raw"], cmd=cmd, exit_code=res["exit"], duration=res["duration"], budget=args.budget,
+                     cwd=args.cwd or "")
+    sys.stdout.write(d.text + "\n")
+    return int(res["exit"] or 0)
+
+
+def cmd_pipe(args: argparse.Namespace) -> int:
+    raw = sys.stdin.read()
+    exit_code = None
+    m = re.search(r"\n?__SKIMREAD_EXIT=(\d+)\s*$", raw)
+    if m:
+        exit_code = int(m.group(1))
+        raw = raw[: m.start()]
+    d = build_digest(raw, cmd=args.cmd or "", exit_code=exit_code, budget=args.budget)
+    sys.stdout.write(d.text + "\n")
+    return exit_code or 0
+
+
+def cmd_show(args: argparse.Namespace) -> int:
+    print(format_show(args.id, args.range, args.grep, args.context, args.all))
+    return 0
+
+
+def cmd_outline(args: argparse.Namespace) -> int:
+    for p in args.paths:
+        if not Path(p).is_file():
+            print(f"skimread: {p}: not a file", file=sys.stderr)
+            continue
+        print(render_outline(analyze(p), max_entries=100000 if args.all else args.max))
+    return 0
+
+
+def cmd_read(args: argparse.Namespace) -> int:
+    r = parse_range(args.range, 10 ** 9) if args.range else None
+    print(read_range(args.path, r[0] if r else None, r[1] if r else None))
+    return 0
+
+
+def cmd_hook(args: argparse.Namespace) -> int:
+    from .hooks import handle
+    try:
+        payload = json.loads(sys.stdin.read() or "{}")
+    except ValueError:
+        return 0
+    try:
+        out = handle(payload)
+    except Exception as e:  # never break the agent's tool call
+        print(f"skimread hook error: {e}", file=sys.stderr)
+        return 0
+    if out:
+        sys.stdout.write(json.dumps(out))
+    return 0
+
+
+def cmd_setup(args: argparse.Namespace) -> int:
+    from . import setup as st
+    if args.target == "claude":
+        scope = "project" if args.project else ("local" if args.local else "global")
+        _, msg = st.install(scope=scope, wrap=args.wrap, uninstall=args.uninstall)
+        print(msg)
+    elif args.target == "print":
+        print(st.snippet(args.wrap))
+    elif args.target == "agents":
+        print(st.agents_snippet())
+    elif args.target == "mcp":
+        print(st.mcp_snippet())
+    return 0
+
+
+def cmd_config(args: argparse.Namespace) -> int:
+    cfg = store.load_config()
+    if args.key and args.value is not None:
+        if args.key not in cfg:
+            print(f"skimread: unknown key {args.key}; keys: {', '.join(cfg)}", file=sys.stderr)
+            return 2
+        cur = cfg[args.key]
+        try:
+            cfg[args.key] = type(cur)(args.value) if not isinstance(cur, bool) else args.value.lower() in ("1", "true", "yes")
+        except ValueError:
+            print(f"skimread: {args.key} expects {type(cur).__name__}", file=sys.stderr)
+            return 2
+        store.save_config(cfg)
+    if args.key:
+        print(f"{args.key} = {cfg[args.key]}")
+    else:
+        for k, v in cfg.items():
+            print(f"{k} = {v}")
+        print(f"(config file: {store.home() / 'config.json'})")
+    return 0
+
+
+def build_parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(prog="skimread", description="Skim before you read: lossless context guard for AI coding agents.")
+    p.add_argument("--version", action="version", version=f"skimread {__version__}")
+    sub = p.add_subparsers(dest="cmd")
+
+    r = sub.add_parser("run", help="run a command and print a lossless digest of its output")
+    r.add_argument("-c", help="command line to run through the shell")
+    r.add_argument("--cwd")
+    r.add_argument("--timeout", type=float)
+    r.add_argument("--budget", type=int, help="digest size in characters")
+    r.add_argument("argv", nargs=argparse.REMAINDER)
+    r.set_defaults(fn=cmd_run)
+
+    pi = sub.add_parser("pipe", help="digest stdin (cmd 2>&1 | skimread pipe)")
+    pi.add_argument("--cmd", help="command name, for format detection and the header")
+    pi.add_argument("--budget", type=int)
+    pi.set_defaults(fn=cmd_pipe)
+
+    s = sub.add_parser("show", help="show stored full output: all, a range, or grep matches")
+    s.add_argument("id", type=int)
+    s.add_argument("range", nargs="?")
+    s.add_argument("--grep", "-g")
+    s.add_argument("-C", "--context", type=int, default=2)
+    s.add_argument("--all", action="store_true")
+    s.set_defaults(fn=cmd_show)
+
+    l = sub.add_parser("list", help="list stored runs")
+    l.add_argument("-n", type=int, default=20)
+    l.set_defaults(fn=lambda a: (print(format_list(a.n)), 0)[1])
+
+    o = sub.add_parser("outline", help="structural outline of a file with line ranges")
+    o.add_argument("paths", nargs="+")
+    o.add_argument("--max", type=int, default=70)
+    o.add_argument("--all", action="store_true")
+    o.set_defaults(fn=cmd_outline)
+
+    rd = sub.add_parser("read", help="print a line range of a file with line numbers")
+    rd.add_argument("path")
+    rd.add_argument("range", nargs="?")
+    rd.set_defaults(fn=cmd_read)
+
+    stt = sub.add_parser("stats", help="tokens and money saved so far")
+    stt.add_argument("--price", type=float, help="USD per million input tokens")
+    stt.add_argument("--json", action="store_true")
+    stt.set_defaults(fn=lambda a: (print(format_stats(a.price, a.json)), 0)[1])
+
+    h = sub.add_parser("hook", help="Claude Code PreToolUse hook (JSON on stdin)")
+    h.set_defaults(fn=cmd_hook)
+
+    se = sub.add_parser("setup", help="install into Claude Code, or print snippets")
+    se.add_argument("target", choices=["claude", "print", "agents", "mcp"])
+    se.add_argument("--project", action="store_true", help="write .claude/settings.json in the current project")
+    se.add_argument("--local", action="store_true", help="write .claude/settings.local.json (not committed)")
+    se.add_argument("--wrap", choices=["noisy", "all", "off"], default="noisy",
+                    help="which Bash commands to route through skimread (default: noisy)")
+    se.add_argument("--uninstall", action="store_true")
+    se.set_defaults(fn=cmd_setup)
+
+    m = sub.add_parser("mcp", help="run as an MCP server on stdio")
+    m.set_defaults(fn=lambda a: __import__("skimread.mcp", fromlist=["serve"]).serve())
+
+    c = sub.add_parser("config", help="show or set configuration")
+    c.add_argument("key", nargs="?")
+    c.add_argument("value", nargs="?")
+    c.set_defaults(fn=cmd_config)
+
+    pr = sub.add_parser("prune", help="delete old stored outputs")
+    pr.set_defaults(fn=lambda a: (store.prune(), print("pruned"), 0)[2])
+
+    v = sub.add_parser("version")
+    v.set_defaults(fn=lambda a: (print(f"skimread {__version__}"), 0)[1])
+    return p
+
+
+def main(argv: Optional[List[str]] = None) -> int:
+    argv = list(sys.argv[1:] if argv is None else argv)
+    if argv and argv[0] not in SUBCOMMANDS and not argv[0].startswith("-"):
+        argv = ["run", "--"] + argv  # `skimread pytest -q` == `skimread run -- pytest -q`
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    if not args.cmd:
+        parser.print_help()
+        return 0
+    try:
+        return int(args.fn(args) or 0)
+    except KeyboardInterrupt:
+        return 130
+    except BrokenPipeError:
+        return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

skimread/digest.py

@@ -0,0 +1,134 @@
+"""Turn one command's raw output into a budgeted, lossless digest."""
+
+from __future__ import annotations
+
+import shlex
+import subprocess
+import sys
+import time
+from dataclasses import dataclass
+from typing import Dict, List, Optional
+
+from . import formats, store
+from .ansi import normalize
+from .reduce import Line, fit_budget, render, to_lines
+from .tokens import estimate_tokens, fmt_bytes, fmt_tokens
+
+MAX_CAPTURE = 32 * 1024 * 1024
+
+
+@dataclass
+class Digest:
+    text: str
+    run_id: Optional[int]
+    fmt: str
+    lines_in: int
+    bytes_in: int
+    tokens_in: int
+    tokens_out: int
+
+
+def build_digest(raw: str, cmd: str = "", exit_code: Optional[int] = None, duration: Optional[float] = None,
+                 budget: Optional[int] = None, save: bool = True, cwd: str = "") -> Digest:
+    cfg = store.load_config()
+    budget = budget or int(cfg["budget"])
+    norm = normalize(raw)
+    lines: List[Line] = to_lines(norm)
+    n_lines = len(lines)
+    bytes_in = len(raw.encode("utf-8", errors="replace"))
+    tokens_in = estimate_tokens(norm)
+
+    run_id: Optional[int] = None
+    if save:
+        run_id = store.save_run(norm, {
+            "cmd": cmd, "cwd": cwd, "exit": exit_code, "lines": n_lines, "bytes": bytes_in,
+            "tokens_in": tokens_in, "duration": duration,
+        })
+    handle = f"skimread show {run_id}" if run_id else "skimread show <id>"
+
+    fmt, reducer = formats.detect(cmd, lines)
+    ctx = formats.Ctx(cmd=cmd, budget=budget, handle=handle)
+    reduced = reducer(list(lines), ctx)
+    fitted = fit_budget(reduced, budget, handle=handle)
+    body = render(fitted)
+
+    tokens_out = estimate_tokens(body)
+    status = ""
+    if exit_code is not None:
+        status = f"exit {exit_code}"
+        if duration is not None:
+            status += f" in {duration:.1f}s"
+    shown = len(fitted)
+    shown_cmd = cmd if len(cmd) <= 300 else cmd[:280] + " …"
+    header_bits = [f"$ {shown_cmd}" if cmd else "$ (stdin)"]
+    if status:
+        header_bits.append(status)
+    header = "  ·  ".join(header_bits)
+    if n_lines <= shown and body == norm:
+        footer = f"[skimread #{run_id}: {n_lines} lines, shown in full]" if run_id else ""
+    else:
+        footer = (f"[skimread #{run_id}: {n_lines} lines, ~{fmt_tokens(tokens_in)} tok → {fmt_tokens(tokens_out)}. "
+                  f"Full/slice/search: {handle} [A-B | --grep RE]]") if run_id else f"[skimread: {n_lines} lines → digest]"
+    text = header + "\n" + body + ("\n" + footer if footer else "")
+    if run_id:
+        try:
+            store.log_event({"kind": "run", "id": run_id, "tokens_in": tokens_in, "lines": n_lines,
+                             "tokens_out": estimate_tokens(text), "fmt": fmt, "cmd": cmd, "exit": exit_code})
+        except OSError:
+            pass
+    return Digest(text, run_id, fmt, n_lines, bytes_in, tokens_in, estimate_tokens(text))
+
+
+def run_command(cmd: str, cwd: Optional[str] = None, timeout: Optional[float] = None,
+                shell: bool = True) -> Dict:
+    """Run a command with merged stdout/stderr, colours off. Returns raw output + exit code."""
+    import os
+    env = dict(os.environ)
+    env.update({"NO_COLOR": "1", "FORCE_COLOR": "0", "TERM": "dumb", "CLICOLOR": "0", "PY_COLORS": "0",
+                "CARGO_TERM_COLOR": "never", "SKIMREAD_ACTIVE": "1"})
+    env.pop("COLORTERM", None)
+    start = time.time()
+    try:
+        if shell:
+            bash = _bash()
+            proc = subprocess.Popen(cmd if bash is None else [bash, "-c", cmd], shell=bash is None,
+                                    stdout=subprocess.PIPE, stderr=subprocess.STDOUT, stdin=subprocess.DEVNULL,
+                                    cwd=cwd, env=env)
+        else:
+            proc = subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+                                    stdin=subprocess.DEVNULL, cwd=cwd, env=env)
+    except FileNotFoundError as e:
+        return {"raw": f"skimread: cannot run command: {e}\n", "exit": 127, "duration": 0.0, "timed_out": False}
+    chunks: List[bytes] = []
+    size = 0
+    timed_out = False
+    assert proc.stdout is not None
+    try:
+        while True:
+            if timeout and time.time() - start > timeout:
+                proc.kill()
+                timed_out = True
+                break
+            chunk = proc.stdout.read1(65536) if hasattr(proc.stdout, "read1") else proc.stdout.read(65536)
+            if not chunk:
+                break
+            chunks.append(chunk)
+            size += len(chunk)
+            if size > MAX_CAPTURE:
+                # keep the tail; the head is almost never what an agent needs
+                while size > MAX_CAPTURE // 2 and len(chunks) > 1:
+                    size -= len(chunks.pop(0))
+    finally:
+        proc.wait()
+    raw = b"".join(chunks).decode("utf-8", errors="replace")
+    if timed_out:
+        raw += f"\nskimread: command timed out after {timeout:.0f}s and was killed\n"
+    return {"raw": raw, "exit": proc.returncode if not timed_out else 124,
+            "duration": time.time() - start, "timed_out": timed_out}
+
+
+def _bash() -> Optional[str]:
+    import shutil
+    if sys.platform.startswith("win"):
+        return None
+    return shutil.which("bash") or shutil.which("sh")