sift-cli 1.0.0__py3-none-any.whl

sift/fallback.py

@@ -0,0 +1,98 @@
+"""What to show when nobody could be asked.
+
+There is no key, or no network, or the endpoint is busy, or the reply made no
+sense. The command has already run and its output is already on disk. Something
+has to be shown, and it has to be shown without a model.
+
+The temptation here is to be clever: look for the word "error", recognise a
+stack trace, score lines by how alarming they look. That is exactly the design
+this project was rewritten to get rid of. Patterns like those are a list of
+languages wearing a disguise -- they work for English and for the half-dozen
+formats whoever wrote them happened to know, and they quietly fail for Turkish,
+for Japanese, for a tool that shipped last week. Worse, they fail while looking
+confident.
+
+So this makes no claim about meaning at all. It shows the beginning and the end
+and marks what it skipped. The beginning says what was run; the end says how it
+came out. That is true of a compiler, a test runner, an installer, a shell
+script, in every language a person or a machine writes in, and it needs to know
+nothing about any of them.
+
+It is worse than a model. It is meant to be. What it must never be is wrong, and
+a rule that makes no claim cannot make a false one.
+"""
+
+from __future__ import annotations
+
+from sift import lines as text_lines
+from sift import records
+from sift.capture import Capture
+from sift.distill import View, render
+
+# The first lines are usually the invocation and the first thing to go wrong.
+HEAD = 10
+
+# The last lines are the result: the summary, the exit, the reason. In every
+# command-line tradition, the ending is where the answer lives.
+TAIL = 40
+
+# A run that failed says why near the end, and tends to say more of it: a
+# traceback, a diagnostic, a list of what did not build.
+TAIL_WHEN_FAILED = 80
+
+
+def ends(total: int, *, head: int = HEAD, tail: int = TAIL) -> set[int]:
+    """The first `head` and the last `tail` lines, or all of them if that is fewer."""
+    if total <= head + tail:
+        return set(range(1, total + 1))
+    return set(range(1, head + 1)) | set(range(total - tail + 1, total + 1))
+
+
+def from_lines(
+    lines: list[str],
+    handle: str,
+    *,
+    tail: int = TAIL,
+    first: int = 1,
+    unit: str = "line",
+) -> View:
+    """The ends of any numbered text, marked with what lies between them.
+
+    A source file gets the same treatment as a capture, and for the same reason:
+    the moment this function starts telling the two apart it has begun keeping a
+    list of what things are, which is the list this project was rewritten to be
+    rid of. The beginning and the end of a file are a poor outline. They are not
+    a wrong one.
+    """
+    total = len(lines)
+    if not total:
+        return View(handle, "", 0, 0, None, 0, unit=unit)
+
+    # `ends` counts from one because it is arithmetic about a length, not about
+    # a capture. Where these lines sit in the run is the caller's business, and
+    # it is added here, once, rather than taught to the arithmetic.
+    chosen = {number + first - 1 for number in ends(total, head=HEAD, tail=tail)}
+    return View(
+        handle=handle,
+        text=render(lines, chosen, handle, first, unit),
+        kept=len(chosen),
+        total=total,
+        model=None,
+        asks=0,
+        unit=unit,
+    )
+
+
+def fallback(capture: Capture) -> View:
+    """A view built without asking anything, and honest about being one.
+
+    `model` is left empty, which is how the caller can tell this apart from a
+    view a model chose. A reader who is told which lines were picked, and by
+    what, can decide whether to go and read the rest.
+    """
+    tail = TAIL_WHEN_FAILED if capture.meta.failed else TAIL
+    text = capture.text()
+    found = records.of(text)
+    if found is not None:
+        return from_lines(found, capture.handle, tail=tail, unit="record")
+    return from_lines(text_lines.of(text), capture.handle, tail=tail)

sift/hook.py

@@ -0,0 +1,275 @@
+"""Catching the shell commands a client runs on its own.
+
+The MCP server can only distil what it was asked to distil. Everything else a
+client does with a shell -- and a coding agent does a great deal -- lands in the
+conversation whole. This closes that gap without a proxy: the client is asked to
+send its shell commands here first, `sift` runs them, and what comes back is a
+view.
+
+Two decisions, and the first is the one worth arguing about.
+
+**Everything is routed. Nothing decides whether a command "looks noisy".**
+
+That rule was tempting and it is exactly the mistake this project was rewritten
+to avoid. A list of commands worth intercepting is a list of tools wearing a
+disguise: it would know `pytest` and `cargo` and `npm`, be wrong about the
+in-house script, and be confidently silent about the one that printed forty
+thousand lines. And it cannot be right in principle -- how much a command prints
+is not knowable before it runs.
+
+Routing everything costs nothing, because a view of a short output *is* that
+output: the budget only takes hold when there is more than the budget. Twelve
+lines in, twelve lines out.
+
+**It fails open, and that is the third rule again.** A bug here, an unreadable
+event, a command that could not be started -- every one of them ends with the
+client running the command itself, exactly as it would have. A gate that breaks
+a shell is worse than no gate, and this one is allowed to break.
+
+`SIFT_HOOK=0` turns it off for someone who wants their shell back untouched.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import json
+import os
+from pathlib import Path
+
+from sift.capture import run
+from sift.view import best_view, footer
+
+# What this returns when it has nothing to say. An empty answer means "carry on
+# as you were", which is the only safe thing to say when something went wrong.
+PASS = {}
+
+
+def wanted() -> bool:
+    """Whether shell commands should be routed here at all."""
+    return (os.environ.get("SIFT_HOOK") or "1").strip().lower() not in {
+        "0",
+        "false",
+        "no",
+        "off",
+        "",
+    }
+
+
+def answer(event: dict) -> dict:
+    """What to tell the client about one shell command it is about to run.
+
+    The event is whatever the client put on stdin, and it is treated as data
+    from somewhere else: every field is checked before it is used, and anything
+    unexpected means *carry on*, never an error. A hook that raises on a shape
+    it did not expect takes the shell down with it.
+    """
+    if not wanted():
+        return PASS
+    if not isinstance(event, dict):
+        return PASS
+    if event.get("tool_name") != "Bash":
+        return PASS
+
+    given = event.get("tool_input")
+    command = given.get("command") if isinstance(given, dict) else None
+    if not isinstance(command, str) or not command.strip():
+        return PASS
+
+    try:
+        capture = run([command], shell=True)
+        view, who = best_view(capture)
+    except Exception:  # a bug here must not cost the caller their shell
+        return PASS
+
+    said = view.text + "\n\n" + footer(capture, view, who) if view.text else footer(
+        capture, view, who
+    )
+    return {
+        "hookSpecificOutput": {
+            "hookEventName": "PreToolUse",
+            "permissionDecision": "deny",
+            "permissionDecisionReason": said,
+        }
+    }
+
+
+# Where the client keeps the settings this would be written into, and the one
+# line that would be written. `SIFT_SETTINGS` moves it, which is how this is
+# tested without touching the file somebody actually uses.
+SETTINGS = "~/.claude/settings.json"
+COMMAND = "sift hook"
+EVENT = "PreToolUse"
+MATCHER = "Bash"
+
+# What somebody is told before they are asked. Everything it gives and
+# everything it costs, in the order somebody deciding would want them.
+OFFER = """\
+sift can also catch the shell commands the client runs on its own.
+
+Right now sift only sees what you or the client explicitly hand it. A coding
+agent runs a great deal of shell besides that, and all of it lands in the
+conversation whole -- and is re-sent on every turn after.
+
+With this on, every shell command the client runs goes through sift first: it
+runs the command, keeps every byte, and hands back the lines that mattered.
+Nothing decides which commands are "worth" catching, because how much a command
+prints is not knowable before it runs. Twelve lines in, twelve lines out.
+
+What it costs, honestly:
+
+  * A command that outruns the client's hook timeout is killed there, and the
+    client then runs it itself -- so a very long command can run twice. Keep
+    this in mind for anything that should not happen twice.
+  * Every caught command costs one model request.
+
+It fails open: a bug in it leaves your shell exactly as it was, and
+`SIFT_HOOK=0` switches it off without touching your settings again.
+
+This would add one line to {where}:
+
+    {event} / {matcher} -> {command}
+
+Nothing already in that file is changed or removed."""
+
+
+def settings_file() -> Path:
+    """The settings file this writes into, with `SIFT_SETTINGS` overriding."""
+    return Path(os.environ.get("SIFT_SETTINGS") or SETTINGS).expanduser()
+
+
+def offer() -> str:
+    """The notice, with the real paths filled in."""
+    return OFFER.format(
+        where=settings_file(), event=EVENT, matcher=MATCHER, command=COMMAND
+    )
+
+
+def _entries(settings: dict) -> list | None:
+    """The list this would be added to, or None if the file is not that shape.
+
+    Refusing an unfamiliar shape rather than reshaping it is the whole of the
+    safety here. This writes into a file somebody else owns, which may hold
+    hooks they depend on; a merge that is not certain what it is merging into
+    should not merge.
+    """
+    hooks = settings.get("hooks", {})
+    if not isinstance(hooks, dict):
+        return None
+    found = hooks.setdefault(EVENT, [])
+    return found if isinstance(found, list) else None
+
+
+def installed(settings: dict | None = None) -> bool:
+    """Whether the client is already routing its shell commands here."""
+    if settings is None:
+        settings = read_settings() or {}
+    entries = _entries(dict(settings))
+    if entries is None:
+        return False
+    for entry in entries:
+        if not isinstance(entry, dict):
+            continue
+        for one in entry.get("hooks", []) or []:
+            if isinstance(one, dict) and COMMAND in str(one.get("command", "")):
+                return True
+    return False
+
+
+def read_settings() -> dict | None:
+    """What is in the settings file, {} if there is none, None if it is not JSON."""
+    path = settings_file()
+    if not path.is_file():
+        return {}
+    try:
+        found = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError):
+        return None
+    return found if isinstance(found, dict) else None
+
+
+def install() -> tuple[bool, str]:
+    """Add the routing, without disturbing anything already there.
+
+    A copy of the original is kept beside it the first time, because this edits
+    a file this tool does not own and did not write.
+    """
+    settings = read_settings()
+    if settings is None:
+        return False, f"sift: {settings_file()} is not JSON this can add to safely."
+    if installed(settings):
+        return True, "sift: the shell is already routed here. Nothing to do."
+
+    entries = _entries(settings)
+    if entries is None:
+        return False, f"sift: {settings_file()} has hooks in a shape this cannot merge."
+
+    path = settings_file()
+    backup = path.with_suffix(path.suffix + ".before-sift")
+    if path.is_file() and not backup.exists():
+        with contextlib.suppress(OSError):
+            backup.write_text(path.read_text(encoding="utf-8"), encoding="utf-8")
+
+    for entry in entries:
+        if isinstance(entry, dict) and entry.get("matcher") == MATCHER:
+            mine = entry.setdefault("hooks", [])
+            if isinstance(mine, list):
+                mine.append({"type": "command", "command": COMMAND})
+                break
+            return False, f"sift: the {MATCHER} entry has hooks this cannot merge."
+    else:
+        entries.append(
+            {"matcher": MATCHER, "hooks": [{"type": "command", "command": COMMAND}]}
+        )
+
+    settings.setdefault("hooks", {})[EVENT] = entries
+    try:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(
+            json.dumps(settings, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
+        )
+    except OSError as exc:
+        return False, f"sift: could not write {path} ({exc})"
+
+    kept = f" The file as it was is in {backup.name}." if backup.exists() else ""
+    return True, (
+        f"sift: the shell is routed here now. Restart the client for it to take"
+        f" effect.{kept}\n"
+        f"      Undo with `sift hook --uninstall`, or switch it off for one"
+        f" session with SIFT_HOOK=0."
+    )
+
+
+def uninstall() -> tuple[bool, str]:
+    """Take the routing out again, and leave everything else exactly as it was."""
+    settings = read_settings()
+    if settings is None:
+        return False, f"sift: {settings_file()} is not JSON this can edit safely."
+    if not installed(settings):
+        return True, "sift: the shell was not routed here. Nothing to do."
+
+    entries = _entries(settings) or []
+    for entry in entries:
+        if not isinstance(entry, dict):
+            continue
+        mine = entry.get("hooks")
+        if isinstance(mine, list):
+            entry["hooks"] = [
+                one
+                for one in mine
+                if not (isinstance(one, dict) and COMMAND in str(one.get("command", "")))
+            ]
+    # An entry whose only hook was this one is removed; one that had others keeps
+    # them. Leaving an empty matcher behind would be leaving litter in somebody
+    # else's file.
+    settings["hooks"][EVENT] = [
+        entry
+        for entry in entries
+        if not (isinstance(entry, dict) and entry.get("hooks") == [])
+    ]
+    try:
+        settings_file().write_text(
+            json.dumps(settings, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
+        )
+    except OSError as exc:
+        return False, f"sift: could not write {settings_file()} ({exc})"
+    return True, "sift: the shell is no longer routed here."

sift/lines.py

@@ -0,0 +1,37 @@
+"""What counts as a line.
+
+`str.splitlines()` splits on more than anyone else does: form feed, vertical
+tab, the file and group separators, NEL (U+0085), and the Unicode line and
+paragraph separators (U+2028, U+2029). Every one of those turns up in real
+command output. U+0085 comes out of EBCDIC conversions, which is how a mainframe
+job log reaches a terminal. U+2028 comes out of tooling that pasted a string it
+read from somewhere else. A form feed is how a good deal of older software
+starts a new page.
+
+When they turn up, a capture gains lines that nothing else agrees exist. `wc -l`
+would not count them, the terminal did not show them, and `sift peek 40 50`
+would answer with different text than the editor open beside it. For a tool
+whose whole promise is that the line you were shown is the line that is there,
+that is not a small disagreement.
+
+So a line here ends at a newline, and at nothing else. A carriage return before
+it belongs to the ending rather than to the content -- which is what a terminal
+does with CRLF, and what every other tool means by a line.
+
+A carriage return anywhere *else* is left exactly where it is. A progress bar
+that redraws itself nine hundred times is one line in the terminal, and it is
+one line here too, rather than nine hundred lines of a number counting up.
+"""
+
+from __future__ import annotations
+
+
+def of(text: str) -> list[str]:
+    """The lines of `text`, counted the way the rest of the world counts them."""
+    if not text:
+        return []
+    found = text.split("\n")
+    if found[-1] == "":
+        # A trailing newline ends the last line; it does not begin another one.
+        found.pop()
+    return [line[:-1] if line.endswith("\r") else line for line in found]

sift/many.py

@@ -0,0 +1,51 @@
+"""Several questions at once, and the one number that keeps them honest.
+
+Everything else in this tool answers one question about one thing. This runs
+several of those side by side: four log files digested together, three running
+commands read in one call, a directory outlined in the time one file used to
+take.
+
+The whole module is a dozen lines, because the hard part is not here. Asking in
+parallel is easy; asking in parallel *without multiplying the load* is what took
+a day to learn, and that lesson lives one layer down. Every ask in this project
+passes through `distill.in_flight`, a single gate sized by `SIFT_WORKERS`. So
+this file may start as many jobs as it likes and the number of requests actually
+in the air is still the number the machine was told to allow.
+
+That division of labour is deliberate. A ceiling that each caller has to work
+out for itself is a ceiling that the next caller will get wrong -- and the next
+caller is always the one written six months later by someone who never read the
+note. The gate cannot be got wrong by arithmetic, because there is none to do.
+
+What this file does own is the promise that **the answers come back in the order
+the questions were asked**. Which of them finishes first is a fact about the
+network, and a result that reordered itself by network timing would be a
+different answer to the same question every time it was asked.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Sequence
+from concurrent.futures import ThreadPoolExecutor
+
+from sift.distill import workers
+
+
+def together[T](jobs: Sequence[Callable[[], T]]) -> list[T]:
+    """Run every job at the same time and return what they returned, in order.
+
+    One job is run directly rather than through a pool, because a pool for one
+    job is a thread and a queue doing what a call does.
+
+    An exception from a job comes back out of here, at the position that job
+    would have had. Swallowing it would leave the caller a list with a hole in
+    it and nothing to say what happened -- and every caller above this one
+    already has a net of its own that turns a failure into a view.
+    """
+    if not jobs:
+        return []
+    if len(jobs) == 1:
+        return [jobs[0]()]
+
+    with ThreadPoolExecutor(max_workers=min(workers(), len(jobs))) as pool:
+        return list(pool.map(lambda job: job(), jobs))

sift/memory.py

@@ -0,0 +1,94 @@
+"""What this machine already knows about the commands run on it.
+
+Every other module here asks a model something. This one asks nothing, and that
+is the design rather than an omission.
+
+The question is *how often was this run here, how did it go, and which failures
+keep coming back* -- and every part of that is counting. A model asked to count
+would be slower, cost a request, and be wrong sometimes; there is no judgement
+in the question for it to supply. Reaching for it anyway would be the same
+mistake as a rules engine deciding which lines matter: using the wrong tool
+because it is the one the project is proud of.
+
+So the rule this file stands on is the mirror of the rest of the project: **the
+model decides what cannot be computed, and nothing else.**
+
+What it can answer is bounded by what is still on disk. A capture that was swept
+up took its record with it, because the record lives beside the bytes rather
+than in a ledger of its own -- which is what makes deleting a capture actually
+delete it. This module says how far back it can see rather than implying it
+remembers everything.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from dataclasses import dataclass
+
+from sift import store
+
+
+@dataclass(frozen=True)
+class Habit:
+    """One command, and everything this machine has seen of it."""
+
+    command: str
+    runs: int
+    failures: int
+    last_ending: str
+    last_at: float
+    cwds: tuple[str, ...]
+
+    @property
+    def never_worked(self) -> bool:
+        """True when every run of this command here has failed.
+
+        Worth its own name because it is the one shape that answers a question
+        somebody actually asks: *is this thing broken, or is it me?* A command
+        that has failed four times out of four in this directory is not a flaky
+        test.
+        """
+        return self.runs > 0 and self.failures == self.runs
+
+
+def habits(term: str | None = None, cwd: str | None = None, limit: int = 200) -> list[Habit]:
+    """What has been run here, most recent first.
+
+    `term` narrows to commands containing it, `cwd` to a directory. Both are
+    plain text rather than patterns: this is a memory, not a search engine, and
+    somebody typing `pytest` should not have to think about what the letters
+    mean to a regular expression.
+    """
+    seen: dict[str, list[store.Meta]] = {}
+    for meta in store.recent(limit):
+        written = " ".join(meta.command)
+        if term and term not in written:
+            continue
+        if cwd and meta.cwd != cwd:
+            continue
+        seen.setdefault(written, []).append(meta)
+
+    found = [_habit(written, runs) for written, runs in seen.items()]
+    found.sort(key=lambda one: one.last_at, reverse=True)
+    return found
+
+
+def _habit(written: str, runs: Sequence[store.Meta]) -> Habit:
+    newest = max(runs, key=lambda meta: meta.started_at)
+    return Habit(
+        command=written,
+        runs=len(runs),
+        failures=sum(1 for meta in runs if meta.failed),
+        last_ending="timed out" if newest.timed_out else f"exit {newest.exit_code}",
+        last_at=newest.started_at,
+        cwds=tuple(sorted({meta.cwd for meta in runs})),
+    )
+
+
+def reach(limit: int = 200) -> int:
+    """How many runs are still on disk to remember at all.
+
+    Printed with the answer, because "this has never failed here" means one thing
+    after four hundred runs and nothing at all after two.
+    """
+    return len(store.recent(limit))