sift-cli 1.0.0__py3-none-any.whl

sift/peek.py

@@ -0,0 +1,161 @@
+"""Getting back exactly what was there.
+
+Every view `sift` produces names the capture it came from, and every gap in it
+says how many lines it covers. `peek` is the other half of that promise: the way
+back. Without it, a view would be a claim you have to take on trust; with it, a
+view is a starting point and the original is one call away.
+
+It returns bytes turned to text and nothing else -- no re-wrapping, no trimming,
+no highlighting. A caller asking to see line 412 is usually asking because
+something did not add up, and at that moment anything helpful is in the way.
+"""
+
+from __future__ import annotations
+
+import re
+import time
+from dataclasses import dataclass
+from pathlib import Path
+
+from sift import lines as text_lines
+from sift import store
+from sift.distill import render
+
+
+@dataclass(frozen=True)
+class Peek:
+    handle: str
+    text: str
+    first_line: int
+    last_line: int
+    total_lines: int
+    byte_count: int
+    # How many lines the caller's pattern matched, when there was one. Not the
+    # number shown: context is added around each match and the whole answer is
+    # capped, so a reader needs both numbers to know whether they saw them all.
+    matched: int = 0
+
+    @property
+    def is_whole(self) -> bool:
+        return self.first_line == 1 and self.last_line == self.total_lines
+
+
+def bytes_of(handle: str) -> bytes:
+    """The bytes behind a handle: a capture's if there is one, otherwise a file's.
+
+    `sift run` writes a handle into its gap marker and `sift outline` writes a
+    path, and both markers end the same way -- *sift peek X for any of them*. So
+    the way back takes either. The store is asked first, so that a file sitting
+    in the working directory under some capture's name can never answer for that
+    capture; only a handle that was never captured falls through to disk.
+    """
+    try:
+        return store.read_raw(handle)
+    except FileNotFoundError:
+        removed = store.gone(handle)
+        if removed is not None:
+            when = time.strftime("%Y-%m-%d", time.localtime(removed.removed_at))
+            raise FileNotFoundError(
+                f"capture {handle!r} was removed by sift gc on {when}"
+                f" ({removed.byte_count:,} bytes)"
+            ) from None
+        found = Path(handle)
+        if not found.is_file():
+            raise
+        return found.read_bytes()
+
+
+# How many lines a search may return before it stops being a way back into a
+# capture and becomes a second copy of it. The caller can raise it; the default
+# is here so that a pattern matching everything cannot answer with everything.
+FOUND_CAP = 200
+
+
+def peek(
+    handle: str,
+    start: int | None = None,
+    end: int | None = None,
+    grep: str | None = None,
+    around: int = 3,
+    cap: int = FOUND_CAP,
+) -> Peek:
+    """Lines `start`..`end` of a capture or a file, 1-based and inclusive.
+
+    Out-of-range asks are clamped rather than refused. Someone reading a view saw
+    "3,914 lines not shown" and guessed at a number; answering with the nearest
+    real lines is more use than an error about the guess.
+
+    With `grep`, the range becomes the window to search rather than the answer:
+    every line matching the pattern comes back, with `around` lines on each side
+    of it so the match can be read. Nothing here judges -- the pattern is the
+    caller's, the matching is exact, and the lines are still the file's own. This
+    is the search a view sends you to when its gap says "3,914 lines not shown"
+    and you already know the word you are looking for.
+    """
+    raw = bytes_of(handle)
+    text = raw.decode("utf-8", errors="replace")
+    lines = text_lines.of(text)
+    total = len(lines)
+
+    if total == 0:
+        return Peek(handle, "", 0, 0, 0, len(raw))
+
+    first = 1 if start is None else max(1, min(start, total))
+    last = total if end is None else max(first, min(end, total))
+
+    if grep is None:
+        return Peek(
+            handle=handle,
+            text="\n".join(lines[first - 1 : last]),
+            first_line=first,
+            last_line=last,
+            total_lines=total,
+            byte_count=len(raw),
+        )
+
+    hits = _found(lines, grep, first, last)
+    chosen = _with_context(hits, around, cap, first, last)
+    if not chosen:
+        return Peek(handle, "", first, first, total, len(raw), matched=0)
+
+    return Peek(
+        handle=handle,
+        text=render(lines, chosen, handle),
+        first_line=min(chosen),
+        last_line=max(chosen),
+        total_lines=total,
+        byte_count=len(raw),
+        matched=len(hits),
+    )
+
+
+def _found(lines: list[str], grep: str, first: int, last: int) -> list[int]:
+    """Every line in the window that the caller's pattern matches.
+
+    A pattern that will not compile is searched for as plain text, the same rule
+    `keep` follows and for the same reason: someone who typed `main()` meant
+    those characters, and answering a mistyped group with an empty result would
+    look exactly like a word that is not there.
+    """
+    try:
+        found = re.compile(grep)
+    except re.error:
+        return [n for n in range(first, last + 1) if grep in lines[n - 1]]
+    return [n for n in range(first, last + 1) if found.search(lines[n - 1])]
+
+
+def _with_context(hits: list[int], around: int, cap: int, first: int, last: int) -> set[int]:
+    """The matches, the lines around them, and no more than `cap` of anything.
+
+    The cap is applied match by match rather than to the finished set, so what
+    comes back is the *first* matches complete with their context instead of
+    every match with its context cut off. A truncated context is worse than a
+    missing match: the reader cannot tell it was truncated.
+    """
+    chosen: set[int] = set()
+    for hit in hits:
+        nearby = set(range(max(first, hit - around), min(last, hit + around) + 1))
+        if len(chosen | nearby) > cap and chosen:
+            break
+        chosen |= nearby
+    return chosen

sift/privacy.py

@@ -0,0 +1,145 @@
+"""What leaves this machine, and what is held back.
+
+Everything else here is about what a reader is shown. This module is about the
+other direction. Exactly one thing ever leaves: the text of a question. This
+decides what that text may contain, and whether it is sent at all.
+
+Two switches and one rule.
+
+**`SIFT_NO_MODEL`** turns sending off entirely. Nothing is asked and every view
+falls back to the deterministic one -- worse, and offered honestly as worse. A
+tool that cannot be told "not from this machine" is a tool that cannot be used
+on the machines where it would help most.
+
+**`SIFT_MASK=0`** turns masking off, for someone who has decided their output
+holds nothing worth hiding and would rather a model saw all of it.
+
+And the rule: **a credential is replaced on the way out, and never in what is
+shown.** The second half is what makes this cheap. A model is only ever asked
+for line numbers; the lines are printed from the local file, byte for byte. So a
+line masked on the way out is still shown to the reader in full. Masking costs
+the model a little context and costs the reader nothing, which is why it is on
+by default.
+
+This is the one place in this project where patterns are allowed, and `fallback`
+refuses them in the strongest terms, so the difference is worth stating. There a
+pattern would be *judging* -- deciding which lines matter, in languages it half
+knows, and being confidently wrong. Here nothing is judged. A pattern that fires
+wrongly costs a masked token in a prompt; a pattern that fails to fire leaks a
+key. Those two mistakes are not comparable, so the rule is: when in doubt, mask.
+
+What is **not** masked matters just as much. Long is not the same as secret. A
+commit hash, a checksum, a base64 payload in a build log are all long, all high
+entropy, and all harmless; a tool that redacted every one of them would hand the
+model a page of `[redacted]` and call it privacy. So what is matched here is the
+shape credentials actually have, and a bare secret shaped like nothing in
+particular will get through. That is a real limit, stated rather than papered
+over: masking reduces what leaks by accident, and `SIFT_NO_MODEL` is the switch
+that guarantees nothing leaves at all.
+"""
+
+from __future__ import annotations
+
+import os
+import re
+
+# Short, plain, and obviously not a value. It travels in a prompt, so it costs
+# the model a token and tells it that something was there.
+REDACTED = "[redacted]"
+
+_NO = {"0", "false", "no", "off", ""}
+
+
+def sending_on() -> bool:
+    """Whether a model may be asked anything at all.
+
+    Off is the easy state to reach: `SIFT_NO_MODEL` set to anything except a
+    plain no means no sending. A privacy switch that only works when spelled
+    exactly right is a switch that fails open, and this one has to fail closed.
+    """
+    return (os.environ.get("SIFT_NO_MODEL") or "").strip().lower() in _NO
+
+
+def masking_on() -> bool:
+    """Whether credentials are replaced before a question is sent.
+
+    On unless switched off, and here a value nobody recognises leaves it on.
+    Both defaults point the same way: towards less leaving the machine.
+    """
+    return (os.environ.get("SIFT_MASK") or "1").strip().lower() not in _NO
+
+
+# Each rule is a pattern and which group of it holds the secret. Group 0 means
+# the whole match is the secret; a numbered group keeps the surrounding text,
+# which is what tells a reader -- and a model -- what was hidden and where.
+_RULES: list[tuple[re.Pattern[str], int]] = [
+    # A JSON Web Token. The first segment always begins this way, because it is
+    # base64 of a JSON object that opens with a quoted key.
+    (re.compile(r"\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{4,}"), 0),
+    # Tokens that carry their own prefix. Nothing else looks like these, so
+    # matching them costs nothing and missing them costs a key.
+    (re.compile(r"\b(?:AKIA|ASIA|ABIA|ACCA|A3T[A-Z0-9])[A-Z0-9]{16}\b"), 0),
+    (re.compile(r"\bgh[pousr]_[A-Za-z0-9]{20,}\b"), 0),
+    (re.compile(r"\bxox[abprs]-[A-Za-z0-9-]{10,}"), 0),
+    (re.compile(r"\bsk-[A-Za-z0-9_-]{20,}\b"), 0),
+    (re.compile(r"\bnvapi-[A-Za-z0-9_-]{20,}\b"), 0),
+    (re.compile(r"\bAIza[0-9A-Za-z_-]{35,}"), 0),
+    (re.compile(r"\bglpat-[A-Za-z0-9_-]{20,}\b"), 0),
+    (re.compile(r"\bhf_[A-Za-z0-9]{20,}\b"), 0),
+    # An authorization header, whatever scheme it names.
+    (re.compile(r"(?i)\b(?:bearer|basic|token)\s+([A-Za-z0-9+/=._~-]{12,})"), 1),
+    # A password inside a connection string: scheme://user:secret@host
+    (re.compile(r"\b[a-zA-Z][a-zA-Z0-9+.-]*://[^\s:/@]+:([^\s@/]{3,})@"), 1),
+    # An assignment whose name says what it holds. The names are English because
+    # configuration and protocols are, whatever language the project around them
+    # is written in -- but this rule is a bonus rather than the floor, and cannot
+    # be complete. The shapes above are what this module actually rests on.
+    (
+        re.compile(
+            r"(?i)\b(?:password|passwd|pwd|secret|token|api[_-]?key|access[_-]?key"
+            r"|client[_-]?secret|auth[_-]?token|credential)s?[\"']?\s*[:=]\s*"
+            r"[\"']?([^\s\"',;)]{4,})"
+        ),
+        1,
+    ),
+    # A line that is nothing but base64: the body of a PEM block. Three things
+    # narrow it, and each was earned. Anchored to the whole line, because a long
+    # base64 run *inside* a sentence is a payload someone is debugging and
+    # masking it would help nobody. And mixed case with a digit, because
+    # "x" * 5000 is a line of x's -- a test in this project produces exactly
+    # that, and an earlier version of this rule redacted it. Long is not the
+    # same as dense, and neither is the same as secret.
+    (
+        re.compile(
+            r"^(?=[A-Za-z0-9+/]*[a-z])(?=[A-Za-z0-9+/]*[A-Z])(?=[A-Za-z0-9+/]*[0-9])"
+            r"[A-Za-z0-9+/]{40,}={0,2}$"
+        ),
+        0,
+    ),
+]
+
+
+def mask(text: str) -> str:
+    """The same text with anything credential-shaped replaced.
+
+    Applied to one line at a time by its caller, and that is deliberate: a
+    replacement that spanned lines would take the line numbers with it, and the
+    numbers are the only thing a model is ever asked to give back.
+    """
+    if not masking_on():
+        return text
+    for pattern, group in _RULES:
+        text = pattern.sub(lambda found, g=group: _hidden(found, g), text)
+    return text
+
+
+def _hidden(found: re.Match[str], group: int) -> str:
+    """The match with its secret part replaced and the rest of it left alone."""
+    whole = found.group(0)
+    if group == 0:
+        return REDACTED
+    value = found.group(group)
+    if not value:
+        return whole
+    cut = found.start(group) - found.start(0)
+    return whole[:cut] + REDACTED + whole[cut + len(value) :]

sift/records.py

@@ -0,0 +1,95 @@
+"""When a line is not a unit anybody can choose.
+
+Everything else here counts in lines, and for command output that is right: a
+line is what the writer meant as one thing and what a reader's eye stops on.
+
+A JSON array is the case where it is simply false. Written by a machine for
+another machine, it very often has no newlines at all -- four hundred kilobytes
+on line 1 -- and then there is exactly one unit to choose from and choosing it
+means showing all of it. Written out with indentation it is worse in a quieter
+way: the lines exist, but a line of one is `"id": 3,`, which is true, chosen,
+byte for byte from the file, and of no use to anybody. Neither shape is helped
+by a better question. The unit is wrong.
+
+So when the text is a JSON array, the record is the unit and everything else
+stays exactly as it was. The model still answers with numbers and nothing else,
+the text still comes out of the source unchanged, and a gap still says how many
+of them it covers. Only what is being counted changes.
+
+**Nothing is re-serialised.** A record is a slice of the original text, offset
+to offset, so the whitespace, key order and number formatting a reader sees are
+the ones in the file. Handing back `json.dumps` of a parsed value would be this
+tool writing a line, which is the one thing it does not do -- and it would be
+wrong quietly: `1.0` comes back `1.0`, but `1E2` comes back `100.0`.
+
+**Only a top-level array.** An object wrapping one (`{"results": [...]}`) is
+left as lines, and that is a decision rather than an omission: picking which of
+an object's arrays holds "the records" is a guess about somebody else's schema,
+and a guess that decides what a reader sees is the thing this project was
+rewritten to be rid of. A caller who knows better has `--keep`.
+"""
+
+from __future__ import annotations
+
+import json
+
+# One record is not a choice, and a view of it is the text itself. Below two,
+# lines are no worse and the machinery is not worth entering.
+FEWEST = 2
+
+_SPACE = " \t\n\r"
+
+
+def of(text: str) -> list[str] | None:
+    """The records of a top-level JSON array, or None if that is not what this is.
+
+    Each record is `text[start:end]` of the source. The scan is a single pass
+    with the standard decoder, which is what makes it exact rather than a
+    bracket-counter that a brace inside a string would fool.
+
+    Returns None rather than raising, everywhere: this is asked of every capture
+    and almost every one of them is not JSON. "No" is the ordinary answer here,
+    not a failure, and it costs one character to reach.
+    """
+    at = _past_space(text, 0)
+    if at >= len(text) or text[at] != "[":
+        return None
+
+    decoder = json.JSONDecoder()
+    found: list[str] = []
+    at = _past_space(text, at + 1)
+
+    while True:
+        if at >= len(text):
+            return None  # the array never closed; this is not a whole document
+        if text[at] == "]" and not found:
+            return None  # an empty array has nothing to choose between
+        try:
+            _, end = decoder.raw_decode(text, at)
+        except ValueError:
+            return None
+        found.append(text[at:end])
+
+        at = _past_space(text, end)
+        if at >= len(text):
+            return None
+        if text[at] == "]":
+            break
+        if text[at] != ",":
+            return None
+        at = _past_space(text, at + 1)
+
+    # A document that goes on after the array is not an array. Refusing it keeps
+    # the promise the rest of this file rests on: every byte of the source is in
+    # exactly one record, so a count of records folded away is a count of the
+    # whole thing.
+    if _past_space(text, at + 1) != len(text):
+        return None
+    return found if len(found) >= FEWEST else None
+
+
+def _past_space(text: str, at: int) -> int:
+    """The next index at or after `at` that is not JSON whitespace."""
+    while at < len(text) and text[at] in _SPACE:
+        at += 1
+    return at