sift-cli 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sift/__init__.py +18 -0
- sift/answers.py +175 -0
- sift/background.py +444 -0
- sift/capture.py +240 -0
- sift/cli.py +820 -0
- sift/digest.py +101 -0
- sift/distill.py +670 -0
- sift/fallback.py +98 -0
- sift/hook.py +275 -0
- sift/lines.py +37 -0
- sift/many.py +51 -0
- sift/memory.py +94 -0
- sift/model.py +433 -0
- sift/outline.py +117 -0
- sift/peek.py +161 -0
- sift/privacy.py +145 -0
- sift/records.py +95 -0
- sift/server.py +552 -0
- sift/store.py +499 -0
- sift/tools.py +76 -0
- sift/view.py +317 -0
- sift/watch.py +166 -0
- sift_cli-1.0.0.dist-info/METADATA +326 -0
- sift_cli-1.0.0.dist-info/RECORD +27 -0
- sift_cli-1.0.0.dist-info/WHEEL +4 -0
- sift_cli-1.0.0.dist-info/entry_points.txt +3 -0
- sift_cli-1.0.0.dist-info/licenses/LICENSE +21 -0
sift/cli.py
ADDED
|
@@ -0,0 +1,820 @@
|
|
|
1
|
+
"""The command line: run something, read what mattered, go and read the rest.
|
|
2
|
+
|
|
3
|
+
Five commands that do something, because each one is a promise that has to keep
|
|
4
|
+
working in every language and every shell -- and two that report on them:
|
|
5
|
+
|
|
6
|
+
sift run -- pytest -q run it, show the lines that mattered
|
|
7
|
+
sift run --background -- make start it, get the prompt back
|
|
8
|
+
sift follow a3f1 what it has said since you last looked
|
|
9
|
+
sift stop a3f1 end it, and write down how it ended
|
|
10
|
+
sift outline src/parser.rs what a file declares, without its bodies
|
|
11
|
+
sift peek a3f1 200 260 the capture itself, byte for byte
|
|
12
|
+
sift list what is running, and what has been run
|
|
13
|
+
sift stats what the shortening cost, and what it saved
|
|
14
|
+
|
|
15
|
+
`outline` is the same machine asking a different question. Nothing in it knows
|
|
16
|
+
one language from another, and there is no list of suffixes deciding what it
|
|
17
|
+
will look at: the command word already said what you wanted.
|
|
18
|
+
|
|
19
|
+
The arguments are read by hand rather than with `argparse`. This is not
|
|
20
|
+
stubbornness: `sift run -- pytest -x --lf` hands `sift` a command that has flags
|
|
21
|
+
of its own, and any parser clever enough to be helpful is clever enough to eat
|
|
22
|
+
them. Everything after the command word is passed through untouched.
|
|
23
|
+
|
|
24
|
+
The rule this file exists to keep is the third one in the README: **nothing can
|
|
25
|
+
break your command.** No key, no network, a busy endpoint, a reply full of
|
|
26
|
+
nonsense, a bug in the distiller -- every one of them ends with the output on
|
|
27
|
+
screen and the command's own exit code coming back out. The model is an
|
|
28
|
+
improvement on this tool's behaviour, never a requirement of it.
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
from __future__ import annotations
|
|
32
|
+
|
|
33
|
+
import contextlib
|
|
34
|
+
import json
|
|
35
|
+
import os
|
|
36
|
+
import sys
|
|
37
|
+
import time
|
|
38
|
+
|
|
39
|
+
from sift import answers, store
|
|
40
|
+
from sift import hook as hook_module
|
|
41
|
+
from sift.background import alive, launch, seen, stop, unread, wait_for
|
|
42
|
+
from sift.capture import Capture, run
|
|
43
|
+
from sift.distill import BUDGET
|
|
44
|
+
from sift.hook import answer as hook_answer
|
|
45
|
+
from sift.memory import habits, reach
|
|
46
|
+
from sift.model import find_key, sending_on
|
|
47
|
+
from sift.peek import FOUND_CAP, peek
|
|
48
|
+
from sift.tools import BY_NAME, KNOWN, command_for
|
|
49
|
+
from sift.view import (
|
|
50
|
+
best_digest,
|
|
51
|
+
best_digests,
|
|
52
|
+
best_follow,
|
|
53
|
+
best_outline,
|
|
54
|
+
best_view,
|
|
55
|
+
ending,
|
|
56
|
+
follow_footer,
|
|
57
|
+
footer,
|
|
58
|
+
outline_footer,
|
|
59
|
+
peek_footer,
|
|
60
|
+
state,
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
USAGE = """sift -- run a command, keep every byte, show the lines that matter
|
|
64
|
+
|
|
65
|
+
sift run [--timeout SECONDS] [--shell] [--background] [--cwd DIR]
|
|
66
|
+
[--budget LINES] [--keep PATTERN] [--] COMMAND...
|
|
67
|
+
sift follow [HANDLE] [--all] [--wait SECONDS]
|
|
68
|
+
sift stop [HANDLE]
|
|
69
|
+
sift outline [--budget LINES] [--keep PATTERN] PATH
|
|
70
|
+
sift digest [--budget LINES] [--keep PATTERN] PATH...
|
|
71
|
+
sift peek HANDLE|PATH [FIRST] [LAST] [--grep PATTERN] [--around N] [--max N]
|
|
72
|
+
sift list [COUNT]
|
|
73
|
+
sift hook answer one shell-command event on stdin
|
|
74
|
+
sift hook --install [--yes] route the client's own shell here too
|
|
75
|
+
sift hook --uninstall and take it back out
|
|
76
|
+
sift tools which dense tools this machine has
|
|
77
|
+
sift tool NAME [ARGS...] run one of them, distilled
|
|
78
|
+
sift memory [TERM] [--here]
|
|
79
|
+
sift stats [COUNT]
|
|
80
|
+
sift gc [DAYS] remove captures older than that
|
|
81
|
+
|
|
82
|
+
Everything after COMMAND is passed to it unchanged. Use -- when the command
|
|
83
|
+
has flags that look like sift's own.
|
|
84
|
+
|
|
85
|
+
--keep shows every line matching PATTERN whatever else was chosen, and whatever
|
|
86
|
+
the budget says. It is your pattern, not one this tool guessed at."""
|
|
87
|
+
|
|
88
|
+
# What a shell reports when a command was killed for running too long, and what
|
|
89
|
+
# `timeout(1)` returns. Borrowed rather than invented: scripts already know it.
|
|
90
|
+
TIMED_OUT = 124
|
|
91
|
+
|
|
92
|
+
# What a shell reports when the command could not be found or could not be run.
|
|
93
|
+
CANNOT_RUN = 127
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
# What a person is told when nobody has finished setting this up.
|
|
97
|
+
#
|
|
98
|
+
# Loud, because the failure it describes is a quiet one. Without a key the tool
|
|
99
|
+
# still runs, still keeps every byte, still returns the exit code -- the third
|
|
100
|
+
# rule holds -- and hands back the first ten lines and the last forty. That view
|
|
101
|
+
# is honest and it is much worse, and it looks exactly like a good one: the same
|
|
102
|
+
# shape, the same gap markers, the same confidence. Somebody who does not know
|
|
103
|
+
# this happened will conclude the tool is not much use.
|
|
104
|
+
#
|
|
105
|
+
# On stderr, so that piping a view somewhere is unaffected by it.
|
|
106
|
+
NO_KEY = """\
|
|
107
|
+
┌──────────────────────────────────────────────────────────────────────┐
|
|
108
|
+
│ sift has no API key, so no model chose these lines. │
|
|
109
|
+
│ You are seeing the beginning and the end of the output, and that is │
|
|
110
|
+
│ all this can do unaided. It works; it works much worse. │
|
|
111
|
+
│ │
|
|
112
|
+
│ The key is yours to add, and free: │
|
|
113
|
+
│ export SIFT_API_KEY=... │
|
|
114
|
+
│ or put it in ~/.config/nvidia/api_key │
|
|
115
|
+
│ get one at https://build.nvidia.com │
|
|
116
|
+
│ │
|
|
117
|
+
│ Meant to run without a model? SIFT_NO_MODEL=1 says so, and silences │
|
|
118
|
+
│ this. │
|
|
119
|
+
└──────────────────────────────────────────────────────────────────────┘"""
|
|
120
|
+
|
|
121
|
+
# The commands that would have asked a model. The rest -- peek, list, stats,
|
|
122
|
+
# memory, gc, tools -- answer out of what is already on disk, and warning about
|
|
123
|
+
# a key they were never going to use is noise.
|
|
124
|
+
NEEDS_A_MODEL = frozenset({"run", "follow", "outline", "digest", "tool", "hook"})
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _warn_unset(word: str) -> None:
|
|
128
|
+
"""Say it, once, before the view rather than after it.
|
|
129
|
+
|
|
130
|
+
`SIFT_NO_MODEL=1` silences this and that is the point of the switch: it is
|
|
131
|
+
the difference between somebody who decided and somebody who has not
|
|
132
|
+
finished. Nagging the first about the second is how a warning gets ignored.
|
|
133
|
+
"""
|
|
134
|
+
if word in NEEDS_A_MODEL and sending_on() and find_key() is None:
|
|
135
|
+
print(NO_KEY, file=sys.stderr)
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def main(argv: list[str] | None = None) -> int:
|
|
139
|
+
_speak_utf8()
|
|
140
|
+
args = list(sys.argv[1:] if argv is None else argv)
|
|
141
|
+
if not args or args[0] in ("-h", "--help", "help"):
|
|
142
|
+
print(USAGE)
|
|
143
|
+
return 0
|
|
144
|
+
word, rest = args[0], args[1:]
|
|
145
|
+
_warn_unset(word)
|
|
146
|
+
if word == "run":
|
|
147
|
+
code = _run(rest)
|
|
148
|
+
_mention_hook()
|
|
149
|
+
return code
|
|
150
|
+
if word == "follow":
|
|
151
|
+
return _follow(rest)
|
|
152
|
+
if word == "stop":
|
|
153
|
+
return _stop(rest)
|
|
154
|
+
if word == "outline":
|
|
155
|
+
return _outline(rest)
|
|
156
|
+
if word == "digest":
|
|
157
|
+
return _digest(rest)
|
|
158
|
+
if word == "peek":
|
|
159
|
+
return _peek(rest)
|
|
160
|
+
if word == "list":
|
|
161
|
+
return _list(rest)
|
|
162
|
+
if word == "hook":
|
|
163
|
+
return _hook(rest)
|
|
164
|
+
if word == "tools":
|
|
165
|
+
return _tools()
|
|
166
|
+
if word == "tool":
|
|
167
|
+
return _tool(rest)
|
|
168
|
+
if word == "memory":
|
|
169
|
+
return _memory(rest)
|
|
170
|
+
if word == "stats":
|
|
171
|
+
return _stats(rest)
|
|
172
|
+
if word == "gc":
|
|
173
|
+
return _gc(rest)
|
|
174
|
+
print(f"sift: no such command: {word}\n\n{USAGE}", file=sys.stderr)
|
|
175
|
+
return 2
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def _speak_utf8() -> None:
|
|
179
|
+
"""Say what the command said, whatever the console was set up to expect.
|
|
180
|
+
|
|
181
|
+
A capture can hold any language, and the gap marker is drawn with `─` and
|
|
182
|
+
`·`. A console still set to a legacy code page -- the default on Windows --
|
|
183
|
+
raises on the first character it cannot encode, which would lose the output
|
|
184
|
+
to a detail of the terminal rather than anything about the command. Encoding
|
|
185
|
+
with replacement loses a glyph; not doing this loses the run.
|
|
186
|
+
"""
|
|
187
|
+
for stream in (sys.stdout, sys.stderr):
|
|
188
|
+
with contextlib.suppress(AttributeError, OSError, ValueError):
|
|
189
|
+
stream.reconfigure(encoding="utf-8", errors="replace")
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def _run(args: list[str]) -> int:
|
|
193
|
+
timeout: float | None = None
|
|
194
|
+
shell = False
|
|
195
|
+
detached = False
|
|
196
|
+
where: str | None = None
|
|
197
|
+
budget: int | None = None
|
|
198
|
+
keep: str | None = None
|
|
199
|
+
while args:
|
|
200
|
+
if args[0] == "--timeout" and len(args) > 1:
|
|
201
|
+
timeout = _seconds(args[1])
|
|
202
|
+
args = args[2:]
|
|
203
|
+
elif args[0] == "--cwd" and len(args) > 1:
|
|
204
|
+
where = args[1]
|
|
205
|
+
args = args[2:]
|
|
206
|
+
elif args[0] == "--budget" and len(args) > 1:
|
|
207
|
+
budget = _line_number(args[1])
|
|
208
|
+
args = args[2:]
|
|
209
|
+
elif args[0] == "--keep" and len(args) > 1:
|
|
210
|
+
keep = args[1]
|
|
211
|
+
args = args[2:]
|
|
212
|
+
elif args[0] == "--shell":
|
|
213
|
+
shell = True
|
|
214
|
+
args = args[1:]
|
|
215
|
+
elif args[0] == "--background":
|
|
216
|
+
detached = True
|
|
217
|
+
args = args[1:]
|
|
218
|
+
elif args[0] == "--":
|
|
219
|
+
args = args[1:]
|
|
220
|
+
break
|
|
221
|
+
else:
|
|
222
|
+
break
|
|
223
|
+
|
|
224
|
+
if not args:
|
|
225
|
+
print(f"sift: run needs a command\n\n{USAGE}", file=sys.stderr)
|
|
226
|
+
return 2
|
|
227
|
+
|
|
228
|
+
if detached:
|
|
229
|
+
return _background(args, shell=shell, timeout=timeout, cwd=where)
|
|
230
|
+
|
|
231
|
+
try:
|
|
232
|
+
capture = run(args, timeout=timeout, shell=shell, cwd=where)
|
|
233
|
+
except OSError as exc:
|
|
234
|
+
print(f"sift: {exc}", file=sys.stderr)
|
|
235
|
+
return CANNOT_RUN
|
|
236
|
+
|
|
237
|
+
try:
|
|
238
|
+
_show(capture, budget, keep)
|
|
239
|
+
except Exception as exc: # the view is optional; the output is not
|
|
240
|
+
print(f"sift: {type(exc).__name__}: {exc}", file=sys.stderr)
|
|
241
|
+
_last_resort(capture)
|
|
242
|
+
return _exit_code(capture)
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
def _background(
|
|
246
|
+
command: list[str], *, shell: bool, timeout: float | None, cwd: str | None = None
|
|
247
|
+
) -> int:
|
|
248
|
+
"""Start the command, say where to find it, and give the prompt back.
|
|
249
|
+
|
|
250
|
+
The handle goes to stdout on its own so it can be caught in a variable; the
|
|
251
|
+
advice goes to stderr with everything else this tool says about itself.
|
|
252
|
+
|
|
253
|
+
A timeout is refused rather than ignored. There is nobody here to enforce
|
|
254
|
+
one -- the point of this flag is that nothing waits -- and quietly dropping
|
|
255
|
+
a limit the caller asked for is how a build runs all night.
|
|
256
|
+
"""
|
|
257
|
+
if timeout is not None:
|
|
258
|
+
print(
|
|
259
|
+
"sift: --background and --timeout do not go together: nothing is waiting"
|
|
260
|
+
f" to enforce it. Use sift stop when you have seen enough.\n\n{USAGE}",
|
|
261
|
+
file=sys.stderr,
|
|
262
|
+
)
|
|
263
|
+
return 2
|
|
264
|
+
|
|
265
|
+
try:
|
|
266
|
+
started = launch(command, shell=shell, cwd=cwd)
|
|
267
|
+
except OSError as exc:
|
|
268
|
+
print(f"sift: {exc}", file=sys.stderr)
|
|
269
|
+
return CANNOT_RUN
|
|
270
|
+
|
|
271
|
+
print(started.handle)
|
|
272
|
+
print(
|
|
273
|
+
f"sift {started.handle} · started · sift follow {started.handle}",
|
|
274
|
+
file=sys.stderr,
|
|
275
|
+
)
|
|
276
|
+
return 0
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
def _follow(args: list[str]) -> int:
|
|
280
|
+
"""What a running command has said since the last look, and nothing before it.
|
|
281
|
+
|
|
282
|
+
With no handle it follows the newest run left going, which is what somebody
|
|
283
|
+
who started one thing and walked away actually wants to type.
|
|
284
|
+
"""
|
|
285
|
+
everything = "--all" in args
|
|
286
|
+
args = [word for word in args if word != "--all"]
|
|
287
|
+
wait = 0.0
|
|
288
|
+
if len(args) > 1 and args[0] == "--wait":
|
|
289
|
+
wait = _seconds(args[1]) or 0.0
|
|
290
|
+
args = args[2:]
|
|
291
|
+
elif len(args) > 2 and args[1] == "--wait":
|
|
292
|
+
wait = _seconds(args[2]) or 0.0
|
|
293
|
+
args = args[:1]
|
|
294
|
+
|
|
295
|
+
if everything:
|
|
296
|
+
return _follow_all(wait)
|
|
297
|
+
|
|
298
|
+
handle = args[0] if args else _newest()
|
|
299
|
+
if handle is None:
|
|
300
|
+
print("sift: nothing is running", file=sys.stderr)
|
|
301
|
+
return 1
|
|
302
|
+
|
|
303
|
+
if wait:
|
|
304
|
+
wait_for(handle, wait)
|
|
305
|
+
|
|
306
|
+
running = store.load_running(handle)
|
|
307
|
+
meta = store.load(handle)
|
|
308
|
+
if running is None and meta is None:
|
|
309
|
+
print(f"sift: no such run: {handle}", file=sys.stderr)
|
|
310
|
+
return 1
|
|
311
|
+
|
|
312
|
+
fresh, first, moved = unread(handle)
|
|
313
|
+
view = None
|
|
314
|
+
try:
|
|
315
|
+
view, who = best_follow(handle, fresh, first)
|
|
316
|
+
if view.text:
|
|
317
|
+
print(view.text)
|
|
318
|
+
except Exception as exc: # the view is optional; the output is not
|
|
319
|
+
print(f"sift: {type(exc).__name__}: {exc}", file=sys.stderr)
|
|
320
|
+
for number, line in enumerate(fresh, first):
|
|
321
|
+
print(f"{number:>6} {line}")
|
|
322
|
+
who = f"no view ({type(exc).__name__})"
|
|
323
|
+
|
|
324
|
+
# The cursor moves only after the lines have been printed, whichever way
|
|
325
|
+
# they were printed. Marking them read before that would lose them for good
|
|
326
|
+
# to a failure that has nothing to do with the command.
|
|
327
|
+
seen(handle, moved)
|
|
328
|
+
|
|
329
|
+
if view is not None:
|
|
330
|
+
note = follow_footer(handle, view, who, first, state(running, meta))
|
|
331
|
+
print(note, file=sys.stderr)
|
|
332
|
+
if meta is not None and running is None:
|
|
333
|
+
return _exit_code(Capture(meta))
|
|
334
|
+
return 0
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
def _follow_all(wait: float = 0.0) -> int:
|
|
338
|
+
"""Every command still going, in one look.
|
|
339
|
+
|
|
340
|
+
An agent supervising three builds should not have to ask three times and
|
|
341
|
+
carry three answers. Waiting is done once, on whichever run speaks first,
|
|
342
|
+
because waiting on each in turn would add their timeouts together.
|
|
343
|
+
"""
|
|
344
|
+
running = store.started()
|
|
345
|
+
if not running:
|
|
346
|
+
print("sift: nothing is running", file=sys.stderr)
|
|
347
|
+
return 1
|
|
348
|
+
|
|
349
|
+
if wait:
|
|
350
|
+
deadline = store.now() + wait
|
|
351
|
+
while store.now() < deadline:
|
|
352
|
+
if any(unread(r.handle)[0] for r in running):
|
|
353
|
+
break
|
|
354
|
+
time.sleep(0.1)
|
|
355
|
+
|
|
356
|
+
for one in running:
|
|
357
|
+
_follow([one.handle])
|
|
358
|
+
return 0
|
|
359
|
+
|
|
360
|
+
|
|
361
|
+
def _stop(args: list[str]) -> int:
|
|
362
|
+
"""End a run and make sure it ends up with an ending written down."""
|
|
363
|
+
handle = args[0] if args else _newest()
|
|
364
|
+
if handle is None:
|
|
365
|
+
print("sift: nothing is running", file=sys.stderr)
|
|
366
|
+
return 1
|
|
367
|
+
|
|
368
|
+
meta = stop(handle)
|
|
369
|
+
if meta is None:
|
|
370
|
+
print(f"sift: no such run: {handle}", file=sys.stderr)
|
|
371
|
+
return 1
|
|
372
|
+
print(
|
|
373
|
+
f"sift {handle} · {ending(meta)} · {meta.byte_count:,} B captured"
|
|
374
|
+
f" · sift follow {handle}",
|
|
375
|
+
file=sys.stderr,
|
|
376
|
+
)
|
|
377
|
+
return 0
|
|
378
|
+
|
|
379
|
+
|
|
380
|
+
def _newest() -> str | None:
|
|
381
|
+
found = store.started()
|
|
382
|
+
return found[0].handle if found else None
|
|
383
|
+
|
|
384
|
+
|
|
385
|
+
def _show(capture: Capture, budget: int | None = None, keep: str | None = None) -> None:
|
|
386
|
+
view, who = best_view(capture, BUDGET if budget is None else budget, keep)
|
|
387
|
+
if view.text:
|
|
388
|
+
print(view.text)
|
|
389
|
+
print(footer(capture, view, who), file=sys.stderr)
|
|
390
|
+
|
|
391
|
+
|
|
392
|
+
def _last_resort(capture: Capture) -> None:
|
|
393
|
+
"""Everything that chooses lines has failed. Show every line instead.
|
|
394
|
+
|
|
395
|
+
This is the floor the third rule stands on. Reaching it means `sift` has a
|
|
396
|
+
bug, and a bug in the part that shortens output must cost the shortening --
|
|
397
|
+
not the output, and not the exit code the caller is about to act on.
|
|
398
|
+
"""
|
|
399
|
+
print(f"sift: showing the capture unchanged ({capture.handle})", file=sys.stderr)
|
|
400
|
+
try:
|
|
401
|
+
sys.stdout.write(capture.text())
|
|
402
|
+
except OSError as exc:
|
|
403
|
+
print(f"sift: the capture is at {store.raw_path(capture.handle)} ({exc})",
|
|
404
|
+
file=sys.stderr)
|
|
405
|
+
|
|
406
|
+
|
|
407
|
+
def _exit_code(capture: Capture) -> int:
|
|
408
|
+
"""The command's own answer, so that wrapping it in `sift` changes nothing.
|
|
409
|
+
|
|
410
|
+
A script that fails when `pytest` fails has to keep failing when it becomes
|
|
411
|
+
`sift run -- pytest`. Anything else would make this tool unusable in the
|
|
412
|
+
place it is most useful.
|
|
413
|
+
"""
|
|
414
|
+
if capture.meta.timed_out:
|
|
415
|
+
return TIMED_OUT
|
|
416
|
+
return capture.meta.exit_code or 0
|
|
417
|
+
|
|
418
|
+
|
|
419
|
+
def _outline(args: list[str]) -> int:
|
|
420
|
+
budget, keep, args = _shown_how(args)
|
|
421
|
+
if not args:
|
|
422
|
+
print(f"sift: outline needs a path\n\n{USAGE}", file=sys.stderr)
|
|
423
|
+
return 2
|
|
424
|
+
|
|
425
|
+
path = args[0]
|
|
426
|
+
try:
|
|
427
|
+
view, who = best_outline(path, budget, keep)
|
|
428
|
+
except OSError as exc: # the file itself cannot be read; there is no view
|
|
429
|
+
print(f"sift: {exc}", file=sys.stderr)
|
|
430
|
+
return 1
|
|
431
|
+
|
|
432
|
+
if view.text:
|
|
433
|
+
print(view.text)
|
|
434
|
+
print(outline_footer(path, view, who), file=sys.stderr)
|
|
435
|
+
return 0
|
|
436
|
+
|
|
437
|
+
|
|
438
|
+
def _digest(args: list[str]) -> int:
|
|
439
|
+
"""A file somebody else produced, read for what is in it.
|
|
440
|
+
|
|
441
|
+
Shaped like `_outline` because it is the same machinery asked a different
|
|
442
|
+
question, and the two must not drift: a caller who learns one has learned
|
|
443
|
+
the other.
|
|
444
|
+
"""
|
|
445
|
+
budget, keep, args = _shown_how(args)
|
|
446
|
+
if not args:
|
|
447
|
+
print(f"sift: digest needs a path\n\n{USAGE}", file=sys.stderr)
|
|
448
|
+
return 2
|
|
449
|
+
|
|
450
|
+
if len(args) == 1:
|
|
451
|
+
try:
|
|
452
|
+
view, who = best_digest(args[0], budget, keep)
|
|
453
|
+
except OSError as exc: # the file itself cannot be read; there is no view
|
|
454
|
+
print(f"sift: {exc}", file=sys.stderr)
|
|
455
|
+
return 1
|
|
456
|
+
if view.text:
|
|
457
|
+
print(view.text)
|
|
458
|
+
print(outline_footer(args[0], view, who), file=sys.stderr)
|
|
459
|
+
return 0
|
|
460
|
+
|
|
461
|
+
# Several paths are asked about at the same time. Each keeps its own footer,
|
|
462
|
+
# because a reader with four digests in front of them needs to know which
|
|
463
|
+
# one they are looking at and which of them nobody could reach a model for.
|
|
464
|
+
worst = 0
|
|
465
|
+
for path, view, who in best_digests(args, budget, keep):
|
|
466
|
+
if view.text:
|
|
467
|
+
print(view.text)
|
|
468
|
+
print(outline_footer(path, view, who), file=sys.stderr)
|
|
469
|
+
if who.startswith("unreadable"):
|
|
470
|
+
worst = 1
|
|
471
|
+
return worst
|
|
472
|
+
|
|
473
|
+
|
|
474
|
+
def _shown_how(args: list[str]) -> tuple[int | None, str | None, list[str]]:
|
|
475
|
+
"""The two words a caller may put before a path, and what is left after them."""
|
|
476
|
+
budget: int | None = None
|
|
477
|
+
keep: str | None = None
|
|
478
|
+
while len(args) > 1 and args[0] in ("--budget", "--keep"):
|
|
479
|
+
if args[0] == "--budget":
|
|
480
|
+
budget = _line_number(args[1])
|
|
481
|
+
else:
|
|
482
|
+
keep = args[1]
|
|
483
|
+
args = args[2:]
|
|
484
|
+
return budget, keep, args
|
|
485
|
+
|
|
486
|
+
|
|
487
|
+
def _peek(args: list[str]) -> int:
|
|
488
|
+
grep: str | None = None
|
|
489
|
+
around = 3
|
|
490
|
+
cap = FOUND_CAP
|
|
491
|
+
kept: list[str] = []
|
|
492
|
+
while args:
|
|
493
|
+
if args[0] == "--grep" and len(args) > 1:
|
|
494
|
+
grep = args[1]
|
|
495
|
+
args = args[2:]
|
|
496
|
+
elif args[0] == "--around" and len(args) > 1:
|
|
497
|
+
around = _line_number(args[1]) or 0
|
|
498
|
+
args = args[2:]
|
|
499
|
+
elif args[0] == "--max" and len(args) > 1:
|
|
500
|
+
cap = _line_number(args[1]) or FOUND_CAP
|
|
501
|
+
args = args[2:]
|
|
502
|
+
else:
|
|
503
|
+
kept.append(args[0])
|
|
504
|
+
args = args[1:]
|
|
505
|
+
args = kept
|
|
506
|
+
|
|
507
|
+
if not args:
|
|
508
|
+
print(f"sift: peek needs a handle\n\n{USAGE}", file=sys.stderr)
|
|
509
|
+
return 2
|
|
510
|
+
first = _line_number(args[1]) if len(args) > 1 else None
|
|
511
|
+
last = _line_number(args[2]) if len(args) > 2 else None
|
|
512
|
+
try:
|
|
513
|
+
found = peek(args[0], first, last, grep, around, cap)
|
|
514
|
+
except (OSError, ValueError) as exc:
|
|
515
|
+
print(f"sift: {exc}", file=sys.stderr)
|
|
516
|
+
return 1
|
|
517
|
+
if found.text:
|
|
518
|
+
print(found.text)
|
|
519
|
+
print(peek_footer(found), file=sys.stderr)
|
|
520
|
+
return 0
|
|
521
|
+
|
|
522
|
+
|
|
523
|
+
def _list(args: list[str]) -> int:
|
|
524
|
+
"""What is running, then what has been run.
|
|
525
|
+
|
|
526
|
+
Running first because it is the part that can still be acted on. Their size
|
|
527
|
+
is read off the file rather than from a record, since the record of how big
|
|
528
|
+
a capture ended up is written when it ends.
|
|
529
|
+
"""
|
|
530
|
+
limit = _line_number(args[0]) if args else None
|
|
531
|
+
for running in store.started():
|
|
532
|
+
path = store.raw_path(running.handle)
|
|
533
|
+
size = path.stat().st_size if path.is_file() else 0
|
|
534
|
+
state = "running" if alive(running) else "lost"
|
|
535
|
+
print(f"{running.handle} {state:>9} {size:>10,} B {' '.join(running.command)}")
|
|
536
|
+
for meta in store.recent(limit or 20):
|
|
537
|
+
written = " ".join(meta.command)
|
|
538
|
+
print(f"{meta.handle} {ending(meta):>9} {meta.byte_count:>10,} B {written}")
|
|
539
|
+
return 0
|
|
540
|
+
|
|
541
|
+
|
|
542
|
+
def _hook(args: list[str]) -> int:
|
|
543
|
+
"""Answer one shell-command event, or set up the routing that sends them.
|
|
544
|
+
|
|
545
|
+
Answering is what a client calls; the two flags are what a person types. They
|
|
546
|
+
live on the same word because they are the same subject, and somebody who has
|
|
547
|
+
just read about `sift hook` should not have to find out that setting it up is
|
|
548
|
+
called something else.
|
|
549
|
+
|
|
550
|
+
Everything about this is written to fail open. Unreadable input, an
|
|
551
|
+
unexpected shape, a bug underneath -- each of them prints an empty answer,
|
|
552
|
+
which the client reads as *carry on*, and the command runs exactly as it
|
|
553
|
+
would have. A gate that breaks a shell is worse than no gate.
|
|
554
|
+
"""
|
|
555
|
+
if "--install" in args:
|
|
556
|
+
return _install_hook(yes="--yes" in args)
|
|
557
|
+
if "--uninstall" in args:
|
|
558
|
+
done, said = hook_module.uninstall()
|
|
559
|
+
print(said, file=sys.stderr if not done else sys.stdout)
|
|
560
|
+
return 0 if done else 1
|
|
561
|
+
|
|
562
|
+
try:
|
|
563
|
+
event = json.loads(sys.stdin.read() or "{}")
|
|
564
|
+
except (OSError, ValueError):
|
|
565
|
+
event = {}
|
|
566
|
+
|
|
567
|
+
try:
|
|
568
|
+
said = hook_answer(event)
|
|
569
|
+
except Exception: # the shell is not allowed to depend on this working
|
|
570
|
+
said = {}
|
|
571
|
+
|
|
572
|
+
print(json.dumps(said, ensure_ascii=False))
|
|
573
|
+
return 0
|
|
574
|
+
|
|
575
|
+
|
|
576
|
+
def _tools() -> int:
|
|
577
|
+
"""What each of them replaces, and whether this machine has it.
|
|
578
|
+
|
|
579
|
+
Nothing is installed from here. The list is a list of programs, not a
|
|
580
|
+
dependency: a machine that has none of them runs everything else in this
|
|
581
|
+
tool exactly as well.
|
|
582
|
+
"""
|
|
583
|
+
for one in KNOWN:
|
|
584
|
+
mark = "here" if one.here else "not here"
|
|
585
|
+
print(f" {one.name:<5} {mark:<9} {one.known_as:<12} replaces {one.replaces}")
|
|
586
|
+
return 0
|
|
587
|
+
|
|
588
|
+
|
|
589
|
+
def _tool(args: list[str]) -> int:
|
|
590
|
+
"""Run one of the dense tools and show what mattered in what it printed.
|
|
591
|
+
|
|
592
|
+
It is distilled like anything else, because that is the point: these
|
|
593
|
+
commands answer a question without opening a file, and then print more than
|
|
594
|
+
anyone can read.
|
|
595
|
+
"""
|
|
596
|
+
if not args:
|
|
597
|
+
print(f"sift: tool needs a name\n\n{USAGE}", file=sys.stderr)
|
|
598
|
+
return 2
|
|
599
|
+
|
|
600
|
+
line = command_for(args[0], args[1:])
|
|
601
|
+
if line is None:
|
|
602
|
+
known = ", ".join(one.name for one in KNOWN)
|
|
603
|
+
print(f"sift: no such tool: {args[0]} (there is {known})", file=sys.stderr)
|
|
604
|
+
return 2
|
|
605
|
+
|
|
606
|
+
known = BY_NAME[args[0]]
|
|
607
|
+
if not known.here:
|
|
608
|
+
print(
|
|
609
|
+
f"sift: {known.known_as} is not on this machine. It is what `{known.name}`"
|
|
610
|
+
f" runs, and it replaces {known.replaces}.",
|
|
611
|
+
file=sys.stderr,
|
|
612
|
+
)
|
|
613
|
+
return CANNOT_RUN
|
|
614
|
+
|
|
615
|
+
try:
|
|
616
|
+
capture = run(line)
|
|
617
|
+
except OSError as exc:
|
|
618
|
+
print(f"sift: {exc}", file=sys.stderr)
|
|
619
|
+
return CANNOT_RUN
|
|
620
|
+
|
|
621
|
+
try:
|
|
622
|
+
_show(capture)
|
|
623
|
+
except Exception as exc: # the view is optional; the output is not
|
|
624
|
+
print(f"sift: {type(exc).__name__}: {exc}", file=sys.stderr)
|
|
625
|
+
_last_resort(capture)
|
|
626
|
+
return _exit_code(capture)
|
|
627
|
+
|
|
628
|
+
|
|
629
|
+
def _memory(args: list[str]) -> int:
|
|
630
|
+
"""What has been run here before, and how it went.
|
|
631
|
+
|
|
632
|
+
No model is asked. The question is counting, and a model asked to count is
|
|
633
|
+
slower, costs a request and is sometimes wrong -- the same misuse as a rules
|
|
634
|
+
engine deciding which lines matter, pointed the other way.
|
|
635
|
+
"""
|
|
636
|
+
here = "--here" in args
|
|
637
|
+
rest = [word for word in args if word != "--here"]
|
|
638
|
+
term = rest[0] if rest else None
|
|
639
|
+
|
|
640
|
+
found = habits(term, os.getcwd() if here else None)
|
|
641
|
+
if not found:
|
|
642
|
+
seen = reach()
|
|
643
|
+
where = " here" if here else ""
|
|
644
|
+
print(f"sift: nothing{where} matches that, out of {seen:,} runs still on disk.")
|
|
645
|
+
return 0
|
|
646
|
+
|
|
647
|
+
print(f"{'runs':>5} {'failed':>6} {'last':>9} command")
|
|
648
|
+
for one in found:
|
|
649
|
+
mark = " ← never worked here" if one.never_worked else ""
|
|
650
|
+
print(
|
|
651
|
+
f"{one.runs:>5} {one.failures:>6} {one.last_ending:>9} {one.command}{mark}"
|
|
652
|
+
)
|
|
653
|
+
|
|
654
|
+
seen = reach()
|
|
655
|
+
print(f"\nout of {seen:,} runs still on disk; a capture that was removed took its record")
|
|
656
|
+
return 0
|
|
657
|
+
|
|
658
|
+
|
|
659
|
+
def _stats(args: list[str]) -> int:
|
|
660
|
+
"""What the shortening cost and what it saved, over the runs it was used on.
|
|
661
|
+
|
|
662
|
+
Per run and then in total, because the two say different things. One run is
|
|
663
|
+
a claim about one command; the total is the only number that answers the
|
|
664
|
+
question somebody installing this actually has, which is whether the tool is
|
|
665
|
+
worth the asks it spends.
|
|
666
|
+
"""
|
|
667
|
+
limit = _line_number(args[0]) if args else None
|
|
668
|
+
found = store.savings(limit or 20)
|
|
669
|
+
if not found:
|
|
670
|
+
print("sift: no view has been built yet, so there is nothing to add up.")
|
|
671
|
+
return 0
|
|
672
|
+
|
|
673
|
+
print(
|
|
674
|
+
f"{'handle':8} {'captured':>12} {'shown':>10} {'part':>6}"
|
|
675
|
+
f" {'asks':>4} {'tokens':>8} command"
|
|
676
|
+
)
|
|
677
|
+
raw = shown = spent = counted = 0
|
|
678
|
+
for meta, saving in found:
|
|
679
|
+
raw += saving.raw_bytes
|
|
680
|
+
shown += saving.shown_bytes
|
|
681
|
+
spent += saving.tokens
|
|
682
|
+
counted += 1 if saving.tokens else 0
|
|
683
|
+
cost = f"{saving.tokens:>8,}" if saving.tokens else f"{'—':>8}"
|
|
684
|
+
print(
|
|
685
|
+
f"{saving.handle:8} {saving.raw_bytes:>10,} B {saving.shown_bytes:>8,} B"
|
|
686
|
+
f" {saving.part:>5.1f}% {saving.asks:>4} {cost} {' '.join(meta.command)}"
|
|
687
|
+
)
|
|
688
|
+
|
|
689
|
+
part = shown * 100 / raw if raw else 0.0
|
|
690
|
+
word = "run" if len(found) == 1 else "runs"
|
|
691
|
+
print(
|
|
692
|
+
f"\n{len(found)} {word} · {raw:,} B captured · {shown:,} B shown"
|
|
693
|
+
f" · {part:.1f}% of it · the other {100 - part:.1f}% is on disk, not gone"
|
|
694
|
+
)
|
|
695
|
+
# Two numbers, and they are not the same kind of number. The share above is
|
|
696
|
+
# this tool's own arithmetic over bytes it holds, so it is exact. The cost
|
|
697
|
+
# below is the endpoint's count of its own tokens, so it is measured rather
|
|
698
|
+
# than estimated -- and a run that nobody counted is left out of it and said
|
|
699
|
+
# so, instead of being filled in with bytes divided by four.
|
|
700
|
+
if spent:
|
|
701
|
+
missing = len(found) - counted
|
|
702
|
+
unsaid = f", {missing} not counted" if missing else ""
|
|
703
|
+
print(f"cost {spent:,} tokens, as the endpoint counted them{unsaid}")
|
|
704
|
+
return 0
|
|
705
|
+
|
|
706
|
+
|
|
707
|
+
def _gc(args: list[str]) -> int:
|
|
708
|
+
"""Remove captures older than an age, and say what went and what it freed.
|
|
709
|
+
|
|
710
|
+
Typed, never automatic. Every other command here can be run without thinking
|
|
711
|
+
about it because none of them destroy anything; this one does, so it is only
|
|
712
|
+
ever this tool doing what somebody asked, at the moment they asked it.
|
|
713
|
+
|
|
714
|
+
What it leaves behind is a stone naming the handle and the date. A gap marker
|
|
715
|
+
printed a fortnight ago still says `sift peek 9f2c41ab`, and the person
|
|
716
|
+
following it deserves "that was removed on the 8th" rather than the answer
|
|
717
|
+
they would get for a handle they made up.
|
|
718
|
+
"""
|
|
719
|
+
days = _line_number(args[0]) if args else None
|
|
720
|
+
if days is None and args:
|
|
721
|
+
print(f"sift: gc takes a number of days\n\n{USAGE}", file=sys.stderr)
|
|
722
|
+
return 2
|
|
723
|
+
age = float(days) if days is not None else store.keep_days()
|
|
724
|
+
|
|
725
|
+
swept = store.sweep(age * 86_400.0)
|
|
726
|
+
forgotten, freed_answers = answers.forget(age * 86_400.0)
|
|
727
|
+
if not swept and not forgotten:
|
|
728
|
+
print(f"sift: nothing here is older than {age:g} days.")
|
|
729
|
+
return 0
|
|
730
|
+
|
|
731
|
+
freed = 0
|
|
732
|
+
for one in swept:
|
|
733
|
+
freed += one.byte_count
|
|
734
|
+
said = " ".join(one.command) or "(a run that was interrupted)"
|
|
735
|
+
print(f"{one.handle:8} {one.byte_count:>12,} B {said}")
|
|
736
|
+
word = "capture" if len(swept) == 1 else "captures"
|
|
737
|
+
print(f"\n{len(swept)} {word} removed, {freed:,} B freed.")
|
|
738
|
+
if forgotten:
|
|
739
|
+
remembered = "answer" if forgotten == 1 else "answers"
|
|
740
|
+
print(f"{forgotten} remembered {remembered} dropped, {freed_answers:,} B.")
|
|
741
|
+
return 0
|
|
742
|
+
|
|
743
|
+
|
|
744
|
+
|
|
745
|
+
|
|
746
|
+
def _install_hook(yes: bool) -> int:
|
|
747
|
+
"""Show what it would do, ask, and only then do it.
|
|
748
|
+
|
|
749
|
+
Asked rather than assumed, because this writes into a file the caller owns
|
|
750
|
+
and this tool does not. `--yes` is for the case where nobody is there to
|
|
751
|
+
answer; without a terminal and without it, this refuses rather than deciding
|
|
752
|
+
on somebody's behalf.
|
|
753
|
+
"""
|
|
754
|
+
print(hook_module.offer())
|
|
755
|
+
print()
|
|
756
|
+
if not yes:
|
|
757
|
+
if not sys.stdin.isatty():
|
|
758
|
+
print(
|
|
759
|
+
"sift: nobody is here to answer. Add --yes to install it anyway.",
|
|
760
|
+
file=sys.stderr,
|
|
761
|
+
)
|
|
762
|
+
return 1
|
|
763
|
+
try:
|
|
764
|
+
said = input("Add it? [y/N] ").strip().lower()
|
|
765
|
+
except (EOFError, KeyboardInterrupt):
|
|
766
|
+
said = ""
|
|
767
|
+
if said not in ("y", "yes"):
|
|
768
|
+
print("sift: left alone.")
|
|
769
|
+
return 0
|
|
770
|
+
|
|
771
|
+
done, said = hook_module.install()
|
|
772
|
+
print(said, file=sys.stdout if done else sys.stderr)
|
|
773
|
+
return 0 if done else 1
|
|
774
|
+
|
|
775
|
+
|
|
776
|
+
# Said once, ever, and then never again.
|
|
777
|
+
#
|
|
778
|
+
# The alternative to mentioning it is that nobody finds it: this is the part of
|
|
779
|
+
# the tool that pays most and the only part that has to be turned on, and a
|
|
780
|
+
# feature nobody knows about is a feature nobody has. The alternative to saying
|
|
781
|
+
# it once is nagging, which is how a message stops being read.
|
|
782
|
+
TOLD = "told-about-hook"
|
|
783
|
+
|
|
784
|
+
|
|
785
|
+
def _mention_hook() -> None:
|
|
786
|
+
"""Tell the reader, one time, that the client's own shell can come here too."""
|
|
787
|
+
if hook_module.installed() or not hook_module.wanted():
|
|
788
|
+
return
|
|
789
|
+
# Not while there is a more important thing to do. Somebody on their first
|
|
790
|
+
# run without a key is already being told to go and get one; spending the
|
|
791
|
+
# single mention this ever makes on the same screen would waste it on
|
|
792
|
+
# somebody who has not seen the tool work yet.
|
|
793
|
+
if sending_on() and find_key() is None:
|
|
794
|
+
return
|
|
795
|
+
marker = store.home() / TOLD
|
|
796
|
+
if marker.exists():
|
|
797
|
+
return
|
|
798
|
+
with contextlib.suppress(OSError):
|
|
799
|
+
marker.parent.mkdir(parents=True, exist_ok=True)
|
|
800
|
+
marker.touch()
|
|
801
|
+
print(
|
|
802
|
+
"\nsift: your client runs shell commands of its own, and those still land"
|
|
803
|
+
"\n in the conversation whole. `sift hook --install` explains what"
|
|
804
|
+
"\n routing them here would give, and asks before changing anything."
|
|
805
|
+
"\n (said once)",
|
|
806
|
+
file=sys.stderr,
|
|
807
|
+
)
|
|
808
|
+
|
|
809
|
+
def _seconds(written: str) -> float | None:
|
|
810
|
+
try:
|
|
811
|
+
return float(written)
|
|
812
|
+
except ValueError:
|
|
813
|
+
return None
|
|
814
|
+
|
|
815
|
+
|
|
816
|
+
def _line_number(written: str) -> int | None:
|
|
817
|
+
try:
|
|
818
|
+
return int(written)
|
|
819
|
+
except ValueError:
|
|
820
|
+
return None
|