dingo-pack 0.4.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
dingo_pack.py
ADDED
|
@@ -0,0 +1,4355 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# SPDX-License-Identifier: MIT
|
|
3
|
+
"""dingo-pack - heavy-job slots, memory admission and a session roster for many agents on one machine.
|
|
4
|
+
|
|
5
|
+
A pack of dingoes eats in order. dingo-pack makes a dozen coding agents on one
|
|
6
|
+
machine take turns at heavy work (test suites, model loads, big builds) instead of
|
|
7
|
+
starting them all at once and running the machine out of memory, and keeps a
|
|
8
|
+
roster of the agent sessions so that after a crash you can bring the whole pack back.
|
|
9
|
+
|
|
10
|
+
Everyday (like a browser's tab saver):
|
|
11
|
+
dingo-pack save remember the open sessions (the hooks do this by themselves)
|
|
12
|
+
dingo-pack close-all remember them and close the ones that are idle and waiting / done
|
|
13
|
+
dingo-pack restore [NAME...] bring back the last batch: what close-all closed, or what a crash lost
|
|
14
|
+
|
|
15
|
+
Advanced -- slots and memory:
|
|
16
|
+
dingo-pack run --heavy [--mem 6G] [--limit 9G] [--tree-limit 9G] -- CMD...
|
|
17
|
+
wait for a slot + enough free memory, run CMD under limits
|
|
18
|
+
dingo-pack lease [--mem 4G] [--for 45m] [--note TEXT]
|
|
19
|
+
hold a slot for work you cannot wrap (an in-agent review, ...)
|
|
20
|
+
dingo-pack release [LEASE_ID...] release your leases (default: all of this session's)
|
|
21
|
+
dingo-pack who who holds slots, who is queuing, memory now
|
|
22
|
+
dingo-pack run --io -- CMD... big reads / writes on a slow disk, in their own lane (1 slot)
|
|
23
|
+
dingo-pack top [--watch S] memory by session
|
|
24
|
+
dingo-pack doctor one checkup: memory, OOM kills, mounts, WSL, slots, roster, ghosts
|
|
25
|
+
dingo-pack ghosts [--reap] processes whose session is gone (reap: humans only)
|
|
26
|
+
dingo-pack log [-n N] recent job lines
|
|
27
|
+
|
|
28
|
+
Advanced -- roster (Claude Code sessions):
|
|
29
|
+
dingo-pack save [--hook] [--every SECONDS] snapshot live sessions to a local roster
|
|
30
|
+
dingo-pack mark STATE [--until WHEN] [NOTE] self-report: running (= midway) | waiting | blocked | done
|
|
31
|
+
dingo-pack list [--since HOURS] [--json] sessions from the roster, grouped for resuming
|
|
32
|
+
dingo-pack resume NAME... | --midway | --lost [--via wt|tmux|print] [--yes] [--dry-run]
|
|
33
|
+
dingo-pack stow [--close] note live sessions; --close closes idle waiting / done ones (humans)
|
|
34
|
+
dingo-pack unstow reopen what stow closed
|
|
35
|
+
dingo-pack one-screen overview
|
|
36
|
+
|
|
37
|
+
Standard library only. Python >= 3.8, Linux/macOS/WSL (needs fcntl).
|
|
38
|
+
"""
|
|
39
|
+
from __future__ import annotations
|
|
40
|
+
|
|
41
|
+
import argparse
|
|
42
|
+
import errno
|
|
43
|
+
try:
|
|
44
|
+
import fcntl
|
|
45
|
+
except ImportError:
|
|
46
|
+
import sys
|
|
47
|
+
print("dingo-pack: native Windows is not supported (POSIX fcntl is required). "
|
|
48
|
+
"Run Python and dingo-pack inside WSL2 or use Linux/macOS.", file=sys.stderr)
|
|
49
|
+
raise SystemExit(2)
|
|
50
|
+
import glob
|
|
51
|
+
import json
|
|
52
|
+
import os
|
|
53
|
+
import re
|
|
54
|
+
import shlex
|
|
55
|
+
import shutil
|
|
56
|
+
import signal
|
|
57
|
+
import socket
|
|
58
|
+
import subprocess
|
|
59
|
+
import sys
|
|
60
|
+
import time
|
|
61
|
+
|
|
62
|
+
__version__ = "0.4.2"
|
|
63
|
+
|
|
64
|
+
EXIT_OK = 0
|
|
65
|
+
EXIT_USAGE = 2
|
|
66
|
+
EXIT_TIMEOUT = 75 # EX_TEMPFAIL: no slot / not enough memory within --timeout
|
|
67
|
+
EXIT_TREE_LIMIT = 137 # the job's process tree went over --tree-limit and was stopped
|
|
68
|
+
|
|
69
|
+
GIB = 1024 ** 3
|
|
70
|
+
MIB = 1024 ** 2
|
|
71
|
+
|
|
72
|
+
NONLOCAL_FS = {"9p", "drvfs", "nfs", "nfs4", "cifs", "smb3", "smbfs", "fuse",
|
|
73
|
+
"fuseblk", "fuse.sshfs", "v9fs", "virtiofs"}
|
|
74
|
+
|
|
75
|
+
STATES = ("midway", "running", "waiting", "blocked", "done") # running = midway
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
# ---------------------------------------------------------------- helpers
|
|
79
|
+
|
|
80
|
+
def eprint(*a):
|
|
81
|
+
print(*a, file=sys.stderr, flush=True)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def now():
|
|
85
|
+
return time.time()
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def hhmm(t):
|
|
89
|
+
return time.strftime("%m-%d %H:%M", time.localtime(t)) if t else "-"
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def state_dir():
|
|
93
|
+
d = os.environ.get("DINGO_HOME")
|
|
94
|
+
if not d:
|
|
95
|
+
base = os.environ.get("XDG_STATE_HOME") or os.path.expanduser("~/.local/state")
|
|
96
|
+
d = os.path.join(base, "dingo-pack")
|
|
97
|
+
os.makedirs(d, exist_ok=True)
|
|
98
|
+
return d
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def sub_dir(name):
|
|
102
|
+
d = os.path.join(state_dir(), name)
|
|
103
|
+
os.makedirs(d, exist_ok=True)
|
|
104
|
+
return d
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def claude_home():
|
|
108
|
+
return (os.environ.get("DINGO_CLAUDE_HOME") or os.environ.get("CLAUDE_CONFIG_DIR")
|
|
109
|
+
or os.path.expanduser("~/.claude"))
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
AGENT_MARKERS = ("CLAUDECODE", "CLAUDE_PID", "CLAUDE_CODE_SESSION_ID", "CLAUDE_CODE_ENTRYPOINT",
|
|
113
|
+
"WOMBAT_NO_ESCAPES", "DINGO_NO_ESCAPES")
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def is_agent():
|
|
117
|
+
"""True if this process looks like it was started by an AI agent (any marker set to
|
|
118
|
+
something other than empty or 0). Used to keep `stow --close` and `ghosts --reap` human-only."""
|
|
119
|
+
return any(os.environ.get(k, "") not in ("", "0") for k in AGENT_MARKERS)
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def fs_type(path):
|
|
123
|
+
"""Best-effort filesystem type of `path` from /proc/mounts (Linux only)."""
|
|
124
|
+
try:
|
|
125
|
+
path = os.path.realpath(path)
|
|
126
|
+
best, best_t = "", None
|
|
127
|
+
with open("/proc/mounts") as f:
|
|
128
|
+
for line in f:
|
|
129
|
+
parts = line.split()
|
|
130
|
+
if len(parts) < 3:
|
|
131
|
+
continue
|
|
132
|
+
mnt = parts[1].replace("\\040", " ")
|
|
133
|
+
if (path == mnt or path.startswith(mnt.rstrip("/") + "/")) and len(mnt) > len(best):
|
|
134
|
+
best, best_t = mnt, parts[2]
|
|
135
|
+
return best_t
|
|
136
|
+
except OSError:
|
|
137
|
+
return None
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def check_local_state():
|
|
141
|
+
t = fs_type(state_dir())
|
|
142
|
+
if t in NONLOCAL_FS:
|
|
143
|
+
eprint(f"dingo-pack: state dir {state_dir()} is on a {t} mount; flock there is not "
|
|
144
|
+
"reliable. Set DINGO_HOME to a local disk.")
|
|
145
|
+
sys.exit(EXIT_USAGE)
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def write_atomic(path, text):
|
|
149
|
+
tmp = f"{path}.tmp.{os.getpid()}"
|
|
150
|
+
try:
|
|
151
|
+
with open(tmp, "w", encoding="utf-8") as f:
|
|
152
|
+
f.write(text)
|
|
153
|
+
f.flush()
|
|
154
|
+
os.fsync(f.fileno())
|
|
155
|
+
os.replace(tmp, path)
|
|
156
|
+
except BaseException:
|
|
157
|
+
try:
|
|
158
|
+
os.unlink(tmp)
|
|
159
|
+
except OSError:
|
|
160
|
+
pass
|
|
161
|
+
raise
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def read_json(path, default=None):
|
|
165
|
+
try:
|
|
166
|
+
with open(path, encoding="utf-8") as f:
|
|
167
|
+
return json.load(f)
|
|
168
|
+
except (OSError, ValueError):
|
|
169
|
+
return default
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
SIZE_RE = re.compile(r"^\s*(\d+(?:\.\d+)?)\s*([kmgt]?)i?b?\s*$", re.I)
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def parse_size(s):
|
|
176
|
+
"""'6G' / '512M' / '1.5g' / '0' -> bytes (int). Bare numbers are GiB."""
|
|
177
|
+
if s is None:
|
|
178
|
+
return None
|
|
179
|
+
if isinstance(s, (int, float)):
|
|
180
|
+
return int(s * GIB)
|
|
181
|
+
m = SIZE_RE.match(str(s))
|
|
182
|
+
if not m:
|
|
183
|
+
raise argparse.ArgumentTypeError(f"bad size: {s!r} (use e.g. 6G, 512M)")
|
|
184
|
+
n, unit = float(m.group(1)), m.group(2).lower()
|
|
185
|
+
mult = {"": GIB, "k": 1024, "m": MIB, "g": GIB, "t": 1024 * GIB}[unit]
|
|
186
|
+
return int(n * mult)
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def parse_duration(s):
|
|
190
|
+
"""'45m' / '2h' / '90s' / '30' (minutes) -> seconds."""
|
|
191
|
+
m = re.match(r"^\s*(\d+(?:\.\d+)?)\s*([smhd]?)\s*$", str(s), re.I)
|
|
192
|
+
if not m:
|
|
193
|
+
raise argparse.ArgumentTypeError(f"bad duration: {s!r} (use e.g. 45m, 2h)")
|
|
194
|
+
return float(m.group(1)) * {"": 60, "s": 1, "m": 60, "h": 3600, "d": 86400}[m.group(2).lower()]
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def fmt_duration(sec):
|
|
198
|
+
return f"{sec:.0f} s" if sec < 120 else f"{sec / 60:.0f} min"
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def fmt_size(b):
|
|
202
|
+
if b is None:
|
|
203
|
+
return "-"
|
|
204
|
+
return f"{b / GIB:.1f}G" if b >= GIB else f"{b / MIB:.0f}M"
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def config():
|
|
208
|
+
"""User config ~/.config/dingo-pack/config.json, overridden by env vars."""
|
|
209
|
+
path = os.environ.get("DINGO_CONFIG") or os.path.join(
|
|
210
|
+
os.environ.get("XDG_CONFIG_HOME") or os.path.expanduser("~/.config"), "dingo-pack", "config.json")
|
|
211
|
+
cfg = read_json(path, {}) or {}
|
|
212
|
+
env = {"slots": "DINGO_SLOTS", "default_mem": "DINGO_DEFAULT_MEM", "reserve": "DINGO_RESERVE"}
|
|
213
|
+
for k, e in env.items():
|
|
214
|
+
if os.environ.get(e):
|
|
215
|
+
cfg[k] = os.environ[e]
|
|
216
|
+
return cfg
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def local_project_config(cwd):
|
|
220
|
+
"""Per-project settings from the user config only (local disk; never blocks)."""
|
|
221
|
+
return (config().get("projects") or {}).get(os.path.normpath(cwd)) or {}
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def project_config(cwd):
|
|
225
|
+
"""Per-project roster settings: <cwd>/.dingo-pack.json, else config()['projects'][cwd].
|
|
226
|
+
May touch the project directory, which can be a slow or hung network mount."""
|
|
227
|
+
c = read_json(os.path.join(cwd, ".dingo-pack.json"))
|
|
228
|
+
if c is None:
|
|
229
|
+
c = local_project_config(cwd)
|
|
230
|
+
return c
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def session_label(args=None):
|
|
234
|
+
s = getattr(args, "session", None) if args else None
|
|
235
|
+
s = s or os.environ.get("DINGO_SESSION")
|
|
236
|
+
if s:
|
|
237
|
+
return s
|
|
238
|
+
pid = os.environ.get("CLAUDE_PID")
|
|
239
|
+
if pid and pid.isdigit():
|
|
240
|
+
e = read_json(os.path.join(claude_home(), "sessions", f"{pid}.json"), {}) or {}
|
|
241
|
+
if e.get("nameSource") == "user" and e.get("name"):
|
|
242
|
+
return e["name"]
|
|
243
|
+
return os.environ.get("CLAUDE_CODE_SESSION_ID", "")[:8] or f"{socket.gethostname()}:{os.getppid()}"
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
# ---------------------------------------------------------------- processes and memory
|
|
247
|
+
|
|
248
|
+
def meminfo():
|
|
249
|
+
"""(MemTotal, MemAvailable) in bytes. DINGO_FAKE_MEMINFO points at a test file."""
|
|
250
|
+
path = os.environ.get("DINGO_FAKE_MEMINFO", "/proc/meminfo")
|
|
251
|
+
vals = {}
|
|
252
|
+
try:
|
|
253
|
+
with open(path) as f:
|
|
254
|
+
for line in f:
|
|
255
|
+
k, _, rest = line.partition(":")
|
|
256
|
+
parts = rest.split()
|
|
257
|
+
if parts:
|
|
258
|
+
vals[k.strip()] = int(parts[0]) * 1024
|
|
259
|
+
return vals.get("MemTotal"), vals.get("MemAvailable", vals.get("MemFree"))
|
|
260
|
+
except OSError:
|
|
261
|
+
pass
|
|
262
|
+
if sys.platform == "darwin":
|
|
263
|
+
try:
|
|
264
|
+
total = int(subprocess.run(["sysctl", "-n", "hw.memsize"], capture_output=True,
|
|
265
|
+
text=True).stdout.strip())
|
|
266
|
+
out = subprocess.run(["vm_stat"], capture_output=True, text=True).stdout
|
|
267
|
+
page = int(re.search(r"page size of (\d+)", out).group(1))
|
|
268
|
+
pages = 0
|
|
269
|
+
for key in ("Pages free", "Pages inactive", "Pages speculative"):
|
|
270
|
+
m = re.search(key + r":\s+(\d+)", out)
|
|
271
|
+
if m:
|
|
272
|
+
pages += int(m.group(1))
|
|
273
|
+
return total, pages * page
|
|
274
|
+
except (OSError, ValueError, AttributeError):
|
|
275
|
+
pass
|
|
276
|
+
return None, None
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
def proc_table():
|
|
280
|
+
"""{pid: (ppid, rss_bytes)} for all processes."""
|
|
281
|
+
table = {}
|
|
282
|
+
if os.path.isdir("/proc/self"):
|
|
283
|
+
page = os.sysconf("SC_PAGE_SIZE")
|
|
284
|
+
for d in os.listdir("/proc"):
|
|
285
|
+
if not d.isdigit():
|
|
286
|
+
continue
|
|
287
|
+
try:
|
|
288
|
+
with open(f"/proc/{d}/stat") as f:
|
|
289
|
+
st = f.read()
|
|
290
|
+
ppid = int(st[st.rindex(")") + 2:].split()[1])
|
|
291
|
+
with open(f"/proc/{d}/statm") as f:
|
|
292
|
+
rss = int(f.read().split()[1]) * page
|
|
293
|
+
table[int(d)] = (ppid, rss)
|
|
294
|
+
except (OSError, ValueError, IndexError):
|
|
295
|
+
continue
|
|
296
|
+
return table
|
|
297
|
+
out = subprocess.run(["ps", "-A", "-o", "pid=,ppid=,rss="], capture_output=True, text=True).stdout
|
|
298
|
+
for line in out.splitlines():
|
|
299
|
+
p = line.split()
|
|
300
|
+
if len(p) == 3:
|
|
301
|
+
table[int(p[0])] = (int(p[1]), int(p[2]) * 1024)
|
|
302
|
+
return table
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
def tree_pids(root, table=None):
|
|
306
|
+
table = table if table is not None else proc_table()
|
|
307
|
+
kids = {}
|
|
308
|
+
for pid, (ppid, _) in table.items():
|
|
309
|
+
kids.setdefault(ppid, []).append(pid)
|
|
310
|
+
out, stack = [], [root]
|
|
311
|
+
while stack:
|
|
312
|
+
p = stack.pop()
|
|
313
|
+
if p in table:
|
|
314
|
+
out.append(p)
|
|
315
|
+
stack.extend(kids.get(p, []))
|
|
316
|
+
return out
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
def proc_pss(pid):
|
|
320
|
+
"""Proportional set size in bytes (shared pages split between their users), or None."""
|
|
321
|
+
try:
|
|
322
|
+
with open(f"/proc/{pid}/smaps_rollup") as f:
|
|
323
|
+
for line in f:
|
|
324
|
+
if line.startswith("Pss:"):
|
|
325
|
+
return int(line.split()[1]) * 1024
|
|
326
|
+
except (OSError, ValueError, IndexError):
|
|
327
|
+
pass
|
|
328
|
+
return None
|
|
329
|
+
|
|
330
|
+
|
|
331
|
+
def tree_rss(root, table=None):
|
|
332
|
+
"""Memory of the process tree under `root`. Uses PSS where the kernel provides it, so pages
|
|
333
|
+
a forked worker shares with its parent are counted once; falls back to RSS (which counts
|
|
334
|
+
them once per process and so overestimates fork-heavy jobs)."""
|
|
335
|
+
table = table if table is not None else proc_table()
|
|
336
|
+
total = 0
|
|
337
|
+
for p in tree_pids(root, table):
|
|
338
|
+
v = proc_pss(p)
|
|
339
|
+
total += v if v is not None else table.get(p, (0, 0))[1]
|
|
340
|
+
return total
|
|
341
|
+
|
|
342
|
+
|
|
343
|
+
def proc_start(pid):
|
|
344
|
+
"""Process start time in clock ticks since boot (Linux), or None."""
|
|
345
|
+
try:
|
|
346
|
+
with open(f"/proc/{pid}/stat") as f:
|
|
347
|
+
st = f.read()
|
|
348
|
+
return int(st[st.rindex(")") + 2:].split()[19])
|
|
349
|
+
except (OSError, ValueError, IndexError):
|
|
350
|
+
return None
|
|
351
|
+
|
|
352
|
+
|
|
353
|
+
def pid_alive(pid, start=None):
|
|
354
|
+
"""True if pid exists (and, when `start` is known and /proc is there, started then)."""
|
|
355
|
+
if not pid:
|
|
356
|
+
return False
|
|
357
|
+
try:
|
|
358
|
+
os.kill(int(pid), 0)
|
|
359
|
+
except ProcessLookupError:
|
|
360
|
+
return False
|
|
361
|
+
except PermissionError:
|
|
362
|
+
pass
|
|
363
|
+
except (OSError, ValueError):
|
|
364
|
+
return False
|
|
365
|
+
try:
|
|
366
|
+
with open(f"/proc/{int(pid)}/stat") as f:
|
|
367
|
+
st = f.read()
|
|
368
|
+
if st[st.rindex(")") + 2:].split()[0] in ("Z", "X"):
|
|
369
|
+
return False # exited, not yet reaped by its parent
|
|
370
|
+
except (OSError, ValueError, IndexError):
|
|
371
|
+
pass
|
|
372
|
+
if start not in (None, ""):
|
|
373
|
+
cur = proc_start(int(pid))
|
|
374
|
+
if cur is not None and str(cur) != str(start):
|
|
375
|
+
return False # pid was reused by another process
|
|
376
|
+
return True
|
|
377
|
+
|
|
378
|
+
|
|
379
|
+
def boot_time():
|
|
380
|
+
if os.environ.get("DINGO_FAKE_BTIME"):
|
|
381
|
+
return float(os.environ["DINGO_FAKE_BTIME"])
|
|
382
|
+
try:
|
|
383
|
+
with open("/proc/stat") as f:
|
|
384
|
+
for line in f:
|
|
385
|
+
if line.startswith("btime"):
|
|
386
|
+
return float(line.split()[1])
|
|
387
|
+
except OSError:
|
|
388
|
+
pass
|
|
389
|
+
if sys.platform == "darwin":
|
|
390
|
+
try:
|
|
391
|
+
out = subprocess.run(["sysctl", "-n", "kern.boottime"], capture_output=True, text=True).stdout
|
|
392
|
+
return float(re.search(r"sec = (\d+)", out).group(1))
|
|
393
|
+
except (OSError, AttributeError, ValueError):
|
|
394
|
+
pass
|
|
395
|
+
return None
|
|
396
|
+
|
|
397
|
+
|
|
398
|
+
def owner_process():
|
|
399
|
+
"""(pid, start) of the agent process this command runs under: Claude Code sets CLAUDE_PID."""
|
|
400
|
+
pid = os.environ.get("DINGO_OWNER_PID") or os.environ.get("CLAUDE_PID")
|
|
401
|
+
if pid and pid.isdigit() and pid_alive(int(pid)):
|
|
402
|
+
return int(pid), proc_start(int(pid))
|
|
403
|
+
return None, None
|
|
404
|
+
|
|
405
|
+
|
|
406
|
+
# ---------------------------------------------------------------- slots
|
|
407
|
+
|
|
408
|
+
class Deadline(Exception):
|
|
409
|
+
pass
|
|
410
|
+
|
|
411
|
+
|
|
412
|
+
def _alarm(signum, frame):
|
|
413
|
+
raise Deadline()
|
|
414
|
+
|
|
415
|
+
|
|
416
|
+
LANE_DIRS = {"heavy": "slots", "io": "io-slots"}
|
|
417
|
+
BACKFILL_MAX = int(os.environ.get("DINGO_BACKFILL_MAX", "3"))
|
|
418
|
+
THREAD_VARS = ("OMP_NUM_THREADS", "MKL_NUM_THREADS", "OPENBLAS_NUM_THREADS", "NUMEXPR_NUM_THREADS",
|
|
419
|
+
"NUMEXPR_MAX_THREADS", "VECLIB_MAXIMUM_THREADS", "BLIS_NUM_THREADS")
|
|
420
|
+
DEFAULT_THREADS = {"heavy": 8, "io": 1, "lease": 2}
|
|
421
|
+
|
|
422
|
+
|
|
423
|
+
def cpu_total(cfg=None):
|
|
424
|
+
cfg = cfg if cfg is not None else config()
|
|
425
|
+
v = os.environ.get("DINGO_CPU_SLOTS") or cfg.get("cpu_slots")
|
|
426
|
+
return int(v) if v else (os.cpu_count() or 1)
|
|
427
|
+
|
|
428
|
+
|
|
429
|
+
def cpu_used(exclude=None):
|
|
430
|
+
"""CPU slots taken by admitted holders (each counts its --threads)."""
|
|
431
|
+
used = 0
|
|
432
|
+
for i, h in holders():
|
|
433
|
+
if (h.get("lane"), i) == exclude or h.get("op") == "waiting for memory":
|
|
434
|
+
continue
|
|
435
|
+
used += int(h.get("threads") or 0)
|
|
436
|
+
return used
|
|
437
|
+
|
|
438
|
+
|
|
439
|
+
def loadavg():
|
|
440
|
+
try:
|
|
441
|
+
with open(os.environ.get("DINGO_FAKE_LOADAVG", "/proc/loadavg")) as f:
|
|
442
|
+
return float(f.read().split()[0])
|
|
443
|
+
except (OSError, ValueError, IndexError):
|
|
444
|
+
return None
|
|
445
|
+
|
|
446
|
+
|
|
447
|
+
def n_cpus():
|
|
448
|
+
return int(os.environ.get("DINGO_FAKE_NCPU") or os.cpu_count() or 1)
|
|
449
|
+
|
|
450
|
+
|
|
451
|
+
def clear_memhead(pid):
|
|
452
|
+
path = os.path.join(state_dir(), "memhead.json")
|
|
453
|
+
h = read_json(path)
|
|
454
|
+
if h and h.get("pid") == pid:
|
|
455
|
+
try:
|
|
456
|
+
os.unlink(path)
|
|
457
|
+
except OSError:
|
|
458
|
+
pass
|
|
459
|
+
|
|
460
|
+
|
|
461
|
+
def lane_size(lane, cfg, override=None):
|
|
462
|
+
if override:
|
|
463
|
+
return int(override)
|
|
464
|
+
if lane == "io":
|
|
465
|
+
return int(os.environ.get("DINGO_IO_SLOTS") or cfg.get("io_slots", 1))
|
|
466
|
+
return int(cfg.get("slots", 6))
|
|
467
|
+
|
|
468
|
+
|
|
469
|
+
class Slots:
|
|
470
|
+
"""N flock'ed slot files plus one queue lock; the head of the queue is always served next."""
|
|
471
|
+
|
|
472
|
+
def __init__(self, n, lane="heavy", threads=0):
|
|
473
|
+
self.n = max(1, int(n))
|
|
474
|
+
self.lane = lane
|
|
475
|
+
self.threads = int(threads or 0)
|
|
476
|
+
self.dir = sub_dir(LANE_DIRS[lane])
|
|
477
|
+
self.qfd = None
|
|
478
|
+
self.fd = None
|
|
479
|
+
self.index = None
|
|
480
|
+
|
|
481
|
+
def slot_path(self, i):
|
|
482
|
+
return os.path.join(self.dir, f"slot-{i}")
|
|
483
|
+
|
|
484
|
+
def holder_path(self, i):
|
|
485
|
+
return self.slot_path(i) + ".holder"
|
|
486
|
+
|
|
487
|
+
def _lock_queue(self, deadline):
|
|
488
|
+
self.qfd = os.open(os.path.join(self.dir, "queue"), os.O_RDWR | os.O_CREAT, 0o644)
|
|
489
|
+
if deadline is None:
|
|
490
|
+
fcntl.flock(self.qfd, fcntl.LOCK_EX)
|
|
491
|
+
return
|
|
492
|
+
left = deadline - now()
|
|
493
|
+
if left <= 0:
|
|
494
|
+
raise Deadline()
|
|
495
|
+
old = signal.signal(signal.SIGALRM, _alarm)
|
|
496
|
+
signal.setitimer(signal.ITIMER_REAL, left)
|
|
497
|
+
try:
|
|
498
|
+
fcntl.flock(self.qfd, fcntl.LOCK_EX)
|
|
499
|
+
except Deadline:
|
|
500
|
+
try: # the alarm may have raced a successful flock
|
|
501
|
+
fcntl.flock(self.qfd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
502
|
+
except OSError:
|
|
503
|
+
raise Deadline()
|
|
504
|
+
finally:
|
|
505
|
+
signal.setitimer(signal.ITIMER_REAL, 0)
|
|
506
|
+
signal.signal(signal.SIGALRM, old)
|
|
507
|
+
|
|
508
|
+
def _try_slot(self):
|
|
509
|
+
for i in range(self.n):
|
|
510
|
+
fd = os.open(self.slot_path(i), os.O_RDWR | os.O_CREAT, 0o644)
|
|
511
|
+
try:
|
|
512
|
+
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
513
|
+
except OSError as e:
|
|
514
|
+
os.close(fd)
|
|
515
|
+
if e.errno in (errno.EWOULDBLOCK, errno.EAGAIN, errno.EACCES):
|
|
516
|
+
continue
|
|
517
|
+
raise
|
|
518
|
+
self.fd, self.index = fd, i
|
|
519
|
+
return True
|
|
520
|
+
return False
|
|
521
|
+
|
|
522
|
+
def acquire(self, est, reserve, timeout, waiting_rec, quiet=False):
|
|
523
|
+
"""Block until we hold a slot and memory admits `est` bytes. Returns (slot_wait_s, mem_wait_s).
|
|
524
|
+
|
|
525
|
+
Slots go first come first served (queue lock). The queue lock is dropped as soon as we
|
|
526
|
+
have a slot, so jobs behind us can take the next slots. Memory admission is a separate
|
|
527
|
+
step, serialised by one lock: whoever is admitted writes its holder record ("starting",
|
|
528
|
+
with its estimate) before that lock is released, so the next check counts it. A job that
|
|
529
|
+
fits may go ahead of one waiting for memory (backfill), but the one waiting -- the
|
|
530
|
+
memory head -- can be passed at most BACKFILL_MAX times; after that everybody waits."""
|
|
531
|
+
deadline = now() + timeout if timeout else None
|
|
532
|
+
wpath = os.path.join(sub_dir("waiting"), f"{os.getpid()}.json")
|
|
533
|
+
write_atomic(wpath, json.dumps(dict(waiting_rec, pid=os.getpid(), since=now(), phase="queue")))
|
|
534
|
+
t0 = now()
|
|
535
|
+
noted = 0.0
|
|
536
|
+
try:
|
|
537
|
+
self._lock_queue(deadline)
|
|
538
|
+
while not self._try_slot():
|
|
539
|
+
if deadline and now() > deadline:
|
|
540
|
+
raise Deadline()
|
|
541
|
+
if not quiet and now() - noted > 60:
|
|
542
|
+
eprint(f"dingo-pack: all {self.n} slots busy; waiting ({int(now() - t0)} s)")
|
|
543
|
+
noted = now()
|
|
544
|
+
time.sleep(0.2)
|
|
545
|
+
t1 = now()
|
|
546
|
+
write_atomic(wpath, json.dumps(dict(waiting_rec, pid=os.getpid(), since=t0, phase="memory")))
|
|
547
|
+
self.write_holder(dict(waiting_rec, pid=os.getpid(), start=proc_start(os.getpid()), since=now(),
|
|
548
|
+
op="waiting for memory"))
|
|
549
|
+
os.close(self.qfd) # the next in line may take the next slot meanwhile
|
|
550
|
+
self.qfd = None
|
|
551
|
+
noted = 0.0
|
|
552
|
+
while True:
|
|
553
|
+
ok, why = self._admit_step(est, reserve)
|
|
554
|
+
if ok:
|
|
555
|
+
break
|
|
556
|
+
if deadline and now() > deadline:
|
|
557
|
+
raise Deadline()
|
|
558
|
+
if not quiet and now() - noted > 60:
|
|
559
|
+
eprint(f"dingo-pack: waiting for memory: {why}")
|
|
560
|
+
noted = now()
|
|
561
|
+
time.sleep(float(os.environ.get("DINGO_MEM_POLL_S", "2")))
|
|
562
|
+
return t1 - t0, now() - t1
|
|
563
|
+
except BaseException:
|
|
564
|
+
clear_memhead(os.getpid())
|
|
565
|
+
self.release()
|
|
566
|
+
raise
|
|
567
|
+
finally:
|
|
568
|
+
try:
|
|
569
|
+
os.unlink(wpath)
|
|
570
|
+
except OSError:
|
|
571
|
+
pass
|
|
572
|
+
if self.qfd is not None:
|
|
573
|
+
os.close(self.qfd) # closing drops the queue flock; the next in line goes
|
|
574
|
+
self.qfd = None
|
|
575
|
+
|
|
576
|
+
def _admit_step(self, est, reserve):
|
|
577
|
+
"""One admission attempt under the admission lock. Returns (admitted, why_not)."""
|
|
578
|
+
me, start = os.getpid(), proc_start(os.getpid())
|
|
579
|
+
lk = os.open(os.path.join(state_dir(), "admit.lock"), os.O_RDWR | os.O_CREAT, 0o644)
|
|
580
|
+
try:
|
|
581
|
+
fcntl.flock(lk, fcntl.LOCK_EX)
|
|
582
|
+
path = os.path.join(state_dir(), "memhead.json")
|
|
583
|
+
head = read_json(path)
|
|
584
|
+
if head and not pid_alive(head.get("pid"), head.get("start")):
|
|
585
|
+
head = None
|
|
586
|
+
ok, why = admit(est, reserve, exclude=(self.lane, self.index), n=self.n)
|
|
587
|
+
if ok and self.threads:
|
|
588
|
+
total, used = cpu_total(), cpu_used(exclude=(self.lane, self.index))
|
|
589
|
+
if used + self.threads > total:
|
|
590
|
+
ok, why = False, f"CPU slots: {used} of {total} taken, this job needs {self.threads}"
|
|
591
|
+
mine = bool(head) and head.get("pid") == me
|
|
592
|
+
if ok and head and not mine:
|
|
593
|
+
if head.get("skips", 0) >= BACKFILL_MAX:
|
|
594
|
+
ok, why = False, (f"{head.get('session', '?')} has waited for memory and was passed "
|
|
595
|
+
f"{head.get('skips')} times; it goes first")
|
|
596
|
+
else:
|
|
597
|
+
head["skips"] = head.get("skips", 0) + 1 # we go ahead of it (backfill)
|
|
598
|
+
write_atomic(path, json.dumps(head))
|
|
599
|
+
if ok:
|
|
600
|
+
if mine:
|
|
601
|
+
try:
|
|
602
|
+
os.unlink(path)
|
|
603
|
+
except OSError:
|
|
604
|
+
pass
|
|
605
|
+
self.write_holder(dict(read_json(self.holder_path(self.index), {}) or {}, pid=me, start=start,
|
|
606
|
+
since=now(), op="starting"))
|
|
607
|
+
elif not head:
|
|
608
|
+
write_atomic(path, json.dumps({"pid": me, "start": start, "est": est, "skips": 0, "since": now(),
|
|
609
|
+
"session": (read_json(self.holder_path(self.index), {}) or {})
|
|
610
|
+
.get("session")}))
|
|
611
|
+
return ok, why
|
|
612
|
+
finally:
|
|
613
|
+
os.close(lk)
|
|
614
|
+
|
|
615
|
+
def write_holder(self, rec):
|
|
616
|
+
write_atomic(self.holder_path(self.index), json.dumps(rec))
|
|
617
|
+
|
|
618
|
+
def release(self):
|
|
619
|
+
"""Give the slot back. flock(LOCK_UN) releases the lock for every holder of this open
|
|
620
|
+
file description, so a background grandchild that inherited it cannot keep the slot."""
|
|
621
|
+
if self.fd is not None:
|
|
622
|
+
try:
|
|
623
|
+
fcntl.flock(self.fd, fcntl.LOCK_UN)
|
|
624
|
+
except OSError:
|
|
625
|
+
pass
|
|
626
|
+
os.close(self.fd)
|
|
627
|
+
self.fd, self.index = None, None
|
|
628
|
+
|
|
629
|
+
def close_only(self):
|
|
630
|
+
"""Drop our descriptor but keep the lock (another process holds the same description)."""
|
|
631
|
+
if self.fd is not None:
|
|
632
|
+
os.close(self.fd)
|
|
633
|
+
self.fd, self.index = None, None
|
|
634
|
+
|
|
635
|
+
|
|
636
|
+
def slot_held(path):
|
|
637
|
+
"""True if someone holds the flock on `path` (tested without taking it for long)."""
|
|
638
|
+
try:
|
|
639
|
+
fd = os.open(path, os.O_RDWR)
|
|
640
|
+
except OSError:
|
|
641
|
+
return False
|
|
642
|
+
try:
|
|
643
|
+
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
644
|
+
fcntl.flock(fd, fcntl.LOCK_UN)
|
|
645
|
+
return False
|
|
646
|
+
except OSError:
|
|
647
|
+
return True
|
|
648
|
+
finally:
|
|
649
|
+
os.close(fd)
|
|
650
|
+
|
|
651
|
+
|
|
652
|
+
def holders(lane=None):
|
|
653
|
+
"""[(index, holder_record)] for slots that are actually held, in one lane or all of
|
|
654
|
+
them; each record carries its "lane"."""
|
|
655
|
+
out = []
|
|
656
|
+
for ln, sub in LANE_DIRS.items():
|
|
657
|
+
if lane and ln != lane:
|
|
658
|
+
continue
|
|
659
|
+
d = sub_dir(sub)
|
|
660
|
+
for p in sorted(glob.glob(os.path.join(d, "slot-*"))):
|
|
661
|
+
if p.endswith(".holder") or ".tmp." in p:
|
|
662
|
+
continue
|
|
663
|
+
try:
|
|
664
|
+
i = int(p.rsplit("-", 1)[1])
|
|
665
|
+
except ValueError:
|
|
666
|
+
continue
|
|
667
|
+
if slot_held(p):
|
|
668
|
+
h = read_json(p + ".holder", {}) or {}
|
|
669
|
+
h["lane"] = ln
|
|
670
|
+
out.append((i, h))
|
|
671
|
+
return out
|
|
672
|
+
|
|
673
|
+
|
|
674
|
+
def admit(est, reserve, exclude=None, n=None):
|
|
675
|
+
"""Memory admission: available - (what running jobs have not yet used) >= est + reserve."""
|
|
676
|
+
total, avail = meminfo()
|
|
677
|
+
if avail is None:
|
|
678
|
+
return True, "memory unknown on this platform; admitting"
|
|
679
|
+
table = proc_table()
|
|
680
|
+
unmet = 0
|
|
681
|
+
for i, h in holders(): # memory is shared by every lane
|
|
682
|
+
if (h.get("lane"), i) == exclude or h.get("op") == "waiting for memory":
|
|
683
|
+
continue # not admitted yet: promises nothing
|
|
684
|
+
jp = h.get("job_pid") # starting / lease: nothing of ours is using it yet
|
|
685
|
+
used = tree_rss(jp, table) if jp else 0
|
|
686
|
+
unmet += max(0, commitment(h) - used)
|
|
687
|
+
need = est + reserve
|
|
688
|
+
ok = avail - unmet >= need
|
|
689
|
+
return ok, (f"available {fmt_size(avail)} - promised to running jobs {fmt_size(unmet)} "
|
|
690
|
+
f"< need {fmt_size(est)} + reserve {fmt_size(reserve)}")
|
|
691
|
+
|
|
692
|
+
|
|
693
|
+
PEAK_AFTER_S = float(os.environ.get("DINGO_PEAK_AFTER_S", "60"))
|
|
694
|
+
|
|
695
|
+
|
|
696
|
+
def commitment(h):
|
|
697
|
+
"""How much memory a holder is counted for: its estimate until it has run PEAK_AFTER_S,
|
|
698
|
+
then 1.2 x the peak actually measured (a job that said 2G but uses 300M stops blocking others)."""
|
|
699
|
+
est = h.get("est") or 0
|
|
700
|
+
if h.get("job_pid") and h.get("peak") and now() - (h.get("job_since") or now()) >= PEAK_AFTER_S:
|
|
701
|
+
return int(h["peak"] * 1.2)
|
|
702
|
+
return est
|
|
703
|
+
|
|
704
|
+
|
|
705
|
+
def reserve_bytes(cfg):
|
|
706
|
+
if cfg.get("reserve"):
|
|
707
|
+
return parse_size(cfg["reserve"])
|
|
708
|
+
total, _ = meminfo()
|
|
709
|
+
return max(GIB, int((total or 0) * 0.10))
|
|
710
|
+
|
|
711
|
+
|
|
712
|
+
# ---------------------------------------------------------------- log
|
|
713
|
+
|
|
714
|
+
def log_path():
|
|
715
|
+
return os.path.join(state_dir(), "log.jsonl")
|
|
716
|
+
|
|
717
|
+
|
|
718
|
+
SECRETISH = re.compile(r"(?i)(token|secret|passw|apikey|api_key|bearer)")
|
|
719
|
+
|
|
720
|
+
|
|
721
|
+
def cmd_summary(argv):
|
|
722
|
+
parts = []
|
|
723
|
+
for a in argv:
|
|
724
|
+
parts.append("***" if SECRETISH.search(a) and ("=" in a or len(a) > 24) else a)
|
|
725
|
+
s = shlex.join(parts) if hasattr(shlex, "join") else " ".join(parts)
|
|
726
|
+
return s[:120]
|
|
727
|
+
|
|
728
|
+
|
|
729
|
+
def write_log(rec):
|
|
730
|
+
try:
|
|
731
|
+
with open(log_path(), "a", encoding="utf-8") as f:
|
|
732
|
+
f.write(json.dumps(rec, ensure_ascii=False) + "\n")
|
|
733
|
+
except OSError as e:
|
|
734
|
+
eprint(f"dingo-pack: could not write log: {e}")
|
|
735
|
+
|
|
736
|
+
|
|
737
|
+
# ---------------------------------------------------------------- run / lease
|
|
738
|
+
|
|
739
|
+
def resolve_sizes(args, cfg):
|
|
740
|
+
est = args.mem if args.mem is not None else parse_size(cfg.get("default_mem", "1G"))
|
|
741
|
+
limit = args.limit if args.limit is not None else int(est * 1.5)
|
|
742
|
+
tree = args.tree_limit if getattr(args, "tree_limit", None) is not None else limit
|
|
743
|
+
return est, limit, tree
|
|
744
|
+
|
|
745
|
+
|
|
746
|
+
def check_fits(est, reserve):
|
|
747
|
+
total, _ = meminfo()
|
|
748
|
+
if total and est + reserve > total:
|
|
749
|
+
eprint(f"dingo-pack: --mem {fmt_size(est)} + reserve {fmt_size(reserve)} is more than this "
|
|
750
|
+
f"machine has ({fmt_size(total)}); it would wait forever.")
|
|
751
|
+
sys.exit(EXIT_USAGE)
|
|
752
|
+
|
|
753
|
+
|
|
754
|
+
def kill_tree(root, sig):
|
|
755
|
+
for p in reversed(tree_pids(root)):
|
|
756
|
+
try:
|
|
757
|
+
os.kill(p, sig)
|
|
758
|
+
except OSError:
|
|
759
|
+
pass
|
|
760
|
+
|
|
761
|
+
|
|
762
|
+
def memory_diagnostic(raw_rc, limit, tree, kind, stopped, peak):
|
|
763
|
+
"""Describe evidence without treating every failure or SIGSEGV as an OOM."""
|
|
764
|
+
sig = -raw_rc if raw_rc < 0 else raw_rc - 128 if raw_rc in (137, 139) else None
|
|
765
|
+
if sig is not None:
|
|
766
|
+
try:
|
|
767
|
+
signal_name = signal.Signals(sig).name
|
|
768
|
+
except ValueError:
|
|
769
|
+
signal_name = "unknown"
|
|
770
|
+
exit_info = f"{sig} ({signal_name})"
|
|
771
|
+
if raw_rc >= 0:
|
|
772
|
+
exit_info += "; inferred from shell exit status"
|
|
773
|
+
else:
|
|
774
|
+
exit_info = "none"
|
|
775
|
+
if stopped:
|
|
776
|
+
cause = "process-tree limit exceeded (confirmed by dingo-pack)"
|
|
777
|
+
elif limit and raw_rc in (-9, -11, 137, 139):
|
|
778
|
+
cause = "possible memory limit hit; the exit alone does not prove OOM"
|
|
779
|
+
elif raw_rc in (-9, -11, 137, 139):
|
|
780
|
+
cause = "possible system OOM or another signal source; check system logs"
|
|
781
|
+
else:
|
|
782
|
+
cause = "cause unknown; check the child's error (e.g. MemoryError / allocation failure)"
|
|
783
|
+
return (f"dingo-pack: memory diagnostic: configured per-process {kind} limit={fmt_size(limit)}; "
|
|
784
|
+
f"tree limit={fmt_size(tree)}; sampled tree peak={fmt_size(peak)}; "
|
|
785
|
+
f"exit signal={exit_info}; child exit={raw_rc}. {cause}. "
|
|
786
|
+
"For a large CSV, try chunked reading; check available memory before raising --mem/--limit. "
|
|
787
|
+
"RLIMIT_DATA/AS is not RSS; a short allocation spike may fall between samples.")
|
|
788
|
+
|
|
789
|
+
|
|
790
|
+
def cmd_run(args):
|
|
791
|
+
if not args.heavy and not args.io:
|
|
792
|
+
eprint("dingo-pack: say --heavy (CPU / memory) or --io (big reads and writes on a slow disk)")
|
|
793
|
+
return EXIT_USAGE
|
|
794
|
+
if args.heavy and args.io:
|
|
795
|
+
eprint("dingo-pack: --heavy or --io, not both")
|
|
796
|
+
return EXIT_USAGE
|
|
797
|
+
cmd = args.cmd
|
|
798
|
+
if cmd and cmd[0] == "--":
|
|
799
|
+
cmd = cmd[1:]
|
|
800
|
+
if not cmd:
|
|
801
|
+
eprint("dingo-pack run: no command after --")
|
|
802
|
+
return EXIT_USAGE
|
|
803
|
+
check_local_state()
|
|
804
|
+
cfg = config()
|
|
805
|
+
lane = "io" if args.io else "heavy"
|
|
806
|
+
if lane == "io" and args.mem is None:
|
|
807
|
+
args.mem = parse_size(cfg.get("io_default_mem", "512M"))
|
|
808
|
+
est, limit, tree = resolve_sizes(args, cfg)
|
|
809
|
+
reserve = reserve_bytes(cfg)
|
|
810
|
+
check_fits(est, reserve)
|
|
811
|
+
threads = args.threads if args.threads is not None else DEFAULT_THREADS[lane]
|
|
812
|
+
if threads > cpu_total(cfg):
|
|
813
|
+
eprint(f"dingo-pack: --threads {threads} is more than this machine's {cpu_total(cfg)} CPU slots")
|
|
814
|
+
return EXIT_USAGE
|
|
815
|
+
slots = Slots(lane_size(lane, cfg, args.slots), lane, threads)
|
|
816
|
+
who = session_label(args)
|
|
817
|
+
rec = {"t": now(), "op": "run", "lane": lane, "threads": threads, "session": who, "cmd": cmd_summary(cmd), "est": est,
|
|
818
|
+
"limit": limit, "tree_limit": tree, "host": socket.gethostname()}
|
|
819
|
+
try:
|
|
820
|
+
slot_wait, mem_wait = slots.acquire(est, reserve, args.timeout,
|
|
821
|
+
{"session": who, "cmd": rec["cmd"], "est": est, "threads": threads})
|
|
822
|
+
except Deadline:
|
|
823
|
+
rec.update(rc=EXIT_TIMEOUT, error="timeout waiting for a slot / memory")
|
|
824
|
+
write_log(rec)
|
|
825
|
+
eprint(f"dingo-pack: gave up after {args.timeout:.0f} s (no slot or not enough memory)")
|
|
826
|
+
return EXIT_TIMEOUT
|
|
827
|
+
except KeyboardInterrupt:
|
|
828
|
+
return 130
|
|
829
|
+
rec.update(slot=slots.index, slot_wait_s=round(slot_wait, 2), mem_wait_s=round(mem_wait, 2))
|
|
830
|
+
|
|
831
|
+
def preexec():
|
|
832
|
+
if limit:
|
|
833
|
+
import resource
|
|
834
|
+
kind = resource.RLIMIT_AS if args.limit_kind == "as" else resource.RLIMIT_DATA
|
|
835
|
+
try:
|
|
836
|
+
resource.setrlimit(kind, (limit, limit))
|
|
837
|
+
except (ValueError, OSError):
|
|
838
|
+
pass
|
|
839
|
+
|
|
840
|
+
t_start = now()
|
|
841
|
+
try:
|
|
842
|
+
# The job inherits the slot lock: if dingo-pack itself is killed, the slot stays
|
|
843
|
+
# taken until the job is gone too.
|
|
844
|
+
env = dict(os.environ)
|
|
845
|
+
if threads:
|
|
846
|
+
env.update({k: str(threads) for k in THREAD_VARS})
|
|
847
|
+
child = subprocess.Popen(cmd, preexec_fn=preexec, close_fds=True, pass_fds=(slots.fd,), env=env)
|
|
848
|
+
except OSError as e:
|
|
849
|
+
slots.release()
|
|
850
|
+
rec.update(rc=127, error=str(e))
|
|
851
|
+
write_log(rec)
|
|
852
|
+
eprint(f"dingo-pack: cannot start {cmd[0]}: {e}")
|
|
853
|
+
return 127
|
|
854
|
+
holder = {"session": who, "op": "run", "cmd": rec["cmd"], "est": est, "threads": threads, "pid": os.getpid(),
|
|
855
|
+
"start": proc_start(os.getpid()), "job_pid": child.pid, "since": t_start, "job_since": t_start,
|
|
856
|
+
"limit": limit, "tree_limit": tree, "peak": 0}
|
|
857
|
+
slots.write_holder(holder)
|
|
858
|
+
peak_written = 0.0
|
|
859
|
+
|
|
860
|
+
def forward(signum, frame):
|
|
861
|
+
kill_tree(child.pid, signum)
|
|
862
|
+
for s in (signal.SIGTERM, signal.SIGHUP):
|
|
863
|
+
signal.signal(s, forward)
|
|
864
|
+
signal.signal(signal.SIGINT, lambda s, f: None) # the child gets ^C from the terminal itself
|
|
865
|
+
|
|
866
|
+
peak, stopped, t_term = 0, False, None
|
|
867
|
+
grace = float(os.environ.get("DINGO_KILL_GRACE", "10"))
|
|
868
|
+
interval = float(os.environ.get("DINGO_SAMPLE_S", "0.5"))
|
|
869
|
+
while True:
|
|
870
|
+
try:
|
|
871
|
+
rc = child.wait(timeout=interval)
|
|
872
|
+
break
|
|
873
|
+
except subprocess.TimeoutExpired:
|
|
874
|
+
pass
|
|
875
|
+
r = tree_rss(child.pid)
|
|
876
|
+
peak = max(peak, r)
|
|
877
|
+
if peak > holder["peak"] and now() - peak_written > 2: # others count us by our peak after a while
|
|
878
|
+
holder["peak"] = peak
|
|
879
|
+
peak_written = now()
|
|
880
|
+
try:
|
|
881
|
+
slots.write_holder(holder)
|
|
882
|
+
except OSError:
|
|
883
|
+
pass
|
|
884
|
+
if tree and r > tree and not stopped:
|
|
885
|
+
stopped, t_term = True, now()
|
|
886
|
+
eprint(f"dingo-pack: process tree uses {fmt_size(r)} > --tree-limit {fmt_size(tree)}; stopping it")
|
|
887
|
+
kill_tree(child.pid, signal.SIGTERM)
|
|
888
|
+
elif stopped and now() - t_term > grace:
|
|
889
|
+
kill_tree(child.pid, signal.SIGKILL)
|
|
890
|
+
import resource
|
|
891
|
+
ru = resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss
|
|
892
|
+
ru_bytes = ru if sys.platform == "darwin" else ru * 1024
|
|
893
|
+
slots.release()
|
|
894
|
+
raw_rc = rc
|
|
895
|
+
if raw_rc or stopped:
|
|
896
|
+
eprint(memory_diagnostic(raw_rc, limit, tree, args.limit_kind, stopped, peak))
|
|
897
|
+
if rc < 0:
|
|
898
|
+
rc = 128 - rc
|
|
899
|
+
if stopped:
|
|
900
|
+
rc = EXIT_TREE_LIMIT
|
|
901
|
+
rec.update(rc=rc, child_rc=raw_rc, limit_kind=args.limit_kind,
|
|
902
|
+
wall_s=round(now() - t_start, 2), peak_tree_rss=peak, peak_proc_rss=ru_bytes,
|
|
903
|
+
stopped_by_tree_limit=stopped)
|
|
904
|
+
write_log(rec)
|
|
905
|
+
if stopped:
|
|
906
|
+
eprint(f"dingo-pack: stopped by --tree-limit: peak {fmt_size(peak)} / limit {fmt_size(tree)} (exit 137)")
|
|
907
|
+
return rc
|
|
908
|
+
|
|
909
|
+
|
|
910
|
+
def lease_daemon(slots, rec, until, owner_pid, owner_start):
|
|
911
|
+
"""Runs in the forked child: hold the slot until release / expiry / the owner dies."""
|
|
912
|
+
os.setsid()
|
|
913
|
+
devnull = os.open(os.devnull, os.O_RDWR)
|
|
914
|
+
for fd in (0, 1, 2):
|
|
915
|
+
os.dup2(devnull, fd)
|
|
916
|
+
for s in (signal.SIGHUP, signal.SIGINT):
|
|
917
|
+
signal.signal(s, signal.SIG_IGN)
|
|
918
|
+
stop = {"why": None}
|
|
919
|
+
|
|
920
|
+
def on_term(signum, frame):
|
|
921
|
+
stop["why"] = "released"
|
|
922
|
+
signal.signal(signal.SIGTERM, on_term)
|
|
923
|
+
rec = dict(rec, pid=os.getpid(), start=proc_start(os.getpid()))
|
|
924
|
+
slots.write_holder(rec)
|
|
925
|
+
while stop["why"] is None:
|
|
926
|
+
if now() >= until:
|
|
927
|
+
stop["why"] = "expired"
|
|
928
|
+
elif owner_pid and not pid_alive(owner_pid, owner_start):
|
|
929
|
+
stop["why"] = "owner exited"
|
|
930
|
+
else:
|
|
931
|
+
time.sleep(float(os.environ.get("DINGO_LEASE_POLL_S", "2")))
|
|
932
|
+
slots.release()
|
|
933
|
+
write_log(dict(rec, op="lease-end", t=now(), why=stop["why"], held_s=round(now() - rec["since"], 1)))
|
|
934
|
+
os._exit(0)
|
|
935
|
+
|
|
936
|
+
|
|
937
|
+
def cmd_lease(args):
|
|
938
|
+
check_local_state()
|
|
939
|
+
cfg = config()
|
|
940
|
+
est = args.mem if args.mem is not None else parse_size(cfg.get("default_mem", "1G"))
|
|
941
|
+
reserve = reserve_bytes(cfg)
|
|
942
|
+
check_fits(est, reserve)
|
|
943
|
+
threads = args.threads if args.threads is not None else DEFAULT_THREADS["lease"]
|
|
944
|
+
slots = Slots(lane_size("heavy", cfg, args.slots), "heavy", threads)
|
|
945
|
+
who = session_label(args)
|
|
946
|
+
owner_pid, owner_start = owner_process()
|
|
947
|
+
if not owner_pid:
|
|
948
|
+
eprint("dingo-pack: no agent process found (CLAUDE_PID / DINGO_OWNER_PID); the lease ends "
|
|
949
|
+
"only by `release` or --for")
|
|
950
|
+
try:
|
|
951
|
+
slot_wait, mem_wait = slots.acquire(est, reserve, args.timeout,
|
|
952
|
+
{"session": who, "cmd": "lease: " + (args.note or ""), "est": est,
|
|
953
|
+
"threads": threads})
|
|
954
|
+
except Deadline:
|
|
955
|
+
write_log({"t": now(), "op": "lease", "session": who, "rc": EXIT_TIMEOUT})
|
|
956
|
+
eprint("dingo-pack: no slot / memory within --timeout")
|
|
957
|
+
return EXIT_TIMEOUT
|
|
958
|
+
t = now()
|
|
959
|
+
rec = {"session": who, "op": "lease", "cmd": "lease: " + (args.note or ""), "est": est, "threads": threads,
|
|
960
|
+
"since": t,
|
|
961
|
+
"until": t + args.duration, "owner_pid": owner_pid, "owner_start": owner_start,
|
|
962
|
+
"slot": slots.index}
|
|
963
|
+
write_log(dict(rec, t=t, slot_wait_s=round(slot_wait, 2), mem_wait_s=round(mem_wait, 2)))
|
|
964
|
+
r, w = os.pipe()
|
|
965
|
+
pid = os.fork()
|
|
966
|
+
if pid == 0:
|
|
967
|
+
os.close(r)
|
|
968
|
+
os.write(w, b"x")
|
|
969
|
+
os.close(w)
|
|
970
|
+
try:
|
|
971
|
+
lease_daemon(slots, rec, t + args.duration, owner_pid, owner_start)
|
|
972
|
+
finally:
|
|
973
|
+
os._exit(1)
|
|
974
|
+
os.close(w)
|
|
975
|
+
os.read(r, 1)
|
|
976
|
+
os.close(r)
|
|
977
|
+
idx = slots.index
|
|
978
|
+
slots.close_only() # the lease daemon holds the same open file description; the lock stays
|
|
979
|
+
for _ in range(50): # wait for the holder file with the daemon pid
|
|
980
|
+
h = read_json(slots.holder_path(idx), {}) or {}
|
|
981
|
+
if h.get("pid") == pid:
|
|
982
|
+
break
|
|
983
|
+
time.sleep(0.02)
|
|
984
|
+
print(f"lease {idx}:{pid} held for {fmt_duration(args.duration)} "
|
|
985
|
+
f"(release: dingo-pack release {idx}:{pid})")
|
|
986
|
+
return EXIT_OK
|
|
987
|
+
|
|
988
|
+
|
|
989
|
+
def cmd_release(args):
|
|
990
|
+
owner_pid, _ = owner_process()
|
|
991
|
+
n = 0
|
|
992
|
+
for i, h in holders():
|
|
993
|
+
if h.get("op") != "lease":
|
|
994
|
+
continue
|
|
995
|
+
lid = f"{i}:{h.get('pid')}"
|
|
996
|
+
if args.ids:
|
|
997
|
+
if lid not in args.ids:
|
|
998
|
+
continue
|
|
999
|
+
elif not (owner_pid and h.get("owner_pid") == owner_pid) and h.get("session") != session_label(args):
|
|
1000
|
+
continue
|
|
1001
|
+
try:
|
|
1002
|
+
if not pid_alive(h.get("pid"), h.get("start")):
|
|
1003
|
+
eprint(f"dingo-pack: {lid}: that lease's process is gone; nothing to release")
|
|
1004
|
+
continue
|
|
1005
|
+
os.kill(int(h["pid"]), signal.SIGTERM)
|
|
1006
|
+
n += 1
|
|
1007
|
+
print(f"released {lid}")
|
|
1008
|
+
except (OSError, KeyError, ValueError) as e:
|
|
1009
|
+
eprint(f"dingo-pack: cannot release {lid}: {e}")
|
|
1010
|
+
if not n:
|
|
1011
|
+
print("no lease to release")
|
|
1012
|
+
return EXIT_OK
|
|
1013
|
+
|
|
1014
|
+
|
|
1015
|
+
def promised(table=None):
|
|
1016
|
+
"""Memory that admitted jobs said they would use and have not used yet."""
|
|
1017
|
+
table = table if table is not None else proc_table()
|
|
1018
|
+
unmet = 0
|
|
1019
|
+
for i, h in holders():
|
|
1020
|
+
if h.get("op") == "waiting for memory":
|
|
1021
|
+
continue
|
|
1022
|
+
jp = h.get("job_pid")
|
|
1023
|
+
used = tree_rss(jp, table) if jp else 0
|
|
1024
|
+
unmet += max(0, commitment(h) - used)
|
|
1025
|
+
return unmet
|
|
1026
|
+
|
|
1027
|
+
|
|
1028
|
+
def ghost_summary():
|
|
1029
|
+
"""Cached by `save` (a /proc scan is too slow for every hook call)."""
|
|
1030
|
+
return read_json(os.path.join(state_dir(), "ghosts.json"), {}) or {}
|
|
1031
|
+
|
|
1032
|
+
|
|
1033
|
+
def status_info(args=None):
|
|
1034
|
+
cfg = config()
|
|
1035
|
+
total, avail = meminfo()
|
|
1036
|
+
table = proc_table()
|
|
1037
|
+
hs = holders()
|
|
1038
|
+
reserve = reserve_bytes(cfg)
|
|
1039
|
+
prom = promised(table)
|
|
1040
|
+
dm = parse_size(cfg.get("default_mem", "1G"))
|
|
1041
|
+
lanes = {}
|
|
1042
|
+
for ln in LANE_DIRS:
|
|
1043
|
+
n = lane_size(ln, cfg, getattr(args, "slots", None) if ln == "heavy" else None)
|
|
1044
|
+
lanes[ln] = {"held": sum(1 for _, h in hs if h["lane"] == ln), "n": n}
|
|
1045
|
+
free_slots = max(0, lanes["heavy"]["n"] - lanes["heavy"]["held"])
|
|
1046
|
+
room = (avail - reserve - prom) if avail is not None else None
|
|
1047
|
+
k = free_slots if room is None else max(0, min(free_slots, int(room // dm) if dm else free_slots))
|
|
1048
|
+
ct, cu = cpu_total(cfg), cpu_used()
|
|
1049
|
+
k = max(0, min(k, (ct - cu) // DEFAULT_THREADS["heavy"]))
|
|
1050
|
+
la, nc = loadavg(), n_cpus()
|
|
1051
|
+
waiting = []
|
|
1052
|
+
for p in glob.glob(os.path.join(sub_dir("waiting"), "*.json")):
|
|
1053
|
+
w = read_json(p, {}) or {}
|
|
1054
|
+
if pid_alive(w.get("pid")):
|
|
1055
|
+
waiting.append(w)
|
|
1056
|
+
rows = []
|
|
1057
|
+
for i, h in hs:
|
|
1058
|
+
pid = h.get("job_pid") or h.get("pid")
|
|
1059
|
+
rows.append(dict(h, slot=i, now=tree_rss(pid, table) if pid else None))
|
|
1060
|
+
return {"lanes": lanes, "cpu": {"used": cu, "total": ct}, "load": la, "ncpu": nc,
|
|
1061
|
+
"overloaded": bool(la is not None and la > 1.5 * nc),
|
|
1062
|
+
"total": total, "avail": avail, "reserve": reserve, "promised": prom,
|
|
1063
|
+
"default_mem": dm, "suggest_parallel": k, "holders": rows, "waiting": waiting,
|
|
1064
|
+
"ghosts": ghost_summary()}
|
|
1065
|
+
|
|
1066
|
+
|
|
1067
|
+
def cmd_who(args):
|
|
1068
|
+
info = status_info(args)
|
|
1069
|
+
if getattr(args, "json", False):
|
|
1070
|
+
print(json.dumps(info, ensure_ascii=False, default=str))
|
|
1071
|
+
return EXIT_OK
|
|
1072
|
+
cfg = config()
|
|
1073
|
+
total, avail = info["total"], info["avail"]
|
|
1074
|
+
table = proc_table()
|
|
1075
|
+
held = holders()
|
|
1076
|
+
L = info["lanes"]
|
|
1077
|
+
print(f"slots: {L['heavy']['held']} / {L['heavy']['n']} heavy, {L['io']['held']} / {L['io']['n']} io, "
|
|
1078
|
+
f"CPU {info['cpu']['used']} / {info['cpu']['total']} "
|
|
1079
|
+
f"· memory available {fmt_size(avail)} of {fmt_size(total)} "
|
|
1080
|
+
f"· promised {fmt_size(info['promised'])} · reserve {fmt_size(reserve_bytes(cfg))}")
|
|
1081
|
+
for i, h in held:
|
|
1082
|
+
pid = h.get("job_pid") or h.get("pid")
|
|
1083
|
+
rss = tree_rss(pid, table) if pid else None
|
|
1084
|
+
extra = f" until {hhmm(h['until'])}" if h.get("until") else ""
|
|
1085
|
+
print(f" [{h['lane']}:{i}] {h.get('session', '?'):<18} {h.get('op', '?'):<5} {int(now() - h.get('since', now()))} s"
|
|
1086
|
+
f" est {fmt_size(h.get('est'))} now {fmt_size(rss)}{extra} {h.get('cmd', '')}")
|
|
1087
|
+
waiting = []
|
|
1088
|
+
for p in glob.glob(os.path.join(sub_dir("waiting"), "*.json")):
|
|
1089
|
+
w = read_json(p, {}) or {}
|
|
1090
|
+
if pid_alive(w.get("pid")):
|
|
1091
|
+
waiting.append(w)
|
|
1092
|
+
for k, w in enumerate(sorted(waiting, key=lambda w: w.get("since", 0)), 1):
|
|
1093
|
+
print(f" queue #{k} {w.get('session', '?'):<18} waiting {int(now() - w.get('since', now()))} s "
|
|
1094
|
+
f"for {w.get('phase')} est {fmt_size(w.get('est'))} {w.get('cmd', '')}")
|
|
1095
|
+
if not held and not waiting:
|
|
1096
|
+
print(" (nobody)")
|
|
1097
|
+
return EXIT_OK
|
|
1098
|
+
|
|
1099
|
+
|
|
1100
|
+
def cmd_log(args):
|
|
1101
|
+
try:
|
|
1102
|
+
with open(log_path(), encoding="utf-8") as f:
|
|
1103
|
+
lines = f.readlines()[-args.n:]
|
|
1104
|
+
except OSError:
|
|
1105
|
+
lines = []
|
|
1106
|
+
for line in lines:
|
|
1107
|
+
print(line.rstrip())
|
|
1108
|
+
return EXIT_OK
|
|
1109
|
+
|
|
1110
|
+
|
|
1111
|
+
# ---------------------------------------------------------------- roster
|
|
1112
|
+
|
|
1113
|
+
DEFAULT_NAME_RE = r"(?<![A-Za-z0-9])([A-Z][0-9]?-\d{4}-[^\s,,。::;;))]+)"
|
|
1114
|
+
DEFAULT_PROMPT = ("The machine crashed and was restarted. Re-read the end of your log and your last status "
|
|
1115
|
+
"line, check which steps already finished, then continue the interrupted step without "
|
|
1116
|
+
"redoing finished work.")
|
|
1117
|
+
|
|
1118
|
+
|
|
1119
|
+
def roster_dir():
|
|
1120
|
+
return sub_dir("roster")
|
|
1121
|
+
|
|
1122
|
+
|
|
1123
|
+
def first_user_text(path, limit_lines=400):
|
|
1124
|
+
try:
|
|
1125
|
+
with open(path, encoding="utf-8", errors="replace") as f:
|
|
1126
|
+
for k, line in enumerate(f):
|
|
1127
|
+
if k > limit_lines:
|
|
1128
|
+
break
|
|
1129
|
+
try:
|
|
1130
|
+
d = json.loads(line)
|
|
1131
|
+
except ValueError:
|
|
1132
|
+
continue
|
|
1133
|
+
if d.get("type") != "user" or d.get("isMeta") or d.get("isSidechain"):
|
|
1134
|
+
continue
|
|
1135
|
+
c = (d.get("message") or {}).get("content")
|
|
1136
|
+
if isinstance(c, list):
|
|
1137
|
+
c = " ".join(x.get("text", "") for x in c if isinstance(x, dict) and x.get("type") == "text")
|
|
1138
|
+
if isinstance(c, str) and c.strip() and not c.lstrip().startswith("<command-"):
|
|
1139
|
+
return c
|
|
1140
|
+
except OSError:
|
|
1141
|
+
pass
|
|
1142
|
+
return ""
|
|
1143
|
+
|
|
1144
|
+
|
|
1145
|
+
def find_transcript(sid):
|
|
1146
|
+
hits = glob.glob(os.path.join(claude_home(), "projects", "*", f"{sid}.jsonl"))
|
|
1147
|
+
return max(hits, key=lambda p: os.path.getmtime(p)) if hits else None
|
|
1148
|
+
|
|
1149
|
+
|
|
1150
|
+
def status_line(cwd, name, pcfg, listing=None):
|
|
1151
|
+
"""Last line matching status_regex in the newest file matching log_glob ({name}, {date}).
|
|
1152
|
+
|
|
1153
|
+
`listing` caches directory listings for one roster build: the logs may sit on a slow
|
|
1154
|
+
network / 9p mount, so each directory is listed once, not once per session."""
|
|
1155
|
+
lg, rx = pcfg.get("log_glob"), pcfg.get("status_regex")
|
|
1156
|
+
if not lg or not rx or not name:
|
|
1157
|
+
return None, None
|
|
1158
|
+
rel = lg.replace("{date}", "*")
|
|
1159
|
+
d, base = os.path.split(os.path.join(cwd, rel))
|
|
1160
|
+
listing = {} if listing is None else listing
|
|
1161
|
+
if d not in listing:
|
|
1162
|
+
try:
|
|
1163
|
+
listing[d] = os.listdir(d)
|
|
1164
|
+
except OSError:
|
|
1165
|
+
listing[d] = []
|
|
1166
|
+
import fnmatch
|
|
1167
|
+
pat = base.replace("{name}", glob.escape(name))
|
|
1168
|
+
files = [os.path.join(d, f) for f in listing[d] if fnmatch.fnmatchcase(f, pat)]
|
|
1169
|
+
if not files:
|
|
1170
|
+
return None, None
|
|
1171
|
+
path = max(files, key=lambda p: os.path.getmtime(p))
|
|
1172
|
+
try:
|
|
1173
|
+
with open(path, "rb") as f:
|
|
1174
|
+
f.seek(0, 2)
|
|
1175
|
+
size = f.tell()
|
|
1176
|
+
f.seek(max(0, size - 65536))
|
|
1177
|
+
tail = f.read().decode("utf-8", "replace")
|
|
1178
|
+
except OSError:
|
|
1179
|
+
return path, None
|
|
1180
|
+
r = re.compile(rx)
|
|
1181
|
+
last = None
|
|
1182
|
+
lines = tail.splitlines()
|
|
1183
|
+
for k, line in enumerate(lines):
|
|
1184
|
+
if r.search(line):
|
|
1185
|
+
last = line.strip()
|
|
1186
|
+
if last.startswith("#"): # "## 状态行(…)" with the text below it
|
|
1187
|
+
nxt = next((x.strip() for x in lines[k + 1:k + 4] if x.strip()), "")
|
|
1188
|
+
last = (last.lstrip("# ") + " " + nxt).strip()
|
|
1189
|
+
return path, last
|
|
1190
|
+
|
|
1191
|
+
|
|
1192
|
+
def read_registry():
|
|
1193
|
+
"""Claude Code's per-process session files. Returns (entries_by_sid, recognised)."""
|
|
1194
|
+
d = os.path.join(claude_home(), "sessions")
|
|
1195
|
+
out, seen_files, ok_files = {}, 0, 0
|
|
1196
|
+
for p in glob.glob(os.path.join(d, "*.json")):
|
|
1197
|
+
seen_files += 1
|
|
1198
|
+
e = read_json(p)
|
|
1199
|
+
if not isinstance(e, dict) or "sessionId" not in e or "pid" not in e:
|
|
1200
|
+
continue
|
|
1201
|
+
ok_files += 1
|
|
1202
|
+
e["_alive"] = pid_alive(e.get("pid"), e.get("procStart"))
|
|
1203
|
+
sid = e["sessionId"]
|
|
1204
|
+
prev = out.get(sid)
|
|
1205
|
+
key = (e["_alive"], e.get("updatedAt") or 0)
|
|
1206
|
+
if prev is None or key > (prev["_alive"], prev.get("updatedAt") or 0):
|
|
1207
|
+
out[sid] = e
|
|
1208
|
+
return out, (seen_files == 0 or ok_files > 0)
|
|
1209
|
+
|
|
1210
|
+
|
|
1211
|
+
def transcripts_fallback(hours=12):
|
|
1212
|
+
out = {}
|
|
1213
|
+
cut = now() - hours * 3600
|
|
1214
|
+
for p in glob.glob(os.path.join(claude_home(), "projects", "*", "*.jsonl")):
|
|
1215
|
+
try:
|
|
1216
|
+
m = os.path.getmtime(p)
|
|
1217
|
+
except OSError:
|
|
1218
|
+
continue
|
|
1219
|
+
if m >= cut:
|
|
1220
|
+
sid = os.path.basename(p)[:-6]
|
|
1221
|
+
out[sid] = {"sessionId": sid, "cwd": None, "_alive": None, "_transcript": p}
|
|
1222
|
+
return out
|
|
1223
|
+
|
|
1224
|
+
|
|
1225
|
+
def guess_state(cc_status):
|
|
1226
|
+
return {"busy": "midway", "waiting": "midway", "shell": "midway", "idle": "idle"}.get(cc_status, "unknown")
|
|
1227
|
+
|
|
1228
|
+
|
|
1229
|
+
def load_roster():
|
|
1230
|
+
return read_json(os.path.join(roster_dir(), "roster.json"), {"sessions": {}}) or {"sessions": {}}
|
|
1231
|
+
|
|
1232
|
+
|
|
1233
|
+
def build_roster(old):
|
|
1234
|
+
reg, recognised = read_registry()
|
|
1235
|
+
fallback = not recognised
|
|
1236
|
+
if fallback:
|
|
1237
|
+
reg = transcripts_fallback()
|
|
1238
|
+
t, bt = now(), boot_time()
|
|
1239
|
+
sessions = dict(old.get("sessions") or {})
|
|
1240
|
+
names_cache = old.get("names") or {}
|
|
1241
|
+
marks_dir = sub_dir("marks")
|
|
1242
|
+
for sid, e in reg.items():
|
|
1243
|
+
s = dict(sessions.get(sid) or {})
|
|
1244
|
+
tr = e.get("_transcript") or find_transcript(sid)
|
|
1245
|
+
cwd = e.get("cwd") or s.get("cwd")
|
|
1246
|
+
if not cwd and tr:
|
|
1247
|
+
cwd = s.get("cwd")
|
|
1248
|
+
name = None
|
|
1249
|
+
if e.get("nameSource") == "user" and e.get("name"):
|
|
1250
|
+
name = e["name"]
|
|
1251
|
+
if not name:
|
|
1252
|
+
name = names_cache.get(sid)
|
|
1253
|
+
if not name and tr:
|
|
1254
|
+
pcfg = local_project_config(cwd) if cwd else {}
|
|
1255
|
+
m = re.search(pcfg.get("name_regex") or DEFAULT_NAME_RE, first_user_text(tr)[:600])
|
|
1256
|
+
if m:
|
|
1257
|
+
name = m.group(1)
|
|
1258
|
+
names_cache[sid] = name
|
|
1259
|
+
name = name or e.get("name") or sid[:8]
|
|
1260
|
+
alive = e.get("_alive")
|
|
1261
|
+
s.update(sid=sid, name=name, cwd=cwd, transcript=tr, pid=e.get("pid"),
|
|
1262
|
+
cc_status=e.get("status"), registry_recognised=not fallback)
|
|
1263
|
+
if e.get("startedAt"):
|
|
1264
|
+
s["started_at"] = e["startedAt"] / 1000.0
|
|
1265
|
+
if tr:
|
|
1266
|
+
try:
|
|
1267
|
+
s["last_activity"] = os.path.getmtime(tr)
|
|
1268
|
+
except OSError:
|
|
1269
|
+
pass
|
|
1270
|
+
if alive:
|
|
1271
|
+
s["alive"], s["last_seen"] = True, t
|
|
1272
|
+
s.pop("ended", None)
|
|
1273
|
+
elif alive is None: # registry not recognised: unknown, keep it
|
|
1274
|
+
s["alive"] = False
|
|
1275
|
+
s.setdefault("ended", "unknown")
|
|
1276
|
+
s["last_seen"] = max(s.get("last_seen") or 0, s.get("last_activity") or 0)
|
|
1277
|
+
elif alive is False:
|
|
1278
|
+
if s.get("alive") or "ended" not in s:
|
|
1279
|
+
# it was alive at our last look (or we never saw it alive): when did it go?
|
|
1280
|
+
ls = s.get("last_seen") or s.get("last_activity") or e.get("updatedAt", 0) / 1000
|
|
1281
|
+
s["last_seen"] = ls
|
|
1282
|
+
s["ended"] = "reboot" if (bt and ls and ls < bt) else "closed"
|
|
1283
|
+
s["alive"] = False
|
|
1284
|
+
mk = read_json(os.path.join(marks_dir, f"{sid}.json"))
|
|
1285
|
+
if mk:
|
|
1286
|
+
s["mark"] = mk
|
|
1287
|
+
sessions[sid] = s
|
|
1288
|
+
# sessions that vanished from the registry entirely
|
|
1289
|
+
for sid, s in sessions.items():
|
|
1290
|
+
if sid not in reg and s.get("alive"):
|
|
1291
|
+
s["alive"] = False
|
|
1292
|
+
s["ended"] = "reboot" if (bt and s.get("last_seen", 0) < bt) else "closed"
|
|
1293
|
+
# prune entries not seen for a week
|
|
1294
|
+
sessions = {k: v for k, v in sessions.items() if t - (v.get("last_seen") or 0) < 7 * 86400}
|
|
1295
|
+
return {"version": 1, "saved_at": t, "boot_time": bt, "host": socket.gethostname(),
|
|
1296
|
+
"registry_recognised": not fallback, "sessions": sessions, "names": names_cache}
|
|
1297
|
+
|
|
1298
|
+
|
|
1299
|
+
def state_of(s):
|
|
1300
|
+
mk = s.get("mark") or {}
|
|
1301
|
+
if mk.get("state"):
|
|
1302
|
+
return mk["state"], False
|
|
1303
|
+
return guess_state(s.get("cc_status")), True
|
|
1304
|
+
|
|
1305
|
+
|
|
1306
|
+
def render_md(r):
|
|
1307
|
+
lines = [f"# dingo-pack roster · saved {hhmm(r['saved_at'])} on {r.get('host')}", ""]
|
|
1308
|
+
if not r.get("registry_recognised", True):
|
|
1309
|
+
lines += ["> Claude Code session registry not recognised; built from transcripts only.", ""]
|
|
1310
|
+
lines += ["| alive | name | state | until / note | last activity | status line | resume |",
|
|
1311
|
+
"|---|---|---|---|---|---|---|"]
|
|
1312
|
+
rows = sorted(r["sessions"].values(), key=lambda s: (not s.get("alive"), -(s.get("last_activity") or 0)))
|
|
1313
|
+
for s in rows:
|
|
1314
|
+
st, guessed = state_of(s)
|
|
1315
|
+
mk = s.get("mark") or {}
|
|
1316
|
+
note = " ".join(x for x in (mk.get("until") or "", mk.get("note") or "") if x)
|
|
1317
|
+
alive = "yes" if s.get("alive") else f"no ({s.get('ended', '?')})"
|
|
1318
|
+
cmd = f"cd {shlex.quote(s.get('cwd') or '.')} && claude --resume {s['sid']}"
|
|
1319
|
+
sl = (s.get("status_line") or "").replace("|", "\\|")[:160]
|
|
1320
|
+
lines.append(f"| {alive} | {s.get('name')} | {st}{' (guessed)' if guessed else ''} | {note} | "
|
|
1321
|
+
f"{hhmm(s.get('last_activity'))} | {sl} | `{cmd}` |")
|
|
1322
|
+
return "\n".join(lines) + "\n"
|
|
1323
|
+
|
|
1324
|
+
|
|
1325
|
+
def save_roster(quiet=False, min_gap=0.0):
|
|
1326
|
+
rd = roster_dir()
|
|
1327
|
+
lk = os.open(os.path.join(rd, "lock"), os.O_RDWR | os.O_CREAT, 0o644)
|
|
1328
|
+
try:
|
|
1329
|
+
try:
|
|
1330
|
+
fcntl.flock(lk, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
1331
|
+
except OSError:
|
|
1332
|
+
return None # someone else is saving right now; their snapshot will do
|
|
1333
|
+
old = load_roster()
|
|
1334
|
+
if min_gap and now() - (old.get("saved_at") or 0) < min_gap:
|
|
1335
|
+
return old
|
|
1336
|
+
r = build_roster(old)
|
|
1337
|
+
body = json.dumps(r, ensure_ascii=False, indent=1, sort_keys=True)
|
|
1338
|
+
write_atomic(os.path.join(rd, "roster.json"), body)
|
|
1339
|
+
write_atomic(os.path.join(rd, "roster.md"), render_md(r))
|
|
1340
|
+
sig = json.dumps({k: (v.get("alive"), v.get("cc_status"), (v.get("mark") or {}).get("state"),
|
|
1341
|
+
v.get("status_line")) for k, v in r["sessions"].items()}, sort_keys=True)
|
|
1342
|
+
sigp = os.path.join(rd, "last.sig")
|
|
1343
|
+
try:
|
|
1344
|
+
with open(sigp) as f:
|
|
1345
|
+
same = f.read() == sig
|
|
1346
|
+
except OSError:
|
|
1347
|
+
same = False
|
|
1348
|
+
if not same:
|
|
1349
|
+
snaps = sub_dir(os.path.join("roster", "snapshots"))
|
|
1350
|
+
write_atomic(os.path.join(snaps, time.strftime("%Y%m%d-%H%M%S") + f"-{os.getpid()}.json"), body)
|
|
1351
|
+
write_atomic(sigp, sig)
|
|
1352
|
+
keep = int(os.environ.get("DINGO_KEEP_SNAPSHOTS", "20"))
|
|
1353
|
+
for old_snap in sorted(glob.glob(os.path.join(snaps, "*.json")))[:-keep]:
|
|
1354
|
+
try:
|
|
1355
|
+
os.unlink(old_snap)
|
|
1356
|
+
except OSError:
|
|
1357
|
+
pass
|
|
1358
|
+
finally:
|
|
1359
|
+
os.close(lk)
|
|
1360
|
+
r = add_status_lines(r) or r
|
|
1361
|
+
st, gs = run_timed(lambda: scan_ghosts(r), 5)
|
|
1362
|
+
if st == "ok":
|
|
1363
|
+
loose = [g for g in gs if not g["kept"]]
|
|
1364
|
+
try:
|
|
1365
|
+
write_atomic(os.path.join(state_dir(), "ghosts.json"), json.dumps(
|
|
1366
|
+
{"t": now(), "n": len(loose), "pss": sum(g["pss"] for g in loose),
|
|
1367
|
+
"kept": len(gs) - len(loose)}))
|
|
1368
|
+
except OSError:
|
|
1369
|
+
pass
|
|
1370
|
+
if not quiet:
|
|
1371
|
+
alive = sum(1 for s in r["sessions"].values() if s.get("alive"))
|
|
1372
|
+
print(f"roster saved: {alive} alive, {len(r['sessions'])} known -> {os.path.join(rd, 'roster.md')}")
|
|
1373
|
+
return r
|
|
1374
|
+
|
|
1375
|
+
|
|
1376
|
+
def add_status_lines(r):
|
|
1377
|
+
"""Phase 2 of a save: read status lines from project logs (possibly on a network mount).
|
|
1378
|
+
Runs *without* the roster lock, so a read stuck on a hung mount cannot freeze the roster;
|
|
1379
|
+
a second lock keeps at most one such reader alive. Results are merged under the roster lock."""
|
|
1380
|
+
rd = roster_dir()
|
|
1381
|
+
nl = os.open(os.path.join(rd, "netlock"), os.O_RDWR | os.O_CREAT, 0o644)
|
|
1382
|
+
try:
|
|
1383
|
+
try:
|
|
1384
|
+
fcntl.flock(nl, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
1385
|
+
except OSError:
|
|
1386
|
+
return None # another reader is busy (or stuck); skip
|
|
1387
|
+
listing, found = {}, {}
|
|
1388
|
+
for sid, s in r["sessions"].items():
|
|
1389
|
+
if not s.get("alive") and s.get("status_line"):
|
|
1390
|
+
continue # keep the last line we read while it was alive
|
|
1391
|
+
cwd, name = s.get("cwd"), s.get("name")
|
|
1392
|
+
if cwd and name:
|
|
1393
|
+
lp, line = status_line(cwd, name, project_config(cwd), listing)
|
|
1394
|
+
if lp and (line is not None or not s.get("status_line")):
|
|
1395
|
+
found[sid] = (lp, line) # a read error keeps the last good line
|
|
1396
|
+
if not found:
|
|
1397
|
+
return None
|
|
1398
|
+
lk = os.open(os.path.join(rd, "lock"), os.O_RDWR | os.O_CREAT, 0o644)
|
|
1399
|
+
try:
|
|
1400
|
+
try:
|
|
1401
|
+
fcntl.flock(lk, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
1402
|
+
except OSError:
|
|
1403
|
+
return None
|
|
1404
|
+
cur = load_roster()
|
|
1405
|
+
changed = False
|
|
1406
|
+
for sid, (lp, line) in found.items():
|
|
1407
|
+
s = cur["sessions"].get(sid)
|
|
1408
|
+
if s is not None and (s.get("log"), s.get("status_line")) != (lp, line):
|
|
1409
|
+
if line != s.get("status_line"):
|
|
1410
|
+
s["status_changed_at"] = now()
|
|
1411
|
+
s["log"], s["status_line"] = lp, line
|
|
1412
|
+
changed = True
|
|
1413
|
+
if changed:
|
|
1414
|
+
write_atomic(os.path.join(rd, "roster.json"),
|
|
1415
|
+
json.dumps(cur, ensure_ascii=False, indent=1, sort_keys=True))
|
|
1416
|
+
write_atomic(os.path.join(rd, "roster.md"), render_md(cur))
|
|
1417
|
+
return cur
|
|
1418
|
+
finally:
|
|
1419
|
+
os.close(lk)
|
|
1420
|
+
finally:
|
|
1421
|
+
os.close(nl)
|
|
1422
|
+
|
|
1423
|
+
|
|
1424
|
+
def cmd_save(args):
|
|
1425
|
+
if args.hook:
|
|
1426
|
+
# Called from a Claude Code hook on every turn: return at once, never fail, never print.
|
|
1427
|
+
# The save itself runs in a detached child, so a hung network mount (status lines are
|
|
1428
|
+
# read from the project's logs) can never stall the agent's turn.
|
|
1429
|
+
try:
|
|
1430
|
+
sys.stdin.read()
|
|
1431
|
+
except (OSError, ValueError):
|
|
1432
|
+
pass
|
|
1433
|
+
if os.environ.get("DINGO_HOOK_FOREGROUND") != "1":
|
|
1434
|
+
try:
|
|
1435
|
+
if os.fork() > 0:
|
|
1436
|
+
return EXIT_OK
|
|
1437
|
+
except OSError:
|
|
1438
|
+
return EXIT_OK
|
|
1439
|
+
os.setsid()
|
|
1440
|
+
devnull = os.open(os.devnull, os.O_RDWR)
|
|
1441
|
+
for fd in (0, 1, 2):
|
|
1442
|
+
os.dup2(devnull, fd)
|
|
1443
|
+
rc = EXIT_OK
|
|
1444
|
+
try:
|
|
1445
|
+
signal.signal(signal.SIGALRM, _alarm)
|
|
1446
|
+
signal.alarm(int(os.environ.get("DINGO_HOOK_MAX_S", "30")))
|
|
1447
|
+
save_roster(quiet=True, min_gap=float(os.environ.get("DINGO_HOOK_MIN_GAP_S", "15")))
|
|
1448
|
+
except BaseException as e: # noqa: BLE001 - a hook must not break the agent's turn
|
|
1449
|
+
try:
|
|
1450
|
+
write_log({"t": now(), "op": "save-hook-error", "error": repr(e)[:300]})
|
|
1451
|
+
except Exception: # noqa: BLE001
|
|
1452
|
+
pass
|
|
1453
|
+
if os.environ.get("DINGO_HOOK_FOREGROUND") != "1":
|
|
1454
|
+
os._exit(rc)
|
|
1455
|
+
return rc
|
|
1456
|
+
check_local_state()
|
|
1457
|
+
if args.every:
|
|
1458
|
+
while True:
|
|
1459
|
+
save_roster(quiet=True)
|
|
1460
|
+
time.sleep(args.every)
|
|
1461
|
+
save_roster(quiet=False)
|
|
1462
|
+
return EXIT_OK
|
|
1463
|
+
|
|
1464
|
+
|
|
1465
|
+
def my_session_id(args):
|
|
1466
|
+
sid = getattr(args, "sid", None) or os.environ.get("CLAUDE_CODE_SESSION_ID")
|
|
1467
|
+
if sid:
|
|
1468
|
+
return sid
|
|
1469
|
+
pid, _ = owner_process()
|
|
1470
|
+
if pid:
|
|
1471
|
+
e = read_json(os.path.join(claude_home(), "sessions", f"{pid}.json"), {}) or {}
|
|
1472
|
+
return e.get("sessionId")
|
|
1473
|
+
return None
|
|
1474
|
+
|
|
1475
|
+
|
|
1476
|
+
def cmd_mark(args):
|
|
1477
|
+
sid = my_session_id(args)
|
|
1478
|
+
if not sid:
|
|
1479
|
+
eprint("dingo-pack mark: cannot tell which session this is (no CLAUDE_CODE_SESSION_ID); pass --sid")
|
|
1480
|
+
return EXIT_USAGE
|
|
1481
|
+
if args.state == "waiting" and not args.until:
|
|
1482
|
+
eprint("dingo-pack mark waiting: tip: add --until WHEN so `list` can sort by when to resume")
|
|
1483
|
+
state = "midway" if args.state == "running" else args.state
|
|
1484
|
+
rec = {"state": state, "until": args.until, "note": " ".join(args.note)[:300], "t": now()}
|
|
1485
|
+
write_atomic(os.path.join(sub_dir("marks"), f"{sid}.json"), json.dumps(rec, ensure_ascii=False))
|
|
1486
|
+
save_roster(quiet=True)
|
|
1487
|
+
print(f"marked {sid[:8]} {state}" + (f" until {args.until}" if args.until else ""))
|
|
1488
|
+
return EXIT_OK
|
|
1489
|
+
|
|
1490
|
+
|
|
1491
|
+
GROUPS = [("stowed", "📦 stowed - bring back with: dingo-pack unstow"), ("midway", "🔴 midway - resume now"), ("waiting", "⏰ waiting for a time - resume when due"),
|
|
1492
|
+
("blocked", "⛔ blocked on a person"), ("idle", "💤 idle (no self-report) - check before resuming"),
|
|
1493
|
+
("unknown", "❔ unknown"), ("done", "✅ done - no need to resume")]
|
|
1494
|
+
|
|
1495
|
+
|
|
1496
|
+
UNTIL_TIME = re.compile(r"(\d{1,2})[::.](\d{2})")
|
|
1497
|
+
|
|
1498
|
+
|
|
1499
|
+
def until_key(until):
|
|
1500
|
+
"""Sort key for --until text: times of day first (earliest first), then other text, then none.
|
|
1501
|
+
Only the first HH:MM in the text is used; the day is not parsed."""
|
|
1502
|
+
if not until:
|
|
1503
|
+
return (2, 0, "")
|
|
1504
|
+
m = UNTIL_TIME.search(until)
|
|
1505
|
+
if m and int(m.group(1)) < 24 and int(m.group(2)) < 60:
|
|
1506
|
+
return (0, int(m.group(1)) * 60 + int(m.group(2)), until)
|
|
1507
|
+
return (1, 0, until)
|
|
1508
|
+
|
|
1509
|
+
|
|
1510
|
+
def select(r, since_h, include_closed):
|
|
1511
|
+
cut = now() - since_h * 3600
|
|
1512
|
+
out = []
|
|
1513
|
+
for s in r["sessions"].values():
|
|
1514
|
+
if (s.get("last_seen") or 0) < cut:
|
|
1515
|
+
continue
|
|
1516
|
+
if s.get("alive"):
|
|
1517
|
+
continue
|
|
1518
|
+
if s.get("ended") == "closed" and not include_closed:
|
|
1519
|
+
continue
|
|
1520
|
+
out.append(s)
|
|
1521
|
+
return out
|
|
1522
|
+
|
|
1523
|
+
|
|
1524
|
+
def cmd_list(args):
|
|
1525
|
+
r = save_roster(quiet=True) if not args.no_refresh else load_roster()
|
|
1526
|
+
if r is None:
|
|
1527
|
+
r = load_roster()
|
|
1528
|
+
if args.at:
|
|
1529
|
+
snaps = sorted(glob.glob(os.path.join(roster_dir(), "snapshots", "*.json")))
|
|
1530
|
+
pick = [p for p in snaps if os.path.basename(p)[:15].replace("-", "") <= args.at.replace("-", "").replace(":", "")]
|
|
1531
|
+
if pick:
|
|
1532
|
+
r = read_json(pick[-1], r)
|
|
1533
|
+
stowed = set(stowed_sessions(r))
|
|
1534
|
+
rows = [s for s in select(r, args.since, True)
|
|
1535
|
+
if args.closed or s.get("ended") != "closed" or s["sid"] in stowed]
|
|
1536
|
+
alive_now = [s for s in r["sessions"].values() if s.get("alive")]
|
|
1537
|
+
if args.json:
|
|
1538
|
+
print(json.dumps({"lost": rows, "alive": alive_now}, ensure_ascii=False, indent=1))
|
|
1539
|
+
return EXIT_OK
|
|
1540
|
+
print(f"roster {hhmm(r.get('saved_at'))} · {len(alive_now)} alive now · {len(rows)} not running "
|
|
1541
|
+
f"(seen in the last {args.since:g} h{', incl. closed' if args.closed else ''})")
|
|
1542
|
+
if not r.get("registry_recognised", True):
|
|
1543
|
+
print(" ⚠ Claude Code session registry not recognised; names / cwd may be missing")
|
|
1544
|
+
for key, title in GROUPS:
|
|
1545
|
+
grp = [s for s in rows if ("stowed" if s["sid"] in stowed else state_of(s)[0]) == key]
|
|
1546
|
+
if not grp:
|
|
1547
|
+
continue
|
|
1548
|
+
print(f"\n{title}")
|
|
1549
|
+
grp.sort(key=lambda s: (until_key((s.get("mark") or {}).get("until")), -(s.get("last_activity") or 0)))
|
|
1550
|
+
for s in grp:
|
|
1551
|
+
st, guessed = state_of(s)
|
|
1552
|
+
mk = s.get("mark") or {}
|
|
1553
|
+
bits = [f" {s.get('name', '?'):<24} last {hhmm(s.get('last_activity'))}",
|
|
1554
|
+
f"ended {s.get('ended', '?')}"]
|
|
1555
|
+
if guessed:
|
|
1556
|
+
bits.append(f"(guessed from '{s.get('cc_status')}')")
|
|
1557
|
+
if mk.get("until"):
|
|
1558
|
+
bits.append(f"until {mk['until']}")
|
|
1559
|
+
print(" · ".join(bits))
|
|
1560
|
+
if mk.get("note"):
|
|
1561
|
+
print(f" note: {mk['note']}")
|
|
1562
|
+
if s.get("status_line"):
|
|
1563
|
+
print(f" status: {s['status_line'][:150]}")
|
|
1564
|
+
print(f" cd {shlex.quote(s.get('cwd') or '.')} && claude --resume {s['sid']}")
|
|
1565
|
+
if alive_now:
|
|
1566
|
+
print("\nalive now: " + ", ".join(sorted(s.get("name", "?") + ("" if s.get("transcript") else " (empty)")
|
|
1567
|
+
for s in alive_now)))
|
|
1568
|
+
return EXIT_OK
|
|
1569
|
+
|
|
1570
|
+
|
|
1571
|
+
def launcher(s, prompt):
|
|
1572
|
+
d = sub_dir("resume")
|
|
1573
|
+
path = os.path.join(d, f"{s['sid']}.sh")
|
|
1574
|
+
claude = os.environ.get("DINGO_CLAUDE_BIN", "claude")
|
|
1575
|
+
body = ("#!/bin/bash\n"
|
|
1576
|
+
f"# dingo-pack resume {s.get('name')} ({hhmm(now())})\n"
|
|
1577
|
+
f"cd {shlex.quote(s.get('cwd') or os.path.expanduser('~'))} || exit 1\n"
|
|
1578
|
+
f"exec {shlex.quote(claude)} --resume {shlex.quote(s['sid'])} {shlex.quote(prompt)}\n")
|
|
1579
|
+
write_atomic(path, body)
|
|
1580
|
+
os.chmod(path, 0o755)
|
|
1581
|
+
return path
|
|
1582
|
+
|
|
1583
|
+
|
|
1584
|
+
def pick_via(v):
|
|
1585
|
+
if v:
|
|
1586
|
+
return v
|
|
1587
|
+
if os.environ.get("WSL_DISTRO_NAME") and shutil.which("wt.exe"):
|
|
1588
|
+
return "wt"
|
|
1589
|
+
if os.environ.get("TMUX") and shutil.which("tmux"):
|
|
1590
|
+
return "tmux"
|
|
1591
|
+
return "print"
|
|
1592
|
+
|
|
1593
|
+
|
|
1594
|
+
def open_tab(via, s, path):
|
|
1595
|
+
title = re.sub(r"[^\w.\-]", "_", s.get("name") or s["sid"][:8])
|
|
1596
|
+
if via == "wt":
|
|
1597
|
+
argv = ["wt.exe", "-w", "0", "nt", "--title", title, "wsl.exe", "-d",
|
|
1598
|
+
os.environ.get("WSL_DISTRO_NAME", ""), "--", "bash", "-l", path]
|
|
1599
|
+
elif via == "tmux":
|
|
1600
|
+
argv = ["tmux", "new-window", "-n", title, "bash", "-l", path]
|
|
1601
|
+
else:
|
|
1602
|
+
print(f" bash -l {shlex.quote(path)} # {s.get('name')}")
|
|
1603
|
+
return 0
|
|
1604
|
+
return subprocess.run(argv).returncode
|
|
1605
|
+
|
|
1606
|
+
|
|
1607
|
+
def cmd_resume(args):
|
|
1608
|
+
r = save_roster(quiet=True) or load_roster()
|
|
1609
|
+
rows = select(r, args.since, True)
|
|
1610
|
+
if args.names:
|
|
1611
|
+
want = set(args.names)
|
|
1612
|
+
rows = [s for s in r["sessions"].values() if s.get("name") in want or s["sid"] in want
|
|
1613
|
+
or s["sid"][:8] in want]
|
|
1614
|
+
missing = want - {s.get("name") for s in rows} - {s["sid"] for s in rows} - {s["sid"][:8] for s in rows}
|
|
1615
|
+
for m in sorted(missing):
|
|
1616
|
+
eprint(f"dingo-pack: {m}: not in the roster")
|
|
1617
|
+
elif args.midway:
|
|
1618
|
+
rows = [s for s in rows if s.get("ended") != "closed" and state_of(s)[0] == "midway"]
|
|
1619
|
+
elif args.lost:
|
|
1620
|
+
rows = [s for s in rows if s.get("ended") != "closed" and state_of(s)[0] != "done"]
|
|
1621
|
+
else:
|
|
1622
|
+
eprint("dingo-pack resume: name sessions, or say --midway / --lost")
|
|
1623
|
+
return EXIT_USAGE
|
|
1624
|
+
return reopen(rows, args, prompt_key="resume_prompt", default_prompt=DEFAULT_PROMPT)
|
|
1625
|
+
|
|
1626
|
+
|
|
1627
|
+
def reopen(rows, args, prompt_key, default_prompt):
|
|
1628
|
+
"""Open each session in `rows` with `claude --resume`, in batches. Skips live ones."""
|
|
1629
|
+
reg, _ = read_registry()
|
|
1630
|
+
todo = []
|
|
1631
|
+
for s in sorted(rows, key=lambda s: -(s.get("last_activity") or 0)):
|
|
1632
|
+
e = reg.get(s["sid"])
|
|
1633
|
+
if e and e.get("_alive"):
|
|
1634
|
+
print(f"skip {s.get('name')}: already running (pid {e.get('pid')})")
|
|
1635
|
+
continue
|
|
1636
|
+
if not s.get("transcript") or not os.path.exists(s["transcript"]):
|
|
1637
|
+
eprint(f"skip {s.get('name')}: transcript not found")
|
|
1638
|
+
continue
|
|
1639
|
+
todo.append(s)
|
|
1640
|
+
if not todo:
|
|
1641
|
+
print("nothing to resume")
|
|
1642
|
+
return EXIT_OK
|
|
1643
|
+
via = "print" if args.dry_run else pick_via(args.via)
|
|
1644
|
+
batch = max(1, int(args.batch or config().get("resume_batch", 4)))
|
|
1645
|
+
gap = float(os.environ.get("DINGO_RESUME_GAP_S", "5"))
|
|
1646
|
+
print(f"resuming {len(todo)} via {via}, {batch} at a time")
|
|
1647
|
+
for k, s in enumerate(todo):
|
|
1648
|
+
if k and k % batch == 0:
|
|
1649
|
+
rest = ", ".join(x.get("name", "?") for x in todo[k:])
|
|
1650
|
+
if args.yes:
|
|
1651
|
+
pass
|
|
1652
|
+
elif sys.stdin.isatty():
|
|
1653
|
+
input(f"next batch: {rest} - press Enter to continue (Ctrl-C to stop) ")
|
|
1654
|
+
else:
|
|
1655
|
+
print(f"stopped after a batch of {batch}; still to resume: {rest}")
|
|
1656
|
+
return EXIT_OK
|
|
1657
|
+
prompt = args.prompt or project_config(s.get("cwd") or ".").get(prompt_key) or default_prompt
|
|
1658
|
+
path = launcher(s, prompt)
|
|
1659
|
+
print(f" {s.get('name')}: {s['sid']} in {s.get('cwd')}")
|
|
1660
|
+
rc = open_tab(via, s, path)
|
|
1661
|
+
if rc:
|
|
1662
|
+
eprint(f"dingo-pack: opening a tab for {s.get('name')} failed (exit {rc}); run: bash -l {path}")
|
|
1663
|
+
write_log({"t": now(), "op": "resume", "session": s.get("name"), "sid": s["sid"], "via": via, "rc": rc})
|
|
1664
|
+
if via != "print" and k + 1 < len(todo):
|
|
1665
|
+
time.sleep(gap)
|
|
1666
|
+
return EXIT_OK
|
|
1667
|
+
|
|
1668
|
+
|
|
1669
|
+
# ---------------------------------------------------------------- ghosts (scan)
|
|
1670
|
+
|
|
1671
|
+
def proc_environ(pid, keys=(b"CLAUDE_PID", b"CLAUDE_CODE_SESSION_ID", b"DINGO_SCOPE_TAG")):
|
|
1672
|
+
try:
|
|
1673
|
+
with open(f"/proc/{pid}/environ", "rb") as f:
|
|
1674
|
+
raw = f.read()
|
|
1675
|
+
except OSError:
|
|
1676
|
+
return {}
|
|
1677
|
+
out = {}
|
|
1678
|
+
for item in raw.split(b"\0"):
|
|
1679
|
+
k, sep, v = item.partition(b"=")
|
|
1680
|
+
if sep and k in keys:
|
|
1681
|
+
out[k.decode()] = v.decode("utf-8", "replace")
|
|
1682
|
+
return out
|
|
1683
|
+
|
|
1684
|
+
|
|
1685
|
+
def proc_cmdline(pid):
|
|
1686
|
+
try:
|
|
1687
|
+
with open(f"/proc/{pid}/cmdline", "rb") as f:
|
|
1688
|
+
return f.read().replace(b"\0", b" ").decode("utf-8", "replace").strip()
|
|
1689
|
+
except OSError:
|
|
1690
|
+
return ""
|
|
1691
|
+
|
|
1692
|
+
|
|
1693
|
+
def proc_detached(pid):
|
|
1694
|
+
"""True if the process looks deliberately left running: SIGHUP ignored (nohup), its own
|
|
1695
|
+
session leader (setsid / daemon), or a terminal multiplexer."""
|
|
1696
|
+
try:
|
|
1697
|
+
with open(f"/proc/{pid}/status") as f:
|
|
1698
|
+
for line in f:
|
|
1699
|
+
if line.startswith("SigIgn:") and int(line.split()[1], 16) & 1:
|
|
1700
|
+
return True
|
|
1701
|
+
with open(f"/proc/{pid}/stat") as f:
|
|
1702
|
+
st = f.read()
|
|
1703
|
+
comm = st[st.index("(") + 1:st.rindex(")")]
|
|
1704
|
+
fields = st[st.rindex(")") + 2:].split()
|
|
1705
|
+
if int(fields[3]) == int(pid): # session id == pid
|
|
1706
|
+
return True
|
|
1707
|
+
if comm.startswith(("tmux", "screen")):
|
|
1708
|
+
return True
|
|
1709
|
+
except (OSError, ValueError, IndexError):
|
|
1710
|
+
pass
|
|
1711
|
+
return False
|
|
1712
|
+
|
|
1713
|
+
|
|
1714
|
+
def scan_ghosts(roster=None):
|
|
1715
|
+
"""Process trees whose Claude Code session is gone. Returns a list of groups:
|
|
1716
|
+
{owner_pid, sid, name, pids, pss, cmd, age_s, kept, why_kept}."""
|
|
1717
|
+
if not os.path.isdir("/proc/self"):
|
|
1718
|
+
return []
|
|
1719
|
+
table = proc_table()
|
|
1720
|
+
reg = {}
|
|
1721
|
+
for p in glob.glob(os.path.join(claude_home(), "sessions", "*.json")):
|
|
1722
|
+
e = read_json(p)
|
|
1723
|
+
if isinstance(e, dict) and "pid" in e:
|
|
1724
|
+
reg[int(e["pid"])] = e
|
|
1725
|
+
live_claude = {pid for pid, e in reg.items() if pid_alive(pid, e.get("procStart"))}
|
|
1726
|
+
present = [pid for pid in reg if pid in table]
|
|
1727
|
+
if present and not live_claude:
|
|
1728
|
+
# every registry pid that exists fails the start-time check: the format probably changed.
|
|
1729
|
+
# Fall back to "the pid exists" rather than calling every session's children ghosts.
|
|
1730
|
+
live_claude = set(present)
|
|
1731
|
+
reg = {pid: dict(e, procStart=None) for pid, e in reg.items()}
|
|
1732
|
+
protected = set()
|
|
1733
|
+
for c in live_claude:
|
|
1734
|
+
protected.update(tree_pids(c, table))
|
|
1735
|
+
roster = roster or load_roster()
|
|
1736
|
+
by_pid = {}
|
|
1737
|
+
for s in (roster.get("sessions") or {}).values():
|
|
1738
|
+
if s.get("pid"):
|
|
1739
|
+
by_pid[int(s["pid"])] = s
|
|
1740
|
+
me = os.getpid()
|
|
1741
|
+
# Narrowing only: with DINGO_SCOPE_TAG set, look at nothing but processes that carry the same
|
|
1742
|
+
# tag in their environment (the self-test uses this so it never touches real processes).
|
|
1743
|
+
scope = os.environ.get("DINGO_SCOPE_TAG")
|
|
1744
|
+
p = me
|
|
1745
|
+
while p and p in table and p not in protected: # our own ancestors are never ghosts
|
|
1746
|
+
protected.add(p)
|
|
1747
|
+
p = table[p][0]
|
|
1748
|
+
ghosts = {}
|
|
1749
|
+
for pid in table:
|
|
1750
|
+
if pid in protected or pid == me:
|
|
1751
|
+
continue
|
|
1752
|
+
env = proc_environ(pid)
|
|
1753
|
+
if scope and env.get("DINGO_SCOPE_TAG") != scope:
|
|
1754
|
+
continue # scoped scan (self-test): only processes carrying the tag
|
|
1755
|
+
cp = env.get("CLAUDE_PID")
|
|
1756
|
+
if not cp or not cp.isdigit():
|
|
1757
|
+
continue
|
|
1758
|
+
cp = int(cp)
|
|
1759
|
+
if pid == cp:
|
|
1760
|
+
continue
|
|
1761
|
+
e = reg.get(cp)
|
|
1762
|
+
if e is not None:
|
|
1763
|
+
owner_ok = pid_alive(cp, e.get("procStart"))
|
|
1764
|
+
else: # registry file gone or kept elsewhere:
|
|
1765
|
+
owner_ok = cp in table # can't verify, so never call it a ghost
|
|
1766
|
+
if owner_ok:
|
|
1767
|
+
continue # its session is still there
|
|
1768
|
+
ghosts.setdefault(cp, set()).add(pid)
|
|
1769
|
+
out = []
|
|
1770
|
+
bt = boot_time() or 0
|
|
1771
|
+
tick = os.sysconf("SC_CLK_TCK") if hasattr(os, "sysconf") else 100
|
|
1772
|
+
kids = {}
|
|
1773
|
+
for pid, (pp, _) in table.items():
|
|
1774
|
+
kids.setdefault(pp, []).append(pid)
|
|
1775
|
+
split = {}
|
|
1776
|
+
for cp, pids in ghosts.items():
|
|
1777
|
+
roots = [p for p in pids if table.get(p, (0, 0))[0] not in pids]
|
|
1778
|
+
for root in roots:
|
|
1779
|
+
tree, stack = set(), [root]
|
|
1780
|
+
while stack:
|
|
1781
|
+
q = stack.pop()
|
|
1782
|
+
if q in pids and q not in tree:
|
|
1783
|
+
tree.add(q)
|
|
1784
|
+
stack.extend(kids.get(q, []))
|
|
1785
|
+
key = (cp, proc_detached(root))
|
|
1786
|
+
split.setdefault(key, set()).update(tree)
|
|
1787
|
+
for (cp, _detached), pids in split.items():
|
|
1788
|
+
roots = [p for p in pids if table.get(p, (0, 0))[0] not in pids]
|
|
1789
|
+
s = by_pid.get(cp) or {}
|
|
1790
|
+
e = reg.get(cp) or {}
|
|
1791
|
+
sid = e.get("sessionId") or s.get("sid") or ""
|
|
1792
|
+
name = (e.get("name") if e.get("nameSource") == "user" else None) or s.get("name") or (sid[:8] or f"pid {cp}")
|
|
1793
|
+
why = []
|
|
1794
|
+
if any(proc_detached(p) for p in roots):
|
|
1795
|
+
why.append("detached (nohup / setsid / tmux)")
|
|
1796
|
+
mk = (s.get("mark") or {}).get("state")
|
|
1797
|
+
if mk == "waiting":
|
|
1798
|
+
why.append("its session said it is waiting")
|
|
1799
|
+
starts = [proc_start(p) for p in roots]
|
|
1800
|
+
age = max((now() - (bt + st / tick)) for st in starts if st) if any(starts) and bt else None
|
|
1801
|
+
total = 0
|
|
1802
|
+
for p in pids:
|
|
1803
|
+
v = proc_pss(p)
|
|
1804
|
+
total += v if v is not None else table.get(p, (0, 0))[1]
|
|
1805
|
+
out.append({"owner_pid": cp, "sid": sid, "name": name, "pids": sorted(pids), "roots": sorted(roots),
|
|
1806
|
+
"starts": {str(p): proc_start(p) for p in pids},
|
|
1807
|
+
"pss": total, "cmd": proc_cmdline(min(roots))[:100] if roots else "",
|
|
1808
|
+
"age_s": age, "kept": bool(why), "why_kept": why})
|
|
1809
|
+
out.sort(key=lambda g: -g["pss"])
|
|
1810
|
+
return out
|
|
1811
|
+
|
|
1812
|
+
|
|
1813
|
+
def cmd_ghosts(args):
|
|
1814
|
+
groups = scan_ghosts()
|
|
1815
|
+
if args.json and not args.reap:
|
|
1816
|
+
print(json.dumps(groups, ensure_ascii=False, indent=1))
|
|
1817
|
+
return EXIT_OK
|
|
1818
|
+
if not groups:
|
|
1819
|
+
print("no ghosts: every process started by a Claude Code session belongs to a live one")
|
|
1820
|
+
return EXIT_OK
|
|
1821
|
+
total = sum(g["pss"] for g in groups)
|
|
1822
|
+
print(f"{len(groups)} ghost tree(s), {fmt_size(total)} in all")
|
|
1823
|
+
for k, g in enumerate(groups, 1):
|
|
1824
|
+
age = fmt_duration(g["age_s"]) if g.get("age_s") else "?"
|
|
1825
|
+
tag = f" [kept: {', '.join(g['why_kept'])}]" if g["kept"] else ""
|
|
1826
|
+
print(f" #{k} {g['name']:<24} {len(g['pids'])} proc {fmt_size(g['pss']):>6} up {age:<8} {g['cmd']}{tag}")
|
|
1827
|
+
if not args.reap:
|
|
1828
|
+
print("reap with: dingo-pack ghosts --reap [#...] (human only)")
|
|
1829
|
+
return EXIT_OK
|
|
1830
|
+
return reap(groups, args)
|
|
1831
|
+
|
|
1832
|
+
|
|
1833
|
+
def reap(groups, args):
|
|
1834
|
+
if is_agent():
|
|
1835
|
+
eprint("dingo-pack ghosts --reap is for humans: refused inside an agent session.")
|
|
1836
|
+
return EXIT_USAGE
|
|
1837
|
+
pick = set(args.ids or [])
|
|
1838
|
+
todo = []
|
|
1839
|
+
for k, g in enumerate(groups, 1):
|
|
1840
|
+
named = f"#{k}" in pick or str(k) in pick
|
|
1841
|
+
if pick and not named:
|
|
1842
|
+
continue
|
|
1843
|
+
if g["kept"] and not named:
|
|
1844
|
+
print(f" skip #{k} {g['name']}: {', '.join(g['why_kept'])} (name it to reap)")
|
|
1845
|
+
continue
|
|
1846
|
+
todo.append((k, g))
|
|
1847
|
+
for k, g in todo:
|
|
1848
|
+
if not args.yes:
|
|
1849
|
+
if not sys.stdin.isatty():
|
|
1850
|
+
print(f" skip #{k}: not confirmed (use --yes)")
|
|
1851
|
+
continue
|
|
1852
|
+
if input(f" reap #{k} {g['name']} ({len(g['pids'])} processes, {fmt_size(g['pss'])})? [y/N] ").strip().lower() != "y":
|
|
1853
|
+
continue
|
|
1854
|
+
# check again right before acting; one dead session can have two groups (kept / not)
|
|
1855
|
+
fresh = {(x["owner_pid"], tuple(x["roots"])): x for x in scan_ghosts()}
|
|
1856
|
+
g2 = fresh.get((g["owner_pid"], tuple(g["roots"])))
|
|
1857
|
+
if not g2:
|
|
1858
|
+
print(f" #{k}: no longer a ghost; left alone")
|
|
1859
|
+
continue
|
|
1860
|
+
victims = [p for p in g["pids"] if p in g2["pids"] and g["starts"].get(str(p)) == g2["starts"].get(str(p))]
|
|
1861
|
+
starts = g2["starts"]
|
|
1862
|
+
for p in victims:
|
|
1863
|
+
try:
|
|
1864
|
+
os.kill(p, signal.SIGTERM)
|
|
1865
|
+
except OSError:
|
|
1866
|
+
pass
|
|
1867
|
+
deadline = now() + float(os.environ.get("DINGO_KILL_GRACE", "10"))
|
|
1868
|
+
while now() < deadline and any(pid_alive(p, starts.get(str(p))) for p in victims):
|
|
1869
|
+
time.sleep(0.2)
|
|
1870
|
+
for p in victims:
|
|
1871
|
+
killed = "term"
|
|
1872
|
+
if pid_alive(p, starts.get(str(p))):
|
|
1873
|
+
try:
|
|
1874
|
+
os.kill(p, signal.SIGKILL)
|
|
1875
|
+
killed = "kill"
|
|
1876
|
+
except OSError:
|
|
1877
|
+
pass
|
|
1878
|
+
write_log({"t": now(), "op": "reap", "pid": p, "how": killed, "session": g["name"], "sid": g["sid"],
|
|
1879
|
+
"owner_pid": g["owner_pid"], "why": "CLAUDE_PID owner gone, outside every live session",
|
|
1880
|
+
"cmd": proc_cmdline(p)[:120]})
|
|
1881
|
+
print(f" reaped #{k} {g['name']}: {len(victims)} process(es)")
|
|
1882
|
+
return EXIT_OK
|
|
1883
|
+
|
|
1884
|
+
|
|
1885
|
+
# ---------------------------------------------------------------- doctor
|
|
1886
|
+
|
|
1887
|
+
def find_cmd(name):
|
|
1888
|
+
"""shutil.which without the Windows directories WSL appends to PATH (each lookup there
|
|
1889
|
+
costs a 9p round trip, and hangs if the C: drive is gone)."""
|
|
1890
|
+
for d in os.environ.get("PATH", "").split(os.pathsep):
|
|
1891
|
+
if not d or d.startswith("/mnt/"):
|
|
1892
|
+
continue
|
|
1893
|
+
p = os.path.join(d, name)
|
|
1894
|
+
if os.path.isfile(p) and os.access(p, os.X_OK):
|
|
1895
|
+
return p
|
|
1896
|
+
return None
|
|
1897
|
+
|
|
1898
|
+
|
|
1899
|
+
def run_timed(fn, timeout):
|
|
1900
|
+
"""Run fn() in a daemon thread; ('ok', value) | ('error', exc) | ('timeout', None).
|
|
1901
|
+
A call stuck on a dead network mount cannot be interrupted; we just stop waiting."""
|
|
1902
|
+
import threading
|
|
1903
|
+
box = {}
|
|
1904
|
+
|
|
1905
|
+
def go():
|
|
1906
|
+
try:
|
|
1907
|
+
box["v"] = ("ok", fn())
|
|
1908
|
+
except Exception as e: # noqa: BLE001
|
|
1909
|
+
box["v"] = ("error", e)
|
|
1910
|
+
th = threading.Thread(target=go, daemon=True)
|
|
1911
|
+
th.start()
|
|
1912
|
+
th.join(timeout)
|
|
1913
|
+
return box.get("v", ("timeout", None))
|
|
1914
|
+
|
|
1915
|
+
|
|
1916
|
+
DMESG_RE = re.compile(r"^\[(\w{3} \w{3}\s+\d+ \d\d:\d\d:\d\d \d{4})\]\s*(.*)$")
|
|
1917
|
+
OOM_RE = re.compile(r"(?i)out of memory|oom-kill|oom_reaper|page allocation failure|9pnet.*(error|fail)")
|
|
1918
|
+
NET_FS = ("9p", "drvfs", "nfs", "nfs4", "cifs", "smb3", "smbfs", "fuse.sshfs", "v9fs", "virtiofs")
|
|
1919
|
+
|
|
1920
|
+
|
|
1921
|
+
def dmesg_lines():
|
|
1922
|
+
fake = os.environ.get("DINGO_FAKE_DMESG")
|
|
1923
|
+
if fake:
|
|
1924
|
+
with open(fake, encoding="utf-8", errors="replace") as f:
|
|
1925
|
+
return f.read().splitlines()
|
|
1926
|
+
r = subprocess.run(["dmesg", "-T"], capture_output=True, text=True, timeout=5)
|
|
1927
|
+
if r.returncode:
|
|
1928
|
+
raise OSError(r.stderr.strip() or f"dmesg exit {r.returncode}")
|
|
1929
|
+
return r.stdout.splitlines()
|
|
1930
|
+
|
|
1931
|
+
|
|
1932
|
+
def probe_mount(path):
|
|
1933
|
+
if os.path.isdir(path):
|
|
1934
|
+
os.listdir(path)
|
|
1935
|
+
else: # (self-test only) a FIFO stands in for a hung mount
|
|
1936
|
+
with open(path) as f:
|
|
1937
|
+
f.read(1)
|
|
1938
|
+
return True
|
|
1939
|
+
|
|
1940
|
+
|
|
1941
|
+
def mounts():
|
|
1942
|
+
path = os.environ.get("DINGO_FAKE_MOUNTS", "/proc/mounts")
|
|
1943
|
+
out = []
|
|
1944
|
+
try:
|
|
1945
|
+
with open(path) as f:
|
|
1946
|
+
for line in f:
|
|
1947
|
+
p = line.split()
|
|
1948
|
+
if len(p) >= 3 and p[2] in NET_FS:
|
|
1949
|
+
out.append((p[1].replace("\\040", " "), p[2]))
|
|
1950
|
+
except OSError:
|
|
1951
|
+
pass
|
|
1952
|
+
return out
|
|
1953
|
+
|
|
1954
|
+
|
|
1955
|
+
def find_wslconfig():
|
|
1956
|
+
fake = os.environ.get("DINGO_FAKE_WSLCONFIG")
|
|
1957
|
+
if fake is not None:
|
|
1958
|
+
return [fake] if fake and os.path.exists(fake) else []
|
|
1959
|
+
return glob.glob("/mnt/c/Users/*/.wslconfig")
|
|
1960
|
+
|
|
1961
|
+
|
|
1962
|
+
def is_wsl():
|
|
1963
|
+
if os.environ.get("DINGO_FAKE_WSL") is not None:
|
|
1964
|
+
return os.environ["DINGO_FAKE_WSL"] == "1"
|
|
1965
|
+
try:
|
|
1966
|
+
with open("/proc/version") as f:
|
|
1967
|
+
return "microsoft" in f.read().lower()
|
|
1968
|
+
except OSError:
|
|
1969
|
+
return False
|
|
1970
|
+
|
|
1971
|
+
|
|
1972
|
+
def doctor_checks(args):
|
|
1973
|
+
"""Yields (level, name, message, hint). level: ok | warn | bad | info."""
|
|
1974
|
+
cfg = config()
|
|
1975
|
+
total, avail = meminfo()
|
|
1976
|
+
reserve = reserve_bytes(cfg)
|
|
1977
|
+
if avail is None:
|
|
1978
|
+
yield ("info", "memory", "free memory unknown on this platform", "")
|
|
1979
|
+
else:
|
|
1980
|
+
lvl = "bad" if avail < reserve else "warn" if avail < 2 * reserve else "ok"
|
|
1981
|
+
yield (lvl, "memory", f"{fmt_size(avail)} free of {fmt_size(total)} (reserve {fmt_size(reserve)})",
|
|
1982
|
+
"close idle sessions or reap ghosts" if lvl != "ok" else "")
|
|
1983
|
+
la, nc = loadavg(), n_cpus()
|
|
1984
|
+
if la is not None:
|
|
1985
|
+
r_ = la / nc
|
|
1986
|
+
lvl = "bad" if r_ > 3 else "warn" if r_ > 1.5 else "ok"
|
|
1987
|
+
yield (lvl, "cpu", f"load {la:.1f} on {nc} cores ({r_:.1f} per core)",
|
|
1988
|
+
"too many threads at once: run heavy jobs with `dingo-pack run --heavy --threads N`, "
|
|
1989
|
+
"set OMP_NUM_THREADS etc." if lvl != "ok" else "")
|
|
1990
|
+
# OOM / 9p errors in the kernel log
|
|
1991
|
+
st, val = run_timed(dmesg_lines, 6)
|
|
1992
|
+
if st != "ok":
|
|
1993
|
+
yield ("info", "kernel log", f"cannot read dmesg ({val if st == 'error' else 'timed out'})",
|
|
1994
|
+
"run `sudo dmesg -T | grep -i oom` yourself")
|
|
1995
|
+
else:
|
|
1996
|
+
hits = []
|
|
1997
|
+
for line in val:
|
|
1998
|
+
m = DMESG_RE.match(line)
|
|
1999
|
+
text = m.group(2) if m else line
|
|
2000
|
+
if OOM_RE.search(text):
|
|
2001
|
+
t = None
|
|
2002
|
+
if m:
|
|
2003
|
+
try:
|
|
2004
|
+
t = time.mktime(time.strptime(re.sub(r"\s+", " ", m.group(1)), "%a %b %d %H:%M:%S %Y"))
|
|
2005
|
+
except ValueError:
|
|
2006
|
+
t = None
|
|
2007
|
+
hits.append((t, text.strip()))
|
|
2008
|
+
recent = [h for h in hits if h[0] is None or now() - h[0] < 86400]
|
|
2009
|
+
if recent:
|
|
2010
|
+
last = recent[-3:]
|
|
2011
|
+
yield ("warn", "kernel log", f"{len(recent)} OOM / page-allocation / 9p line(s) in the last 24 h; latest: "
|
|
2012
|
+
+ " | ".join(f"{hhmm(t)} {x[:70]}" for t, x in last),
|
|
2013
|
+
"see the WSL notes in REFERENCE.md")
|
|
2014
|
+
else:
|
|
2015
|
+
yield ("ok", "kernel log", "no OOM kills in the last 24 h", "")
|
|
2016
|
+
# network / 9p mounts answer?
|
|
2017
|
+
ms = mounts()
|
|
2018
|
+
if not ms:
|
|
2019
|
+
yield ("ok", "mounts", "no network / 9p mounts", "")
|
|
2020
|
+
for mp, fstype in ms:
|
|
2021
|
+
st, val = run_timed(lambda mp=mp: probe_mount(mp), float(os.environ.get("DINGO_PROBE_TIMEOUT_S", "3")))
|
|
2022
|
+
if st == "ok":
|
|
2023
|
+
yield ("ok", "mounts", f"{mp} ({fstype}) answers", "")
|
|
2024
|
+
elif st == "timeout":
|
|
2025
|
+
yield ("bad", "mounts", f"{mp} ({fstype}) does not answer within 3 s",
|
|
2026
|
+
"probably dropped; remount it (WSL: `wsl.exe -u root mount -t drvfs ...`)")
|
|
2027
|
+
else:
|
|
2028
|
+
yield ("bad", "mounts", f"{mp} ({fstype}): {val}", "remount it")
|
|
2029
|
+
# WSL settings
|
|
2030
|
+
if is_wsl():
|
|
2031
|
+
st, files = run_timed(find_wslconfig, 3)
|
|
2032
|
+
if st != "ok":
|
|
2033
|
+
yield ("warn", "wsl", "cannot look for .wslconfig (C: drive not answering?)", "")
|
|
2034
|
+
elif not files:
|
|
2035
|
+
yield ("warn", "wsl", "no .wslconfig: WSL uses its default memory and swap",
|
|
2036
|
+
"set [wsl2] memory= and swap= (see REFERENCE.md), then `wsl --shutdown`")
|
|
2037
|
+
else:
|
|
2038
|
+
txt = ""
|
|
2039
|
+
st2, txt = run_timed(lambda: open(files[0], encoding="utf-8", errors="replace").read(), 3)
|
|
2040
|
+
txt = txt if st2 == "ok" else ""
|
|
2041
|
+
got = {k: v for k, v in re.findall(r"(?im)^\s*(memory|swap)\s*=\s*(\S+)", txt or "")}
|
|
2042
|
+
if "memory" in {k.lower() for k in got}:
|
|
2043
|
+
yield ("ok", "wsl", f".wslconfig sets {', '.join(f'{k}={v}' for k, v in got.items())}", "")
|
|
2044
|
+
else:
|
|
2045
|
+
yield ("warn", "wsl", ".wslconfig has no memory= line", "add [wsl2] memory= and swap=")
|
|
2046
|
+
mf = os.environ.get("DINGO_FAKE_MINFREE")
|
|
2047
|
+
try:
|
|
2048
|
+
mfk = int(mf) if mf else int(open("/proc/sys/vm/min_free_kbytes").read())
|
|
2049
|
+
yield ("ok" if mfk >= 131072 else "warn", "wsl", f"vm.min_free_kbytes = {mfk}",
|
|
2050
|
+
"" if mfk >= 131072 else "raise to 262144 (keeps memory free for 9p)")
|
|
2051
|
+
except (OSError, ValueError):
|
|
2052
|
+
pass
|
|
2053
|
+
# slots held by orphans
|
|
2054
|
+
orphans = []
|
|
2055
|
+
for i, h in holders():
|
|
2056
|
+
pid, jp = h.get("pid"), h.get("job_pid")
|
|
2057
|
+
if not pid_alive(pid) and not (jp and pid_alive(jp)):
|
|
2058
|
+
orphans.append(f"[{i}] {h.get('session', '?')} {h.get('cmd', '')[:40]}")
|
|
2059
|
+
if orphans:
|
|
2060
|
+
yield ("warn", "slots", "held although the job is gone: " + "; ".join(orphans),
|
|
2061
|
+
"a leftover child still has the slot open; find it with `lsof <state>/slots/slot-N`")
|
|
2062
|
+
else:
|
|
2063
|
+
yield ("ok", "slots", f"{len(holders())} held, none orphaned", "")
|
|
2064
|
+
# roster: lost sessions nobody brought back
|
|
2065
|
+
r = load_roster()
|
|
2066
|
+
lost = [s for s in select(r, 24.0, False) if state_of(s)[0] not in ("done",)]
|
|
2067
|
+
if lost:
|
|
2068
|
+
yield ("warn", "roster", f"{len(lost)} session(s) gone and not resumed: "
|
|
2069
|
+
+ ", ".join(s.get("name", "?") for s in lost[:8]), "dingo-pack list / resume")
|
|
2070
|
+
else:
|
|
2071
|
+
yield ("ok", "roster", "no lost sessions in the last 24 h", "")
|
|
2072
|
+
# ghosts
|
|
2073
|
+
st, gs = run_timed(lambda: scan_ghosts(r), 10)
|
|
2074
|
+
if st != "ok":
|
|
2075
|
+
yield ("info", "ghosts", "process scan did not finish in 10 s (a process stuck on a dead mount?)", "")
|
|
2076
|
+
gs = None
|
|
2077
|
+
if gs is not None:
|
|
2078
|
+
loose = [g for g in gs if not g["kept"]]
|
|
2079
|
+
kept = [g for g in gs if g["kept"]]
|
|
2080
|
+
gb = sum(g["pss"] for g in loose)
|
|
2081
|
+
if loose:
|
|
2082
|
+
yield ("bad" if gb > GIB else "warn", "ghosts", f"{len(loose)} tree(s) whose session is gone, {fmt_size(gb)}",
|
|
2083
|
+
"dingo-pack ghosts (and --reap, human only)")
|
|
2084
|
+
else:
|
|
2085
|
+
yield ("ok", "ghosts", "none", "")
|
|
2086
|
+
if kept:
|
|
2087
|
+
yield ("info", "ghosts", f"{len(kept)} kept on purpose (nohup / setsid / tmux / waiting), "
|
|
2088
|
+
f"{fmt_size(sum(g['pss'] for g in kept))}", "")
|
|
2089
|
+
# git locks, through wombat-gate if it is installed
|
|
2090
|
+
wg = None
|
|
2091
|
+
if args.git:
|
|
2092
|
+
st_w, wg = run_timed(lambda: find_cmd("wombat-gate"), 3)
|
|
2093
|
+
wg = wg if st_w == "ok" else None
|
|
2094
|
+
repos = sorted({s.get("cwd") for s in r.get("sessions", {}).values() if s.get("alive") and s.get("cwd")})
|
|
2095
|
+
if args.git and wg:
|
|
2096
|
+
for cwd in repos:
|
|
2097
|
+
st, res = run_timed(lambda cwd=cwd: subprocess.run(
|
|
2098
|
+
[wg, "doctor", "--json"], cwd=cwd, capture_output=True, text=True, timeout=20), 25)
|
|
2099
|
+
if st != "ok" or res.returncode not in (0, 1):
|
|
2100
|
+
yield ("info", "git locks", f"{cwd}: wombat-gate doctor not available", "")
|
|
2101
|
+
continue
|
|
2102
|
+
try:
|
|
2103
|
+
for item in json.loads(res.stdout).get("checks", []):
|
|
2104
|
+
yield (item.get("level", "info"), "git locks", f"{cwd}: {item.get('message', '')}",
|
|
2105
|
+
item.get("hint", ""))
|
|
2106
|
+
except ValueError:
|
|
2107
|
+
yield ("info", "git locks", f"{cwd}: unreadable wombat-gate doctor output", "")
|
|
2108
|
+
|
|
2109
|
+
|
|
2110
|
+
ICON = {"ok": "✅", "warn": "⚠️ ", "bad": "⛔", "info": "ℹ️ "}
|
|
2111
|
+
|
|
2112
|
+
|
|
2113
|
+
def cmd_doctor(args):
|
|
2114
|
+
rows = list(doctor_checks(args))
|
|
2115
|
+
if args.json:
|
|
2116
|
+
print(json.dumps({"checks": [{"level": l, "name": n, "message": m, "hint": h} for l, n, m, h in rows]},
|
|
2117
|
+
ensure_ascii=False, indent=1))
|
|
2118
|
+
else:
|
|
2119
|
+
for l, n, m, h in rows:
|
|
2120
|
+
print(f"{ICON[l]} {n:<10} {m}" + (f"\n → {h}" if h else ""))
|
|
2121
|
+
bad = any(l == "bad" for l, *_ in rows)
|
|
2122
|
+
sys.stdout.flush()
|
|
2123
|
+
os._exit(1 if bad else 0) # a probe thread may still be stuck on a dead mount
|
|
2124
|
+
|
|
2125
|
+
|
|
2126
|
+
# ---------------------------------------------------------------- stow / unstow
|
|
2127
|
+
|
|
2128
|
+
DEFAULT_UNSTOW_PROMPT = ("You were put away for a planned pause and are back now. Re-read the end of your log "
|
|
2129
|
+
"and your last status line, then carry on where you stopped.")
|
|
2130
|
+
|
|
2131
|
+
|
|
2132
|
+
def stows_dir():
|
|
2133
|
+
return sub_dir("stows")
|
|
2134
|
+
|
|
2135
|
+
|
|
2136
|
+
def latest_stow():
|
|
2137
|
+
files = sorted(glob.glob(os.path.join(stows_dir(), "*.json")))
|
|
2138
|
+
return (read_json(files[-1]) or {}) if files else {}
|
|
2139
|
+
|
|
2140
|
+
|
|
2141
|
+
def stowed_sessions(roster=None, days=7):
|
|
2142
|
+
"""Sessions some stow closed and that have not been seen alive since: {sid: record}."""
|
|
2143
|
+
roster = roster or load_roster()
|
|
2144
|
+
cut = now() - days * 86400
|
|
2145
|
+
out = {}
|
|
2146
|
+
for f in sorted(glob.glob(os.path.join(stows_dir(), "*.json"))):
|
|
2147
|
+
rec = read_json(f) or {}
|
|
2148
|
+
if (rec.get("t") or 0) < cut:
|
|
2149
|
+
continue
|
|
2150
|
+
for x in rec.get("sessions", []):
|
|
2151
|
+
if x.get("closed"):
|
|
2152
|
+
out[x["sid"]] = x
|
|
2153
|
+
for sid in list(out):
|
|
2154
|
+
s = roster.get("sessions", {}).get(sid) or {}
|
|
2155
|
+
closed_at = out[sid].get("closed_at") or 0
|
|
2156
|
+
if s.get("alive") or (s.get("last_seen") or 0) > closed_at + 30:
|
|
2157
|
+
del out[sid] # it came back after that stow
|
|
2158
|
+
return out
|
|
2159
|
+
|
|
2160
|
+
|
|
2161
|
+
def cmd_stow(args):
|
|
2162
|
+
r = save_roster(quiet=True) or load_roster()
|
|
2163
|
+
me = my_session_id(args)
|
|
2164
|
+
reg, _ = read_registry()
|
|
2165
|
+
alive = [s for s in r["sessions"].values() if s.get("alive")]
|
|
2166
|
+
rec = {"t": now(), "host": socket.gethostname(), "sessions": []}
|
|
2167
|
+
plan = []
|
|
2168
|
+
for s in sorted(alive, key=lambda s: s.get("name") or ""):
|
|
2169
|
+
st, _guessed = state_of(s)
|
|
2170
|
+
mk = s.get("mark") or {}
|
|
2171
|
+
if mk.get("t") and (s.get("last_activity") or 0) > mk["t"] + 120 and st in ("waiting", "done"):
|
|
2172
|
+
st = "stale mark" # it kept working after saying waiting / done
|
|
2173
|
+
e = reg.get(s["sid"]) or {}
|
|
2174
|
+
closable = (e.get("status") == "idle" and st in ("waiting", "done") and s["sid"] != me
|
|
2175
|
+
and e.get("_alive"))
|
|
2176
|
+
rec["sessions"].append({"sid": s["sid"], "name": s.get("name"), "cwd": s.get("cwd"), "state": st,
|
|
2177
|
+
"cc_status": s.get("cc_status"), "mark": s.get("mark"),
|
|
2178
|
+
"status_line": s.get("status_line"), "transcript": s.get("transcript"),
|
|
2179
|
+
"closed": False})
|
|
2180
|
+
plan.append((s, e, closable))
|
|
2181
|
+
if args.close and not args.dry_run and is_agent():
|
|
2182
|
+
eprint("dingo-pack stow --close closes other sessions: run it yourself, not from inside an agent.")
|
|
2183
|
+
return EXIT_USAGE
|
|
2184
|
+
print(f"stowing {len(alive)} live session(s){' (dry run)' if args.dry_run else ''}:")
|
|
2185
|
+
for idx, (s, e, closable) in enumerate(plan):
|
|
2186
|
+
st = rec["sessions"][idx]["state"]
|
|
2187
|
+
action = "keep running"
|
|
2188
|
+
if args.close and closable:
|
|
2189
|
+
action = "close"
|
|
2190
|
+
if not args.dry_run:
|
|
2191
|
+
pid = e.get("pid")
|
|
2192
|
+
fresh = read_json(os.path.join(claude_home(), "sessions", f"{pid}.json"), {}) or {}
|
|
2193
|
+
try:
|
|
2194
|
+
if fresh.get("sessionId") != s["sid"] or fresh.get("status") != "idle":
|
|
2195
|
+
action = f"keep running (now {fresh.get('status') or 'gone'})"
|
|
2196
|
+
elif pid_alive(pid, e.get("procStart")):
|
|
2197
|
+
os.kill(int(pid), signal.SIGTERM)
|
|
2198
|
+
deadline = now() + float(os.environ.get("DINGO_STOW_WAIT_S", "10"))
|
|
2199
|
+
while now() < deadline and pid_alive(pid, e.get("procStart")):
|
|
2200
|
+
time.sleep(0.2)
|
|
2201
|
+
if pid_alive(pid, e.get("procStart")):
|
|
2202
|
+
action = "asked to close, still running"
|
|
2203
|
+
else:
|
|
2204
|
+
action = "closed"
|
|
2205
|
+
rec["sessions"][idx]["closed"] = True
|
|
2206
|
+
rec["sessions"][idx]["closed_at"] = now()
|
|
2207
|
+
except OSError as ex:
|
|
2208
|
+
action = f"could not close ({ex})"
|
|
2209
|
+
write_log({"t": now(), "op": "stow-close", "session": s.get("name"), "sid": s["sid"],
|
|
2210
|
+
"pid": pid, "result": action})
|
|
2211
|
+
elif args.close:
|
|
2212
|
+
why = "this session" if s["sid"] == me else \
|
|
2213
|
+
"busy" if s.get("cc_status") != "idle" else f"state {st}"
|
|
2214
|
+
action = f"keep running ({why})"
|
|
2215
|
+
print(f" {s.get('name', '?'):<24} {st:<8} {s.get('cc_status') or '?':<7} {action}")
|
|
2216
|
+
if not args.dry_run:
|
|
2217
|
+
name = time.strftime("%Y%m%d-%H%M%S") + f"-{os.getpid()}.json"
|
|
2218
|
+
write_atomic(os.path.join(stows_dir(), name), json.dumps(rec, ensure_ascii=False, indent=1))
|
|
2219
|
+
print(f"stow list saved ({name}); bring them back with: dingo-pack unstow")
|
|
2220
|
+
return EXIT_OK
|
|
2221
|
+
|
|
2222
|
+
|
|
2223
|
+
def cmd_close_all(args):
|
|
2224
|
+
"""Front door: remember every session and close the ones it is safe to close."""
|
|
2225
|
+
args.close = True
|
|
2226
|
+
return cmd_stow(args)
|
|
2227
|
+
|
|
2228
|
+
|
|
2229
|
+
def cmd_restore(args):
|
|
2230
|
+
"""Front door: bring back the last batch -- what close-all closed, or, if nothing was
|
|
2231
|
+
closed that way, the sessions lost when the machine went down."""
|
|
2232
|
+
r = save_roster(quiet=True) or load_roster()
|
|
2233
|
+
if stowed_sessions(r) or args.names:
|
|
2234
|
+
return cmd_unstow(args)
|
|
2235
|
+
lost = [s for s in select(r, 24.0, False) if state_of(s)[0] != "done"]
|
|
2236
|
+
if not lost:
|
|
2237
|
+
print("nothing to restore: no closed or lost sessions in the last 24 h")
|
|
2238
|
+
return EXIT_OK
|
|
2239
|
+
print(f"nothing closed by close-all; restoring {len(lost)} session(s) lost when the machine went down")
|
|
2240
|
+
return reopen(lost, args, prompt_key="resume_prompt", default_prompt=DEFAULT_PROMPT)
|
|
2241
|
+
|
|
2242
|
+
|
|
2243
|
+
def cmd_unstow(args):
|
|
2244
|
+
r = save_roster(quiet=True) or load_roster()
|
|
2245
|
+
pool = dict(stowed_sessions(r))
|
|
2246
|
+
if args.all or args.names:
|
|
2247
|
+
for x in latest_stow().get("sessions", []):
|
|
2248
|
+
pool.setdefault(x["sid"], x)
|
|
2249
|
+
if not pool:
|
|
2250
|
+
print("nothing stowed")
|
|
2251
|
+
return EXIT_OK
|
|
2252
|
+
rows = []
|
|
2253
|
+
want = set(args.names or [])
|
|
2254
|
+
for x in pool.values():
|
|
2255
|
+
if want and x.get("name") not in want and x["sid"] not in want and x["sid"][:8] not in want:
|
|
2256
|
+
continue
|
|
2257
|
+
s = dict(r["sessions"].get(x["sid"]) or {}, **{k: v for k, v in x.items() if v is not None})
|
|
2258
|
+
s["sid"] = x["sid"]
|
|
2259
|
+
rows.append(s)
|
|
2260
|
+
if not rows:
|
|
2261
|
+
print("nothing to bring back (use --all to include sessions stow left running)")
|
|
2262
|
+
return EXIT_OK
|
|
2263
|
+
return reopen(rows, args, prompt_key="unstow_prompt", default_prompt=DEFAULT_UNSTOW_PROMPT)
|
|
2264
|
+
|
|
2265
|
+
|
|
2266
|
+
# ---------------------------------------------------------------- board (0.3, minimal)
|
|
2267
|
+
|
|
2268
|
+
BOX_RE = re.compile(r"(?i)\bbox\s?(\d+)\b")
|
|
2269
|
+
STAMP_RE = re.compile(r"状态[^\d]{0,20}(?:(\d{1,2})-(\d{1,2})\s*)?[^\d]{0,6}(\d{1,2}):(\d{2}|\dx)") # when it was written
|
|
2270
|
+
DATED_T = re.compile(r"(\d{1,2})-(\d{1,2})\s*(?:约\s*)?(\d{1,2}):(\d{2}|\dx)")
|
|
2271
|
+
# An expected finish needs a word that says so: "预计 / 约 … 完 / ETA / 跑完" and a time
|
|
2272
|
+
# within a short window after it; inside that window a Berlin time ("=柏林 HH:MM") wins.
|
|
2273
|
+
ETA_KEY = re.compile(r"(?i)预计|约(?=[^,。;;]{0,20}完)|\bETA\b|跑完|完成于")
|
|
2274
|
+
ETA_BEFORE = re.compile(r"(\d{1,2}):(\d{2}|\dx)\s*前")
|
|
2275
|
+
BERLIN_T = re.compile(r"[==]\s*柏林\s*(\d{1,2}):(\d{2}|\dx)")
|
|
2276
|
+
ANY_T = re.compile(r"(\d{1,2}):(\d{2}|\dx)")
|
|
2277
|
+
|
|
2278
|
+
|
|
2279
|
+
def _on(mon, day, hh, mm, ref):
|
|
2280
|
+
"""MM-DD HH:MM in ref's year (local time), or None."""
|
|
2281
|
+
try:
|
|
2282
|
+
mon, day, hh = int(mon), int(day), int(hh)
|
|
2283
|
+
mm = int(mm[0]) * 10 if str(mm).endswith("x") else int(mm)
|
|
2284
|
+
if not (1 <= mon <= 12 and 1 <= day <= 31 and hh <= 23 and mm <= 59):
|
|
2285
|
+
return None
|
|
2286
|
+
return time.mktime((time.localtime(ref).tm_year, mon, day, hh, mm, 0, 0, 0, -1))
|
|
2287
|
+
except (ValueError, OverflowError):
|
|
2288
|
+
return None
|
|
2289
|
+
|
|
2290
|
+
|
|
2291
|
+
def _at(hh, mm, ref):
|
|
2292
|
+
"""Today's HH:MM (local time) relative to ref; None if not a valid time."""
|
|
2293
|
+
hh = int(hh)
|
|
2294
|
+
mm = int(mm[0]) * 10 if str(mm).endswith("x") else int(mm) # "19:3x" = about 19:30
|
|
2295
|
+
if hh > 23 or mm > 59:
|
|
2296
|
+
return None
|
|
2297
|
+
lt = time.localtime(ref)
|
|
2298
|
+
return time.mktime((lt.tm_year, lt.tm_mon, lt.tm_mday, hh, mm, 0, 0, 0, -1))
|
|
2299
|
+
|
|
2300
|
+
|
|
2301
|
+
def parse_status(line, written):
|
|
2302
|
+
"""(box, eta, written_at) from a status line. Only the listed forms count; otherwise None."""
|
|
2303
|
+
line = line or ""
|
|
2304
|
+
m = BOX_RE.search(line)
|
|
2305
|
+
box = f"box{m.group(1)}" if m else None
|
|
2306
|
+
w = written
|
|
2307
|
+
st = _stamp(line)
|
|
2308
|
+
if st:
|
|
2309
|
+
if st.group(1): # "10-05 22:0x": the date says which day
|
|
2310
|
+
t = _on(st.group(1), st.group(2), st.group(3), st.group(4), written)
|
|
2311
|
+
if t is not None:
|
|
2312
|
+
w = t
|
|
2313
|
+
else:
|
|
2314
|
+
t = _at(st.group(3), st.group(4), written)
|
|
2315
|
+
if t is not None:
|
|
2316
|
+
w = t - 86400 if t > written + 600 else t # a stamp later than we saw it: yesterday
|
|
2317
|
+
eta = None
|
|
2318
|
+
hit = None
|
|
2319
|
+
dated = None
|
|
2320
|
+
for k in ETA_KEY.finditer(line):
|
|
2321
|
+
win = line[k.end():k.end() + 40]
|
|
2322
|
+
dated = DATED_T.search(win)
|
|
2323
|
+
hit = dated or BERLIN_T.search(win) or ANY_T.search(win)
|
|
2324
|
+
if hit:
|
|
2325
|
+
break
|
|
2326
|
+
if not hit:
|
|
2327
|
+
hit = ETA_BEFORE.search(line)
|
|
2328
|
+
if hit is not None and hit is dated:
|
|
2329
|
+
eta = _on(dated.group(1), dated.group(2), dated.group(3), dated.group(4), w)
|
|
2330
|
+
elif hit:
|
|
2331
|
+
eta = _at(hit.group(1), hit.group(2), w)
|
|
2332
|
+
if eta is not None and eta < w:
|
|
2333
|
+
eta += 86400 # earlier than when it was written: tomorrow
|
|
2334
|
+
return box, eta, w
|
|
2335
|
+
|
|
2336
|
+
|
|
2337
|
+
def _stamp(line):
|
|
2338
|
+
"""The status line's own time: its '=柏林 HH:MM' if it gives one, never a 'box HH:MM' (Berlin + 6 h)."""
|
|
2339
|
+
line = line or ""
|
|
2340
|
+
st = STAMP_RE.search(line)
|
|
2341
|
+
if not st:
|
|
2342
|
+
return None
|
|
2343
|
+
head = line[st.start():st.end() + 24]
|
|
2344
|
+
b = re.search(r"[==]\s*柏林\s*(?:(\d{1,2})-(\d{1,2})\s*)?(\d{1,2}):(\d{2}|\dx)", head)
|
|
2345
|
+
if b:
|
|
2346
|
+
return b
|
|
2347
|
+
if re.search(r"box\s*(?:\d{1,2}-\d{1,2}\s*)?$", line[:st.start(3)]):
|
|
2348
|
+
return None # only a box time: cannot tell Berlin from it
|
|
2349
|
+
return st
|
|
2350
|
+
|
|
2351
|
+
|
|
2352
|
+
def stamp_ahead(line, seen):
|
|
2353
|
+
"""Minutes by which a status line's own time is later than when dingo first saw the line
|
|
2354
|
+
(a time written without reading `date`), or None. Undated: only up to 6 h ahead counts."""
|
|
2355
|
+
st = _stamp(line)
|
|
2356
|
+
if not st or not seen:
|
|
2357
|
+
return None
|
|
2358
|
+
if st.group(1):
|
|
2359
|
+
t = _on(st.group(1), st.group(2), st.group(3), st.group(4), seen)
|
|
2360
|
+
else:
|
|
2361
|
+
t = _at(st.group(3), st.group(4), seen)
|
|
2362
|
+
if t is not None and t - seen > 6 * 3600:
|
|
2363
|
+
return None # yesterday's stamp, not a future one
|
|
2364
|
+
if t is None:
|
|
2365
|
+
return None
|
|
2366
|
+
ahead = (t - seen) / 60 # "11:2x" counts as 11:20, the earliest it can mean
|
|
2367
|
+
return int(ahead) if ahead > 10 else None
|
|
2368
|
+
|
|
2369
|
+
|
|
2370
|
+
def board_rows(r, stale_h):
|
|
2371
|
+
rows = []
|
|
2372
|
+
t = now()
|
|
2373
|
+
for s in sorted(r["sessions"].values(), key=lambda s: (not s.get("alive"), s.get("name") or "")):
|
|
2374
|
+
if not s.get("alive") and t - (s.get("last_seen") or 0) > 86400:
|
|
2375
|
+
continue
|
|
2376
|
+
st, guessed = state_of(s)
|
|
2377
|
+
changed = s.get("status_changed_at") or s.get("last_seen") or t
|
|
2378
|
+
box, eta, written = parse_status(s.get("status_line"), changed)
|
|
2379
|
+
flags = []
|
|
2380
|
+
if s.get("alive") and eta and eta < t and st != "done":
|
|
2381
|
+
flags.append("ask now") # its own expected time has passed
|
|
2382
|
+
if s.get("alive") and s.get("status_line") and s.get("status_changed_at") \
|
|
2383
|
+
and t - s["status_changed_at"] > stale_h * 3600 and st != "done":
|
|
2384
|
+
flags.append("may have stopped")
|
|
2385
|
+
ahead = stamp_ahead(s.get("status_line"), s.get("status_changed_at"))
|
|
2386
|
+
if s.get("alive") and ahead:
|
|
2387
|
+
flags.append(f"time {ahead} min ahead") # the line's own time is later than when it was written
|
|
2388
|
+
rows.append({"name": s.get("name"), "sid": s["sid"][:8], "alive": bool(s.get("alive")),
|
|
2389
|
+
"state": st + ("?" if guessed else ""), "started": s.get("started_at"),
|
|
2390
|
+
"last_activity": s.get("last_activity"), "status": s.get("status_line"),
|
|
2391
|
+
"status_changed": s.get("status_changed_at"), "box": box, "eta": eta,
|
|
2392
|
+
"log": s.get("log"), "flags": flags})
|
|
2393
|
+
return rows
|
|
2394
|
+
|
|
2395
|
+
|
|
2396
|
+
def board_html(rows, title):
|
|
2397
|
+
import html as H
|
|
2398
|
+
esc = lambda x: H.escape(str(x if x is not None else ""))
|
|
2399
|
+
trs = []
|
|
2400
|
+
for b in rows:
|
|
2401
|
+
cls = "warn" if b["flags"] else ("" if b["alive"] else "gone")
|
|
2402
|
+
trs.append("<tr class='%s'><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td>"
|
|
2403
|
+
"<td class='st'>%s</td></tr>" % (cls, esc(b["name"]), esc(b["state"]), esc(b["box"]),
|
|
2404
|
+
esc(hhmm(b["eta"]) if b["eta"] else ""), esc(", ".join(b["flags"])),
|
|
2405
|
+
esc(hhmm(b["last_activity"])), esc(hhmm(b["started"])),
|
|
2406
|
+
esc((b["status"] or "")[:160]) +
|
|
2407
|
+
"".join("<br>⏳ " + esc(w[:160]) for w in b.get("waits") or []) +
|
|
2408
|
+
(f"<br>tokens today {esc(fmt_tok(b['tokens_today']))} · last input "
|
|
2409
|
+
f"{esc(fmt_tok(b.get('last_input')))}" if b.get("tokens_today") is not None else "")))
|
|
2410
|
+
return ("<!doctype html><html><head><meta charset='utf-8'><meta http-equiv='refresh' content='30'>"
|
|
2411
|
+
"<meta name='viewport' content='width=device-width,initial-scale=1'><title>%s</title><style>"
|
|
2412
|
+
"body{font:14px system-ui,sans-serif;margin:16px;background:#fff;color:#222}"
|
|
2413
|
+
"table{border-collapse:collapse;width:100%%}td,th{border-bottom:1px solid #ddd;padding:4px 6px;"
|
|
2414
|
+
"text-align:left;vertical-align:top}tr.warn{background:#fff3cd}tr.gone{color:#999}td.st{font-size:12px}"
|
|
2415
|
+
"@media (prefers-color-scheme:dark){body{background:#111;color:#ddd}tr.warn{background:#3a3000}"
|
|
2416
|
+
"td,th{border-color:#333}}</style></head><body><h3>%s</h3><table><tr><th>session</th><th>state</th>"
|
|
2417
|
+
"<th>box</th><th>expected</th><th>flags</th><th>last activity</th><th>started</th><th>status line</th>"
|
|
2418
|
+
"</tr>%s</table></body></html>") % (esc(title), esc(title), "".join(trs))
|
|
2419
|
+
|
|
2420
|
+
|
|
2421
|
+
def board_tasks(rows):
|
|
2422
|
+
"""Add the task-manager facts to board rows: waits, spin and long-context flags, tokens today."""
|
|
2423
|
+
try:
|
|
2424
|
+
tk = {t["sid"][:8]: t for t in session_tasks(sample_s=0) if t.get("sid")}
|
|
2425
|
+
except Exception as e: # the board must still show if /proc reading fails
|
|
2426
|
+
eprint(f"dingo-pack board: task facts unavailable ({e})")
|
|
2427
|
+
return rows
|
|
2428
|
+
for b in rows:
|
|
2429
|
+
t = tk.get(b["sid"])
|
|
2430
|
+
if not t:
|
|
2431
|
+
continue
|
|
2432
|
+
if any(w["spin"] for w in t["waits"]):
|
|
2433
|
+
b["flags"].append("spin?")
|
|
2434
|
+
if t["ctx_long"]:
|
|
2435
|
+
b["flags"].append("context long")
|
|
2436
|
+
b["waits"] = [w["text"] + (f" · 🌀 {w['spin']}" if w["spin"] else "") for w in t["waits"]]
|
|
2437
|
+
tok = t.get("tokens")
|
|
2438
|
+
b["tokens_today"] = sum(tok[k] for k in ("input", "cache_write", "cache_read", "output")) if tok else None
|
|
2439
|
+
b["last_input"] = tok.get("last_input") if tok else None
|
|
2440
|
+
return rows
|
|
2441
|
+
|
|
2442
|
+
|
|
2443
|
+
def cmd_board(args):
|
|
2444
|
+
while True:
|
|
2445
|
+
r = save_roster(quiet=True) or load_roster()
|
|
2446
|
+
rows = board_tasks(board_rows(r, args.stale_h))
|
|
2447
|
+
if args.json:
|
|
2448
|
+
print(json.dumps(rows, ensure_ascii=False, indent=1))
|
|
2449
|
+
elif args.html:
|
|
2450
|
+
write_atomic(args.html, board_html(rows, f"dingo-pack board · {time.strftime('%H:%M')}"))
|
|
2451
|
+
print(f"board written: {args.html} ({len(rows)} sessions)")
|
|
2452
|
+
else:
|
|
2453
|
+
if args.watch:
|
|
2454
|
+
sys.stdout.write("\033[2J\033[H")
|
|
2455
|
+
print(f"dingo-pack board · {time.strftime('%H:%M')} · {sum(1 for b in rows if b['alive'])} alive")
|
|
2456
|
+
print(f" {'session':<24} {'state':<9} {'box':<6} {'expected':<11} {'flags':<20} status")
|
|
2457
|
+
for b in rows:
|
|
2458
|
+
print(f" {(b['name'] or '?')[:24]:<24} {b['state']:<9} {(b['box'] or ''):<6} "
|
|
2459
|
+
f"{(hhmm(b['eta']) if b['eta'] else ''):<11} {', '.join(b['flags']):<20} "
|
|
2460
|
+
f"{(b['status'] or '')[:70]}" + ("" if b["alive"] else " (gone)"))
|
|
2461
|
+
for w in b.get("waits") or []:
|
|
2462
|
+
print(f" {'':<24} ⏳ {w[:110]}")
|
|
2463
|
+
cache = read_json(os.path.join(state_dir(), "boxes.json"), {}) or {}
|
|
2464
|
+
if cache:
|
|
2465
|
+
t0 = min(r.get("probed_at") or 0 for r in cache.values())
|
|
2466
|
+
print(f"\nboxes (dingo-pack boxes, read {hhmm(t0)}; run it again to refresh)")
|
|
2467
|
+
hist = boxes_history()
|
|
2468
|
+
print_boxes([box_row(r, hist) for r in sorted(cache.values(), key=lambda r: (len(r["name"]), r["name"]))])
|
|
2469
|
+
if not args.watch:
|
|
2470
|
+
return EXIT_OK
|
|
2471
|
+
sys.stdout.flush()
|
|
2472
|
+
time.sleep(args.watch)
|
|
2473
|
+
|
|
2474
|
+
|
|
2475
|
+
# ---------------------------------------------------------------- 0.3: names, cards, lint, queue, box claims
|
|
2476
|
+
|
|
2477
|
+
NAME_RULE = re.compile(r"^([A-Z])(\d*)-(\d{4})-(\S+)$")
|
|
2478
|
+
|
|
2479
|
+
|
|
2480
|
+
def today_mmdd():
|
|
2481
|
+
return time.strftime("%m%d")
|
|
2482
|
+
|
|
2483
|
+
|
|
2484
|
+
def cmd_name(args):
|
|
2485
|
+
"""Next session name for a letter today: L-MMDD-x for the first, then L2-, L3-...;
|
|
2486
|
+
warns if a session of that letter is alive now (one per letter at a time)."""
|
|
2487
|
+
L = args.letter.upper()
|
|
2488
|
+
if not re.fullmatch(r"[A-Z]", L):
|
|
2489
|
+
eprint("dingo-pack name: the letter must be one of A-Z")
|
|
2490
|
+
return EXIT_USAGE
|
|
2491
|
+
r = save_roster(quiet=True) or load_roster()
|
|
2492
|
+
mmdd = args.date or today_mmdd()
|
|
2493
|
+
used, alive = [], []
|
|
2494
|
+
for s_ in r["sessions"].values():
|
|
2495
|
+
m = NAME_RULE.match(s_.get("name") or "")
|
|
2496
|
+
if not m or m.group(1) != L:
|
|
2497
|
+
continue
|
|
2498
|
+
if m.group(3) == mmdd:
|
|
2499
|
+
used.append(int(m.group(2) or 1))
|
|
2500
|
+
if s_.get("alive"):
|
|
2501
|
+
alive.append(s_["name"])
|
|
2502
|
+
n = max(used) + 1 if used else 1
|
|
2503
|
+
suffix = args.what + (f"-{args.suffix}" if args.suffix else "")
|
|
2504
|
+
name = f"{L}{n if n > 1 else ''}-{mmdd}-{suffix}"
|
|
2505
|
+
print(name)
|
|
2506
|
+
if alive:
|
|
2507
|
+
eprint(f"dingo-pack: warning: {', '.join(alive)} is still alive -- one session per letter at a time; "
|
|
2508
|
+
"close it first or take another letter")
|
|
2509
|
+
return 3
|
|
2510
|
+
return EXIT_OK
|
|
2511
|
+
|
|
2512
|
+
|
|
2513
|
+
CARD_TEMPLATE = """# {number} · {title}
|
|
2514
|
+
|
|
2515
|
+
> Fixed block: assigned to · machine · GPU yes / no · machine lease · read first · may write only · may ask · report format
|
|
2516
|
+
> Duplicate check (item 0): what earlier cards / results already said about this; what is new here.
|
|
2517
|
+
|
|
2518
|
+
1. Main line:
|
|
2519
|
+
2. Which link of the funnel this hits (task · link · how many episodes are lost now · source):
|
|
2520
|
+
3. Why it must be done (what each outcome changes):
|
|
2521
|
+
4. Feasibility first (probe / intervention / takeover used, and its result):
|
|
2522
|
+
5. Can it tell apart (control · noise floor · detectable difference · main metric · criteria written in advance):
|
|
2523
|
+
6. Proof it took effect (real-entry smoke · per-step physical quantities · full config diff · watchdog + test alert):
|
|
2524
|
+
7. Data / expert (our expert succeeds on x / x of the openings we plan to use; failure causes):
|
|
2525
|
+
8. The report must include (success side · failure side · intermediate states · ledger):
|
|
2526
|
+
9. Cost (training / evaluation card-hours separately · machine · expected finish · who reports it can be shut down):
|
|
2527
|
+
10. Decisions for the human:
|
|
2528
|
+
"""
|
|
2529
|
+
|
|
2530
|
+
|
|
2531
|
+
def card_numbers(path, rx):
|
|
2532
|
+
try:
|
|
2533
|
+
with open(path, encoding="utf-8", errors="replace") as f:
|
|
2534
|
+
text = f.read()
|
|
2535
|
+
except OSError:
|
|
2536
|
+
return []
|
|
2537
|
+
return [int(m.group(1)) for m in re.finditer(rx, text)]
|
|
2538
|
+
|
|
2539
|
+
|
|
2540
|
+
def cmd_card(args):
|
|
2541
|
+
"""`card new`: next free number = 1 + max(numbers in the registry file, numbers reserved
|
|
2542
|
+
here); reserves it locally and writes a skeleton. The registry file itself is only read."""
|
|
2543
|
+
pcfg = project_config(os.getcwd())
|
|
2544
|
+
reg = args.registry or pcfg.get("card_registry")
|
|
2545
|
+
rx = pcfg.get("card_number_regex") or r"\b0-(\d{2,4})\b"
|
|
2546
|
+
prefix = pcfg.get("card_prefix") or "0-"
|
|
2547
|
+
resv = os.path.join(state_dir(), "card-reservations.json")
|
|
2548
|
+
lk = os.open(os.path.join(state_dir(), "card.lock"), os.O_RDWR | os.O_CREAT, 0o644)
|
|
2549
|
+
try:
|
|
2550
|
+
fcntl.flock(lk, fcntl.LOCK_EX)
|
|
2551
|
+
taken = read_json(resv, {}) or {}
|
|
2552
|
+
nums = card_numbers(reg, rx) if reg else []
|
|
2553
|
+
nums += [int(k) for k in taken]
|
|
2554
|
+
n = (max(nums) + 1) if nums else 1
|
|
2555
|
+
number = f"{prefix}{n}"
|
|
2556
|
+
taken[str(n)] = {"by": session_label(args), "t": now(), "title": args.title}
|
|
2557
|
+
write_atomic(resv, json.dumps(taken, ensure_ascii=False, indent=1))
|
|
2558
|
+
finally:
|
|
2559
|
+
os.close(lk)
|
|
2560
|
+
body = (open(pcfg["card_template"], encoding="utf-8").read() if pcfg.get("card_template")
|
|
2561
|
+
else CARD_TEMPLATE).format(number=number, title=args.title)
|
|
2562
|
+
if args.out:
|
|
2563
|
+
if os.path.exists(args.out):
|
|
2564
|
+
eprint(f"dingo-pack card new: {args.out} exists; not overwritten")
|
|
2565
|
+
return EXIT_USAGE
|
|
2566
|
+
write_atomic(args.out, body)
|
|
2567
|
+
print(number)
|
|
2568
|
+
if args.out:
|
|
2569
|
+
print(f"skeleton written: {args.out}")
|
|
2570
|
+
print(f"next free number by {'the ledger ' + reg if reg else 'local reservations'}: {number} -- "
|
|
2571
|
+
"register it in the ledger yourself (or ask whoever keeps it); dingo-pack never edits the ledger")
|
|
2572
|
+
return EXIT_OK
|
|
2573
|
+
|
|
2574
|
+
|
|
2575
|
+
LINT_CHECKS = ("name", "letter", "stale", "overdue", "no-status", "future-time")
|
|
2576
|
+
|
|
2577
|
+
|
|
2578
|
+
def lint_findings(r, cfg, stale_h):
|
|
2579
|
+
on = cfg.get("lint") or {}
|
|
2580
|
+
want = [c for c in LINT_CHECKS if on.get(c, True)]
|
|
2581
|
+
alive = [s_ for s_ in r["sessions"].values() if s_.get("alive")]
|
|
2582
|
+
out = []
|
|
2583
|
+
by_letter = {}
|
|
2584
|
+
for s_ in alive:
|
|
2585
|
+
name = s_.get("name") or ""
|
|
2586
|
+
m = NAME_RULE.match(name)
|
|
2587
|
+
if "name" in want and not m and s_.get("transcript"):
|
|
2588
|
+
out.append(("name", name, "does not follow L[n]-MMDD-what"))
|
|
2589
|
+
if m:
|
|
2590
|
+
by_letter.setdefault(m.group(1), []).append(name)
|
|
2591
|
+
if "letter" in want:
|
|
2592
|
+
for L, names in sorted(by_letter.items()):
|
|
2593
|
+
if len(names) > 1:
|
|
2594
|
+
out.append(("letter", ", ".join(sorted(names)), f"{len(names)} sessions of letter {L} alive at once"))
|
|
2595
|
+
for b in board_rows(r, stale_h):
|
|
2596
|
+
if not b["alive"]:
|
|
2597
|
+
continue
|
|
2598
|
+
if "stale" in want and "may have stopped" in b["flags"]:
|
|
2599
|
+
out.append(("stale", b["name"], f"status line unchanged for more than {stale_h:g} h"))
|
|
2600
|
+
if "future-time" in want and any(f.startswith("time ") for f in b["flags"]):
|
|
2601
|
+
out.append(("future-time", b["name"], next(f for f in b["flags"] if f.startswith("time ")) +
|
|
2602
|
+
": status time later than when it was written (read `date` before writing)"))
|
|
2603
|
+
if "overdue" in want and "ask now" in b["flags"]:
|
|
2604
|
+
out.append(("overdue", b["name"], f"expected finish {hhmm(b['eta'])} has passed"))
|
|
2605
|
+
if "no-status" in want and not b["status"] and b["log"] is None and b["state"] not in ("done",):
|
|
2606
|
+
s_ = r["sessions"].get(next((k for k in r["sessions"] if k.startswith(b["sid"])), ""), {})
|
|
2607
|
+
if s_.get("cwd") and project_config(s_["cwd"]).get("log_glob"):
|
|
2608
|
+
out.append(("no-status", b["name"], "no log with a status line found"))
|
|
2609
|
+
return out
|
|
2610
|
+
|
|
2611
|
+
|
|
2612
|
+
def cmd_lint(args):
|
|
2613
|
+
r = save_roster(quiet=True) or load_roster()
|
|
2614
|
+
found = lint_findings(r, config(), args.stale_h)
|
|
2615
|
+
if args.json:
|
|
2616
|
+
print(json.dumps([{"check": c, "who": w, "why": y} for c, w, y in found], ensure_ascii=False, indent=1))
|
|
2617
|
+
else:
|
|
2618
|
+
for c, w, y in found:
|
|
2619
|
+
print(f"⚠️ {c:<9} {w}: {y}")
|
|
2620
|
+
if not found:
|
|
2621
|
+
print("✅ no findings")
|
|
2622
|
+
return 1 if found else EXIT_OK
|
|
2623
|
+
|
|
2624
|
+
|
|
2625
|
+
# queue: "when <condition> => to <who> => send <prompt>", conditions checked locally
|
|
2626
|
+
def queue_path():
|
|
2627
|
+
return os.path.join(state_dir(), "queue.json")
|
|
2628
|
+
|
|
2629
|
+
|
|
2630
|
+
def queue_load():
|
|
2631
|
+
return read_json(queue_path(), {"items": [], "next": 1}) or {"items": [], "next": 1}
|
|
2632
|
+
|
|
2633
|
+
|
|
2634
|
+
def cond_met(cond, r):
|
|
2635
|
+
"""mark:<name>=<state> | file:<path> | at:<HH:MM> | alive:<name> | gone:<name>"""
|
|
2636
|
+
kind, _, arg = cond.partition(":")
|
|
2637
|
+
if kind == "mark":
|
|
2638
|
+
name, _, st = arg.partition("=")
|
|
2639
|
+
for s_ in r["sessions"].values():
|
|
2640
|
+
if s_.get("name") == name:
|
|
2641
|
+
return state_of(s_)[0] == (st or "done") and not state_of(s_)[1]
|
|
2642
|
+
return False
|
|
2643
|
+
if kind == "file":
|
|
2644
|
+
return os.path.exists(os.path.expanduser(arg))
|
|
2645
|
+
if kind == "at":
|
|
2646
|
+
m = re.fullmatch(r"(\d{1,2}):(\d{2})", arg.strip())
|
|
2647
|
+
return bool(m) and now() >= (_at(m.group(1), m.group(2), now()) or 1e18)
|
|
2648
|
+
if kind in ("alive", "gone"):
|
|
2649
|
+
hit = any(s_.get("name") == arg and s_.get("alive") for s_ in r["sessions"].values())
|
|
2650
|
+
return hit if kind == "alive" else not hit
|
|
2651
|
+
raise ValueError(f"unknown condition {cond!r} (mark:, file:, at:, alive:, gone:)")
|
|
2652
|
+
|
|
2653
|
+
|
|
2654
|
+
def cmd_queue(args):
|
|
2655
|
+
lk = os.open(os.path.join(state_dir(), "queue.lock"), os.O_RDWR | os.O_CREAT, 0o644)
|
|
2656
|
+
try:
|
|
2657
|
+
fcntl.flock(lk, fcntl.LOCK_EX)
|
|
2658
|
+
q = queue_load()
|
|
2659
|
+
if args.action == "add":
|
|
2660
|
+
for c in args.when:
|
|
2661
|
+
cond_met(c, {"sessions": {}}) # validates the syntax
|
|
2662
|
+
prompt = open(args.prompt_file, encoding="utf-8").read() if args.prompt_file else (args.prompt or "")
|
|
2663
|
+
if not prompt.strip():
|
|
2664
|
+
eprint("dingo-pack queue add: give --prompt or --prompt-file")
|
|
2665
|
+
return EXIT_USAGE
|
|
2666
|
+
item = {"id": q["next"], "when": args.when, "to": args.to, "prompt": prompt, "added": now(),
|
|
2667
|
+
"by": session_label(args)}
|
|
2668
|
+
q["items"].append(item)
|
|
2669
|
+
q["next"] += 1
|
|
2670
|
+
write_atomic(queue_path(), json.dumps(q, ensure_ascii=False, indent=1))
|
|
2671
|
+
print(f"queued #{item['id']} for {args.to}: when {' and '.join(args.when)}")
|
|
2672
|
+
return EXIT_OK
|
|
2673
|
+
r = save_roster(quiet=True) or load_roster()
|
|
2674
|
+
if args.action == "list":
|
|
2675
|
+
for it in q["items"]:
|
|
2676
|
+
if it.get("sent"):
|
|
2677
|
+
continue
|
|
2678
|
+
ready = all(cond_met(c, r) for c in it["when"])
|
|
2679
|
+
print(f" {'🟢 ready' if ready else '⏳ wait '} #{it['id']:<3} to {it['to']:<20} when {' and '.join(it['when'])}"
|
|
2680
|
+
f" · {it['prompt'].strip().splitlines()[0][:60]}")
|
|
2681
|
+
if not any(not it.get("sent") for it in q["items"]):
|
|
2682
|
+
print(" (queue empty)")
|
|
2683
|
+
return EXIT_OK
|
|
2684
|
+
it = next((x for x in q["items"] if x["id"] == args.id), None)
|
|
2685
|
+
if not it:
|
|
2686
|
+
eprint(f"dingo-pack queue: no item #{args.id}")
|
|
2687
|
+
return EXIT_USAGE
|
|
2688
|
+
if args.action == "show":
|
|
2689
|
+
sys.stdout.write(it["prompt"] if it["prompt"].endswith("\n") else it["prompt"] + "\n")
|
|
2690
|
+
if args.clip and shutil.which("clip.exe"):
|
|
2691
|
+
subprocess.run(["clip.exe"], input=it["prompt"].encode("utf-16le"))
|
|
2692
|
+
eprint("copied to the Windows clipboard")
|
|
2693
|
+
return EXIT_OK
|
|
2694
|
+
if args.action == "sent":
|
|
2695
|
+
it["sent"] = now()
|
|
2696
|
+
write_atomic(queue_path(), json.dumps(q, ensure_ascii=False, indent=1))
|
|
2697
|
+
print(f"#{it['id']} archived as sent")
|
|
2698
|
+
return EXIT_OK
|
|
2699
|
+
finally:
|
|
2700
|
+
os.close(lk)
|
|
2701
|
+
return EXIT_USAGE
|
|
2702
|
+
|
|
2703
|
+
|
|
2704
|
+
# box claims: one line per job in a plain text file on the box. dingo writes
|
|
2705
|
+
# who · task · cards · 预计完成 柏林 MM-DD HH:MM · 完成标志 PATH · 写于 柏林 MM-DD HH:MM
|
|
2706
|
+
# and also reads the free-form lines people write by hand ("I-1005-0-202 · GPU 0:… 起 box 01:43 … 预计柏林 10-06 05:05 完").
|
|
2707
|
+
# A line whose done flag exists is removed by the next claim command run on that box (check / add / wait / sweep / watch);
|
|
2708
|
+
# a line past its expected end + grace, with no compute process on its cards, is "stale": reported and alerted, never deleted.
|
|
2709
|
+
CLAIM_SEP = " · "
|
|
2710
|
+
CLAIM_TIME = re.compile(r"(?:(柏林|box)\s*)?(?:(\d{1,2})-(\d{1,2})\s+)?(?<![\d:])(\d{1,2}):(\d{2}|\dx)(?!\d)")
|
|
2711
|
+
CLAIM_DUR = re.compile(r"预计\s*约?\s*(\d+(?:\.\d+)?)\s*(h|小时|min|分钟)")
|
|
2712
|
+
CLAIM_DONE = re.compile(r"(?:完成标志|done=)\s*(\S*/\S*)") # only a path: "跑完写完成标志" alone is not one
|
|
2713
|
+
CLAIM_GPU = re.compile(r"GPU\s*(\d+(?:\s*[-–~]\s*\d+(?![\d.]*\s*(?:worker|进程|个|集|h|min|分))|\s*[/,,、]\s*\d+"
|
|
2714
|
+
r"|\s+\d+(?![\d.]*\s*(?:worker|进程|个|集|h|min|分)))*)")
|
|
2715
|
+
CLAIM_END_WORDS = ("完", "预计", "ETA", "≈", "until")
|
|
2716
|
+
CLAIM_GRACE_MIN = 30
|
|
2717
|
+
CLAIM_EXIT_BUSY = 3 # someone else's live claim
|
|
2718
|
+
CLAIM_EXIT_STALE = 4 # only stale claims are in the way
|
|
2719
|
+
|
|
2720
|
+
|
|
2721
|
+
def _zone(label):
|
|
2722
|
+
name = {"柏林": os.environ.get("DINGO_BERLIN_TZ", "Europe/Berlin"),
|
|
2723
|
+
"box": os.environ.get("DINGO_BOX_TZ", "Asia/Shanghai")}.get(label)
|
|
2724
|
+
if not name:
|
|
2725
|
+
return None
|
|
2726
|
+
try:
|
|
2727
|
+
from zoneinfo import ZoneInfo
|
|
2728
|
+
return ZoneInfo(name)
|
|
2729
|
+
except Exception: # Python 3.8, or no tz database: never fall back to local time
|
|
2730
|
+
return _FixedZone(label)
|
|
2731
|
+
|
|
2732
|
+
|
|
2733
|
+
class _FixedZone(__import__("datetime").tzinfo):
|
|
2734
|
+
"""Berlin (CET / CEST by the EU rule) or the box (UTC+8) without a tz database."""
|
|
2735
|
+
def __init__(self, label):
|
|
2736
|
+
self.label = label
|
|
2737
|
+
|
|
2738
|
+
def utcoffset(self, d):
|
|
2739
|
+
import datetime as dt
|
|
2740
|
+
if self.label != "柏林":
|
|
2741
|
+
return dt.timedelta(hours=8)
|
|
2742
|
+
y = d.year
|
|
2743
|
+
last_sun = lambda m: max(dt.datetime(y, m, day) for day in range(25, 32) if dt.datetime(y, m, day).weekday() == 6)
|
|
2744
|
+
start, end = last_sun(3).replace(hour=2), last_sun(10).replace(hour=3)
|
|
2745
|
+
naive = d.replace(tzinfo=None)
|
|
2746
|
+
return dt.timedelta(hours=2 if start <= naive < end else 1)
|
|
2747
|
+
|
|
2748
|
+
def dst(self, d):
|
|
2749
|
+
return None
|
|
2750
|
+
|
|
2751
|
+
def tzname(self, d):
|
|
2752
|
+
return self.label
|
|
2753
|
+
|
|
2754
|
+
def fromutc(self, d):
|
|
2755
|
+
import datetime as dt
|
|
2756
|
+
guess = d + dt.timedelta(hours=1 if self.label == "柏林" else 8)
|
|
2757
|
+
return d + self.utcoffset(guess.replace(tzinfo=None))
|
|
2758
|
+
|
|
2759
|
+
|
|
2760
|
+
def _clock(mon, day, hh, mm, tz, ref, after=True):
|
|
2761
|
+
"""A wall-clock time in tz as an epoch. With a date: that date in ref's year. Without: the first
|
|
2762
|
+
occurrence after ref (after=True), or the last one at or before ref (after=False)."""
|
|
2763
|
+
import datetime as dt
|
|
2764
|
+
hh = int(hh)
|
|
2765
|
+
mm = int(mm[0]) * 10 if str(mm).endswith("x") else int(mm)
|
|
2766
|
+
if hh > 23 or mm > 59:
|
|
2767
|
+
return None
|
|
2768
|
+
base = dt.datetime.fromtimestamp(ref, tz) if tz else dt.datetime.fromtimestamp(ref)
|
|
2769
|
+
try:
|
|
2770
|
+
d = base.replace(hour=hh, minute=mm, second=0, microsecond=0)
|
|
2771
|
+
if mon:
|
|
2772
|
+
d = d.replace(month=int(mon), day=int(day))
|
|
2773
|
+
except ValueError:
|
|
2774
|
+
return None
|
|
2775
|
+
t = d.timestamp()
|
|
2776
|
+
if not mon:
|
|
2777
|
+
if after and t < ref - 60:
|
|
2778
|
+
t = (d + dt.timedelta(days=1)).timestamp()
|
|
2779
|
+
elif not after and t > ref + 60:
|
|
2780
|
+
t = (d - dt.timedelta(days=1)).timestamp()
|
|
2781
|
+
return t
|
|
2782
|
+
|
|
2783
|
+
|
|
2784
|
+
def fmt_berlin(t):
|
|
2785
|
+
import datetime as dt
|
|
2786
|
+
tz = _zone("柏林")
|
|
2787
|
+
d = dt.datetime.fromtimestamp(t, tz) if tz else dt.datetime.fromtimestamp(t)
|
|
2788
|
+
return d.strftime("柏林 %m-%d %H:%M")
|
|
2789
|
+
|
|
2790
|
+
|
|
2791
|
+
def claim_times(text):
|
|
2792
|
+
"""Every time in a claim line: (start, end, label, mon, day, hh, mm, kind) with kind start / end / other.
|
|
2793
|
+
"起" next to a time = start; any time after an end word ("预计", "完", "≈" ...), or with "完" / "前" right after = end.
|
|
2794
|
+
"box 04:45(=柏林 22:45)": the two are one instant -- only the Berlin one is kept, with the box one's kind."""
|
|
2795
|
+
out = []
|
|
2796
|
+
zone = None
|
|
2797
|
+
for m in CLAIM_TIME.finditer(text):
|
|
2798
|
+
label = m.group(1)
|
|
2799
|
+
if label:
|
|
2800
|
+
zone = label
|
|
2801
|
+
seg0 = max(text.rfind(c, 0, m.start()) for c in "·;;→,,")
|
|
2802
|
+
pre = text[seg0 + 1:m.start()]
|
|
2803
|
+
post = text[m.end():m.end() + 3]
|
|
2804
|
+
if post.lstrip().startswith("起") or pre.rstrip().endswith("起"):
|
|
2805
|
+
kind = "start"
|
|
2806
|
+
elif any(w in text[:m.start()] for w in CLAIM_END_WORDS) or re.match(r"\s*(前|完)", post):
|
|
2807
|
+
kind = "end"
|
|
2808
|
+
else:
|
|
2809
|
+
kind = "other"
|
|
2810
|
+
item = [m.start(), m.end(), label or zone, m.group(2), m.group(3), m.group(4), m.group(5), kind]
|
|
2811
|
+
gap = text[out[-1][1]:m.start()] if out else None
|
|
2812
|
+
if out and label == "柏林" and gap is not None and len(gap) <= 3 and re.fullmatch(r"\s*[((]?\s*[==]\s*", gap):
|
|
2813
|
+
item[7] = out[-1][7] if out[-1][7] != "other" else kind
|
|
2814
|
+
out[-1] = item # the Berlin equivalent replaces the box time
|
|
2815
|
+
continue
|
|
2816
|
+
out.append(item)
|
|
2817
|
+
return out
|
|
2818
|
+
|
|
2819
|
+
|
|
2820
|
+
def claim_window(text, written):
|
|
2821
|
+
"""(start, end) of a claim line as epochs; either may be None. `written` = when the line was written
|
|
2822
|
+
(its "写于" stamp, else when dingo first saw it, else the file's last change: never earlier than the truth)."""
|
|
2823
|
+
times = claim_times(text)
|
|
2824
|
+
w = re.search(r"写于\s*(柏林|box)?\s*(\d{1,2})-(\d{1,2})\s+(\d{1,2}):(\d{2})", text)
|
|
2825
|
+
if w:
|
|
2826
|
+
written = _clock(w.group(2), w.group(3), w.group(4), w.group(5), _zone(w.group(1) or "柏林"), written or now()) or written
|
|
2827
|
+
times = [x for x in times if not (x[0] >= w.start() and x[1] <= w.end())]
|
|
2828
|
+
written = written or now()
|
|
2829
|
+
starts = [t for t in (_clock(x[3], x[4], x[5], x[6], _zone(x[2]), written, after=False)
|
|
2830
|
+
for x in times if x[7] == "start") if t]
|
|
2831
|
+
start = min(starts) if starts else None
|
|
2832
|
+
ref = start or written
|
|
2833
|
+
ends = [x for x in times if x[7] == "end"]
|
|
2834
|
+
if not ends: # the bare 0.3 form "who · task · cards · until": times in the 4th field
|
|
2835
|
+
parts = text.split("·")
|
|
2836
|
+
if len(parts) >= 4:
|
|
2837
|
+
lo = len("·".join(parts[:3])) + 1
|
|
2838
|
+
hi = lo + len(parts[3])
|
|
2839
|
+
ends = [x for x in times if lo <= x[0] < hi]
|
|
2840
|
+
end_ts = [t for t in (_clock(x[3], x[4], x[5], x[6], _zone(x[2]), ref) for x in ends) if t]
|
|
2841
|
+
end = max(end_ts) if end_ts else None
|
|
2842
|
+
d = CLAIM_DUR.search(text)
|
|
2843
|
+
if d and start:
|
|
2844
|
+
n = float(d.group(1)) * (3600 if d.group(2) in ("h", "小时") else 60)
|
|
2845
|
+
end = max(end or 0, start + n)
|
|
2846
|
+
return start, end
|
|
2847
|
+
|
|
2848
|
+
|
|
2849
|
+
def parse_until(text, written=None):
|
|
2850
|
+
"""End time of a claim's 'until' text (kept for the 0.3 callers)."""
|
|
2851
|
+
return claim_window("预计 " + (text or ""), written)[1]
|
|
2852
|
+
|
|
2853
|
+
|
|
2854
|
+
def claim_cards(text, field=None):
|
|
2855
|
+
if field is not None and re.fullmatch(r"\s*(无卡|不占卡|不用卡)\s*", field or ""):
|
|
2856
|
+
return set()
|
|
2857
|
+
if re.search(r"不占卡|不用卡", text):
|
|
2858
|
+
return set() # copies, downloads: blocks no card (only the same task)
|
|
2859
|
+
if field is not None and re.fullmatch(r"[\d\s,,/、\-–]+", field or ""):
|
|
2860
|
+
return _expand_cards(field) or {"*"}
|
|
2861
|
+
m = CLAIM_GPU.search(text)
|
|
2862
|
+
return _expand_cards(m.group(1)) if m else {"*"}
|
|
2863
|
+
|
|
2864
|
+
|
|
2865
|
+
def _expand_cards(text):
|
|
2866
|
+
"""'0 1 2' / '0,1' / '0-3' / '0–3' -> {'0', '1', '2', '3'}."""
|
|
2867
|
+
out = set()
|
|
2868
|
+
for a, b in re.findall(r"(\d+)(?:\s*[-–~]\s*(\d+))?", text):
|
|
2869
|
+
lo, hi = int(a), int(b) if b else int(a)
|
|
2870
|
+
if hi - lo > 16 or hi < lo:
|
|
2871
|
+
hi = lo
|
|
2872
|
+
out |= {str(i) for i in range(lo, hi + 1)}
|
|
2873
|
+
return out
|
|
2874
|
+
|
|
2875
|
+
|
|
2876
|
+
def card_set(cards):
|
|
2877
|
+
return set(re.findall(r"\d+", cards or "")) or {"*"}
|
|
2878
|
+
|
|
2879
|
+
|
|
2880
|
+
def parse_claim(line, written=None):
|
|
2881
|
+
parts = [x.strip() for x in line.strip().split("·")]
|
|
2882
|
+
if len(parts) < 2 or not parts[0]:
|
|
2883
|
+
return None
|
|
2884
|
+
if re.match(r"[^·]{0,40}?[::]", parts[0]) and not re.match(r"[A-Z]\d*-\d{4}", parts[0]):
|
|
2885
|
+
return None # a filled-in template field ("输出目录:… · …"), not a claim
|
|
2886
|
+
who = parts[0][:48]
|
|
2887
|
+
task = parts[1]
|
|
2888
|
+
field = parts[2] if len(parts) > 2 else None
|
|
2889
|
+
cards = claim_cards(line, field)
|
|
2890
|
+
start, end = claim_window(line.strip(), written)
|
|
2891
|
+
done = CLAIM_DONE.search(line)
|
|
2892
|
+
return {"written_t": claim_written(line), "who": who, "task": task, "cards": ",".join(sorted(cards)), "card_set": cards,
|
|
2893
|
+
"until": parts[3] if len(parts) > 3 else "", "start_t": start, "until_t": end,
|
|
2894
|
+
"done_flag": done.group(1).rstrip(")),;;。") if done else None, "line": line.rstrip("\n")}
|
|
2895
|
+
|
|
2896
|
+
|
|
2897
|
+
def claim_file(args):
|
|
2898
|
+
f = args.file or os.environ.get("DINGO_CLAIM_FILE")
|
|
2899
|
+
if not f:
|
|
2900
|
+
eprint("dingo-pack claim: say --file /root/gpufree-data/CLAIM_boxNN.txt (or set DINGO_CLAIM_FILE)")
|
|
2901
|
+
sys.exit(EXIT_USAGE)
|
|
2902
|
+
return f
|
|
2903
|
+
|
|
2904
|
+
|
|
2905
|
+
def _seen_path(path):
|
|
2906
|
+
return path + ".seen.json"
|
|
2907
|
+
|
|
2908
|
+
|
|
2909
|
+
def _line_key(line):
|
|
2910
|
+
import hashlib
|
|
2911
|
+
return hashlib.sha1(line.strip().encode("utf-8")).hexdigest()[:16]
|
|
2912
|
+
|
|
2913
|
+
|
|
2914
|
+
def _note_seen(path):
|
|
2915
|
+
"""Before dingo rewrites the file (which moves its mtime), remember for each line the latest time it
|
|
2916
|
+
can have been written (the current mtime), so undated times in older lines keep their day."""
|
|
2917
|
+
try:
|
|
2918
|
+
mt = os.path.getmtime(path)
|
|
2919
|
+
with open(path, encoding="utf-8") as f:
|
|
2920
|
+
lines = [x for x in f if x.strip()]
|
|
2921
|
+
except FileNotFoundError:
|
|
2922
|
+
return
|
|
2923
|
+
seen = read_json(_seen_path(path), {}) or {}
|
|
2924
|
+
keys = set()
|
|
2925
|
+
for x in lines:
|
|
2926
|
+
k = _line_key(x)
|
|
2927
|
+
keys.add(k)
|
|
2928
|
+
seen[k] = min(seen.get(k, mt), mt)
|
|
2929
|
+
write_atomic(_seen_path(path), json.dumps({k: v for k, v in seen.items() if k in keys}))
|
|
2930
|
+
|
|
2931
|
+
|
|
2932
|
+
def read_claims(path):
|
|
2933
|
+
try:
|
|
2934
|
+
mt = os.path.getmtime(path)
|
|
2935
|
+
with open(path, encoding="utf-8") as f:
|
|
2936
|
+
lines = f.readlines()
|
|
2937
|
+
except FileNotFoundError:
|
|
2938
|
+
return []
|
|
2939
|
+
seen = read_json(_seen_path(path), {}) or {}
|
|
2940
|
+
out = []
|
|
2941
|
+
for x in lines:
|
|
2942
|
+
if not x.strip() or x.lstrip().startswith("#"):
|
|
2943
|
+
continue
|
|
2944
|
+
c = parse_claim(x, min(seen.get(_line_key(x), mt), mt))
|
|
2945
|
+
if c:
|
|
2946
|
+
c["base"] = os.path.dirname(os.path.abspath(path))
|
|
2947
|
+
out.append(c)
|
|
2948
|
+
return out
|
|
2949
|
+
|
|
2950
|
+
|
|
2951
|
+
def gpu_procs():
|
|
2952
|
+
"""{card index: [(pid, MiB)]} from nvidia-smi, or None when it is not there / fails."""
|
|
2953
|
+
exe = os.environ.get("DINGO_NVSMI", "nvidia-smi")
|
|
2954
|
+
try:
|
|
2955
|
+
g = subprocess.run([exe, "--query-gpu=index,uuid", "--format=csv,noheader"],
|
|
2956
|
+
capture_output=True, text=True, timeout=20)
|
|
2957
|
+
a = subprocess.run([exe, "--query-compute-apps=gpu_uuid,pid,used_memory", "--format=csv,noheader"],
|
|
2958
|
+
capture_output=True, text=True, timeout=20)
|
|
2959
|
+
except (OSError, subprocess.SubprocessError):
|
|
2960
|
+
return None
|
|
2961
|
+
if g.returncode != 0 or a.returncode != 0:
|
|
2962
|
+
return None
|
|
2963
|
+
idx = {}
|
|
2964
|
+
for row in g.stdout.splitlines():
|
|
2965
|
+
p = [x.strip() for x in row.split(",")]
|
|
2966
|
+
if len(p) >= 2:
|
|
2967
|
+
idx[p[1]] = p[0]
|
|
2968
|
+
out = {i: [] for i in idx.values()}
|
|
2969
|
+
for row in a.stdout.splitlines():
|
|
2970
|
+
p = [x.strip() for x in row.split(",")]
|
|
2971
|
+
if len(p) >= 2 and p[0] in idx:
|
|
2972
|
+
mem = re.match(r"\d+", p[2]) if len(p) > 2 else None
|
|
2973
|
+
out[idx[p[0]]].append((p[1], int(mem.group()) if mem else 0))
|
|
2974
|
+
return out
|
|
2975
|
+
|
|
2976
|
+
|
|
2977
|
+
def done_flag_hit(c, base=None):
|
|
2978
|
+
"""The line's done flag exists and is not older than the line (a flag left by an earlier job does not
|
|
2979
|
+
count). A relative path is taken from the claim file's directory, never from the caller's cwd."""
|
|
2980
|
+
flag = c.get("done_flag")
|
|
2981
|
+
if not flag:
|
|
2982
|
+
return False
|
|
2983
|
+
if not os.path.isabs(flag):
|
|
2984
|
+
flag = os.path.join(base or c.get("base") or "/nonexistent", flag)
|
|
2985
|
+
since = c.get("written_t") or c.get("start_t")
|
|
2986
|
+
try: # a literal regular file: no globs, not a directory
|
|
2987
|
+
return os.path.isfile(flag) and (since is None or os.path.getmtime(flag) >= since - 60)
|
|
2988
|
+
except OSError:
|
|
2989
|
+
return False
|
|
2990
|
+
|
|
2991
|
+
|
|
2992
|
+
def claim_written(text):
|
|
2993
|
+
w = re.search(r"写于\s*(柏林|box)?\s*(\d{1,2})-(\d{1,2})\s+(\d{1,2}):(\d{2})", text or "")
|
|
2994
|
+
return _clock(w.group(2), w.group(3), w.group(4), w.group(5), _zone(w.group(1) or "柏林"), now()) if w else None
|
|
2995
|
+
|
|
2996
|
+
|
|
2997
|
+
def claim_state(c, procs, grace_min=CLAIM_GRACE_MIN):
|
|
2998
|
+
"""done (flag is there) / live / overdue (past end + grace, cards still computing) / stale (past end +
|
|
2999
|
+
grace, no compute process on its cards, or nvidia-smi unknown) / open (no end time found)."""
|
|
3000
|
+
if done_flag_hit(c):
|
|
3001
|
+
return "done"
|
|
3002
|
+
if c["until_t"] is None:
|
|
3003
|
+
return "open"
|
|
3004
|
+
if now() <= c["until_t"] + grace_min * 60:
|
|
3005
|
+
return "live"
|
|
3006
|
+
if procs is None:
|
|
3007
|
+
return "stale"
|
|
3008
|
+
cards = procs.keys() if "*" in c["card_set"] else c["card_set"]
|
|
3009
|
+
return "overdue" if any(procs.get(k) for k in cards) else "stale"
|
|
3010
|
+
|
|
3011
|
+
|
|
3012
|
+
CLAIM_STATE_TEXT = {"done": "✅ 完成标志已在(下一次 claim 命令自动删)", "live": "在用", "open": "⚠️ 没写预计完成",
|
|
3013
|
+
"overdue": "⏰ 已超预计、卡上仍有计算进程", "stale": "🟡 疑似过期(已超预计 + 宽限、卡上没有计算进程)"}
|
|
3014
|
+
|
|
3015
|
+
|
|
3016
|
+
def sweep_claims(path):
|
|
3017
|
+
"""Remove the lines whose done flag exists; the removed lines go to <file>.released. Returns them."""
|
|
3018
|
+
lk = os.open(path + ".lock", os.O_CREAT | os.O_RDWR, 0o644)
|
|
3019
|
+
try:
|
|
3020
|
+
fcntl.flock(lk, fcntl.LOCK_EX)
|
|
3021
|
+
base = os.path.dirname(os.path.abspath(path))
|
|
3022
|
+
for _ in range(5):
|
|
3023
|
+
try:
|
|
3024
|
+
st0 = os.stat(path)
|
|
3025
|
+
with open(path, encoding="utf-8") as f:
|
|
3026
|
+
lines = f.readlines()
|
|
3027
|
+
except FileNotFoundError:
|
|
3028
|
+
return []
|
|
3029
|
+
seen = read_json(_seen_path(path), {}) or {}
|
|
3030
|
+
keep, gone = [], []
|
|
3031
|
+
for x in lines:
|
|
3032
|
+
c = parse_claim(x, min(seen.get(_line_key(x), st0.st_mtime), st0.st_mtime)) \
|
|
3033
|
+
if x.strip() and not x.lstrip().startswith("#") else None
|
|
3034
|
+
if c and done_flag_hit(c, base):
|
|
3035
|
+
gone.append(c)
|
|
3036
|
+
else:
|
|
3037
|
+
keep.append(x)
|
|
3038
|
+
if not gone:
|
|
3039
|
+
return []
|
|
3040
|
+
_note_seen(path)
|
|
3041
|
+
st1 = os.stat(path)
|
|
3042
|
+
# people append by hand (echo >>) without our lock: if the file moved since we read it,
|
|
3043
|
+
# read again rather than replace it with a copy that misses their line
|
|
3044
|
+
if (st1.st_size, st1.st_mtime_ns, st1.st_ino) == (st0.st_size, st0.st_mtime_ns, st0.st_ino):
|
|
3045
|
+
write_atomic(path, "".join(keep))
|
|
3046
|
+
break
|
|
3047
|
+
time.sleep(0.2)
|
|
3048
|
+
else:
|
|
3049
|
+
eprint(f"dingo-pack claim: {path} kept changing; did not remove anything this time")
|
|
3050
|
+
return []
|
|
3051
|
+
if gone:
|
|
3052
|
+
with open(path + ".released", "a", encoding="utf-8") as f:
|
|
3053
|
+
for c in gone:
|
|
3054
|
+
f.write(f"{fmt_berlin(now())} 完成标志 {c['done_flag']} 已在 ⇒ 删行:{c['line']}\n")
|
|
3055
|
+
return gone
|
|
3056
|
+
finally:
|
|
3057
|
+
fcntl.flock(lk, fcntl.LOCK_UN)
|
|
3058
|
+
os.close(lk)
|
|
3059
|
+
|
|
3060
|
+
|
|
3061
|
+
def claim_alert(path, alert, msgs):
|
|
3062
|
+
"""One line per stale claim into an alert file the box watchdog already watches (runs/*.alert)."""
|
|
3063
|
+
if not alert or not msgs:
|
|
3064
|
+
return
|
|
3065
|
+
try:
|
|
3066
|
+
with open(alert, encoding="utf-8") as f:
|
|
3067
|
+
old = f.read()
|
|
3068
|
+
except OSError:
|
|
3069
|
+
old = ""
|
|
3070
|
+
msgs = [m for m in msgs if m not in old] # each stale line once: the watchdog alerts on every change
|
|
3071
|
+
if not msgs:
|
|
3072
|
+
return
|
|
3073
|
+
try:
|
|
3074
|
+
with open(alert, "a", encoding="utf-8") as f:
|
|
3075
|
+
for m in msgs:
|
|
3076
|
+
f.write(f"{fmt_berlin(now())} {m}\n")
|
|
3077
|
+
except OSError as e:
|
|
3078
|
+
eprint(f"dingo-pack claim: could not write the alert file {alert}: {e}")
|
|
3079
|
+
|
|
3080
|
+
|
|
3081
|
+
def default_alert(path):
|
|
3082
|
+
runs = os.path.join(os.path.dirname(os.path.abspath(path)), "runs")
|
|
3083
|
+
if os.path.isdir(runs):
|
|
3084
|
+
return os.path.join(runs, "claim_stale_" + os.path.splitext(os.path.basename(path))[0] + ".alert")
|
|
3085
|
+
return None
|
|
3086
|
+
|
|
3087
|
+
|
|
3088
|
+
def claim_blockers(claims, who, cards, task, procs, grace):
|
|
3089
|
+
mine = card_set(cards) if cards else {"*"}
|
|
3090
|
+
out = []
|
|
3091
|
+
for c in claims:
|
|
3092
|
+
if c["who"] == who:
|
|
3093
|
+
continue
|
|
3094
|
+
st = claim_state(c, procs, grace)
|
|
3095
|
+
if st == "done":
|
|
3096
|
+
continue
|
|
3097
|
+
overlap = bool(c["card_set"]) and ("*" in mine or "*" in c["card_set"] or bool(mine & c["card_set"]))
|
|
3098
|
+
same = bool(task) and c["task"].strip() == task.strip()
|
|
3099
|
+
if overlap or same:
|
|
3100
|
+
out.append((c, st, overlap))
|
|
3101
|
+
return out
|
|
3102
|
+
|
|
3103
|
+
|
|
3104
|
+
def stale_msg(c, path):
|
|
3105
|
+
box = os.path.splitext(os.path.basename(path))[0].replace("CLAIM_", "")
|
|
3106
|
+
return (f"{box} CLAIM 疑似过期:{c['who']} 卡 {c['cards']} 预计 {fmt_berlin(c['until_t'])} 已过、"
|
|
3107
|
+
f"卡上没有计算进程;问持有者删行(dingo 不删)")
|
|
3108
|
+
|
|
3109
|
+
|
|
3110
|
+
def cmd_claim(args):
|
|
3111
|
+
path = claim_file(args)
|
|
3112
|
+
who = args.who or session_label(args)
|
|
3113
|
+
grace = args.grace
|
|
3114
|
+
if args.action == "sweep":
|
|
3115
|
+
gone = sweep_claims(path)
|
|
3116
|
+
for c in gone:
|
|
3117
|
+
print(f"removed (done flag {c['done_flag']} is there): {c['line']}")
|
|
3118
|
+
print(f"removed {len(gone)} line(s)")
|
|
3119
|
+
return EXIT_OK
|
|
3120
|
+
if args.action == "list":
|
|
3121
|
+
claims = read_claims(path)
|
|
3122
|
+
procs = gpu_procs()
|
|
3123
|
+
rows = []
|
|
3124
|
+
for c in claims:
|
|
3125
|
+
st = claim_state(c, procs, grace)
|
|
3126
|
+
rows.append(dict({k: v for k, v in c.items() if k != "card_set"}, state=st))
|
|
3127
|
+
if args.json:
|
|
3128
|
+
print(json.dumps({"file": path, "now": now(), "gpu_procs": procs, "claims": rows}, ensure_ascii=False))
|
|
3129
|
+
return EXIT_OK
|
|
3130
|
+
for r in rows:
|
|
3131
|
+
end = fmt_berlin(r["until_t"]) if r["until_t"] else "?"
|
|
3132
|
+
print(f" {r['who']:<22} 卡 {r['cards']:<6} 预计 {end:<16} {CLAIM_STATE_TEXT[r['state']]}")
|
|
3133
|
+
if not rows:
|
|
3134
|
+
print(" (nobody)")
|
|
3135
|
+
if procs is None:
|
|
3136
|
+
print(" (nvidia-smi not available: stale = by time only)")
|
|
3137
|
+
claim_alert(path, args.alert, [stale_msg(c, path) for c, r in zip(claims, rows) if r["state"] == "stale"])
|
|
3138
|
+
return EXIT_OK
|
|
3139
|
+
if args.action == "remove":
|
|
3140
|
+
lk = os.open(path + ".lock", os.O_CREAT | os.O_RDWR, 0o644)
|
|
3141
|
+
try:
|
|
3142
|
+
fcntl.flock(lk, fcntl.LOCK_EX)
|
|
3143
|
+
try:
|
|
3144
|
+
with open(path, encoding="utf-8") as f:
|
|
3145
|
+
lines = f.readlines()
|
|
3146
|
+
except FileNotFoundError:
|
|
3147
|
+
lines = []
|
|
3148
|
+
keep, removed = [], 0
|
|
3149
|
+
for x in lines:
|
|
3150
|
+
c = parse_claim(x) if x.strip() and not x.lstrip().startswith("#") else None
|
|
3151
|
+
if c and c["who"] == who and (not args.task or c["task"].strip() == args.task.strip()):
|
|
3152
|
+
removed += 1
|
|
3153
|
+
else:
|
|
3154
|
+
keep.append(x)
|
|
3155
|
+
if removed:
|
|
3156
|
+
_note_seen(path)
|
|
3157
|
+
write_atomic(path, "".join(keep))
|
|
3158
|
+
finally:
|
|
3159
|
+
fcntl.flock(lk, fcntl.LOCK_UN)
|
|
3160
|
+
os.close(lk)
|
|
3161
|
+
print(f"removed {removed} line(s) of {who}")
|
|
3162
|
+
return EXIT_OK
|
|
3163
|
+
if args.action not in ("check", "add", "wait"):
|
|
3164
|
+
return EXIT_USAGE
|
|
3165
|
+
alert = args.alert or default_alert(path)
|
|
3166
|
+
alerted = set()
|
|
3167
|
+
deadline = now() + parse_duration(args.timeout) if getattr(args, "timeout", None) else None
|
|
3168
|
+
while True:
|
|
3169
|
+
for c in sweep_claims(path):
|
|
3170
|
+
print(f"auto-released (done flag {c['done_flag']} is there): {c['line']}")
|
|
3171
|
+
procs = gpu_procs()
|
|
3172
|
+
blockers = claim_blockers(read_claims(path), who, args.cards, args.task, procs, grace)
|
|
3173
|
+
live = [b for b in blockers if b[1] != "stale"]
|
|
3174
|
+
stale = [b for b in blockers if b[1] == "stale"]
|
|
3175
|
+
if args.action != "wait" or not blockers or (not live and args.take_stale):
|
|
3176
|
+
break
|
|
3177
|
+
new = [stale_msg(c, path) for c, _, _ in stale if c["line"] not in alerted]
|
|
3178
|
+
if not live and new:
|
|
3179
|
+
for m in new:
|
|
3180
|
+
print(f"🟡 {m}")
|
|
3181
|
+
claim_alert(path, alert, new)
|
|
3182
|
+
alerted.update(c["line"] for c, _, _ in stale)
|
|
3183
|
+
if args.stop_on_stale:
|
|
3184
|
+
return CLAIM_EXIT_STALE
|
|
3185
|
+
if deadline and now() >= deadline:
|
|
3186
|
+
print("timed out while waiting for the claim(s) below")
|
|
3187
|
+
break
|
|
3188
|
+
time.sleep(args.every)
|
|
3189
|
+
for c, st, overlap in blockers:
|
|
3190
|
+
if st == "stale":
|
|
3191
|
+
print(f"🟡 stale: {c['line']}")
|
|
3192
|
+
print(f" 预计 {fmt_berlin(c['until_t'])} + 宽限 {grace} min 已过、卡上没有计算进程:问持有者或聊天侧,⛔ 不擅自删")
|
|
3193
|
+
elif overlap:
|
|
3194
|
+
print(f"⛔ in use{' (overdue, cards busy)' if st == 'overdue' else ''}: {c['line']}")
|
|
3195
|
+
else:
|
|
3196
|
+
print(f"⚠️ same task already claimed: {c['line']}")
|
|
3197
|
+
if stale and args.action != "wait":
|
|
3198
|
+
claim_alert(path, alert, [stale_msg(c, path) for c, _, _ in stale])
|
|
3199
|
+
if blockers and not (not live and args.take_stale):
|
|
3200
|
+
print("stop and report (someone else holds this box / these cards)")
|
|
3201
|
+
if args.action == "wait" and deadline and now() >= deadline:
|
|
3202
|
+
return EXIT_TIMEOUT
|
|
3203
|
+
return CLAIM_EXIT_BUSY if live else CLAIM_EXIT_STALE
|
|
3204
|
+
if stale:
|
|
3205
|
+
print("taking over stale claim(s) (--take-stale)")
|
|
3206
|
+
for c in sweep_claims(path): # a flag that appeared after the sweep above
|
|
3207
|
+
print(f"auto-released (done flag {c['done_flag']} is there): {c['line']}")
|
|
3208
|
+
if args.action in ("check", "wait"):
|
|
3209
|
+
print("free")
|
|
3210
|
+
return EXIT_OK
|
|
3211
|
+
if not (args.task and args.cards and args.until):
|
|
3212
|
+
eprint("dingo-pack claim add: give --task, --cards and --until (and --done FLAG, so the line removes itself)")
|
|
3213
|
+
return EXIT_USAGE
|
|
3214
|
+
_, end = claim_window("预计 " + args.until, now())
|
|
3215
|
+
if end is None or end < now() - 60:
|
|
3216
|
+
eprint(f"dingo-pack claim add: cannot read a future time in --until {args.until!r} (write e.g. '柏林 10-06 05:05')")
|
|
3217
|
+
return EXIT_USAGE
|
|
3218
|
+
if args.done and (not os.path.isabs(args.done) or os.path.exists(args.done) or any(ch in args.done for ch in "*?[")):
|
|
3219
|
+
eprint(f"dingo-pack claim add: --done must be an absolute path that does not exist yet (no wildcards): {args.done}")
|
|
3220
|
+
return EXIT_USAGE
|
|
3221
|
+
fields = [who, args.task, args.cards, "预计完成 " + fmt_berlin(end)]
|
|
3222
|
+
if args.done:
|
|
3223
|
+
fields.append("完成标志 " + args.done)
|
|
3224
|
+
else:
|
|
3225
|
+
eprint("⚠️ no --done flag: remove the line yourself when the job ends (dingo-pack claim remove)")
|
|
3226
|
+
fields.append("写于 " + fmt_berlin(now()))
|
|
3227
|
+
line = CLAIM_SEP.join(fields)
|
|
3228
|
+
lk = os.open(path + ".lock", os.O_CREAT | os.O_RDWR, 0o644)
|
|
3229
|
+
try:
|
|
3230
|
+
fcntl.flock(lk, fcntl.LOCK_EX)
|
|
3231
|
+
# again under the lock: another `claim add` may have taken the cards since the check above
|
|
3232
|
+
again = claim_blockers(read_claims(path), who, args.cards, args.task, gpu_procs(), grace)
|
|
3233
|
+
if [b for b in again if b[1] != "stale" or not args.take_stale]:
|
|
3234
|
+
for c, st, _ in again:
|
|
3235
|
+
print(f"⛔ in use (just added): {c['line']}")
|
|
3236
|
+
print("stop and report (someone else holds this box / these cards)")
|
|
3237
|
+
return CLAIM_EXIT_BUSY
|
|
3238
|
+
_note_seen(path)
|
|
3239
|
+
with open(path, "a", encoding="utf-8") as f:
|
|
3240
|
+
f.write(line + "\n")
|
|
3241
|
+
finally:
|
|
3242
|
+
fcntl.flock(lk, fcntl.LOCK_UN)
|
|
3243
|
+
os.close(lk)
|
|
3244
|
+
print(f"claimed: {line}")
|
|
3245
|
+
return EXIT_OK
|
|
3246
|
+
|
|
3247
|
+
|
|
3248
|
+
# ---------------------------------------------------------------- boxes (read-only ssh)
|
|
3249
|
+
# `dingo-pack boxes` sends this very file over one ssh per box (`python3 - _probe`): nothing is installed
|
|
3250
|
+
# or written there; the probe only reads (nvidia-smi, df, du, tmux ls, ps, the watchdog files, CLAIM_*.txt).
|
|
3251
|
+
BOX_STATES = {"ok": "在线有卡", "nogpu": "在线无卡", "fewer": "卡少了", "down": "连不上", "unknown": "没查过"}
|
|
3252
|
+
BOX_BAD = ("nogpu", "fewer", "down")
|
|
3253
|
+
PROBE_PROC = re.compile(r"python|torchrun|\btar\b|rsync|\bdu\b|hf_hub|huggingface-cli")
|
|
3254
|
+
PROBE_SKIP = re.compile(r"jupyter|box_watchdog|_probe|ipykernel|unattended-upgrade|networkd-dispatcher")
|
|
3255
|
+
|
|
3256
|
+
|
|
3257
|
+
def _sh(cmd, timeout=30):
|
|
3258
|
+
try:
|
|
3259
|
+
r = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout)
|
|
3260
|
+
return r.returncode, r.stdout, r.stderr
|
|
3261
|
+
except (OSError, subprocess.SubprocessError) as e:
|
|
3262
|
+
return None, "", str(e)
|
|
3263
|
+
|
|
3264
|
+
|
|
3265
|
+
def probe_here(data, du=True):
|
|
3266
|
+
"""Runs on the box. Everything it reports is read, nothing is written."""
|
|
3267
|
+
r = {"now": now(), "host": socket.gethostname(), "data": data}
|
|
3268
|
+
try:
|
|
3269
|
+
with open("/proc/uptime") as f:
|
|
3270
|
+
r["boot"] = now() - float(f.read().split()[0])
|
|
3271
|
+
except (OSError, ValueError):
|
|
3272
|
+
r["boot"] = None
|
|
3273
|
+
rc, out, err = _sh([os.environ.get("DINGO_NVSMI", "nvidia-smi"), "-L"])
|
|
3274
|
+
r["gpus"] = sum(1 for x in out.splitlines() if x.startswith("GPU ")) if rc == 0 else 0
|
|
3275
|
+
r["gpu_error"] = None if rc == 0 else ((err or out).strip().splitlines() or ["nvidia-smi not found"])[-1][:120]
|
|
3276
|
+
r["gpu_procs"] = gpu_procs() if rc == 0 else None
|
|
3277
|
+
r["disks"] = {}
|
|
3278
|
+
for p in (data, "/"):
|
|
3279
|
+
try:
|
|
3280
|
+
u = shutil.disk_usage(p)
|
|
3281
|
+
r["disks"][p] = {"used": u.used, "total": u.total}
|
|
3282
|
+
except OSError:
|
|
3283
|
+
pass
|
|
3284
|
+
r["du"] = []
|
|
3285
|
+
if du and os.path.isdir(data):
|
|
3286
|
+
rc, out, _ = _sh(["timeout", "90", "du", "-x", "-B1", "--max-depth=1", data], timeout=100)
|
|
3287
|
+
rows = []
|
|
3288
|
+
for x in out.splitlines():
|
|
3289
|
+
p = x.split("\t", 1)
|
|
3290
|
+
if len(p) == 2 and p[0].isdigit() and os.path.normpath(p[1]) != os.path.normpath(data):
|
|
3291
|
+
rows.append((int(p[0]), os.path.basename(p[1])))
|
|
3292
|
+
r["du"] = sorted(rows, reverse=True)[:12]
|
|
3293
|
+
r["du_complete"] = rc == 0
|
|
3294
|
+
rc, out, _ = _sh(["tmux", "ls", "-F", "#{session_name}"])
|
|
3295
|
+
r["tmux"] = out.split() if rc == 0 else []
|
|
3296
|
+
rc, out, _ = _sh(["ps", "-eo", "pid=,etimes=,pcpu=,rss=,args="])
|
|
3297
|
+
procs = []
|
|
3298
|
+
me = {str(os.getpid()), str(os.getppid())}
|
|
3299
|
+
for x in out.splitlines():
|
|
3300
|
+
p = x.split(None, 4)
|
|
3301
|
+
if len(p) == 5 and p[0] not in me and PROBE_PROC.search(p[4]) and not PROBE_SKIP.search(p[4]):
|
|
3302
|
+
procs.append({"pid": p[0], "age_s": int(p[1]) if p[1].isdigit() else None, "cpu": p[2],
|
|
3303
|
+
"rss_mb": int(p[3]) // 1024 if p[3].isdigit() else None, "cmd": p[4][:100]})
|
|
3304
|
+
r["procs"] = procs[:30]
|
|
3305
|
+
wd = os.path.join(data, "watchdog")
|
|
3306
|
+
w = {"present": os.path.isdir(wd)}
|
|
3307
|
+
if w["present"]:
|
|
3308
|
+
try:
|
|
3309
|
+
w["heartbeat_age_s"] = now() - os.path.getmtime(os.path.join(wd, "HEARTBEAT"))
|
|
3310
|
+
except OSError:
|
|
3311
|
+
w["heartbeat_age_s"] = None
|
|
3312
|
+
w["targets"] = sorted(os.path.splitext(os.path.basename(x))[0]
|
|
3313
|
+
for x in glob.glob(os.path.join(wd, "targets.d", "*.json")))
|
|
3314
|
+
try:
|
|
3315
|
+
with open(os.path.join(wd, "alerts.log"), encoding="utf-8", errors="replace") as f:
|
|
3316
|
+
w["alerts_tail"] = [x.rstrip()[:160] for x in f.readlines()[-3:]]
|
|
3317
|
+
except OSError:
|
|
3318
|
+
w["alerts_tail"] = []
|
|
3319
|
+
r["watchdog"] = w
|
|
3320
|
+
claims = []
|
|
3321
|
+
for path in sorted(glob.glob(os.path.join(data, "CLAIM_*.txt"))):
|
|
3322
|
+
for c in read_claims(path):
|
|
3323
|
+
row = {k: v for k, v in c.items() if k != "card_set"}
|
|
3324
|
+
row["state"] = claim_state(c, r["gpu_procs"])
|
|
3325
|
+
row["file"] = os.path.basename(path)
|
|
3326
|
+
claims.append(row)
|
|
3327
|
+
r["claims"] = claims
|
|
3328
|
+
return r
|
|
3329
|
+
|
|
3330
|
+
|
|
3331
|
+
def cmd_probe(args):
|
|
3332
|
+
print(json.dumps(probe_here(args.data, du=not args.no_du), ensure_ascii=False))
|
|
3333
|
+
return EXIT_OK
|
|
3334
|
+
|
|
3335
|
+
|
|
3336
|
+
def boxes_file(args=None):
|
|
3337
|
+
return (getattr(args, "boxes", None) or os.environ.get("DINGO_BOXES")
|
|
3338
|
+
or os.path.join(os.path.expanduser("~/.config/dingo-pack"), "boxes.json"))
|
|
3339
|
+
|
|
3340
|
+
|
|
3341
|
+
def load_boxes(args=None):
|
|
3342
|
+
"""[{"name", "host", "gpus" (expected), "data", ...}] from boxes.json; without one, the box* hosts
|
|
3343
|
+
in ~/.ssh/config (expected card count unknown)."""
|
|
3344
|
+
j = read_json(boxes_file(args), None)
|
|
3345
|
+
if isinstance(j, dict):
|
|
3346
|
+
j = j.get("boxes")
|
|
3347
|
+
if isinstance(j, dict): # {"box22": {"ssh": "box22", "gpus": 4, ...}} (T2's table)
|
|
3348
|
+
j = [dict(v, name=k) for k, v in j.items() if isinstance(v, dict)]
|
|
3349
|
+
if isinstance(j, list):
|
|
3350
|
+
out = []
|
|
3351
|
+
for b in j:
|
|
3352
|
+
if isinstance(b, dict) and b.get("name"):
|
|
3353
|
+
e = dict({"gpus": None, "data": "/root/gpufree-data"}, **b)
|
|
3354
|
+
e["host"] = b.get("host") or b.get("ssh") or b["name"]
|
|
3355
|
+
out.append(e)
|
|
3356
|
+
return sorted(out, key=lambda b: (len(b["name"]), b["name"]))
|
|
3357
|
+
hosts = []
|
|
3358
|
+
try:
|
|
3359
|
+
with open(os.path.expanduser("~/.ssh/config")) as f:
|
|
3360
|
+
for x in f:
|
|
3361
|
+
m = re.match(r"\s*Host\s+(.+)", x)
|
|
3362
|
+
if m:
|
|
3363
|
+
hosts += [h for h in m.group(1).split() if re.fullmatch(r"box\d+", h)]
|
|
3364
|
+
except OSError:
|
|
3365
|
+
pass
|
|
3366
|
+
return [{"name": h, "host": h, "gpus": None, "data": "/root/gpufree-data"}
|
|
3367
|
+
for h in sorted(set(hosts), key=lambda h: int(h[3:]))]
|
|
3368
|
+
|
|
3369
|
+
|
|
3370
|
+
def probe_box(b, du=True, timeout=150):
|
|
3371
|
+
ssh = os.environ.get("DINGO_SSH", "ssh")
|
|
3372
|
+
src = open(os.path.realpath(__file__), encoding="utf-8").read()
|
|
3373
|
+
cmd = "python3 - _probe --data " + shlex.quote(b.get("data") or "/root/gpufree-data") + ("" if du else " --no-du")
|
|
3374
|
+
t0 = now()
|
|
3375
|
+
try:
|
|
3376
|
+
p = subprocess.run([ssh, "-o", "ConnectTimeout=10", "-o", "BatchMode=yes", b["host"], cmd],
|
|
3377
|
+
input=src, capture_output=True, text=True, timeout=timeout)
|
|
3378
|
+
rc, out, err = p.returncode, p.stdout, p.stderr
|
|
3379
|
+
except subprocess.TimeoutExpired:
|
|
3380
|
+
rc, out, err = None, "", f"no answer in {timeout}s"
|
|
3381
|
+
except OSError as e:
|
|
3382
|
+
rc, out, err = None, "", str(e)
|
|
3383
|
+
try:
|
|
3384
|
+
r = json.loads(out.strip().splitlines()[-1]) if rc == 0 and out.strip() else None
|
|
3385
|
+
except ValueError:
|
|
3386
|
+
r = None
|
|
3387
|
+
if r is None:
|
|
3388
|
+
return {"name": b["name"], "state": "down", "error": ((err or out).strip().splitlines() or ["?"])[-1][:160],
|
|
3389
|
+
"probed_at": now(), "took_s": round(now() - t0, 1)}
|
|
3390
|
+
want = b.get("gpus")
|
|
3391
|
+
if not r["gpus"]:
|
|
3392
|
+
st = "nogpu" if want != 0 else "ok"
|
|
3393
|
+
elif want and r["gpus"] < want:
|
|
3394
|
+
st = "fewer"
|
|
3395
|
+
else:
|
|
3396
|
+
st = "ok"
|
|
3397
|
+
r.update(name=b["name"], state=st, expected_gpus=want, probed_at=now(), took_s=round(now() - t0, 1))
|
|
3398
|
+
return r
|
|
3399
|
+
|
|
3400
|
+
|
|
3401
|
+
def boxes_log_path():
|
|
3402
|
+
return os.path.join(state_dir(), "boxes.log.jsonl")
|
|
3403
|
+
|
|
3404
|
+
|
|
3405
|
+
def boxes_log(rec):
|
|
3406
|
+
with open(boxes_log_path(), "a", encoding="utf-8") as f:
|
|
3407
|
+
f.write(json.dumps(rec, ensure_ascii=False) + "\n")
|
|
3408
|
+
|
|
3409
|
+
|
|
3410
|
+
def boxes_history(name=None):
|
|
3411
|
+
out = []
|
|
3412
|
+
try:
|
|
3413
|
+
with open(boxes_log_path(), encoding="utf-8") as f:
|
|
3414
|
+
for x in f:
|
|
3415
|
+
try:
|
|
3416
|
+
e = json.loads(x)
|
|
3417
|
+
except ValueError:
|
|
3418
|
+
continue
|
|
3419
|
+
if name is None or e.get("box") == name:
|
|
3420
|
+
out.append(e)
|
|
3421
|
+
except FileNotFoundError:
|
|
3422
|
+
pass
|
|
3423
|
+
return out
|
|
3424
|
+
|
|
3425
|
+
|
|
3426
|
+
def _gb(n):
|
|
3427
|
+
return f"{n / 2 ** 30:.0f}G" if n is not None else "?"
|
|
3428
|
+
|
|
3429
|
+
|
|
3430
|
+
def box_users(r):
|
|
3431
|
+
"""Who is on the box: claim lines (with their state) + cards computing without a claim line."""
|
|
3432
|
+
out = []
|
|
3433
|
+
covered = set()
|
|
3434
|
+
for c in r.get("claims") or []:
|
|
3435
|
+
mark = {"live": "", "open": "", "overdue": "⏰", "stale": "🟡疑似过期", "done": "✅待删"}.get(c["state"], "")
|
|
3436
|
+
out.append(f"{c['who']}[{c['cards']}]{mark}")
|
|
3437
|
+
if c["state"] in ("live", "open", "overdue"):
|
|
3438
|
+
covered |= set(c["cards"].split(","))
|
|
3439
|
+
for k, ps in sorted((r.get("gpu_procs") or {}).items()):
|
|
3440
|
+
if ps and k not in covered and "*" not in covered:
|
|
3441
|
+
out.append(f"卡{k}:{len(ps)}进程无登记")
|
|
3442
|
+
return out
|
|
3443
|
+
|
|
3444
|
+
|
|
3445
|
+
def box_row(r, hist):
|
|
3446
|
+
fails = [e for e in hist if e.get("box") == r["name"] and (e.get("kind") == "note" or e.get("state") in BOX_BAD)]
|
|
3447
|
+
last = fails[-1] if fails else None
|
|
3448
|
+
lastfail = (f"{hhmm(last['t'])} {last.get('text') or BOX_STATES.get(last.get('state'), '')}"[:60]) if last else ""
|
|
3449
|
+
if r["state"] == "down":
|
|
3450
|
+
return {"box": r["name"], "state": BOX_STATES["down"], "cards": "", "users": "", "disk": "", "dirs": "",
|
|
3451
|
+
"heartbeat": "", "last_fail": lastfail, "error": r.get("error", "")}
|
|
3452
|
+
d = (r.get("disks") or {}).get(r.get("data")) or {}
|
|
3453
|
+
w = r.get("watchdog") or {}
|
|
3454
|
+
hb = w.get("heartbeat_age_s")
|
|
3455
|
+
return {"box": r["name"], "state": BOX_STATES[r["state"]],
|
|
3456
|
+
"cards": f"{r['gpus']}/{r['expected_gpus'] if r.get('expected_gpus') is not None else '?'}",
|
|
3457
|
+
"users": " ".join(box_users(r)), "disk": f"{_gb(d.get('used'))}/{_gb(d.get('total'))}",
|
|
3458
|
+
"dirs": " ".join(f"{n} {_gb(s)}" for s, n in (r.get("du") or [])[:4]),
|
|
3459
|
+
"heartbeat": ("无看门狗" if not w.get("present") else "无心跳" if hb is None else f"{int(hb // 60)}min前"),
|
|
3460
|
+
"last_fail": lastfail, "error": r.get("gpu_error") or ""}
|
|
3461
|
+
|
|
3462
|
+
|
|
3463
|
+
def cmd_boxes(args):
|
|
3464
|
+
w = list(args.words or [])
|
|
3465
|
+
args.action = w.pop(0) if w and w[0] in ("show", "note", "history") else "show"
|
|
3466
|
+
args.names, args.text = (w[:1], " ".join(w[1:])) if args.action == "note" else (w, "")
|
|
3467
|
+
if args.action == "note":
|
|
3468
|
+
if not args.names or not args.text:
|
|
3469
|
+
eprint('dingo-pack boxes note BOX "what happened"')
|
|
3470
|
+
return EXIT_USAGE
|
|
3471
|
+
for n in args.names:
|
|
3472
|
+
boxes_log({"t": now(), "box": n, "kind": "note", "by": session_label(args), "text": args.text})
|
|
3473
|
+
print(f"noted for {', '.join(args.names)}")
|
|
3474
|
+
return EXIT_OK
|
|
3475
|
+
if args.action == "history":
|
|
3476
|
+
for e in boxes_history(args.names[0] if args.names else None):
|
|
3477
|
+
what = e.get("text") or (BOX_STATES.get(e.get("state"), e.get("state", "")) + (f" · {e['detail']}" if e.get("detail") else ""))
|
|
3478
|
+
print(f" {hhmm(e['t'])} {e.get('box', '?'):<7} {e.get('kind', ''):<5} {e.get('by', ''):<16} {what}")
|
|
3479
|
+
return EXIT_OK
|
|
3480
|
+
boxes = load_boxes(args)
|
|
3481
|
+
if args.names:
|
|
3482
|
+
boxes = [b for b in boxes if b["name"] in args.names] + \
|
|
3483
|
+
[{"name": n, "host": n, "gpus": None, "data": "/root/gpufree-data"}
|
|
3484
|
+
for n in args.names if n not in {b["name"] for b in boxes}]
|
|
3485
|
+
if not boxes:
|
|
3486
|
+
eprint(f"dingo-pack boxes: no boxes (write {boxes_file(args)} or add box* hosts to ~/.ssh/config)")
|
|
3487
|
+
return EXIT_USAGE
|
|
3488
|
+
cache_path = os.path.join(state_dir(), "boxes.json")
|
|
3489
|
+
while True:
|
|
3490
|
+
import concurrent.futures as cf
|
|
3491
|
+
with cf.ThreadPoolExecutor(max_workers=min(8, len(boxes))) as ex:
|
|
3492
|
+
res = list(ex.map(lambda b: probe_box(b, du=not args.no_du, timeout=args.ssh_timeout), boxes))
|
|
3493
|
+
cache = read_json(cache_path, {}) or {}
|
|
3494
|
+
for r in res:
|
|
3495
|
+
prev = (cache.get(r["name"]) or {}).get("state")
|
|
3496
|
+
if r["state"] != prev and (r["state"] in BOX_BAD or prev in BOX_BAD):
|
|
3497
|
+
detail = r.get("error") or r.get("gpu_error") or (
|
|
3498
|
+
f"{r.get('gpus')}/{r.get('expected_gpus')} 卡" if r["state"] == "fewer" else "")
|
|
3499
|
+
boxes_log({"t": now(), "box": r["name"], "kind": "auto", "by": "boxes", "state": r["state"],
|
|
3500
|
+
"prev": prev, "detail": detail})
|
|
3501
|
+
cache[r["name"]] = r
|
|
3502
|
+
write_atomic(cache_path, json.dumps(cache, ensure_ascii=False))
|
|
3503
|
+
hist = boxes_history()
|
|
3504
|
+
rows = [box_row(r, hist) for r in res]
|
|
3505
|
+
if args.json:
|
|
3506
|
+
print(json.dumps(res, ensure_ascii=False))
|
|
3507
|
+
else:
|
|
3508
|
+
if args.every:
|
|
3509
|
+
sys.stdout.write("\033[2J\033[H")
|
|
3510
|
+
print(f"dingo-pack boxes · {time.strftime('%H:%M')} · {sum(1 for r in res if r['state'] == 'ok')}/{len(res)} 在线有卡")
|
|
3511
|
+
print_boxes(rows)
|
|
3512
|
+
if not args.every:
|
|
3513
|
+
return EXIT_OK
|
|
3514
|
+
sys.stdout.flush()
|
|
3515
|
+
time.sleep(args.every)
|
|
3516
|
+
|
|
3517
|
+
|
|
3518
|
+
def print_boxes(rows):
|
|
3519
|
+
print(f" {'box':<7} {'状态':<6} {'卡':<5} {'数据盘':<10} {'看门狗':<8} 谁在用 · 主要目录 · 最近开机失败")
|
|
3520
|
+
for b in rows:
|
|
3521
|
+
extra = " · ".join(x for x in (b["users"], b["dirs"], (f"⚠️ {b['last_fail']}" if b["last_fail"] else ""),
|
|
3522
|
+
b["error"]) if x)
|
|
3523
|
+
print(f" {b['box']:<7} {b['state']:<6} {b['cards']:<5} {b['disk']:<10} {b['heartbeat']:<8} {extra}")
|
|
3524
|
+
|
|
3525
|
+
|
|
3526
|
+
# ---------------------------------------------------------------- top / overview
|
|
3527
|
+
|
|
3528
|
+
# ---------------------------------------------------------------- adapters (see ADAPTERS.md)
|
|
3529
|
+
# What dingo-pack needs to know about one kind of agent: its live sessions, its transcripts, its hooks.
|
|
3530
|
+
class ClaudeCodeAdapter:
|
|
3531
|
+
name = "claude-code"
|
|
3532
|
+
owner_env = "CLAUDE_PID"
|
|
3533
|
+
|
|
3534
|
+
def sessions(self):
|
|
3535
|
+
out = []
|
|
3536
|
+
for p in glob.glob(os.path.join(claude_home(), "sessions", "*.json")):
|
|
3537
|
+
e = read_json(p)
|
|
3538
|
+
if not isinstance(e, dict) or "pid" not in e or not pid_alive(e.get("pid"), e.get("procStart")):
|
|
3539
|
+
continue
|
|
3540
|
+
st = e.get("startedAt")
|
|
3541
|
+
out.append({"id": e.get("sessionId"), "pid": int(e["pid"]),
|
|
3542
|
+
"name": e.get("name") if e.get("nameSource") == "user" else None,
|
|
3543
|
+
"cwd": e.get("cwd"), "proc_start": e.get("procStart"), "status": e.get("status"),
|
|
3544
|
+
"started_at": st / 1000 if isinstance(st, (int, float)) and st > 1e11 else st,
|
|
3545
|
+
"fallback_name": e.get("name")})
|
|
3546
|
+
return out
|
|
3547
|
+
|
|
3548
|
+
def transcript(self, session_id):
|
|
3549
|
+
return find_transcript(session_id) if session_id else None
|
|
3550
|
+
|
|
3551
|
+
def subtranscripts(self, session_id, path):
|
|
3552
|
+
return glob.glob(os.path.join(os.path.dirname(path), session_id, "subagents", "*.jsonl")) if path else []
|
|
3553
|
+
|
|
3554
|
+
def usage(self, path, cache):
|
|
3555
|
+
return transcript_usage(path, cache) # keeps its state in cache[path]
|
|
3556
|
+
|
|
3557
|
+
def hooks(self):
|
|
3558
|
+
return "/plugin install dingo-pack@dingo-pack, or the hooks snippet in REFERENCE.md"
|
|
3559
|
+
|
|
3560
|
+
|
|
3561
|
+
BUILTIN_ADAPTERS = {"claude-code": ClaudeCodeAdapter}
|
|
3562
|
+
|
|
3563
|
+
|
|
3564
|
+
def load_adapters(spec=None):
|
|
3565
|
+
"""DINGO_AGENTS: comma-separated built-in names and paths to adapter files (default claude-code)."""
|
|
3566
|
+
out = []
|
|
3567
|
+
for item in (spec or os.environ.get("DINGO_AGENTS") or "claude-code").split(","):
|
|
3568
|
+
item = item.strip()
|
|
3569
|
+
if not item:
|
|
3570
|
+
continue
|
|
3571
|
+
if item in BUILTIN_ADAPTERS:
|
|
3572
|
+
out.append(BUILTIN_ADAPTERS[item]())
|
|
3573
|
+
continue
|
|
3574
|
+
path = os.path.expanduser(item)
|
|
3575
|
+
if not os.path.isfile(path):
|
|
3576
|
+
path = os.path.join(os.path.expanduser("~/.config/dingo-pack/adapters"), item + ".py")
|
|
3577
|
+
try:
|
|
3578
|
+
import importlib.util
|
|
3579
|
+
spec_ = importlib.util.spec_from_file_location("dingo_adapter_" + re.sub(r"\W", "_", item), path)
|
|
3580
|
+
mod = importlib.util.module_from_spec(spec_)
|
|
3581
|
+
spec_.loader.exec_module(mod)
|
|
3582
|
+
out.append(mod.ADAPTER)
|
|
3583
|
+
except Exception as e: # a broken adapter must not take the board down
|
|
3584
|
+
eprint(f"dingo-pack: adapter {item!r} not loaded: {e}")
|
|
3585
|
+
return out
|
|
3586
|
+
|
|
3587
|
+
|
|
3588
|
+
def adapter_usage(ad, sid, day, cache):
|
|
3589
|
+
"""Token totals of one session through its adapter (same shape as session_tokens), or None."""
|
|
3590
|
+
path = ad.transcript(sid)
|
|
3591
|
+
if not path:
|
|
3592
|
+
return None
|
|
3593
|
+
key = lambda p: f"{getattr(ad, 'name', '?')}:{p}"
|
|
3594
|
+
sub_cache = cache.setdefault(key(path), {})
|
|
3595
|
+
c = ad.usage(path, sub_cache)
|
|
3596
|
+
tot = list((c or {}).get("days", {}).get(day, [0] * 5)) if c else [0] * 5
|
|
3597
|
+
sub = [0] * 5
|
|
3598
|
+
for sp_ in ad.subtranscripts(sid, path) or []:
|
|
3599
|
+
sc = ad.usage(sp_, cache.setdefault(key(sp_), {}))
|
|
3600
|
+
if sc:
|
|
3601
|
+
sub = [a + b for a, b in zip(sub, sc.get("days", {}).get(day, [0] * 5))]
|
|
3602
|
+
return {"day": day, "input": tot[0], "cache_write": tot[1], "cache_read": tot[2], "output": tot[3], "turns": tot[4],
|
|
3603
|
+
"last_input": (c or {}).get("last_input"), "sub": sum(sub[:4])}
|
|
3604
|
+
|
|
3605
|
+
|
|
3606
|
+
ADAPTER_KEYS = {"id": str, "pid": int}
|
|
3607
|
+
|
|
3608
|
+
|
|
3609
|
+
def cmd_adapter(args):
|
|
3610
|
+
"""Check adapters against the live machine: shapes, live pids, transcripts, usage numbers, read-only."""
|
|
3611
|
+
bad, n = [], 0
|
|
3612
|
+
for ad in load_adapters(args.agents):
|
|
3613
|
+
nm = getattr(ad, "name", "?")
|
|
3614
|
+
for attr in ("name", "owner_env", "sessions", "transcript", "subtranscripts", "usage", "hooks"):
|
|
3615
|
+
if not hasattr(ad, attr):
|
|
3616
|
+
bad.append(f"{nm}: missing {attr}")
|
|
3617
|
+
if any(b.startswith(f"{nm}: missing") for b in bad):
|
|
3618
|
+
continue
|
|
3619
|
+
try:
|
|
3620
|
+
ss = ad.sessions()
|
|
3621
|
+
except Exception as e:
|
|
3622
|
+
bad.append(f"{nm}: sessions() raised {e!r}")
|
|
3623
|
+
continue
|
|
3624
|
+
if not isinstance(ss, list):
|
|
3625
|
+
bad.append(f"{nm}: sessions() must return a list")
|
|
3626
|
+
continue
|
|
3627
|
+
for s in ss:
|
|
3628
|
+
n += 1
|
|
3629
|
+
if not isinstance(s, dict):
|
|
3630
|
+
bad.append(f"{nm}: a session is not a dict: {s!r}"[:120])
|
|
3631
|
+
continue
|
|
3632
|
+
for k, t in ADAPTER_KEYS.items():
|
|
3633
|
+
if not isinstance(s.get(k), t) or isinstance(s.get(k), bool):
|
|
3634
|
+
bad.append(f"{nm}: session {s.get('id')!r}: {k} must be {t.__name__}, got {s.get(k)!r}"[:160])
|
|
3635
|
+
if isinstance(s.get("pid"), int) and not pid_alive(s["pid"], s.get("proc_start")):
|
|
3636
|
+
bad.append(f"{nm}: session {s.get('id')!r}: pid {s['pid']} is not alive")
|
|
3637
|
+
path = ad.transcript(s.get("id")) if isinstance(s.get("id"), str) else None
|
|
3638
|
+
if path is None:
|
|
3639
|
+
continue
|
|
3640
|
+
if not os.path.isfile(path):
|
|
3641
|
+
bad.append(f"{nm}: session {s.get('id')!r}: transcript {path} does not exist")
|
|
3642
|
+
continue
|
|
3643
|
+
before = (os.path.getmtime(path), os.path.getsize(path))
|
|
3644
|
+
quiet = now() - before[0] > 60 # a transcript being written right now cannot show a read-only breach
|
|
3645
|
+
try:
|
|
3646
|
+
u = ad.usage(path, {})
|
|
3647
|
+
except Exception as e:
|
|
3648
|
+
bad.append(f"{nm}: usage({path}) raised {e!r}")
|
|
3649
|
+
continue
|
|
3650
|
+
if u is not None:
|
|
3651
|
+
days = u.get("days") if isinstance(u, dict) else None
|
|
3652
|
+
if not isinstance(days, dict) or any(not isinstance(v, list) or len(v) != 5 or
|
|
3653
|
+
any(not isinstance(x, int) or x < 0 for x in v) for v in days.values()):
|
|
3654
|
+
bad.append(f"{nm}: usage({os.path.basename(path)}): days must map YYYY-MM-DD to 5 non-negative ints")
|
|
3655
|
+
if quiet and (os.path.getmtime(path), os.path.getsize(path)) != before:
|
|
3656
|
+
bad.append(f"{nm}: reading {os.path.basename(path)} changed it (adapters must only read)")
|
|
3657
|
+
for b in bad:
|
|
3658
|
+
print(f" ✗ {b}")
|
|
3659
|
+
print(f"adapter test: {n} session(s) checked, {len(bad)} problem(s)")
|
|
3660
|
+
return EXIT_OK if not bad else 1
|
|
3661
|
+
|
|
3662
|
+
|
|
3663
|
+
# ---------------------------------------------------------------- task manager: children, waits, tokens
|
|
3664
|
+
CTX_WARN = int(os.environ.get("DINGO_CTX_WARN", "600000")) # last turn's input tokens => "context long, start fresh"
|
|
3665
|
+
SPIN_H = float(os.environ.get("DINGO_SPIN_H", "3")) # a wait older than this whose target never changed
|
|
3666
|
+
AGENT_OUT_H = float(os.environ.get("DINGO_AGENT_OUT_H", "1")) # a waited-on sub-agent output untouched this long
|
|
3667
|
+
CHILD_MIN_S = 60
|
|
3668
|
+
LOOP_RE = re.compile(r"\b(until|while)\s+(.+?);\s*do\b(.*?)\bdone\b", re.S)
|
|
3669
|
+
CLK = os.sysconf("SC_CLK_TCK") if hasattr(os, "sysconf") else 100
|
|
3670
|
+
|
|
3671
|
+
|
|
3672
|
+
def proc_ticks(pid):
|
|
3673
|
+
try:
|
|
3674
|
+
with open(f"/proc/{pid}/stat") as f:
|
|
3675
|
+
st = f.read()
|
|
3676
|
+
x = st[st.rindex(")") + 2:].split()
|
|
3677
|
+
return int(x[11]) + int(x[12])
|
|
3678
|
+
except (OSError, ValueError, IndexError):
|
|
3679
|
+
return None
|
|
3680
|
+
|
|
3681
|
+
|
|
3682
|
+
def proc_age(pid):
|
|
3683
|
+
st, bt = proc_start(pid), boot_time()
|
|
3684
|
+
return now() - (bt + st / CLK) if st is not None and bt else None
|
|
3685
|
+
|
|
3686
|
+
|
|
3687
|
+
def cpu_percent(pids, dt):
|
|
3688
|
+
t0 = {p: proc_ticks(p) for p in pids}
|
|
3689
|
+
time.sleep(dt)
|
|
3690
|
+
out = {}
|
|
3691
|
+
for p in pids:
|
|
3692
|
+
t1 = proc_ticks(p)
|
|
3693
|
+
out[p] = round((t1 - t0[p]) / CLK / dt * 100) if t1 is not None and t0[p] is not None and dt > 0 else None
|
|
3694
|
+
return out
|
|
3695
|
+
|
|
3696
|
+
|
|
3697
|
+
def _unquote(s):
|
|
3698
|
+
s = s.strip()
|
|
3699
|
+
return s[1:-1] if len(s) >= 2 and s[0] == s[-1] and s[0] in "'\"" else s
|
|
3700
|
+
|
|
3701
|
+
|
|
3702
|
+
def parse_wait(cmd):
|
|
3703
|
+
"""A waiting loop in a command line -> {"remote", "kind" (until/while), "cond", "targets": [...]}, or None.
|
|
3704
|
+
targets: {"what": file / grep / pgrep / tmux / pid / other, "arg", "pat", "neg"}."""
|
|
3705
|
+
text = cmd.replace("'\"'\"'", "'")
|
|
3706
|
+
m = LOOP_RE.search(text)
|
|
3707
|
+
if not m or "sleep" not in m.group(3):
|
|
3708
|
+
return None
|
|
3709
|
+
head = text[:m.start()]
|
|
3710
|
+
remote = None
|
|
3711
|
+
r = list(re.finditer(r"(?:^|[\s;&|(])ssh\s", head))
|
|
3712
|
+
if r:
|
|
3713
|
+
toks = head[r[-1].end():].split()
|
|
3714
|
+
i = 0
|
|
3715
|
+
while i < len(toks) and toks[i].startswith("-"):
|
|
3716
|
+
i += 2 if len(toks[i]) == 2 and toks[i][1] in "bcDEeFIiJLlmOopQRSWw" else 1
|
|
3717
|
+
if i < len(toks):
|
|
3718
|
+
remote = toks[i].strip("'\"")
|
|
3719
|
+
vars_ = dict(re.findall(r"\b([A-Za-z_]\w*)=([^\s;'\"]+)", head))
|
|
3720
|
+
cond = m.group(2)
|
|
3721
|
+
for k, v in vars_.items():
|
|
3722
|
+
cond = re.sub(r"\$\{?" + k + r"\b\}?", v, cond)
|
|
3723
|
+
targets = []
|
|
3724
|
+
for clause in re.split(r"\|\||&&", cond):
|
|
3725
|
+
c = re.sub(r"\s*(?:\d|&)?>>?&?\s*\S+", "", clause).strip() # drop redirections (>/dev/null, 2>&1)
|
|
3726
|
+
neg = c.startswith("!")
|
|
3727
|
+
c = c.lstrip("! ").strip()
|
|
3728
|
+
t = None
|
|
3729
|
+
if re.match(r"(\[|test)\s+-[efsd]\s", c):
|
|
3730
|
+
t = {"what": "file", "arg": _unquote(re.sub(r"^(\[|test)\s+-[efsd]\s+", "", c).rstrip("] ").strip())}
|
|
3731
|
+
elif c.startswith("ls "):
|
|
3732
|
+
t = {"what": "file", "arg": _unquote(c.split()[-1])}
|
|
3733
|
+
elif re.match(r"grep\s", c):
|
|
3734
|
+
parts = shlex.split(c, posix=True) if c.count("'") % 2 == 0 and c.count('"') % 2 == 0 else c.split()
|
|
3735
|
+
args_ = [a for a in parts[1:] if not a.startswith("-")]
|
|
3736
|
+
if len(args_) >= 2:
|
|
3737
|
+
t = {"what": "grep", "pat": args_[0], "arg": args_[-1]}
|
|
3738
|
+
elif re.match(r"pgrep\s", c):
|
|
3739
|
+
parts = shlex.split(c, posix=True) if c.count("'") % 2 == 0 and c.count('"') % 2 == 0 else c.split()
|
|
3740
|
+
args_ = [a for a in parts[1:] if not a.startswith("-")]
|
|
3741
|
+
if args_:
|
|
3742
|
+
t = {"what": "pgrep", "pat": args_[-1], "full": "-f" in parts or any(a.startswith("-") and "f" in a for a in parts[1:])}
|
|
3743
|
+
elif "tmux has-session" in c:
|
|
3744
|
+
s = re.search(r"-t\s+=?(\S+)", c)
|
|
3745
|
+
t = {"what": "tmux", "arg": s.group(1).lstrip("=") if s else "?"}
|
|
3746
|
+
elif re.match(r"kill\s+-0\s+\d+", c):
|
|
3747
|
+
t = {"what": "pid", "arg": c.split()[-1]}
|
|
3748
|
+
if t is None:
|
|
3749
|
+
t = {"what": "other", "arg": c[:60]}
|
|
3750
|
+
t["neg"] = neg
|
|
3751
|
+
targets.append(t)
|
|
3752
|
+
to = re.search(r"\btimeout\s+(\d+[smhd]?)\s", head)
|
|
3753
|
+
return {"remote": remote, "kind": m.group(1), "cond": cond.strip()[:200], "targets": targets,
|
|
3754
|
+
"loop": text[m.start():m.end()], "timeout": to.group(1) if to else None}
|
|
3755
|
+
|
|
3756
|
+
|
|
3757
|
+
def wait_text(w):
|
|
3758
|
+
"""'等 box14 上:TRAIN_DONE 出现 / tmux gpu-v199 结束' -- what the loop waits for, in words."""
|
|
3759
|
+
bits = []
|
|
3760
|
+
for t in w["targets"]:
|
|
3761
|
+
# until X: waits for X to become true; while X: waits for X to become false
|
|
3762
|
+
want_true = (w["kind"] == "until") != t["neg"]
|
|
3763
|
+
if t["what"] == "file":
|
|
3764
|
+
bits.append(f"{os.path.basename(t['arg']) or t['arg']} {'出现' if want_true else '消失'}")
|
|
3765
|
+
elif t["what"] == "grep":
|
|
3766
|
+
bits.append(f"{os.path.basename(t['arg'])} 里出现「{t['pat'][:20]}」" if want_true else f"{os.path.basename(t['arg'])} 里「{t['pat'][:20]}」消失")
|
|
3767
|
+
elif t["what"] == "pgrep":
|
|
3768
|
+
bits.append(f"进程「{t['pat'][:30]}」{'起来' if want_true else '结束'}")
|
|
3769
|
+
elif t["what"] == "tmux":
|
|
3770
|
+
bits.append(f"tmux {t['arg']} {'起来' if want_true else '结束'}")
|
|
3771
|
+
elif t["what"] == "pid":
|
|
3772
|
+
bits.append(f"pid {t['arg']} {'在' if want_true else '结束'}")
|
|
3773
|
+
else:
|
|
3774
|
+
bits.append(f"条件 {t['arg']}")
|
|
3775
|
+
return (f"等 {w['remote']} 上:" if w["remote"] else "等 ") + " / ".join(bits)
|
|
3776
|
+
|
|
3777
|
+
|
|
3778
|
+
def display_cmd(cmd):
|
|
3779
|
+
"""The command a Claude Code Bash call runs, without its shell-snapshot wrapper."""
|
|
3780
|
+
m = re.search(r"&& eval '(.*?)' \\?< /dev/null", cmd, re.S)
|
|
3781
|
+
return m.group(1).replace("'\"'\"'", "'") if m and "shell-snapshots" in cmd else cmd
|
|
3782
|
+
|
|
3783
|
+
|
|
3784
|
+
def chain_pids(pid, table):
|
|
3785
|
+
"""pid, its ancestors, and its descendants."""
|
|
3786
|
+
out = set(tree_pids(pid, table))
|
|
3787
|
+
p = pid
|
|
3788
|
+
while p in table and p not in (0, 1):
|
|
3789
|
+
out.add(p)
|
|
3790
|
+
p = table[p][0]
|
|
3791
|
+
return out
|
|
3792
|
+
|
|
3793
|
+
|
|
3794
|
+
def spin_reason(w, pid, age, table, cmdlines):
|
|
3795
|
+
"""Why a waiting loop looks like it will never end, or None (the 10-05 cases, made checkable)."""
|
|
3796
|
+
for t in w["targets"]:
|
|
3797
|
+
waits_for_end = (w["kind"] == "while") != t["neg"]
|
|
3798
|
+
if t["what"] == "pgrep" and waits_for_end and t.get("full"):
|
|
3799
|
+
try:
|
|
3800
|
+
rx = re.compile(t["pat"])
|
|
3801
|
+
except re.error:
|
|
3802
|
+
continue
|
|
3803
|
+
# the shell running the loop has the loop in its own command line: pgrep -f always finds it
|
|
3804
|
+
if rx.search(w["loop"]):
|
|
3805
|
+
cut = f"(外面有 timeout {w['timeout']},到时会被掐断)" if w.get("timeout") else ""
|
|
3806
|
+
return f"永远等不完:pgrep -f「{t['pat'][:30]}」会匹配到跑这个循环的 shell 自己{cut}"
|
|
3807
|
+
if w["remote"]:
|
|
3808
|
+
return None
|
|
3809
|
+
files = [t for t in w["targets"] if t["what"] in ("file", "grep")]
|
|
3810
|
+
if not files or age is None:
|
|
3811
|
+
return None
|
|
3812
|
+
start = now() - age
|
|
3813
|
+
for t in files:
|
|
3814
|
+
path = t["arg"]
|
|
3815
|
+
if "/tasks/" in path and path.endswith(".output"):
|
|
3816
|
+
try:
|
|
3817
|
+
idle = now() - os.path.getmtime(path)
|
|
3818
|
+
except OSError:
|
|
3819
|
+
idle = None
|
|
3820
|
+
if idle is not None and idle > AGENT_OUT_H * 3600:
|
|
3821
|
+
return f"等的子 agent 可能早停了:{os.path.basename(path)} 已 {idle / 3600:.1f} h 没变"
|
|
3822
|
+
if age < SPIN_H * 3600:
|
|
3823
|
+
return None
|
|
3824
|
+
for t in files:
|
|
3825
|
+
# the target itself, or anything next to it (a DONE flag appears only at the end; the job's
|
|
3826
|
+
# logs and checkpoints beside it show it is alive)
|
|
3827
|
+
d = os.path.dirname(t["arg"].split("*")[0]) or "."
|
|
3828
|
+
try:
|
|
3829
|
+
near = [d] + [os.path.join(d, x) for x in os.listdir(d)[:500]]
|
|
3830
|
+
except OSError:
|
|
3831
|
+
near = []
|
|
3832
|
+
if any(os.path.getmtime(x) >= start for x in near if os.path.exists(x)):
|
|
3833
|
+
return None
|
|
3834
|
+
return f"可能空转:等了 {age / 3600:.1f} h,等的文件和它所在目录从开始等到现在都没变过"
|
|
3835
|
+
|
|
3836
|
+
|
|
3837
|
+
def session_tasks(sample_s=None, show_all=False):
|
|
3838
|
+
"""Live sessions with their child processes, waiting loops and spin flags."""
|
|
3839
|
+
if not os.path.isdir("/proc/self"):
|
|
3840
|
+
return []
|
|
3841
|
+
table = proc_table()
|
|
3842
|
+
cmdlines = {p: proc_cmdline(p) for p in table}
|
|
3843
|
+
reg = {}
|
|
3844
|
+
adapters = load_adapters()
|
|
3845
|
+
for ad in adapters:
|
|
3846
|
+
try:
|
|
3847
|
+
for e in ad.sessions():
|
|
3848
|
+
if isinstance(e, dict) and isinstance(e.get("pid"), int):
|
|
3849
|
+
reg[e["pid"]] = (ad, e)
|
|
3850
|
+
except Exception as ex:
|
|
3851
|
+
eprint(f"dingo-pack: adapter {getattr(ad, 'name', '?')}: sessions() failed: {ex}")
|
|
3852
|
+
# children that left the tree (nohup / setsid) still carry the agent's pid in an env var (CLAUDE_PID)
|
|
3853
|
+
envs = sorted({ad.owner_env.encode() for ad in adapters if getattr(ad, "owner_env", None)})
|
|
3854
|
+
owner = {}
|
|
3855
|
+
for p in table:
|
|
3856
|
+
if p in reg or not envs:
|
|
3857
|
+
continue
|
|
3858
|
+
env_ = proc_environ(p, tuple(envs))
|
|
3859
|
+
for k in envs:
|
|
3860
|
+
cp = env_.get(k.decode())
|
|
3861
|
+
if cp and cp.isdigit() and int(cp) in reg and reg[int(cp)][0].owner_env == k.decode():
|
|
3862
|
+
owner[p] = int(cp)
|
|
3863
|
+
for cp in reg:
|
|
3864
|
+
for p in tree_pids(cp, table):
|
|
3865
|
+
if p != cp:
|
|
3866
|
+
owner.setdefault(p, cp)
|
|
3867
|
+
sample = float(os.environ.get("DINGO_SAMPLE_S", sample_s if sample_s is not None else 1.0))
|
|
3868
|
+
tcache = read_json(_tok_cache_path(), {}) or {}
|
|
3869
|
+
cpu = cpu_percent(list(owner), sample) if owner else {}
|
|
3870
|
+
roster = (load_roster().get("sessions") or {})
|
|
3871
|
+
out = []
|
|
3872
|
+
day = time.strftime("%Y-%m-%d")
|
|
3873
|
+
for cp, (ad, e) in reg.items():
|
|
3874
|
+
sid = e.get("id")
|
|
3875
|
+
name = e.get("name") or (roster.get(sid) or {}).get("name") or e.get("fallback_name") or str(cp)
|
|
3876
|
+
kids, waits, seen = [], [], set()
|
|
3877
|
+
for p in sorted(q for q, o in owner.items() if o == cp):
|
|
3878
|
+
age = proc_age(p)
|
|
3879
|
+
cmd = cmdlines.get(p, "")
|
|
3880
|
+
w = parse_wait(cmd)
|
|
3881
|
+
if w:
|
|
3882
|
+
key = (w["remote"], w["cond"])
|
|
3883
|
+
if key not in seen:
|
|
3884
|
+
seen.add(key)
|
|
3885
|
+
waits.append(dict(w, pid=p, age_s=age, text=wait_text(w), spin=spin_reason(w, p, age, table, cmdlines)))
|
|
3886
|
+
continue
|
|
3887
|
+
if not show_all and (age is None or age < CHILD_MIN_S):
|
|
3888
|
+
continue
|
|
3889
|
+
if re.match(r"(sleep|cat|tee|head|tail)\b|ssh -W ", cmd) or not cmd:
|
|
3890
|
+
continue
|
|
3891
|
+
kids.append({"pid": p, "cmd": display_cmd(cmd)[:80], "age_s": age, "cpu": cpu.get(p), "pss": proc_pss(p)})
|
|
3892
|
+
try:
|
|
3893
|
+
tok = adapter_usage(ad, sid, day, tcache) if sid else None
|
|
3894
|
+
except Exception as ex:
|
|
3895
|
+
eprint(f"dingo-pack: adapter {getattr(ad, 'name', '?')}: usage failed: {ex}")
|
|
3896
|
+
tok = None
|
|
3897
|
+
out.append({"name": name, "agent": getattr(ad, "name", "?"), "pid": cp, "sid": sid, "children": kids, "waits": waits, "tokens": tok,
|
|
3898
|
+
"ctx_long": bool(tok and (tok.get("last_input") or 0) > CTX_WARN)})
|
|
3899
|
+
try:
|
|
3900
|
+
write_atomic(_tok_cache_path(), json.dumps(tcache))
|
|
3901
|
+
except OSError:
|
|
3902
|
+
pass
|
|
3903
|
+
return sorted(out, key=lambda s: (not any(w["spin"] for w in s["waits"]), not s["ctx_long"], s["name"]))
|
|
3904
|
+
|
|
3905
|
+
|
|
3906
|
+
# --- tokens: Claude Code transcripts carry message.usage on each assistant line (one message is often
|
|
3907
|
+
# written several times with the same id: count it once). Read incrementally; cache under the state dir.
|
|
3908
|
+
def _tok_cache_path():
|
|
3909
|
+
return os.path.join(state_dir(), "tokens.json")
|
|
3910
|
+
|
|
3911
|
+
|
|
3912
|
+
def _local_day(ts):
|
|
3913
|
+
import datetime as dt
|
|
3914
|
+
try:
|
|
3915
|
+
return dt.datetime.fromisoformat(ts.replace("Z", "+00:00")).astimezone().strftime("%Y-%m-%d")
|
|
3916
|
+
except (ValueError, AttributeError):
|
|
3917
|
+
return None
|
|
3918
|
+
|
|
3919
|
+
|
|
3920
|
+
def transcript_usage(path, cache):
|
|
3921
|
+
try:
|
|
3922
|
+
st = os.stat(path)
|
|
3923
|
+
except OSError:
|
|
3924
|
+
return None
|
|
3925
|
+
c = cache.get(path)
|
|
3926
|
+
if not c or c.get("ino") != st.st_ino or c.get("off", 0) > st.st_size:
|
|
3927
|
+
c = {"ino": st.st_ino, "off": 0, "days": {}, "last_input": None, "last_t": None, "ids": [], "day": None}
|
|
3928
|
+
if c["off"] < st.st_size:
|
|
3929
|
+
with open(path, "rb") as f:
|
|
3930
|
+
f.seek(c["off"])
|
|
3931
|
+
data = f.read()
|
|
3932
|
+
end = data.rfind(b"\n") + 1
|
|
3933
|
+
c["off"] += end
|
|
3934
|
+
for line in data[:end].splitlines():
|
|
3935
|
+
if b'"usage"' not in line or b'"assistant"' not in line:
|
|
3936
|
+
continue
|
|
3937
|
+
try:
|
|
3938
|
+
e = json.loads(line)
|
|
3939
|
+
except ValueError:
|
|
3940
|
+
continue
|
|
3941
|
+
m = e.get("message") or {}
|
|
3942
|
+
u = m.get("usage")
|
|
3943
|
+
if e.get("type") != "assistant" or not isinstance(u, dict):
|
|
3944
|
+
continue
|
|
3945
|
+
mid = m.get("id") or e.get("uuid")
|
|
3946
|
+
if mid in c["ids"]:
|
|
3947
|
+
continue
|
|
3948
|
+
c["ids"] = (c["ids"] + [mid])[-30:]
|
|
3949
|
+
day = _local_day(e.get("timestamp")) or c.get("day") or time.strftime("%Y-%m-%d", time.localtime(st.st_mtime))
|
|
3950
|
+
c["day"] = day
|
|
3951
|
+
v = [int(u.get("input_tokens") or 0), int(u.get("cache_creation_input_tokens") or 0),
|
|
3952
|
+
int(u.get("cache_read_input_tokens") or 0), int(u.get("output_tokens") or 0)]
|
|
3953
|
+
d = c["days"].setdefault(day, [0, 0, 0, 0, 0])
|
|
3954
|
+
for i in range(4):
|
|
3955
|
+
d[i] += v[i]
|
|
3956
|
+
d[4] += 1
|
|
3957
|
+
c["last_input"] = v[0] + v[1] + v[2]
|
|
3958
|
+
c["last_t"] = e.get("timestamp")
|
|
3959
|
+
cache[path] = c
|
|
3960
|
+
return c
|
|
3961
|
+
|
|
3962
|
+
|
|
3963
|
+
def session_tokens(sid, day=None, cache=None, adapter=None):
|
|
3964
|
+
"""{"day", "input", "cache_write", "cache_read", "output", "turns", "last_input", "sub"} or None (no transcript)."""
|
|
3965
|
+
own = cache is None
|
|
3966
|
+
cache = cache if cache is not None else (read_json(_tok_cache_path(), {}) or {})
|
|
3967
|
+
t = adapter_usage(adapter or ClaudeCodeAdapter(), sid, day or time.strftime("%Y-%m-%d"), cache)
|
|
3968
|
+
if own:
|
|
3969
|
+
try:
|
|
3970
|
+
write_atomic(_tok_cache_path(), json.dumps(cache))
|
|
3971
|
+
except OSError:
|
|
3972
|
+
pass
|
|
3973
|
+
return t
|
|
3974
|
+
|
|
3975
|
+
|
|
3976
|
+
def fmt_tok(n):
|
|
3977
|
+
if n is None:
|
|
3978
|
+
return "读不到"
|
|
3979
|
+
return f"{n / 1e6:.1f}M" if n >= 1e6 else f"{n / 1e3:.0f}k"
|
|
3980
|
+
|
|
3981
|
+
|
|
3982
|
+
def cmd_tokens(args):
|
|
3983
|
+
import datetime as dt
|
|
3984
|
+
day = args.day or time.strftime("%Y-%m-%d")
|
|
3985
|
+
try:
|
|
3986
|
+
t0 = time.mktime(dt.datetime.strptime(day, "%Y-%m-%d").timetuple())
|
|
3987
|
+
except ValueError:
|
|
3988
|
+
eprint("dingo-pack tokens: --day YYYY-MM-DD")
|
|
3989
|
+
return EXIT_USAGE
|
|
3990
|
+
cache = read_json(_tok_cache_path(), {}) or {}
|
|
3991
|
+
roster = (load_roster().get("sessions") or {})
|
|
3992
|
+
rows = []
|
|
3993
|
+
for path in glob.glob(os.path.join(claude_home(), "projects", "*", "*.jsonl")):
|
|
3994
|
+
try:
|
|
3995
|
+
if os.path.getmtime(path) < t0:
|
|
3996
|
+
continue
|
|
3997
|
+
except OSError:
|
|
3998
|
+
continue
|
|
3999
|
+
sid = os.path.splitext(os.path.basename(path))[0]
|
|
4000
|
+
t = session_tokens(sid, day, cache)
|
|
4001
|
+
if t and (t["turns"] or t["sub"]):
|
|
4002
|
+
rows.append(((roster.get(sid) or {}).get("name") or sid[:8], t))
|
|
4003
|
+
for ad in load_adapters(): # other agents: their live sessions
|
|
4004
|
+
if getattr(ad, "name", "") == "claude-code":
|
|
4005
|
+
continue
|
|
4006
|
+
try:
|
|
4007
|
+
for e in ad.sessions():
|
|
4008
|
+
t = adapter_usage(ad, e.get("id"), day, cache)
|
|
4009
|
+
if t and (t["turns"] or t["sub"]):
|
|
4010
|
+
rows.append((f"{e.get('name') or e.get('id')} ({ad.name})", t))
|
|
4011
|
+
except Exception as ex:
|
|
4012
|
+
eprint(f"dingo-pack tokens: adapter {getattr(ad, 'name', '?')}: {ex}")
|
|
4013
|
+
write_atomic(_tok_cache_path(), json.dumps(cache))
|
|
4014
|
+
if args.json:
|
|
4015
|
+
print(json.dumps([dict(t, name=n) for n, t in rows], ensure_ascii=False))
|
|
4016
|
+
return EXIT_OK
|
|
4017
|
+
rows.sort(key=lambda r: -(r[1]["input"] + r[1]["cache_write"] + r[1]["cache_read"] + r[1]["output"]))
|
|
4018
|
+
print(f"dingo-pack tokens · {day} · {len(rows)} sessions (main transcript; sub-agents in their own column)")
|
|
4019
|
+
print(f" {'session':<28} {'input':>7} {'cache写':>8} {'cache读':>8} {'output':>7} {'turns':>6} {'子agent':>8} {'上一轮输入':>9}")
|
|
4020
|
+
tot = [0] * 6
|
|
4021
|
+
for n, t in rows:
|
|
4022
|
+
v = [t["input"], t["cache_write"], t["cache_read"], t["output"], t["turns"], t["sub"]]
|
|
4023
|
+
tot = [a + b for a, b in zip(tot, v)]
|
|
4024
|
+
print(f" {n[:28]:<28} {fmt_tok(v[0]):>7} {fmt_tok(v[1]):>8} {fmt_tok(v[2]):>8} {fmt_tok(v[3]):>7} {v[4]:>6} "
|
|
4025
|
+
f"{fmt_tok(v[5]):>8} {fmt_tok(t['last_input']):>9}" + (" ⚠️ 上下文偏长" if (t["last_input"] or 0) > CTX_WARN else ""))
|
|
4026
|
+
print(f" {'合计':<28} {fmt_tok(tot[0]):>7} {fmt_tok(tot[1]):>8} {fmt_tok(tot[2]):>8} {fmt_tok(tot[3]):>7} {tot[4]:>6} {fmt_tok(tot[5]):>8}")
|
|
4027
|
+
return EXIT_OK
|
|
4028
|
+
|
|
4029
|
+
|
|
4030
|
+
def fmt_age(s):
|
|
4031
|
+
if s is None:
|
|
4032
|
+
return "?"
|
|
4033
|
+
return f"{s / 3600:.1f}h" if s >= 3600 else f"{s / 60:.0f}m"
|
|
4034
|
+
|
|
4035
|
+
|
|
4036
|
+
def print_tasks(rows):
|
|
4037
|
+
for s in rows:
|
|
4038
|
+
t = s["tokens"]
|
|
4039
|
+
flags = []
|
|
4040
|
+
if any(w["spin"] for w in s["waits"]):
|
|
4041
|
+
flags.append("🌀空转?")
|
|
4042
|
+
if s["ctx_long"]:
|
|
4043
|
+
flags.append("📜上下文偏长")
|
|
4044
|
+
tok = (f"今天 {fmt_tok(t['input'] + t['cache_write'] + t['cache_read'] + t['output'])} · 上一轮输入 {fmt_tok(t['last_input'])}"
|
|
4045
|
+
if t else "token 读不到")
|
|
4046
|
+
print(f" {s['name'][:30]:<30} {tok} {' '.join(flags)}")
|
|
4047
|
+
for k in s["children"]:
|
|
4048
|
+
print(f" ├ {k['pid']:>7} {fmt_age(k['age_s']):>5} cpu {str(k['cpu']) + '%' if k['cpu'] is not None else '?':>5} "
|
|
4049
|
+
f"{fmt_size(k['pss']):>6} {k['cmd'][:70]}")
|
|
4050
|
+
for w in s["waits"]:
|
|
4051
|
+
print(f" ⏳ {w['pid']:>7} {fmt_age(w['age_s']):>5} {w['text'][:90]}")
|
|
4052
|
+
if w["spin"]:
|
|
4053
|
+
print(f" 🌀 {w['spin']}")
|
|
4054
|
+
|
|
4055
|
+
|
|
4056
|
+
def session_memory():
|
|
4057
|
+
"""[(name, claude_pid, pss of the session's whole tree, n processes)] for live sessions."""
|
|
4058
|
+
table = proc_table()
|
|
4059
|
+
out = []
|
|
4060
|
+
for p in glob.glob(os.path.join(claude_home(), "sessions", "*.json")):
|
|
4061
|
+
e = read_json(p)
|
|
4062
|
+
if not isinstance(e, dict) or not pid_alive(e.get("pid"), e.get("procStart")):
|
|
4063
|
+
continue
|
|
4064
|
+
pid = int(e["pid"])
|
|
4065
|
+
pids = tree_pids(pid, table)
|
|
4066
|
+
name = e.get("name") if e.get("nameSource") == "user" else None
|
|
4067
|
+
if not name:
|
|
4068
|
+
name = ((load_roster().get("sessions") or {}).get(e.get("sessionId")) or {}).get("name") or e.get("name")
|
|
4069
|
+
out.append((name or str(pid), pid, tree_rss(pid, table), len(pids)))
|
|
4070
|
+
return sorted(out, key=lambda x: -x[2])
|
|
4071
|
+
|
|
4072
|
+
|
|
4073
|
+
def cmd_top(args):
|
|
4074
|
+
while True:
|
|
4075
|
+
rows = session_memory()
|
|
4076
|
+
total, avail = meminfo()
|
|
4077
|
+
table = proc_table()
|
|
4078
|
+
inside = set()
|
|
4079
|
+
for _, pid, _, _ in rows:
|
|
4080
|
+
inside.update(tree_pids(pid, table))
|
|
4081
|
+
others = sorted(((rss, p) for p, (_, rss) in table.items() if p not in inside), reverse=True)[:5]
|
|
4082
|
+
g = ghost_summary()
|
|
4083
|
+
out = [f"dingo-pack top · {time.strftime('%H:%M:%S')} · free {fmt_size(avail)} of {fmt_size(total)}"
|
|
4084
|
+
f" · sessions {fmt_size(sum(r[2] for r in rows))} · ghosts {g.get('n', '?')} ({fmt_size(g.get('pss'))})"]
|
|
4085
|
+
out.append(f" {'session':<26} {'memory':>8} {'procs':>6}")
|
|
4086
|
+
for name, pid, mem, n in rows:
|
|
4087
|
+
out.append(f" {name[:26]:<26} {fmt_size(mem):>8} {n:>6}")
|
|
4088
|
+
out.append(" other big processes (RSS):")
|
|
4089
|
+
for rss, p in others:
|
|
4090
|
+
out.append(f" {fmt_size(rss):>8} {p:>7} {proc_cmdline(p)[:60]}")
|
|
4091
|
+
tasks = session_tasks(show_all=args.all)
|
|
4092
|
+
if args.json:
|
|
4093
|
+
print(json.dumps({"memory": rows, "tasks": tasks}, ensure_ascii=False, default=str))
|
|
4094
|
+
return EXIT_OK
|
|
4095
|
+
if args.watch:
|
|
4096
|
+
sys.stdout.write("\033[2J\033[H")
|
|
4097
|
+
print("\n".join(out), flush=True)
|
|
4098
|
+
print(" sessions: children (> 60 s), what they wait for, tokens today:")
|
|
4099
|
+
print_tasks(tasks)
|
|
4100
|
+
sys.stdout.flush()
|
|
4101
|
+
if not args.watch:
|
|
4102
|
+
return EXIT_OK
|
|
4103
|
+
time.sleep(args.watch)
|
|
4104
|
+
|
|
4105
|
+
|
|
4106
|
+
def cmd_overview(args):
|
|
4107
|
+
r = save_roster(quiet=True) or load_roster()
|
|
4108
|
+
info = status_info(args)
|
|
4109
|
+
L = info["lanes"]
|
|
4110
|
+
g = info["ghosts"]
|
|
4111
|
+
alive = sorted((s for s in r["sessions"].values() if s.get("alive")), key=lambda s: s.get("name") or "")
|
|
4112
|
+
print(DINGO.strip("\n"))
|
|
4113
|
+
print(f"\nmemory {fmt_size(info['avail'])} free of {fmt_size(info['total'])} · promised {fmt_size(info['promised'])}"
|
|
4114
|
+
f" · reserve {fmt_size(info['reserve'])}")
|
|
4115
|
+
print(f"cpu {info['cpu']['used']} / {info['cpu']['total']} slots · load {info['load']} on {info['ncpu']} cores"
|
|
4116
|
+
+ (" ⚠ overloaded" if info["overloaded"] else ""))
|
|
4117
|
+
print(f"slots heavy {L['heavy']['held']} / {L['heavy']['n']} · io {L['io']['held']} / {L['io']['n']}"
|
|
4118
|
+
f" · {len(info['waiting'])} waiting · ghosts {g.get('n', '?')} ({fmt_size(g.get('pss'))})")
|
|
4119
|
+
for h in info["holders"]:
|
|
4120
|
+
print(f" [{h['lane']}:{h['slot']}] {h.get('session', '?'):<20} {h.get('cmd', '')[:50]}")
|
|
4121
|
+
print(f"\nsessions ({len(alive)} alive)")
|
|
4122
|
+
for s in alive:
|
|
4123
|
+
st, guessed = state_of(s)
|
|
4124
|
+
mk = s.get("mark") or {}
|
|
4125
|
+
extra = f" until {mk['until']}" if mk.get("until") else ""
|
|
4126
|
+
print(f" {s.get('name', '?')[:26]:<26} {st}{'?' if guessed else ''}{extra}")
|
|
4127
|
+
lost = select(r, 24.0, False)
|
|
4128
|
+
if lost:
|
|
4129
|
+
print(f"\n{len(lost)} lost in the last 24 h: dingo-pack list")
|
|
4130
|
+
print("\nmore: dingo-pack who · top · doctor · list · --help")
|
|
4131
|
+
return EXIT_OK
|
|
4132
|
+
|
|
4133
|
+
|
|
4134
|
+
# ---------------------------------------------------------------- main
|
|
4135
|
+
|
|
4136
|
+
EVERYDAY = """everyday:
|
|
4137
|
+
dingo-pack save remember the open sessions (the hooks also do it)
|
|
4138
|
+
dingo-pack close-all remember them, close the idle ones that are waiting / done
|
|
4139
|
+
dingo-pack restore bring the last batch back in new terminal tabs
|
|
4140
|
+
everything else is advanced: dingo-pack (overview), run, who, doctor, ghosts, list, resume ..."""
|
|
4141
|
+
|
|
4142
|
+
DINGO = r"""
|
|
4143
|
+
/|
|
|
4144
|
+
~\___/ o\_
|
|
4145
|
+
( __) dingo-pack: the pack eats in order
|
|
4146
|
+
|_|--|_|
|
|
4147
|
+
"""
|
|
4148
|
+
|
|
4149
|
+
def main(argv=None):
|
|
4150
|
+
p = argparse.ArgumentParser(prog="dingo-pack", description=DINGO + "\n" + __doc__.split("\n\n")[0],
|
|
4151
|
+
epilog=EVERYDAY, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
4152
|
+
p.add_argument("--version", action="version", version=f"dingo-pack {__version__}")
|
|
4153
|
+
sp = p.add_subparsers(dest="sub")
|
|
4154
|
+
|
|
4155
|
+
def common(q):
|
|
4156
|
+
q.add_argument("--session", help="label for who / log (default: Claude Code session id)")
|
|
4157
|
+
q.add_argument("--slots", type=int, help="number of heavy slots (default 6 / DINGO_SLOTS)")
|
|
4158
|
+
q.add_argument("--timeout", type=parse_duration, default=None,
|
|
4159
|
+
help="give up (exit 75) after this long waiting, e.g. 30m")
|
|
4160
|
+
|
|
4161
|
+
q = sp.add_parser("run", help="run a heavy command in a slot, under memory limits")
|
|
4162
|
+
common(q)
|
|
4163
|
+
q.add_argument("--heavy", action="store_true", required=False)
|
|
4164
|
+
q.add_argument("--io", action="store_true", help="big reads / writes on a slow or network disk (own lane, 1 slot)")
|
|
4165
|
+
q.add_argument("--threads", type=int, help="CPU threads this job may use (sets OMP_NUM_THREADS etc.; "
|
|
4166
|
+
"takes that many CPU slots; default 8 heavy, 1 io)")
|
|
4167
|
+
q.add_argument("--mem", type=parse_size, help="expected peak memory, e.g. 6G (default 1G; configurable)")
|
|
4168
|
+
q.add_argument("--limit", type=parse_size, help="per-process limit (default 1.5 x --mem; 0 = none)")
|
|
4169
|
+
q.add_argument("--limit-kind", choices=("data", "as"), default="data")
|
|
4170
|
+
q.add_argument("--tree-limit", type=parse_size, help="limit on the whole process tree (default = --limit; 0 = none)")
|
|
4171
|
+
q.add_argument("cmd", nargs=argparse.REMAINDER)
|
|
4172
|
+
|
|
4173
|
+
q = sp.add_parser("lease", help="hold a slot for work that is not a shell command")
|
|
4174
|
+
common(q)
|
|
4175
|
+
q.add_argument("--heavy", action="store_true")
|
|
4176
|
+
q.add_argument("--mem", type=parse_size)
|
|
4177
|
+
q.add_argument("--for", dest="duration", type=parse_duration, default=45 * 60.0)
|
|
4178
|
+
q.add_argument("--threads", type=int, help="CPU slots the leased work takes (default 2)")
|
|
4179
|
+
q.add_argument("--note", default="")
|
|
4180
|
+
|
|
4181
|
+
q = sp.add_parser("release", help="release leases")
|
|
4182
|
+
q.add_argument("--session")
|
|
4183
|
+
q.add_argument("ids", nargs="*")
|
|
4184
|
+
|
|
4185
|
+
q = sp.add_parser("who", help="slot holders and the queue")
|
|
4186
|
+
q.add_argument("--slots", type=int)
|
|
4187
|
+
q.add_argument("--json", action="store_true")
|
|
4188
|
+
|
|
4189
|
+
q = sp.add_parser("board", help="one line per session: state, box, expected finish, ask-now / stopped flags")
|
|
4190
|
+
q.add_argument("--json", action="store_true")
|
|
4191
|
+
q.add_argument("--html", help="write a self-refreshing HTML page to this path")
|
|
4192
|
+
q.add_argument("--watch", type=float, help="refresh every N seconds")
|
|
4193
|
+
q.add_argument("--stale-h", type=float, default=3.0, help="flag a status line unchanged this many hours (default 3)")
|
|
4194
|
+
|
|
4195
|
+
q = sp.add_parser("name", help="next session name for a letter today (L-MMDD-what, then L2-, L3- ...)")
|
|
4196
|
+
q.add_argument("letter")
|
|
4197
|
+
q.add_argument("what", help="card number or short word, e.g. 0-196")
|
|
4198
|
+
q.add_argument("--suffix", help="e.g. ds-yi for a session on another account")
|
|
4199
|
+
q.add_argument("--date", help="MMDD (default today)")
|
|
4200
|
+
|
|
4201
|
+
q = sp.add_parser("card", help="card new: next free number, reserved here, with a skeleton")
|
|
4202
|
+
q.add_argument("action", choices=("new",))
|
|
4203
|
+
q.add_argument("title")
|
|
4204
|
+
q.add_argument("--out", help="write the skeleton to this file")
|
|
4205
|
+
q.add_argument("--registry", help="file that lists the numbers already used (read only)")
|
|
4206
|
+
q.add_argument("--session")
|
|
4207
|
+
|
|
4208
|
+
q = sp.add_parser("lint", help="naming, one-per-letter, stale / overdue status lines, missing status")
|
|
4209
|
+
q.add_argument("--json", action="store_true")
|
|
4210
|
+
q.add_argument("--stale-h", type=float, default=3.0)
|
|
4211
|
+
|
|
4212
|
+
q = sp.add_parser("queue", help="prompts waiting for a condition: add / list / show / sent")
|
|
4213
|
+
q.add_argument("action", choices=("add", "list", "show", "sent"))
|
|
4214
|
+
q.add_argument("id", nargs="?", type=int)
|
|
4215
|
+
q.add_argument("--when", action="append", default=[], help="mark:NAME=done | file:PATH | at:HH:MM | alive:NAME | gone:NAME")
|
|
4216
|
+
q.add_argument("--to", default="?")
|
|
4217
|
+
q.add_argument("--prompt")
|
|
4218
|
+
q.add_argument("--prompt-file")
|
|
4219
|
+
q.add_argument("--clip", action="store_true", help="show: also copy to the Windows clipboard")
|
|
4220
|
+
q.add_argument("--session")
|
|
4221
|
+
|
|
4222
|
+
q = sp.add_parser("claim", help="box occupancy file: list / check / add / wait / sweep / remove; done flags release lines, stale lines alert")
|
|
4223
|
+
q.add_argument("action", choices=("list", "check", "add", "remove", "wait", "sweep"))
|
|
4224
|
+
q.add_argument("--file")
|
|
4225
|
+
q.add_argument("--who")
|
|
4226
|
+
q.add_argument("--task")
|
|
4227
|
+
q.add_argument("--cards")
|
|
4228
|
+
q.add_argument("--until", help="expected end, e.g. '柏林 10-06 05:05'")
|
|
4229
|
+
q.add_argument("--done", help="add: done flag (path or glob); once it exists the line is removed by the next claim command")
|
|
4230
|
+
q.add_argument("--grace", type=int, default=CLAIM_GRACE_MIN, help="minutes past the expected end before a line counts as stale")
|
|
4231
|
+
q.add_argument("--alert", help="append stale lines to this alert file (default: <claim dir>/runs/claim_stale_<box>.alert if runs/ exists)")
|
|
4232
|
+
q.add_argument("--every", type=float, default=60, help="wait: seconds between checks")
|
|
4233
|
+
q.add_argument("--timeout", help="wait: give up after this long (e.g. 8h); exit 75")
|
|
4234
|
+
q.add_argument("--take-stale", action="store_true", help="check / wait / add: treat stale lines as free (only if your card says so)")
|
|
4235
|
+
q.add_argument("--stop-on-stale", action="store_true", help="wait: exit 4 as soon as only stale lines are in the way")
|
|
4236
|
+
q.add_argument("--json", action="store_true", help="list: machine-readable")
|
|
4237
|
+
q.add_argument("--session")
|
|
4238
|
+
|
|
4239
|
+
q = sp.add_parser("boxes", help="one line per box over read-only ssh: up / no GPUs / unreachable, cards, claims, disk, watchdog; note / history")
|
|
4240
|
+
q.add_argument("words", nargs="*", help="[box ...] | note BOX TEXT | history [BOX]")
|
|
4241
|
+
q.add_argument("--boxes", help="boxes.json (default $DINGO_BOXES or ~/.config/dingo-pack/boxes.json)")
|
|
4242
|
+
q.add_argument("--no-du", action="store_true", help="skip the per-directory sizes (du runs on the box, up to 90 s)")
|
|
4243
|
+
q.add_argument("--ssh-timeout", type=int, default=150)
|
|
4244
|
+
q.add_argument("--every", type=float, help="repeat every N seconds (only while this machine is on)")
|
|
4245
|
+
q.add_argument("--json", action="store_true")
|
|
4246
|
+
q.add_argument("--session")
|
|
4247
|
+
|
|
4248
|
+
q = sp.add_parser("_probe", help="(internal) what `boxes` runs on a box; reads only")
|
|
4249
|
+
q.add_argument("--data", default="/root/gpufree-data")
|
|
4250
|
+
q.add_argument("--no-du", action="store_true")
|
|
4251
|
+
|
|
4252
|
+
q = sp.add_parser("top", help="task manager: memory, each session's children, what it waits for, spin flags, tokens")
|
|
4253
|
+
q.add_argument("--watch", type=float, help="refresh every N seconds")
|
|
4254
|
+
q.add_argument("--all", action="store_true", help="also children younger than 60 s")
|
|
4255
|
+
q.add_argument("--json", action="store_true")
|
|
4256
|
+
q = sp.add_parser("adapter", help="test agent adapters (DINGO_AGENTS) against this machine; see ADAPTERS.md")
|
|
4257
|
+
q.add_argument("action", choices=("test",))
|
|
4258
|
+
q.add_argument("--agents", help="comma-separated built-in names / adapter files (default $DINGO_AGENTS or claude-code)")
|
|
4259
|
+
q = sp.add_parser("tokens", help="tokens per session for a day, from Claude Code's transcripts")
|
|
4260
|
+
q.add_argument("--day", help="YYYY-MM-DD (default today)")
|
|
4261
|
+
q.add_argument("--json", action="store_true")
|
|
4262
|
+
q = sp.add_parser("log", help="recent log lines")
|
|
4263
|
+
q.add_argument("-n", type=int, default=20)
|
|
4264
|
+
|
|
4265
|
+
q = sp.add_parser("save", help="snapshot live Claude Code sessions to the roster")
|
|
4266
|
+
q.add_argument("--hook", action="store_true", help="hook mode: silent, never fails, rate-limited")
|
|
4267
|
+
q.add_argument("--every", type=float, help="keep saving every N seconds")
|
|
4268
|
+
|
|
4269
|
+
q = sp.add_parser("mark", help="self-report this session's state")
|
|
4270
|
+
q.add_argument("state", choices=STATES)
|
|
4271
|
+
q.add_argument("--until", help="for waiting: when to resume")
|
|
4272
|
+
q.add_argument("--sid", help="session id (default: this Claude Code session)")
|
|
4273
|
+
q.add_argument("note", nargs="*")
|
|
4274
|
+
|
|
4275
|
+
q = sp.add_parser("list", help="sessions to resume, grouped")
|
|
4276
|
+
q.add_argument("--since", type=float, default=24.0, help="hours (default 24)")
|
|
4277
|
+
q.add_argument("--closed", action="store_true", help="also show sessions closed while the machine was up")
|
|
4278
|
+
q.add_argument("--at", help="use the newest snapshot not later than YYYYMMDD-HHMMSS")
|
|
4279
|
+
q.add_argument("--json", action="store_true")
|
|
4280
|
+
q.add_argument("--no-refresh", action="store_true")
|
|
4281
|
+
|
|
4282
|
+
q = sp.add_parser("resume", help="reopen sessions from the roster")
|
|
4283
|
+
q.add_argument("names", nargs="*")
|
|
4284
|
+
q.add_argument("--midway", action="store_true", help="all lost sessions marked / guessed midway")
|
|
4285
|
+
q.add_argument("--lost", action="store_true", help="all sessions lost in a reboot except done")
|
|
4286
|
+
q.add_argument("--since", type=float, default=24.0)
|
|
4287
|
+
q.add_argument("--via", choices=("wt", "tmux", "print"))
|
|
4288
|
+
q.add_argument("--batch", type=int, help="tabs per batch (default 4)")
|
|
4289
|
+
q.add_argument("--prompt", help="first message for each resumed session")
|
|
4290
|
+
q.add_argument("--yes", action="store_true", help="do not pause between batches")
|
|
4291
|
+
q.add_argument("--dry-run", action="store_true")
|
|
4292
|
+
|
|
4293
|
+
q = sp.add_parser("doctor", help="check memory, OOM kills, mounts, WSL settings, slots, roster, ghosts")
|
|
4294
|
+
q.add_argument("--json", action="store_true")
|
|
4295
|
+
q.add_argument("--no-git", dest="git", action="store_false", help="skip wombat-gate doctor on live repos")
|
|
4296
|
+
|
|
4297
|
+
q = sp.add_parser("ghosts", help="process trees left behind by sessions that are gone")
|
|
4298
|
+
q.add_argument("--reap", action="store_true", help="stop them (human only)")
|
|
4299
|
+
q.add_argument("--yes", action="store_true")
|
|
4300
|
+
q.add_argument("--json", action="store_true")
|
|
4301
|
+
q.add_argument("ids", nargs="*", help="#k of the groups to reap (default: all not kept)")
|
|
4302
|
+
|
|
4303
|
+
q = sp.add_parser("close-all", help="remember every session and close the safe ones (idle + waiting / done)")
|
|
4304
|
+
q.add_argument("--dry-run", action="store_true")
|
|
4305
|
+
q.add_argument("--sid", help=argparse.SUPPRESS)
|
|
4306
|
+
|
|
4307
|
+
q = sp.add_parser("restore", help="bring back the last batch (closed by close-all, or lost in a crash)")
|
|
4308
|
+
q.add_argument("names", nargs="*")
|
|
4309
|
+
q.add_argument("--all", action="store_true", help=argparse.SUPPRESS)
|
|
4310
|
+
q.add_argument("--via", choices=("wt", "tmux", "print"))
|
|
4311
|
+
q.add_argument("--batch", type=int)
|
|
4312
|
+
q.add_argument("--prompt")
|
|
4313
|
+
q.add_argument("--yes", action="store_true")
|
|
4314
|
+
q.add_argument("--dry-run", action="store_true")
|
|
4315
|
+
|
|
4316
|
+
q = sp.add_parser("stow", help="(advanced) note the live sessions; with --close, close the idle waiting / done ones")
|
|
4317
|
+
q.add_argument("--close", action="store_true")
|
|
4318
|
+
q.add_argument("--dry-run", action="store_true")
|
|
4319
|
+
q.add_argument("--sid", help=argparse.SUPPRESS)
|
|
4320
|
+
|
|
4321
|
+
q = sp.add_parser("unstow", help="reopen what the last stow closed")
|
|
4322
|
+
q.add_argument("names", nargs="*")
|
|
4323
|
+
q.add_argument("--all", action="store_true", help="also sessions stow left running (if they are gone now)")
|
|
4324
|
+
q.add_argument("--via", choices=("wt", "tmux", "print"))
|
|
4325
|
+
q.add_argument("--batch", type=int)
|
|
4326
|
+
q.add_argument("--prompt")
|
|
4327
|
+
q.add_argument("--yes", action="store_true")
|
|
4328
|
+
q.add_argument("--dry-run", action="store_true")
|
|
4329
|
+
|
|
4330
|
+
args, extra = p.parse_known_args(argv)
|
|
4331
|
+
if extra:
|
|
4332
|
+
# Python < 3.12 drops a positional that follows an option
|
|
4333
|
+
# (`mark done --sid X finished` -> "unrecognized arguments: finished").
|
|
4334
|
+
# Put those words back on the subcommand's tail positional.
|
|
4335
|
+
tail = {"mark": "note", "ghosts": "ids", "release": "ids",
|
|
4336
|
+
"resume": "names", "restore": "names", "unstow": "names"}.get(getattr(args, "sub", None))
|
|
4337
|
+
if tail is None or not hasattr(args, tail):
|
|
4338
|
+
p.error("unrecognized arguments: " + " ".join(extra))
|
|
4339
|
+
setattr(args, tail, list(getattr(args, tail) or []) + extra)
|
|
4340
|
+
if args.sub and not (args.sub == "save" and getattr(args, "hook", False)) and args.sub not in ("_probe", "claim"):
|
|
4341
|
+
check_local_state()
|
|
4342
|
+
fn = {"run": cmd_run, "lease": cmd_lease, "release": cmd_release, "who": cmd_who, "log": cmd_log,
|
|
4343
|
+
"save": cmd_save, "mark": cmd_mark, "list": cmd_list, "resume": cmd_resume,
|
|
4344
|
+
"doctor": cmd_doctor, "ghosts": cmd_ghosts, "stow": cmd_stow, "unstow": cmd_unstow,
|
|
4345
|
+
"top": cmd_top, "close-all": cmd_close_all, "restore": cmd_restore, "board": cmd_board,
|
|
4346
|
+
"name": cmd_name, "card": cmd_card, "lint": cmd_lint, "queue": cmd_queue, "claim": cmd_claim,
|
|
4347
|
+
"boxes": cmd_boxes, "_probe": cmd_probe, "tokens": cmd_tokens,
|
|
4348
|
+
"adapter": cmd_adapter}.get(args.sub)
|
|
4349
|
+
if not fn:
|
|
4350
|
+
return cmd_overview(args)
|
|
4351
|
+
return fn(args)
|
|
4352
|
+
|
|
4353
|
+
|
|
4354
|
+
if __name__ == "__main__":
|
|
4355
|
+
sys.exit(main())
|