phiward 0.2.0b1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. phiward/__init__.py +8 -0
  2. phiward/__main__.py +5 -0
  3. phiward/_toml.py +211 -0
  4. phiward/adapters/__init__.py +105 -0
  5. phiward/adapters/claude.py +429 -0
  6. phiward/broker.py +831 -0
  7. phiward/carryover.py +1613 -0
  8. phiward/claude_settings.py +1307 -0
  9. phiward/cli.py +110 -0
  10. phiward/config.py +936 -0
  11. phiward/confine.py +476 -0
  12. phiward/copying.py +533 -0
  13. phiward/core.py +105 -0
  14. phiward/data/agents-section.md +43 -0
  15. phiward/doctor.py +1132 -0
  16. phiward/fasthook.py +70 -0
  17. phiward/fsinfo.py +241 -0
  18. phiward/gateway.py +159 -0
  19. phiward/githooks.py +248 -0
  20. phiward/guards/__init__.py +0 -0
  21. phiward/guards/protected_paths.py +1138 -0
  22. phiward/hidden.py +281 -0
  23. phiward/init.py +155 -0
  24. phiward/install.py +828 -0
  25. phiward/inventory.py +173 -0
  26. phiward/mcp.py +186 -0
  27. phiward/mcp_config.py +1736 -0
  28. phiward/names.py +5 -0
  29. phiward/packs/dates.toml +17 -0
  30. phiward/packs/genomics.toml +51 -0
  31. phiward/packs/msk.toml +32 -0
  32. phiward/pathid.py +403 -0
  33. phiward/pathmatch.py +98 -0
  34. phiward/paths.py +942 -0
  35. phiward/pin.py +214 -0
  36. phiward/procinfo.py +73 -0
  37. phiward/remote.py +448 -0
  38. phiward/run.py +788 -0
  39. phiward/safeio.py +84 -0
  40. phiward/sanitize.py +691 -0
  41. phiward/selfcheck.py +1506 -0
  42. phiward/service.py +321 -0
  43. phiward/session_note.py +86 -0
  44. phiward/setup_cmd.py +886 -0
  45. phiward/shell.py +504 -0
  46. phiward/wholetool.py +1463 -0
  47. phiward-0.2.0b1.dist-info/METADATA +147 -0
  48. phiward-0.2.0b1.dist-info/RECORD +53 -0
  49. phiward-0.2.0b1.dist-info/WHEEL +5 -0
  50. phiward-0.2.0b1.dist-info/entry_points.txt +3 -0
  51. phiward-0.2.0b1.dist-info/licenses/LICENSE +662 -0
  52. phiward-0.2.0b1.dist-info/licenses/NOTICE +16 -0
  53. phiward-0.2.0b1.dist-info/top_level.txt +1 -0
phiward/remote.py ADDED
@@ -0,0 +1,448 @@
1
+ """HPC requests with the login kept outside.
2
+
3
+ Every request to an HPC host (`copy` to or from a remote, `run --remote`) runs ssh through the
4
+ operator's shared connection: the operator starts it (`kinit`, then `ssh -fN HOST`), and it holds
5
+ the login. phiward's ssh joins it through its control socket, which needs no key and no ticket.
6
+ So the profile of the request reads no credential and has no network except that one socket:
7
+ a key or ticket can never reach a request, and a program on the other end that attacks the
8
+ local ssh or rsync can reach nothing but the HPC host, where the data already is.
9
+
10
+ `[remote] use_shared_connection = false` (user config) keeps the older way: ssh logs in itself,
11
+ so it must read the key or ticket, and the request has the network.
12
+
13
+ Pushes (`copy LOCAL... HOST:/path`) do not use rsync: the broker opens each checked source part
14
+ by part without following a link, walks it by descriptors, and writes a tar stream into
15
+ `ssh HOST tar -x`. Nothing the agent can change is opened by name after the check, and the
16
+ remote side cannot ask for anything (the stream goes one way).
17
+ """
18
+
19
+ import os
20
+ import shlex
21
+ import stat
22
+ import subprocess
23
+ import tarfile
24
+ from dataclasses import dataclass
25
+ from typing import BinaryIO, Callable, List, Optional, Sequence, Tuple
26
+
27
+ from phiward import config
28
+
29
+ SSH_OPTIONS = ["-o", "BatchMode=yes", "-o", "ProxyCommand=none", "-o", "ProxyJump=none",
30
+ "-o", "PermitLocalCommand=no", "-o", "ClearAllForwardings=yes",
31
+ "-o", "ForwardAgent=no", "-o", "ForwardX11=no", "-o", "ForwardX11Trusted=no",
32
+ "-o", "RequestTTY=no", "-T"]
33
+
34
+ START_IT = "start your shared connection first: kinit, then ssh -fN {host} (docs/hpc.md)"
35
+
36
+
37
+ def _ssh() -> Optional[str]:
38
+ from phiward.copying import system_program
39
+ return system_program("ssh")
40
+
41
+
42
+ def _ssh_file(cfg: config.Config) -> str:
43
+ return cfg.ssh_config or "/dev/null"
44
+
45
+
46
+ def shared_socket(cfg: config.Config, host: str, timeout: float = 10) -> Tuple[str, str]:
47
+ """(the control socket of the operator's shared connection to `host`, "") or ("", why not).
48
+ The socket is the one the operator's ssh config names (`ssh -G`, no network), and the
49
+ connection must answer (`ssh -O check`); both run in the broker, outside every profile."""
50
+ ssh = _ssh()
51
+ if ssh is None:
52
+ return "", "there is no system ssh"
53
+ env = {"PATH": "/usr/bin:/bin", "HOME": os.path.expanduser("~")}
54
+ try:
55
+ r = subprocess.run([ssh, "-G", "-F", _ssh_file(cfg), "--", host], capture_output=True,
56
+ text=True, timeout=timeout, env=env, stdin=subprocess.DEVNULL)
57
+ except (OSError, subprocess.SubprocessError):
58
+ return "", "ssh could not read its config"
59
+ sock = ""
60
+ for line in r.stdout.splitlines():
61
+ k, _, v = line.partition(" ")
62
+ if k.lower() == "controlpath":
63
+ sock = v.strip()
64
+ if not cfg.ssh_config:
65
+ return "", ("no [copy] ssh_config is set in the user config. The operator does the "
66
+ "one-time HPC setup (docs/hpc.md), then `kinit` and `ssh -fN " + host + "`")
67
+ if not sock or sock == "none" or not sock.startswith("/"):
68
+ return "", ("[copy] ssh_config names no ControlPath for this host; add one and "
69
+ + START_IT.format(host=host))
70
+ why = socket_problem(sock)
71
+ if why == "missing":
72
+ return "", START_IT.format(host=host)
73
+ if why:
74
+ return "", why
75
+ sock = os.path.realpath(sock)
76
+ try:
77
+ r = subprocess.run([ssh, "-F", _ssh_file(cfg), "-S", sock, "-O", "check", "--", host],
78
+ capture_output=True, timeout=timeout, env=env,
79
+ stdin=subprocess.DEVNULL)
80
+ except (OSError, subprocess.SubprocessError):
81
+ return "", START_IT.format(host=host)
82
+ if r.returncode != 0:
83
+ return "", START_IT.format(host=host)
84
+ return sock, ""
85
+
86
+
87
+ def socket_problem(sock: str) -> str:
88
+ """Why the control socket at `sock` may not be trusted, "missing", or "" (a socket
89
+ the agent made first, where the operator's ssh would find it, would receive a
90
+ push's stream). It must be a socket (not a link to one), owned by the operator, under
91
+ ~/.ssh or /run/user/<uid> (folders no session can read or write), with every folder from
92
+ there down owned by the operator and writable by no one else."""
93
+ try:
94
+ st = os.lstat(sock)
95
+ except OSError:
96
+ return "missing"
97
+ if not stat.S_ISSOCK(st.st_mode) or st.st_uid != os.getuid():
98
+ return ("the shared connection's socket is not a socket of yours; remove it and start "
99
+ "the connection again")
100
+ real = os.path.realpath(sock)
101
+ bases = [os.path.realpath(os.path.join(os.path.expanduser("~"), ".ssh")),
102
+ f"/run/user/{os.getuid()}"]
103
+ base = next((b for b in bases if real.startswith(b + "/")), "")
104
+ if not base:
105
+ return ("the shared connection's ControlPath must be under ~/.ssh (a folder no session "
106
+ "can read or write); see docs/hpc.md")
107
+ d = os.path.dirname(real)
108
+ while True:
109
+ try:
110
+ ds = os.lstat(d)
111
+ except OSError:
112
+ return "missing"
113
+ if ds.st_uid != os.getuid() or ds.st_mode & (stat.S_IWGRP | stat.S_IWOTH):
114
+ return ("a folder above the shared connection's socket is not yours alone (owner or "
115
+ "mode)")
116
+ if d == base:
117
+ return ""
118
+ d = os.path.dirname(d)
119
+
120
+
121
+ def ssh_command(cfg: config.Config, sock: str = "") -> Optional[List[str]]:
122
+ """The ssh argv without the host: the operator's chosen config only, no proxy or local
123
+ command, no forwarding, no terminal, never a prompt; with `sock`, through that socket and
124
+ never a new login."""
125
+ ssh = _ssh()
126
+ if ssh is None:
127
+ return None
128
+ cmd = [ssh, "-F", _ssh_file(cfg)] + SSH_OPTIONS
129
+ if sock:
130
+ cmd += ["-o", "ControlMaster=no", "-S", sock]
131
+ return cmd
132
+
133
+
134
+ @dataclass
135
+ class Route:
136
+ """How a request reaches a host: the ssh argv (without the host), the socket (empty without
137
+ the shared connection) and the profile settings that go with it."""
138
+ ssh: List[str]
139
+ socket: str
140
+ net: bool
141
+ deny_read: List[str]
142
+ allow_read: List[str]
143
+
144
+
145
+ def route(cfg: config.Config, host: str) -> Tuple[Optional[Route], str]:
146
+ """(the route to `host`, "") or (None, why not)."""
147
+ if not cfg.use_shared_connection:
148
+ ssh = ssh_command(cfg)
149
+ if ssh is None:
150
+ return None, "there is no system ssh"
151
+ # the older way: ssh logs in itself, so it reads the credentials and has the network
152
+ return Route(ssh, "", True, [], []), ""
153
+ sock, why = shared_socket(cfg, host)
154
+ if why:
155
+ return None, why
156
+ from phiward import confine
157
+ # on Linux the socket is bound at a fixed name in a fresh /tmp inside the request
158
+ ssh = ssh_command(cfg, confine.LINUX_SOCKET if confine.backend() == "bwrap" else sock)
159
+ if ssh is None:
160
+ return None, "there is no system ssh"
161
+ from phiward import run
162
+ allow = [cfg.ssh_config] if cfg.ssh_config else []
163
+ return Route(ssh, sock, False, run.gateway_hidden(cfg), allow), ""
164
+
165
+
166
+ # -- pushes ------------------------------------------------------------------------------------
167
+
168
+ class PushError(Exception):
169
+ pass
170
+
171
+
172
+ @dataclass
173
+ class Counts:
174
+ files: int = 0
175
+ folders: int = 0
176
+ skipped: int = 0
177
+ size: int = 0
178
+
179
+
180
+ _NOFOLLOW = os.O_NOFOLLOW | os.O_CLOEXEC
181
+
182
+ # files a shell, ssh or a tool on the cluster reads at login or when you enter a folder: a push
183
+ # never plants them, at any depth (final review: a push to a broad prefix could plant
184
+ # ~/.bashrc or ~/.ssh/authorized_keys)
185
+ RUNS_AT_LOGIN = frozenset({
186
+ ".ssh", ".bashrc", ".bash_profile", ".bash_login", ".bash_logout", ".profile", ".zshrc",
187
+ ".zprofile", ".zshenv", ".zlogin", ".zlogout", ".login", ".logout", ".cshrc", ".tcshrc",
188
+ ".kshrc", ".mkshrc", ".inputrc", ".forward", ".rhosts", ".shosts", ".k5login", ".k5users",
189
+ ".pam_environment", ".envrc", ".gnupg", ".netrc", ".config", ".local", ".condarc",
190
+ ".Rprofile", ".Renviron", ".modulerc", ".lmod.d", ".bash_aliases", ".gitconfig", ".ipython",
191
+ ".jupyter", ".vimrc", ".tmux.conf", ".emacs", ".emacs.d", ".conda", ".nextflow", ".julia",
192
+ ".npmrc", ".pypirc", ".Xresources", ".xinitrc", ".xprofile",
193
+ # a repository's config and hooks: git runs what they name when someone runs git there
194
+ ".git"})
195
+
196
+
197
+ def open_by_parts(path: str, forbid: Optional[set] = None) -> int:
198
+ """A read descriptor of the file or folder at the absolute `path`, reached part by part
199
+ without following a link anywhere (OSError otherwise). With `forbid`, a folder on the way
200
+ that lies on one of those devices raises PushError."""
201
+ parts = [p for p in path.split("/") if p]
202
+ if not parts:
203
+ raise PushError("the root folder cannot be pushed")
204
+ fd = os.open("/", os.O_RDONLY | os.O_DIRECTORY | _NOFOLLOW)
205
+ try:
206
+ for p in parts[:-1]:
207
+ nxt = os.open(p, os.O_RDONLY | os.O_DIRECTORY | _NOFOLLOW, dir_fd=fd)
208
+ os.close(fd)
209
+ fd = nxt
210
+ if forbid and os.fstat(fd).st_dev in forbid:
211
+ raise PushError("a source is in /dev, /proc or /sys")
212
+ last = os.open(parts[-1], os.O_RDONLY | os.O_NONBLOCK | _NOFOLLOW, dir_fd=fd)
213
+ if forbid and os.fstat(last).st_dev in forbid:
214
+ os.close(last)
215
+ raise PushError("a source is in /dev, /proc or /sys")
216
+ return last
217
+ finally:
218
+ os.close(fd)
219
+
220
+
221
+ def _walk(fd: int, name: str, counts: Counts, limit: int,
222
+ visit: Optional[Callable[[str, int, os.stat_result], None]], depth: int = 0) -> None:
223
+ """Each folder and plain file under `fd`, by descriptors only: a link, a special file or a
224
+ file with a second name is skipped and counted, never followed."""
225
+ if depth > 256:
226
+ raise PushError("the folder tree is too deep")
227
+ if name and (os.path.basename(name) in RUNS_AT_LOGIN
228
+ or ("/" not in name and name.startswith("."))):
229
+ # a login file at any depth; and every dot-name at the top of the destination, which
230
+ # may be a home folder on the cluster (`.gitconfig`, `.vimrc`)
231
+ counts.skipped += 1
232
+ return
233
+ st = os.fstat(fd)
234
+ if stat.S_ISDIR(st.st_mode):
235
+ counts.folders += 1 if name else 0
236
+ if visit and name:
237
+ visit(name, fd, st)
238
+ for n in sorted(os.listdir(fd)):
239
+ if n in RUNS_AT_LOGIN or (not name and n.startswith(".")):
240
+ counts.skipped += 1
241
+ continue
242
+ try:
243
+ c = os.open(n, os.O_RDONLY | os.O_NONBLOCK | _NOFOLLOW, dir_fd=fd)
244
+ except OSError: # a link (ELOOP), or gone
245
+ counts.skipped += 1
246
+ continue
247
+ try:
248
+ _walk(c, f"{name}/{n}" if name else n, counts, limit, visit, depth + 1)
249
+ finally:
250
+ os.close(c)
251
+ return
252
+ if not stat.S_ISREG(st.st_mode) or st.st_nlink != 1:
253
+ counts.skipped += 1
254
+ return
255
+ counts.files += 1
256
+ counts.size += st.st_size
257
+ if counts.size > limit:
258
+ raise PushError("the push is larger than [copy] push_limit_gib; ask the operator to copy "
259
+ "it")
260
+ if visit:
261
+ visit(name, fd, st)
262
+
263
+
264
+ @dataclass
265
+ class Source:
266
+ """A push source, resolved once when it is checked: `path` has no link on it then, and it
267
+ is opened part by part without following one ever after (a link put on it later makes the
268
+ open fail). `top` is the name it lands under ("" for a folder's contents, as rsync's
269
+ `src/`)."""
270
+ path: str
271
+ top: str
272
+
273
+
274
+ def contents_only(spec: str) -> bool:
275
+ """rsync's rule: `src/` and `src/.` (and `.`) mean the folder's contents."""
276
+ return spec.endswith("/") or spec.rstrip("/").split("/")[-1] == "."
277
+
278
+
279
+ def resolve(sources: Sequence[str], cwd: str) -> List[Source]:
280
+ out = []
281
+ for src in sources:
282
+ full = os.path.realpath(os.path.join(cwd, os.path.expanduser(src)))
283
+ top = "" if contents_only(src) and os.path.isdir(full) else os.path.basename(full)
284
+ out.append(Source(full, top))
285
+ return out
286
+
287
+
288
+ # file systems whose "files" are not files: /dev/fd/N would stream a file the broker has open
289
+ NOT_FILES = ("/dev", "/proc", "/sys")
290
+
291
+
292
+ def _special_devices() -> set:
293
+ out = set()
294
+ for p in NOT_FILES:
295
+ try:
296
+ out.add(os.stat(p).st_dev)
297
+ except OSError:
298
+ pass
299
+ return out
300
+
301
+
302
+ def _open_source(src: "Source") -> int:
303
+ """The source opened without a link on its path, refused when it lies on /dev, /proc or
304
+ /sys: judged by the file system it is on, not by its spelling (the macOS root folder
305
+ ignores case: `/DEV/fd/3`)."""
306
+ # the folders on the way are judged: opening /dev/fd/N gives the file behind it, which is
307
+ # on its own file system, while /dev (in any spelling) is on devfs
308
+ return open_by_parts(src.path, _special_devices())
309
+
310
+
311
+ def survey(sources: Sequence[Source], limit: int) -> Counts:
312
+ """Count what a push would send (and check the limit) without sending anything."""
313
+ counts = Counts()
314
+ for src in sources:
315
+ fd = _open_source(src)
316
+ try:
317
+ _walk(fd, src.top, counts, limit, None)
318
+ finally:
319
+ os.close(fd)
320
+ return counts
321
+
322
+
323
+ def feeder(sources: Sequence[Source], limit: int, counts: Counts) -> Callable[[BinaryIO], None]:
324
+ """A function that writes the tar stream of the sources into ssh's input. It never resolves
325
+ a path again: only the checked ones, opened without following a link."""
326
+ def feed(stream: BinaryIO) -> None:
327
+ with tarfile.open(fileobj=stream, mode="w|", format=tarfile.PAX_FORMAT) as tar:
328
+ def visit(name: str, fd: int, st: os.stat_result) -> None:
329
+ ti = tarfile.TarInfo(name)
330
+ ti.mtime = int(st.st_mtime)
331
+ # folders 0755, files 0644: a pushed file never arrives as a program the operator
332
+ # might run later (a replaced job script)
333
+ ti.mode = 0o755 if stat.S_ISDIR(st.st_mode) else 0o644
334
+ ti.uid = ti.gid = 0
335
+ ti.uname = ti.gname = ""
336
+ if stat.S_ISDIR(st.st_mode):
337
+ ti.type = tarfile.DIRTYPE
338
+ tar.addfile(ti)
339
+ return
340
+ ti.size = st.st_size
341
+ with os.fdopen(os.dup(fd), "rb") as fh:
342
+ os.lseek(fh.fileno(), 0, os.SEEK_SET)
343
+ tar.addfile(ti, fh)
344
+ for src in sources:
345
+ fd = _open_source(src)
346
+ try:
347
+ _walk(fd, src.top, counts, limit, visit)
348
+ finally:
349
+ os.close(fd)
350
+ return feed
351
+
352
+
353
+ # the receiving side of a push, in sh (the login shell may be csh or fish): make the folder,
354
+ # then refuse when it is, by its real path, the home folder or above it (a push there could plant
355
+ # your login files; a name cannot tell where a cluster keeps home folders), then unpack. Only
356
+ # characters that sh, csh and fish all leave alone inside single quotes.
357
+ _RECEIVE = ('mkdir -p -- "$1" && d=$(cd -- "$1" && pwd -P) && h=$(cd && pwd -P) && '
358
+ 'case "$d" in /) exit 3;; esac && case "$h/" in "$d"/*) '
359
+ 'echo "phiward: the destination is your home folder or above it" >&2; exit 3;; esac '
360
+ # keep existing files: --skip-old-files where tar has it (GNU tar 1.28 and later),
361
+ # else -k (bsdtar skips; an older GNU tar keeps them and reports each)
362
+ '&& k=-k && { tar --skip-old-files --version >/dev/null 2>&1 && k=--skip-old-files '
363
+ '|| :; } && exec tar -x "$k" -f - -C "$d"')
364
+
365
+
366
+ def receive_script(path: str) -> str:
367
+ """The remote command of a push: make the folder, check it is not the home folder or above
368
+ it, unpack the stream into it. The path holds only safe characters (copying._SAFE_PATH);
369
+ quoting keeps it one word in sh, csh and fish."""
370
+ return f"sh -c {shlex.quote(_RECEIVE)} sh {shlex.quote(path)}"
371
+
372
+
373
+ # -- pulls -------------------------------------------------------------------------------------
374
+
375
+ def send_script(path: str) -> str:
376
+ """The remote command of a pull: a tar stream of `path` (its contents for `path/`)."""
377
+ if contents_only(path):
378
+ return f"tar -c -f - -C {shlex.quote(os.path.normpath(path))} ."
379
+ p = os.path.normpath(path)
380
+ return (f"tar -c -f - -C {shlex.quote(os.path.dirname(p) or '/')} -- "
381
+ f"{shlex.quote(os.path.basename(p))}")
382
+
383
+
384
+ # "$2" and "$3" are the system mkdir and tar (never found through the request's PATH)
385
+ # The receiving side of a pull, in the gateway profile: a careful extractor in place of tar, the
386
+ # same on every system (bsdtar -k could still change an existing folder's mode). Standard
387
+ # library only; it reads the stream from stdin and writes under argv[1]. What exists is kept;
388
+ # links, special files, `..`, `.claude` are skipped; nothing is followed through a link.
389
+ EXTRACT_PY = r"""
390
+ import os, stat, sys, tarfile
391
+ dest = os.path.realpath(sys.argv[1])
392
+ os.makedirs(dest, mode=0o755, exist_ok=True)
393
+ NF = os.O_NOFOLLOW | os.O_CLOEXEC
394
+ def folder(parts):
395
+ fd = os.open(dest, os.O_RDONLY | os.O_DIRECTORY | NF)
396
+ try:
397
+ for p in parts:
398
+ try:
399
+ os.mkdir(p, 0o755, dir_fd=fd)
400
+ except FileExistsError:
401
+ pass
402
+ nfd = os.open(p, os.O_RDONLY | os.O_DIRECTORY | NF, dir_fd=fd)
403
+ os.close(fd)
404
+ fd = nfd
405
+ return fd
406
+ except OSError:
407
+ os.close(fd)
408
+ return None
409
+ kept = skipped = 0
410
+ with tarfile.open(fileobj=sys.stdin.buffer, mode="r|") as t:
411
+ for m in t:
412
+ parts = [p for p in m.name.split("/") if p not in ("", ".")]
413
+ if not parts or ".." in parts or any(p.casefold() == ".claude" for p in parts):
414
+ skipped += 1
415
+ continue
416
+ if m.isdir():
417
+ fd = folder(parts)
418
+ if fd is None:
419
+ kept += 1
420
+ else:
421
+ os.close(fd)
422
+ continue
423
+ if not m.isreg():
424
+ skipped += 1
425
+ continue
426
+ fd = folder(parts[:-1])
427
+ if fd is None:
428
+ kept += 1
429
+ continue
430
+ try:
431
+ try:
432
+ out = os.open(parts[-1], os.O_WRONLY | os.O_CREAT | os.O_EXCL | NF, 0o644,
433
+ dir_fd=fd)
434
+ except OSError:
435
+ kept += 1
436
+ continue
437
+ src = t.extractfile(m)
438
+ with os.fdopen(out, "wb") as fh:
439
+ while True:
440
+ b = src.read(1 << 20)
441
+ if not b:
442
+ break
443
+ fh.write(b)
444
+ finally:
445
+ os.close(fd)
446
+ sys.stdout.write(f"kept {kept} existing, skipped {skipped}\n")
447
+ """
448
+