phiward 0.2.0b1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. phiward/__init__.py +8 -0
  2. phiward/__main__.py +5 -0
  3. phiward/_toml.py +211 -0
  4. phiward/adapters/__init__.py +105 -0
  5. phiward/adapters/claude.py +429 -0
  6. phiward/broker.py +831 -0
  7. phiward/carryover.py +1613 -0
  8. phiward/claude_settings.py +1307 -0
  9. phiward/cli.py +110 -0
  10. phiward/config.py +936 -0
  11. phiward/confine.py +476 -0
  12. phiward/copying.py +533 -0
  13. phiward/core.py +105 -0
  14. phiward/data/agents-section.md +43 -0
  15. phiward/doctor.py +1132 -0
  16. phiward/fasthook.py +70 -0
  17. phiward/fsinfo.py +241 -0
  18. phiward/gateway.py +159 -0
  19. phiward/githooks.py +248 -0
  20. phiward/guards/__init__.py +0 -0
  21. phiward/guards/protected_paths.py +1138 -0
  22. phiward/hidden.py +281 -0
  23. phiward/init.py +155 -0
  24. phiward/install.py +828 -0
  25. phiward/inventory.py +173 -0
  26. phiward/mcp.py +186 -0
  27. phiward/mcp_config.py +1736 -0
  28. phiward/names.py +5 -0
  29. phiward/packs/dates.toml +17 -0
  30. phiward/packs/genomics.toml +51 -0
  31. phiward/packs/msk.toml +32 -0
  32. phiward/pathid.py +403 -0
  33. phiward/pathmatch.py +98 -0
  34. phiward/paths.py +942 -0
  35. phiward/pin.py +214 -0
  36. phiward/procinfo.py +73 -0
  37. phiward/remote.py +448 -0
  38. phiward/run.py +788 -0
  39. phiward/safeio.py +84 -0
  40. phiward/sanitize.py +691 -0
  41. phiward/selfcheck.py +1506 -0
  42. phiward/service.py +321 -0
  43. phiward/session_note.py +86 -0
  44. phiward/setup_cmd.py +886 -0
  45. phiward/shell.py +504 -0
  46. phiward/wholetool.py +1463 -0
  47. phiward-0.2.0b1.dist-info/METADATA +147 -0
  48. phiward-0.2.0b1.dist-info/RECORD +53 -0
  49. phiward-0.2.0b1.dist-info/WHEEL +5 -0
  50. phiward-0.2.0b1.dist-info/entry_points.txt +3 -0
  51. phiward-0.2.0b1.dist-info/licenses/LICENSE +662 -0
  52. phiward-0.2.0b1.dist-info/licenses/NOTICE +16 -0
  53. phiward-0.2.0b1.dist-info/top_level.txt +1 -0
phiward/run.py ADDED
@@ -0,0 +1,788 @@
1
+ """`phiward run [GRANTS] -- CMD [ARG]...`: the one way to read protected data.
2
+
3
+ The command the agent types is a client only (broker.py): it sends argv to the broker and
4
+ prints what comes back. The broker calls `execute`, which runs CMD in the GATEWAY profile (see
5
+ confine.py): it may read the protected roots; it may write only in a private scratch folder
6
+ (its TMPDIR) and in the folders granted with `--write-to` (protected roots are read only); it
7
+ has no network unless `--net` is given. Its stdin is empty. Everything it
8
+ prints, stdout and stderr, goes through the sanitizer, which also masks the name of every entry
9
+ under the roots.
10
+
11
+ Grants (`--write-to DIR`, `--net`) widen what CMD can do. Nothing asks anyone at run time:
12
+ the operator writes the grants into the user config ahead of time.
13
+ A `--write-to DIR` is resolved once (links followed) and must be one of the `[broker] write_to`
14
+ folders or inside one; `--net` needs `[broker] net = true`. The resolved folder is what the
15
+ profile gets, and the gateway refuses to start if it is no longer a real folder. The program is
16
+ resolved once too: on the operator's PATH, or as a path in the request's folder.
17
+
18
+ `--remote HOST` runs CMD on an HPC host instead, through the operator's own ssh
19
+ setup (`[copy] ssh_config`, the operator's Kerberos ticket and connection): the agent never holds
20
+ them. Only hosts in `[remote] hosts` and programs in `[remote] commands` of the user config are
21
+ allowed; the program is one bare name, and every word reaches the remote shell quoted. Nothing
22
+ confines CMD on the remote side: the list of commands is what limits its effects there. Its
23
+ output is masked like local output, plus the paths under `[remote] roots`.
24
+ """
25
+
26
+ import os
27
+ import re
28
+ import shlex
29
+ import shutil
30
+ import signal
31
+ import stat
32
+ import subprocess
33
+ import sys
34
+ import tempfile
35
+ import threading
36
+ import time
37
+ from dataclasses import dataclass, field
38
+ from typing import BinaryIO, Callable, Dict, Iterator, List, Optional, Sequence, TextIO, Tuple
39
+
40
+ from phiward import config, confine, paths, safeio
41
+ from phiward.sanitize import Sanitizer, SanitizerError
42
+
43
+ MAX_LINE = 64 * 1024 # bytes; a longer output line is withheld, not held in memory
44
+ USAGE = ("usage: phiward run [--write-to DIR]... [--net] [--names-from FILE]... [--table] "
45
+ "-- CMD [ARG]...\n"
46
+ " phiward run --remote HOST [--names-from FILE]... [--table] -- CMD [ARG]...\n")
47
+ HELP = USAGE + """
48
+ Options:
49
+ --write-to DIR let CMD write in DIR. DIR must be one of the `[broker] write_to` folders
50
+ of the user config, or inside one.
51
+ --net let CMD use the network (needs `[broker] net = true`)
52
+ --names-from FILE also mask the names listed in FILE
53
+ --table mask the output as a table (numeric cells are kept)
54
+ --remote HOST run CMD on an HPC host, through your own ssh setup. HOST must be in
55
+ `[remote] hosts`, and CMD in `[remote] commands`.
56
+ -- ends the options; CMD and its arguments follow (required)
57
+ -h, --help show this help
58
+
59
+ Run CMD with read access to the protected data, and print its output masked: protected
60
+ paths, entry names and identifiers become placeholders. This is the one way to work with
61
+ protected data.
62
+
63
+ The command is only a client. The broker runs CMD outside the agent's sandbox, so a broker
64
+ must be running (`phiward broker`). CMD has no shell and an empty stdin. It may write only in a
65
+ private temporary folder and in the folders granted with --write-to (protected roots are read
66
+ only; a granted folder may sit inside one). It has no network unless --net is granted. Nothing asks anyone at run time: the grants come from the
67
+ user config. With `[broker] commands` set, CMD starts in that private folder, not in yours:
68
+ give paths in full. Nothing confines CMD on a remote host; the list of programs limits it
69
+ there.
70
+
71
+ Print counts and summaries, not records: masking hides names and identifiers, not other
72
+ content.
73
+ """
74
+
75
+
76
+ @dataclass
77
+ class RunArgs:
78
+ cmd: List[str] = field(default_factory=list)
79
+ write_to: List[str] = field(default_factory=list)
80
+ net: bool = False
81
+ names_from: List[str] = field(default_factory=list)
82
+ table: bool = False
83
+ remote: str = ""
84
+
85
+ @property
86
+ def has_grants(self) -> bool:
87
+ return bool(self.write_to or self.net)
88
+
89
+
90
+ def parse(argv: Sequence[str]) -> Optional[RunArgs]:
91
+ """The parsed arguments, or None when they are not exactly the documented form."""
92
+ a = RunArgs()
93
+ i = 0
94
+ while i < len(argv):
95
+ w = argv[i]
96
+ if w == "--":
97
+ a.cmd = list(argv[i + 1:])
98
+ return a if a.cmd else None
99
+ if w == "--net":
100
+ a.net = True
101
+ elif w == "--table":
102
+ a.table = True
103
+ elif w == "--remote" and i + 1 < len(argv) and not a.remote:
104
+ a.remote = argv[i + 1]
105
+ i += 1
106
+ elif w in ("--write-to", "--names-from") and i + 1 < len(argv):
107
+ (a.write_to if w == "--write-to" else a.names_from).append(argv[i + 1])
108
+ i += 1
109
+ else:
110
+ return None
111
+ i += 1
112
+ return None # no "--": refused, so the hook and the gateway read the same grants
113
+
114
+
115
+ def _never() -> bool:
116
+ return False
117
+
118
+
119
+ # the broker sets `log.fn` for the request its worker thread runs: what the agent must not see
120
+ # (a command's real exit status) goes there, to the operator's broker log
121
+ log = threading.local()
122
+
123
+
124
+ def _log(msg: str) -> None:
125
+ fn = getattr(log, "fn", None)
126
+ if fn is not None:
127
+ try:
128
+ fn(msg)
129
+ except Exception: # noqa: BLE001 - a log line never fails a request
130
+ pass
131
+
132
+
133
+ # the requests running now (process group, scratch folder): a broker that stops ends them and
134
+ # removes their scratch, which may hold copies of protected data
135
+ _RUNNING: Dict[int, str] = {}
136
+ _RUNNING_LOCK = threading.Lock()
137
+ SCRATCH_PREFIXES = ("run-", "pull-")
138
+ # after its command ends, a request reads what is already in the pipe for at most this long: a
139
+ # process the command left behind may hold the pipe open, but the request ends with its command
140
+ DRAIN_SECONDS = 1.0
141
+
142
+
143
+ def end_all() -> None:
144
+ """The broker stops: end every running request's process group and remove its scratch."""
145
+ with _RUNNING_LOCK:
146
+ running = list(_RUNNING.items())
147
+ for pgid, scratch in running:
148
+ try:
149
+ os.killpg(pgid, signal.SIGKILL)
150
+ except OSError:
151
+ pass
152
+ safeio.remove_tree(scratch)
153
+
154
+
155
+ def sweep_scratch(cache: str) -> int:
156
+ """At the broker's start: remove scratch folders that an earlier broker left (it was killed,
157
+ or the machine stopped). Folders only, never through a link. How many were removed."""
158
+ n = 0
159
+ try:
160
+ entries = list(os.scandir(cache))
161
+ except OSError:
162
+ return 0
163
+ for e in entries:
164
+ if e.name.startswith(SCRATCH_PREFIXES) and e.is_dir(follow_symlinks=False):
165
+ safeio.remove_tree(e.path)
166
+ n += 1
167
+ return n
168
+
169
+
170
+ def _raw_lines(proc: "subprocess.Popen[bytes]", stop: Callable[[], bool]
171
+ ) -> "Iterator[Tuple[bytes, bool]]":
172
+ """The command's output as (line, too_long): read without blocking, ended when the command
173
+ ends (after DRAIN_SECONDS of what is left), at the end of the pipe, or when `stop()` says so
174
+ (a cancel). A line longer than MAX_LINE is given once as (b"", True) and the rest of it
175
+ dropped."""
176
+ import select
177
+ assert proc.stdout is not None
178
+ fd = proc.stdout.fileno()
179
+ os.set_blocking(fd, False)
180
+ buf, skipping, ended = b"", False, 0.0
181
+ while True:
182
+ if stop():
183
+ return
184
+ if not ended and proc.poll() is not None:
185
+ ended = time.monotonic()
186
+ r, _w, _x = select.select([fd], [], [], 0.05 if ended else 0.2)
187
+ data, eof = b"", False
188
+ if r:
189
+ try:
190
+ data = os.read(fd, 65536)
191
+ eof = data == b""
192
+ except BlockingIOError:
193
+ pass
194
+ if ended and (not r or time.monotonic() - ended > DRAIN_SECONDS):
195
+ eof = True
196
+ buf += data
197
+ while True:
198
+ i = buf.find(b"\n")
199
+ if i == -1:
200
+ if len(buf) > MAX_LINE:
201
+ if not skipping:
202
+ yield b"", True
203
+ skipping, buf = True, b""
204
+ break
205
+ line, buf = buf[:i + 1], buf[i + 1:]
206
+ if skipping:
207
+ skipping = False # the end of a line too long to hold
208
+ continue
209
+ if len(line) > MAX_LINE + 1:
210
+ yield b"", True
211
+ continue
212
+ yield line, False
213
+ if eof:
214
+ if buf and not skipping:
215
+ yield buf, False
216
+ return
217
+
218
+
219
+ def run_confined(cmd: Sequence[str], cfg: config.Config, san: Sanitizer, write_to: Sequence[str],
220
+ net: bool, out: TextIO, table: bool = False, label: str = "run",
221
+ env: Optional[Dict[str, str]] = None, cwd: Optional[str] = None,
222
+ cancelled: Callable[[], bool] = _never, deny_read: Sequence[str] = (),
223
+ socket: str = "", allow_read: Sequence[str] = (),
224
+ stdin_feed: Optional[Callable[[BinaryIO], None]] = None,
225
+ quiet: bool = False,
226
+ in_scratch: bool = False) -> int:
227
+ """Run `cmd` in the gateway profile and write its masked output to `out`. `stdin_feed`, when
228
+ given, writes the command's input from the broker (a push's tar stream) in a thread."""
229
+ os.makedirs(cfg.cache_dir, mode=0o700, exist_ok=True)
230
+ scratch = tempfile.mkdtemp(prefix="run-", dir=cfg.cache_dir)
231
+ try:
232
+ # the gateway writes only its scratch folder and the places granted for this request
233
+ # (`write_to`, or a copy's destination): never a protected root or pattern match as
234
+ # such, where it could plant what your own tools run later, or rewrite a names file
235
+ writable = [os.path.realpath(scratch)] + list(write_to)
236
+ argv, fds = confine.gateway_command(list(cmd), writable, [], net,
237
+ deny_read=deny_read, socket=socket,
238
+ allow_read=allow_read)
239
+ env = dict(os.environ if env is None else env, TMPDIR=scratch)
240
+ if in_scratch:
241
+ # a listed program starts in a folder the agent cannot write: no `.Rprofile` or
242
+ # `./helper` of the agent's runs with it
243
+ cwd = scratch
244
+ if socket:
245
+ cwd = "/" # a request through the shared connection needs no working folder
246
+ try:
247
+ proc = subprocess.Popen(argv, stdin=subprocess.PIPE if stdin_feed else subprocess.DEVNULL,
248
+ stdout=subprocess.PIPE,
249
+ stderr=subprocess.STDOUT, env=env, cwd=cwd,
250
+ start_new_session=True, close_fds=True, pass_fds=fds)
251
+ except OSError:
252
+ out.write(f"{label} failed: the command could not start\n")
253
+ return 127
254
+ finally:
255
+ for fd in fds:
256
+ os.close(fd)
257
+ with _RUNNING_LOCK:
258
+ _RUNNING[proc.pid] = scratch
259
+ done = threading.Event()
260
+ feed_failed: List[str] = []
261
+ if stdin_feed is not None:
262
+ def feed() -> None:
263
+ assert proc.stdin is not None
264
+ try:
265
+ stdin_feed(proc.stdin) # type: ignore[misc]
266
+ except Exception as exc: # noqa: BLE001 - reported as a fixed line, no names
267
+ feed_failed.append(str(exc) if type(exc).__name__ == "PushError"
268
+ else type(exc).__name__)
269
+ finally:
270
+ try:
271
+ proc.stdin.close()
272
+ except OSError:
273
+ pass
274
+ feeder_thread = threading.Thread(target=feed, daemon=True)
275
+ feeder_thread.start()
276
+
277
+ def watch() -> None:
278
+ while not done.wait(0.5):
279
+ if cancelled():
280
+ try:
281
+ os.killpg(proc.pid, signal.SIGKILL)
282
+ except OSError:
283
+ pass
284
+ return
285
+
286
+ threading.Thread(target=watch, daemon=True).start()
287
+ assert proc.stdout is not None
288
+ lines = changed = 0
289
+ last, repeats, shown, withheld = None, 0, 0, 0
290
+ cap = max(1, cfg.max_output_lines)
291
+ try:
292
+ for raw, too_long in _raw_lines(proc, cancelled):
293
+ if too_long:
294
+ # a line too long to hold: the rest of it is dropped, a fixed notice shown
295
+ lines += 1
296
+ changed += 1
297
+ if shown >= cap: # the notice counts as a line too
298
+ withheld += 1
299
+ continue
300
+ out.write("[line withheld by the gateway: longer than "
301
+ f"{MAX_LINE // 1024} KiB]\n")
302
+ shown += 1
303
+ last = None
304
+ out.flush()
305
+ continue
306
+ text = raw.decode("utf-8", "replace").rstrip("\r\n")
307
+ clean = san.clean(text, table=table)
308
+ lines += 1
309
+ changed += clean != text
310
+ if quiet:
311
+ # a pull's tar names each member relative to the pulled folder: no mask
312
+ # knows those names; only counted
313
+ continue
314
+ # context, not secrecy: identical lines in a row are shown once
315
+ # with a count, and output past the cap is counted, not shown
316
+ if shown >= cap: # past the cap only counted: no line, no marker
317
+ withheld += 1
318
+ continue
319
+ if clean == last:
320
+ repeats += 1
321
+ continue
322
+ if repeats:
323
+ out.write(f"[{repeats} more identical line(s)]\n")
324
+ shown += 1
325
+ repeats = 0
326
+ if shown >= cap:
327
+ withheld += 1
328
+ continue
329
+ last = clean
330
+ out.write(clean + "\n")
331
+ shown += 1
332
+ out.flush()
333
+ # no counts in these notes: a count is a number the command chooses
334
+ if repeats and shown < cap:
335
+ out.write("[the line above repeats]\n")
336
+ elif repeats:
337
+ withheld += repeats
338
+ if withheld:
339
+ out.write(f"[more lines withheld: the output is longer than {cap} lines; write "
340
+ "results to a file in a granted folder and print a summary]\n")
341
+ code = proc.wait()
342
+ finally:
343
+ done.set()
344
+ try:
345
+ os.killpg(proc.pid, signal.SIGKILL) # nothing it started outlives it
346
+ except OSError:
347
+ pass
348
+ with _RUNNING_LOCK:
349
+ _RUNNING.pop(proc.pid, None)
350
+ if stdin_feed is not None:
351
+ feeder_thread.join(timeout=30)
352
+ if feed_failed:
353
+ out.write(f"{label}: the stream stopped: {feed_failed[0]}\n")
354
+ code = code or 1
355
+ if quiet and lines:
356
+ out.write(f"{label}: message lines from the program withheld (they can name "
357
+ "files)\n")
358
+ # the agent sees success or failure only: an exit status (0-255) and counts are numbers
359
+ # a command chooses, so code written to leak could encode data in them. The real
360
+ # status goes to the operator's broker log. Cancelled stays 130 (phiward's own).
361
+ _log(f"{label}: exit {code}, {lines} line(s), {changed} masked")
362
+ seen = 0 if code == 0 else (130 if cancelled() else 1)
363
+ out.write(f"{label} done: exit={seen}" + ("" if seen != 1 else
364
+ " (the command failed; its exit status is in the operator's broker log)")
365
+ + "\n")
366
+ return seen
367
+ finally:
368
+ safeio.remove_tree(scratch)
369
+
370
+
371
+ def resolve_grant(d: str, cwd: str, cfg: config.Config) -> Optional[str]:
372
+ """The folder a `--write-to` grant names, resolved once, or None when the user config does
373
+ not grant it (it must be a `[broker] write_to` folder or inside one)."""
374
+ from phiward import pathid
375
+ full = os.path.realpath(os.path.join(cwd, os.path.expanduser(d)))
376
+ if not os.path.isdir(full):
377
+ return None
378
+ full = pathid.canon(full) # an allow decision: matched exactly, in canonical spelling
379
+ if not any(config.inside(full, w) for w in cfg.write_to):
380
+ return None
381
+ if config.grant_problem(full, cfg): # inside a write_to folder, but e.g. a root moved in
382
+ return None
383
+ return full
384
+
385
+
386
+ def resolve_program(word: str, cwd: str, env: Dict[str, str]) -> Optional[str]:
387
+ """The program a command's first word names: a path in the request's folder, or a name on
388
+ the operator's PATH (never the agent's)."""
389
+ if "/" in word:
390
+ p = os.path.realpath(os.path.join(cwd, word))
391
+ return p if os.path.isfile(p) and os.access(p, os.X_OK) else None
392
+ found = shutil.which(word, path=env.get("PATH", os.defpath))
393
+ return os.path.realpath(found) if found else None
394
+
395
+
396
+ NOT_LISTED = ("run refused: this program is not in [broker] commands of the config (the "
397
+ "operator lists the programs gateway commands may run: `(cd ~ && phiward setup)`, "
398
+ "or edit the user config, then restart the broker). A program listed by its name "
399
+ "is called by that name alone, not by a path\n")
400
+
401
+
402
+ def _entry_run_path(entry: str, word: str, program: str, cwd: str,
403
+ path_env: str) -> str:
404
+ """The path to run when `entry` (one name of a list, presets expanded) lists the program
405
+ the request names, or "": a bare name runs what it finds on the broker's PATH, called by
406
+ that name alone; a path entry runs as itself (any other name of its file is that file, and a
407
+ multi-call program acts as the entry's name); a folder entry runs the word's name in that
408
+ folder, when the word's real folder is that folder."""
409
+ if "/" not in entry:
410
+ if "/" in word or word != entry:
411
+ return ""
412
+ found = shutil.which(entry, path=path_env)
413
+ return os.path.abspath(found) if found else ""
414
+ e = os.path.expanduser(entry)
415
+ if not e.startswith("/"):
416
+ return "" # a relative entry names nothing (the config check refuses it)
417
+ real = os.path.realpath(e)
418
+ if entry.endswith("/") or os.path.isdir(real):
419
+ given = os.path.join(cwd, word) if "/" in word else (shutil.which(word, path=path_env)
420
+ or "")
421
+ if not given or os.path.realpath(os.path.dirname(given)) != real:
422
+ return ""
423
+ run = os.path.join(real, os.path.basename(given))
424
+ return run if os.path.realpath(run) == program else ""
425
+ # a path entry is a real file: no link and no `..` anywhere on it, so what was checked and
426
+ # approved is what starts (a link could be changed between the check and the start); a
427
+ # link is listed by its name, or through its folder
428
+ if real != os.path.normpath(e) or real != program:
429
+ return ""
430
+ return real
431
+
432
+
433
+ def listed_run_path(word: str, program: str, cfg: config.Config, cwd: str,
434
+ path_env: str) -> str:
435
+ """The path to run under `[broker] commands`, or "" when the program is not listed. With no
436
+ list, the program itself. A program must be in every list that is set (a repo list narrows);
437
+ the path that runs comes from the user config's list when it sets one, never from a repo's."""
438
+ if not cfg.command_lists:
439
+ return program
440
+ chosen = ""
441
+ for lst in cfg.command_lists:
442
+ hit = ""
443
+ for entry in lst:
444
+ names = config.COMMAND_PRESETS.get(entry[1:], ()) if entry.startswith("@") else (entry,)
445
+ for e in names:
446
+ hit = _entry_run_path(e, word, program, cwd, path_env)
447
+ if hit:
448
+ break
449
+ if hit:
450
+ break
451
+ if not hit:
452
+ return ""
453
+ if lst is cfg.user_commands:
454
+ chosen = hit
455
+ # with no user list any program may run anyway: the program itself, never a repo list's path
456
+ return chosen or program
457
+
458
+
459
+ def program_listed(word: str, program: str, cfg: config.Config, cwd: str = "",
460
+ path_env: Optional[str] = None) -> bool:
461
+ """Whether the program may run under `[broker] commands` (`listed_run_path`)."""
462
+ return bool(listed_run_path(word, program, cfg, cwd or os.getcwd(),
463
+ os.environ.get("PATH", os.defpath) if path_env is None
464
+ else path_env))
465
+
466
+
467
+ def execute(argv: Sequence[str], cwd: str, env: Dict[str, str], out: TextIO,
468
+ cancelled: Callable[[], bool] = _never) -> int:
469
+ """Run a request (in the broker). `env` is the command's environment; the config in force
470
+ is the broker's own user config plus the repo config of `cwd` (tighten only)."""
471
+ if list(argv[:1]) in (["-h"], ["--help"]):
472
+ out.write(USAGE)
473
+ return 0
474
+ a = parse(argv)
475
+ if a is None:
476
+ if "--" not in argv:
477
+ out.write("run refused: put `--` before the command, for example "
478
+ "`phiward run -- wc -l FILE`\n")
479
+ out.write(USAGE)
480
+ return 2
481
+ cfg = config.load(cwd)
482
+ if cfg.problem:
483
+ out.write("run refused: the phiward config cannot be used (run phiward doctor)\n")
484
+ return 3
485
+ if a.remote: # needs only [remote]
486
+ return _execute_remote(a, cwd, env, out, cfg, cancelled)
487
+ if not cfg.has_protected_paths:
488
+ out.write("run refused: no protected paths are configured: the operator has not set "
489
+ "phiward up yet (`(cd ~ && phiward setup)`, in their own terminal)\n")
490
+ return 2
491
+ if a.net and not cfg.net:
492
+ out.write("run refused: --net is not granted. The operator sets [broker] net = true in "
493
+ "the user config (`(cd ~ && phiward setup)`, choice 3 \"every setting\", question 5) and restarts "
494
+ "the broker.\n")
495
+ return 2
496
+ write_to = []
497
+ if (not a.write_to and len(cfg.write_to) == 1 and not cfg.write_to_missing
498
+ and any(config._deny_inside(cfg.write_to[0], r) for r in cfg.user_roots)):
499
+ # one write place, inside a protected root: the default, so the agent needs no
500
+ # --write-to for it (what lands there stays protected)
501
+ write_to.append(cfg.write_to[0])
502
+ for d in a.write_to:
503
+ full = resolve_grant(d, cwd, cfg)
504
+ if full is None:
505
+ out.write("run refused: --write-to names a folder that is not granted. It must be an "
506
+ "existing folder in, or inside, [broker] write_to of the user config; the "
507
+ "operator adds one with `(cd ~ && phiward setup)` (its results folder, or in choice 3 "
508
+ "\"every setting\", question 3) and restarts the broker.\n")
509
+ return 2
510
+ write_to.append(full)
511
+ program = resolve_program(a.cmd[0], cwd, env)
512
+ if program is None and cfg.command_lists and "/" in a.cmd[0]:
513
+ # the same answer as for an unlisted program: a path the agent names must not tell
514
+ # whether a file exists there (in a protected folder too)
515
+ out.write(NOT_LISTED)
516
+ return 2
517
+ if program is None:
518
+ out.write("run refused: the program is not on the broker's PATH (or not an executable "
519
+ "file). The broker keeps the PATH of the terminal that started it: the "
520
+ "operator starts it again (or runs `(cd ~ && phiward broker --service --write)` again) "
521
+ "from a terminal where the program works, for example after `conda "
522
+ "activate`; or give the program's full path. A folder "
523
+ "the agent's sandbox may write (a temp folder, or one in allowWrite) is never "
524
+ "searched: the operator installs the program elsewhere.\n")
525
+ return 127
526
+ path_env = env.get("PATH", os.defpath)
527
+ if cfg.command_lists:
528
+ # what runs is the listed entry itself (a bare name: what it finds on the broker's
529
+ # PATH; a path entry: that path; a folder entry: the name in that folder), taken from
530
+ # the user config's list: never a link of the agent's, never a repo list's path
531
+ run_as = listed_run_path(a.cmd[0], program, cfg, cwd, path_env)
532
+ if not run_as:
533
+ out.write(NOT_LISTED)
534
+ return 2
535
+ else:
536
+ run_as = program if "/" in a.cmd[0] else os.path.abspath(
537
+ shutil.which(a.cmd[0], path=path_env) or program)
538
+ cmd = [run_as] + a.cmd[1:]
539
+ why, lines = _sanitizer_options(a, cwd, cfg)
540
+ if why:
541
+ out.write(f"run refused: {why}\n")
542
+ return 2
543
+ try:
544
+ san = Sanitizer.from_config(cfg, entries=True, names_lines=lines)
545
+ except SanitizerError:
546
+ out.write("run refused: the sanitizer cannot start (run phiward doctor)\n")
547
+ return 3
548
+ watch = _Watch(out)
549
+ try:
550
+ listed = bool(cfg.command_lists)
551
+ code = run_confined(cmd, cfg, san, write_to, a.net, watch, a.table, env=env, cwd=cwd,
552
+ cancelled=cancelled, deny_read=gateway_hidden(cfg),
553
+ in_scratch=listed)
554
+ if listed and any(w and not w.startswith(("/", "-")) for w in a.cmd[1:]):
555
+ # said whatever exists there (no file name is looked up for the agent)
556
+ out.write("run: with [broker] commands set, a command starts in a private folder "
557
+ "that is removed afterwards, not in yours: give paths in full, and write "
558
+ "results with --write-to.\n")
559
+ if code != 0 and watch.write_refused:
560
+ out.write("run: the command was refused a path. Gateway commands cannot read the "
561
+ "hidden list (credentials), and write only in a granted folder, named with "
562
+ "--write-to DIR (protected roots are read only). The operator adds a "
563
+ "folder with `(cd ~ && phiward setup)` (its results folder, or in choice 3 \"every "
564
+ "setting\", question 3) and restarts the broker; `phiward doctor` counts "
565
+ "the granted folders.\n")
566
+ if code != 0 and watch.no_network and not a.net:
567
+ out.write("run: the command had no network (by design). `--net` needs "
568
+ "[broker] net = true in the user config.\n")
569
+ return code
570
+ except confine.ConfineError as exc:
571
+ out.write(f"run refused: {exc}\n")
572
+ return 3
573
+ except Exception as exc: # noqa: BLE001 - never print a traceback (it can name paths)
574
+ out.write(f"run failed ({type(exc).__name__})\n")
575
+ return 3
576
+
577
+
578
+ class _Watch:
579
+ """Passes output on unchanged, and notes the words of a refused write or a missing network,
580
+ so `run` can say what to do next."""
581
+ _WRITE = ("Operation not permitted", "Read-only file system", "Permission denied")
582
+ _NET = ("Could not resolve host", "Network is unreachable", "Name or service not known",
583
+ "nodename nor servname", "Temporary failure in name resolution")
584
+
585
+ def __init__(self, out):
586
+ self.out, self.write_refused, self.no_network = out, False, False
587
+
588
+ def write(self, text: str) -> int:
589
+ if any(w in text for w in self._WRITE):
590
+ self.write_refused = True
591
+ if any(w in text for w in self._NET):
592
+ self.no_network = True
593
+ return self.out.write(text)
594
+
595
+ def __getattr__(self, name):
596
+ return getattr(self.out, name)
597
+
598
+
599
+ _HOST = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]*")
600
+ # a csh or tcsh login shell expands `!` even inside single quotes, and a newline ends a quoted
601
+ # word there: no quoting makes these plain text on every login shell
602
+ # `!` (csh and tcsh expand it inside quotes), `\\` (fish reads it as an escape inside single
603
+ # quotes) and control characters cannot be quoted for every login shell: refused
604
+ _UNQUOTABLE = re.compile(r"[!\\\x00-\x1f\x7f]")
605
+
606
+
607
+ TABLE_OFF = ("--table is off: table mode lets bare numbers through (a record number would "
608
+ "pass); the operator turns it on with [sanitize] table = true in the user config")
609
+
610
+
611
+ NAMES_OUTSIDE = ("a --names-from file must be a plain file inside a protected path of the user "
612
+ "config, reached without a link (the broker reads it outside the sandbox)")
613
+
614
+
615
+ def _read_by_parts(path: str) -> str:
616
+ """The text of the regular file at the resolved `path`, opened part by part without
617
+ following a link, and with one name only (a hard link could carry another file in)."""
618
+ flags = os.O_NOFOLLOW | os.O_CLOEXEC
619
+ fd = os.open("/", os.O_RDONLY | os.O_DIRECTORY | flags)
620
+ try:
621
+ parts = [p for p in path.split("/") if p]
622
+ for p in parts[:-1]:
623
+ nxt = os.open(p, os.O_RDONLY | os.O_DIRECTORY | flags, dir_fd=fd)
624
+ os.close(fd)
625
+ fd = nxt
626
+ f = os.open(parts[-1], os.O_RDONLY | os.O_NONBLOCK | flags, dir_fd=fd)
627
+ finally:
628
+ os.close(fd)
629
+ try:
630
+ st = os.fstat(f)
631
+ if not stat.S_ISREG(st.st_mode) or st.st_nlink != 1 or st.st_size > (64 << 20):
632
+ raise OSError("not a plain file")
633
+ chunks, total = [], 0
634
+ while True:
635
+ b = os.read(f, 1 << 20)
636
+ if not b:
637
+ break
638
+ total += len(b)
639
+ if total > (64 << 20): # it grew while it was read
640
+ raise OSError("too large")
641
+ chunks.append(b)
642
+ finally:
643
+ os.close(f)
644
+ return b"".join(chunks).decode("utf-8")
645
+
646
+
647
+ def _sanitizer_options(a: "RunArgs", cwd: str, cfg: config.Config) -> Tuple[str, List[str]]:
648
+ """(why the agent's --table or --names-from cannot be used or "", the names files' lines)
649
+ (what is checked is what is read)."""
650
+ if a.table and not cfg.sanitize_table:
651
+ return TABLE_OFF, []
652
+ pset = paths.ProtectedSet(cfg.user_roots, cfg.user_patterns)
653
+ lines: List[str] = []
654
+ for n in a.names_from:
655
+ full = os.path.realpath(os.path.join(cwd, os.path.expanduser(n)))
656
+ # the broker reads it outside every profile: a file elsewhere (a key, a token) would
657
+ # turn masking into a test of its contents
658
+ if pset.relation(full) != paths.INSIDE:
659
+ return NAMES_OUTSIDE, []
660
+ try:
661
+ text = _read_by_parts(full)
662
+ except (OSError, UnicodeDecodeError):
663
+ return NAMES_OUTSIDE, []
664
+ from phiward.sanitize import STOP_WORDS, name_tokens
665
+ if not name_tokens(text.splitlines(), set(STOP_WORDS)):
666
+ return "a --names-from file yields no name to mask", []
667
+ lines += text.splitlines()
668
+ return "", lines
669
+
670
+
671
+ KRB5_MAX = 2000
672
+
673
+
674
+ def gateway_hidden(cfg: config.Config) -> List[str]:
675
+ """What a local gateway command may not read: the credentials and traces `launch` hides,
676
+ from the broker's own environment (never the request's), the write_to folders aside (a
677
+ grant writes there). Without this, `run -- cat ~/.ssh/id_x` printed the key."""
678
+ from phiward import hidden
679
+ import glob
680
+ out = hidden.paths(dict(os.environ), cfg)
681
+ if sys.platform == "darwin":
682
+ return out # the profile hides every ticket cache by its name (confine)
683
+ # Linux: every cache name there now, with no cap; past a bound the request is refused (a
684
+ # cache pushed past a cap would be readable)
685
+ found: List[str] = []
686
+ for d in ("/tmp", "/var/tmp", "/dev/shm"):
687
+ found += glob.glob(os.path.join(d, "krb5cc*"))
688
+ if len(found) > KRB5_MAX:
689
+ raise confine.ConfineError(
690
+ f"the temp folders hold more than {KRB5_MAX} Kerberos cache names, too many to "
691
+ "hide one by one; the operator removes the extra ones")
692
+ return out + found
693
+
694
+
695
+ def add_remote_roots(san: Sanitizer, cfg: config.Config) -> None:
696
+ """The remote protected paths, masked before every other step (longest first, so a root
697
+ inside another one does not leave a tail); for `run --remote` and for copies to and from a
698
+ remote."""
699
+ from phiward.sanitize import normalize
700
+ if san.first_rules:
701
+ return
702
+ for root in sorted(cfg.remote_roots, key=len, reverse=True):
703
+ san.first_rules.append((remote_root_regex(normalize(root)), "<PROTECTED>"))
704
+
705
+
706
+ def remote_root_regex(root: str) -> "re.Pattern[str]":
707
+ """A remote protected path in any spelling the remote shell treats as the same (`//`,
708
+ `/./`), with the rest of the path up to a tab or the end of the line: a file name may hold
709
+ spaces."""
710
+ sep = r"/(?:/|\./)*"
711
+ parts = [re.escape(p) for p in root.split("/") if p and p != "."]
712
+ if not parts:
713
+ return re.compile(r"(?!)")
714
+ return re.compile(sep + sep.join(parts) + r"(?:/[^\t\n]*|(?=[\s'\":;,)]|$))")
715
+
716
+
717
+ def _remote_argv(rt, host: str, cmd: Sequence[str]) -> List[str]:
718
+ """The ssh argv for a remote run on the route `rt` (remote.route: the operator's ssh config
719
+ only, no proxy or local command, no forwarding, no terminal, never a prompt); every word
720
+ quoted for the remote shell."""
721
+ return rt.ssh + [host, "--", " ".join(shlex.quote(w) for w in cmd)]
722
+
723
+
724
+ def _execute_remote(a: RunArgs, cwd: str, env: Dict[str, str], out: TextIO, cfg: config.Config,
725
+ cancelled: Callable[[], bool]) -> int:
726
+ if a.has_grants:
727
+ out.write("run refused: --remote takes no --write-to or --net\n")
728
+ return 2
729
+ if not _HOST.fullmatch(a.remote) or a.remote not in cfg.remote_hosts:
730
+ out.write("run refused: the host is not in [remote] hosts of the user config\n")
731
+ return 2
732
+ prog = a.cmd[0]
733
+ if "/" in prog or prog not in cfg.remote_commands:
734
+ out.write("run refused: the program is not in [remote] commands of the user config (the "
735
+ "remote side is not confined: the operator lists what may run there)\n")
736
+ return 2
737
+ if any(".." in w.replace("\\", "/").split("/") for w in a.cmd):
738
+ out.write("run refused: a word holds a `..` part; a remote path given that way is not "
739
+ "masked. Give remote paths without `..`\n")
740
+ return 2
741
+ if any(_UNQUOTABLE.search(w) for w in a.cmd):
742
+ out.write("run refused: a word holds a character the remote shell may expand even when "
743
+ "quoted (`!`, `\\`, a newline or another control character)\n")
744
+ return 2
745
+ why, lines = _sanitizer_options(a, cwd, cfg)
746
+ if why:
747
+ out.write(f"run refused: {why}\n")
748
+ return 2
749
+ from phiward import remote as rmt
750
+ rt, why = rmt.route(cfg, a.remote)
751
+ if rt is None:
752
+ out.write(f"run refused: {why}\n")
753
+ return 2
754
+ ssh = _remote_argv(rt, a.remote, a.cmd)
755
+ try:
756
+ san = Sanitizer.from_config(cfg, entries=True, names_lines=lines)
757
+ except SanitizerError:
758
+ out.write("run refused: the sanitizer cannot start (run phiward doctor)\n")
759
+ return 3
760
+ # the remote protected paths, masked first (longest first, so a root inside another one
761
+ # does not leave a tail)
762
+ add_remote_roots(san, cfg)
763
+ try:
764
+ # through the shared connection: no credential, no network but its socket
765
+ return run_confined(ssh, cfg, san, [], rt.net, out, a.table, label="remote run",
766
+ env=env, cwd=cwd, cancelled=cancelled, deny_read=rt.deny_read,
767
+ socket=rt.socket, allow_read=rt.allow_read)
768
+ except confine.ConfineError as exc:
769
+ out.write(f"run refused: {exc}\n")
770
+ return 3
771
+ except Exception as exc: # noqa: BLE001 - never print a traceback
772
+ out.write(f"run failed ({type(exc).__name__})\n")
773
+ return 3
774
+
775
+
776
+ def main(argv: List[str], out: TextIO = sys.stdout) -> int:
777
+ """The client: the request goes to the broker, which runs it outside the agent's sandbox."""
778
+ if argv[:1] in (["-h"], ["--help"]):
779
+ out.write(HELP)
780
+ return 0
781
+ if parse(argv) is None:
782
+ if "--" not in argv:
783
+ out.write("run refused: put `--` before the command, for example "
784
+ "`phiward run -- wc -l FILE`\n")
785
+ out.write(USAGE)
786
+ return 2
787
+ from phiward import broker
788
+ return broker.client("run", list(argv), out)