phiward 0.2.0b1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. phiward/__init__.py +8 -0
  2. phiward/__main__.py +5 -0
  3. phiward/_toml.py +211 -0
  4. phiward/adapters/__init__.py +105 -0
  5. phiward/adapters/claude.py +429 -0
  6. phiward/broker.py +831 -0
  7. phiward/carryover.py +1613 -0
  8. phiward/claude_settings.py +1307 -0
  9. phiward/cli.py +110 -0
  10. phiward/config.py +936 -0
  11. phiward/confine.py +476 -0
  12. phiward/copying.py +533 -0
  13. phiward/core.py +105 -0
  14. phiward/data/agents-section.md +43 -0
  15. phiward/doctor.py +1132 -0
  16. phiward/fasthook.py +70 -0
  17. phiward/fsinfo.py +241 -0
  18. phiward/gateway.py +159 -0
  19. phiward/githooks.py +248 -0
  20. phiward/guards/__init__.py +0 -0
  21. phiward/guards/protected_paths.py +1138 -0
  22. phiward/hidden.py +281 -0
  23. phiward/init.py +155 -0
  24. phiward/install.py +828 -0
  25. phiward/inventory.py +173 -0
  26. phiward/mcp.py +186 -0
  27. phiward/mcp_config.py +1736 -0
  28. phiward/names.py +5 -0
  29. phiward/packs/dates.toml +17 -0
  30. phiward/packs/genomics.toml +51 -0
  31. phiward/packs/msk.toml +32 -0
  32. phiward/pathid.py +403 -0
  33. phiward/pathmatch.py +98 -0
  34. phiward/paths.py +942 -0
  35. phiward/pin.py +214 -0
  36. phiward/procinfo.py +73 -0
  37. phiward/remote.py +448 -0
  38. phiward/run.py +788 -0
  39. phiward/safeio.py +84 -0
  40. phiward/sanitize.py +691 -0
  41. phiward/selfcheck.py +1506 -0
  42. phiward/service.py +321 -0
  43. phiward/session_note.py +86 -0
  44. phiward/setup_cmd.py +886 -0
  45. phiward/shell.py +504 -0
  46. phiward/wholetool.py +1463 -0
  47. phiward-0.2.0b1.dist-info/METADATA +147 -0
  48. phiward-0.2.0b1.dist-info/RECORD +53 -0
  49. phiward-0.2.0b1.dist-info/WHEEL +5 -0
  50. phiward-0.2.0b1.dist-info/entry_points.txt +3 -0
  51. phiward-0.2.0b1.dist-info/licenses/LICENSE +662 -0
  52. phiward-0.2.0b1.dist-info/licenses/NOTICE +16 -0
  53. phiward-0.2.0b1.dist-info/top_level.txt +1 -0
phiward/broker.py ADDED
@@ -0,0 +1,831 @@
1
+ """`phiward broker`: the only process that starts gateway commands.
2
+
3
+ The broker runs in the operator's terminal, outside every sandbox. `phiward run ...` and
4
+ `phiward copy ...` are clients only: they run inside the agent's sandbox, leave a request
5
+ in the spool folder, and print the output that the broker writes back. No shell outside the
6
+ sandbox ever reads a gateway command; the broker gets argv.
7
+
8
+ The spool (`$PHIWARD_SPOOL`, else `~/.phiward-spool`) is writable by the agent, so the
9
+ broker treats everything in it as untrusted data:
10
+ - it opens the folder once (no symlink, owned by the user, mode 0700) and works through that
11
+ descriptor; if the path stops naming that folder, it stops;
12
+ - it reads a request only as a regular file with one link, owned by the user, opened without
13
+ following a link and without blocking, and of bounded size;
14
+ - it creates each file it writes with O_CREAT|O_EXCL|O_NOFOLLOW, mode 0600, so a planted link
15
+ or file makes it skip the request, never write through it;
16
+ - it runs argv with no shell, in the gateway profile, in a new session with no terminal. The
17
+ command's environment is not the request's: the broker builds it from a short allow list of
18
+ its own variables (PATH, HOME, USER, LOGNAME, LANG, LC_*, TZ, TERM), so variables such as
19
+ `BASH_ENV`, `RSYNC_RSH` or `LD_PRELOAD` cannot change what an approved command runs.
20
+
21
+ The broker also writes a status file, `~/.phiward/broker.json` (outside the spool; the
22
+ agent may not write it): the user config, cache folder, roots and patterns that it uses. It
23
+ writes it again when they change, and removes it when it stops. `doctor` reports a difference
24
+ from the agent's own view (the spool too). A reader uses it only while the heartbeat is fresh.
25
+
26
+ Files: the client writes `req-<id>.json` (through a temporary name and a rename); the broker
27
+ claims it by renaming it to `job-<id>.json`, writes the sanitized output to `out-<id>.log`, then
28
+ `done-<id>.json` ({"exit": N}). The client writes `cancel-<id>` to stop a command. The broker
29
+ touches `broker.alive` every second; a client that sees no fresh heartbeat says so and exits 3.
30
+ At most `QUEUE_PER_JOB` requests per worker wait; past that a request is refused ("busy"). A
31
+ request cancelled while it waits never starts.
32
+
33
+ Nothing asks the operator at run time: any prompt on the operator's
34
+ screen or terminal is in reach of the agent's code. Grants come from the user config
35
+ (`[broker] write_to`, `[broker] net`, `[copy] remotes`), which the operator writes ahead of time,
36
+ for example with `phiward setup`.
37
+
38
+ usage:
39
+ phiward broker [--max-jobs N]
40
+ phiward broker --service [--write]
41
+ """
42
+
43
+ import errno
44
+ import json
45
+ import os
46
+ import queue
47
+ import re
48
+ import secrets
49
+ import signal
50
+ import stat
51
+ import sys
52
+ import threading
53
+ import time
54
+ from typing import Any, Dict, List, Optional, TextIO
55
+
56
+ from phiward import config, safeio
57
+
58
+ USAGE = ("usage: phiward broker [--max-jobs N]\n"
59
+ " phiward broker --service [--write]\n")
60
+ HELP = USAGE + """
61
+ Options:
62
+ --max-jobs N run at most N gateway commands at the same time (default 4)
63
+ --service print the login service for the broker, and the command that loads it
64
+ --write with --service: write the service file, then print the commands that load
65
+ it and remove it
66
+ -h, --help show this help
67
+
68
+ The broker runs the gateway commands (`phiward run`, `phiward copy`) outside the agent's
69
+ sandbox, with the grants of your user config. The `phiward run` and `phiward copy` that the
70
+ agent types are only clients: they leave a request in the spool and print the masked output
71
+ that the broker writes back. Without a running broker they fail.
72
+
73
+ Start the broker yourself, outside every agent tool. Either keep it running in a terminal of
74
+ its own (`phiward broker`), or install it once as a login service.
75
+
76
+ The login service:
77
+ - On macOS it is a LaunchAgent (~/Library/LaunchAgents/org.phiward.broker.plist). On Linux it
78
+ is a systemd user unit (~/.config/systemd/user/phiward-broker.service).
79
+ - It starts at login, and starts again if it stops. Its log is in ~/Library/Logs/phiward (macOS)
80
+ or ~/.local/state/phiward (Linux), which no agent session can read. `phiward doctor` shows
81
+ whether the broker runs.
82
+ - `--service` without `--write` prints the file and the command that loads it. It writes
83
+ nothing.
84
+ - `--service --write` writes the file. In your own terminal it then loads the service, or
85
+ restarts it when it runs already (launchctl or systemctl --user); without a terminal it only
86
+ prints those commands.
87
+ - The service gets the PATH of the terminal you install it from (only folders that no one
88
+ else can write). Install it from a terminal where your tools are found, and again after
89
+ your PATH changes.
90
+ - Without a terminal, it refuses when PHIWARD_CONFIG, PHIWARD_SPOOL, XDG_CONFIG_HOME or
91
+ XDG_CACHE_HOME is set.
92
+
93
+ Nothing asks you at run time. The grants come from the user config (`[broker] write_to`,
94
+ `[broker] net`, `[copy] remotes`), which you write ahead of time, for example with
95
+ `phiward setup`.
96
+ """
97
+ HEARTBEAT = 1.0 # seconds between touches of broker.alive
98
+ STALE = 10.0 # a heartbeat older than this means no broker
99
+ MAX_REQUEST = 256 * 1024 # bytes
100
+ MAX_ARGS = 4096
101
+ QUEUE_PER_JOB = 8 # waiting requests per worker; past this, "busy"
102
+ STATUS_SECONDS = 2.0 # how often the broker checks whether its status changed
103
+ GC_SECONDS = 3600 # leftover files older than this are removed
104
+ _ID = r"[0-9a-f]{16,64}"
105
+ _REQ = re.compile(rf"req-({_ID})\.json")
106
+ _LEFTOVER = re.compile(rf"(?:job|out|done|cancel|\.tmp)-{_ID}(?:\.json|\.log)?")
107
+ _ENV_KEEP = re.compile(r"(?:PATH|HOME|USER|LOGNAME|LANG|LC_[A-Z]+|TZ|TERM)")
108
+
109
+ class SpoolError(Exception):
110
+ pass
111
+
112
+
113
+ # ---- shared ------------------------------------------------------------------------------------
114
+
115
+ def _wait_seconds(env: Dict[str, str]) -> float:
116
+ try:
117
+ return max(0.5, float(env.get("PHIWARD_BROKER_WAIT", "10")))
118
+ except ValueError:
119
+ return 10.0
120
+
121
+
122
+ def _alive(spool: str) -> bool:
123
+ try:
124
+ st = os.lstat(os.path.join(spool, "broker.alive"))
125
+ except OSError:
126
+ return False
127
+ return stat.S_ISREG(st.st_mode) and time.time() - st.st_mtime < STALE
128
+
129
+
130
+ # ---- the client (inside the agent's sandbox) ---------------------------------------------------
131
+
132
+ NO_BROKER = ("{name} refused: no phiward broker is running. The operator starts one with "
133
+ "`phiward broker` in a terminal outside the agent (or its login service: "
134
+ "`(cd ~ && phiward broker --service --write)`).\n")
135
+ OTHER_SPOOL = ("{name} refused: the broker runs with the spool {other}, and this shell uses "
136
+ "{spool} (an older phiward used {other}). Ask the operator to restart the broker "
137
+ "(`phiward broker`, or its login service).\n")
138
+
139
+
140
+ def _no_broker(name: str, spool: str) -> str:
141
+ """Why a request was not taken: no broker, or one that runs on another spool (an older
142
+ phiward's, before the spool moved out of ~/.phiward)."""
143
+ other = other_spool(spool)
144
+ if other:
145
+ return OTHER_SPOOL.format(name=name, other=other, spool=spool)
146
+ return NO_BROKER.format(name=name)
147
+
148
+
149
+ def client(name: str, args: List[str], out: TextIO = sys.stdout,
150
+ env: Optional[Dict[str, str]] = None) -> int:
151
+ """Send `phiward <name> <args>` to the broker and print what it writes back."""
152
+ env = dict(os.environ if env is None else env)
153
+ spool = config.spool_dir(env)
154
+ if not _alive(spool):
155
+ out.write(_no_broker(name, spool))
156
+ return 3
157
+ rid = secrets.token_hex(16)
158
+ # no environment: the broker builds the command's environment itself
159
+ body = {"v": 1, "cmd": name, "args": args, "cwd": os.getcwd(), "env": {}}
160
+ tmp = os.path.join(spool, f".tmp-{rid}")
161
+ try:
162
+ fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
163
+ with os.fdopen(fd, "w", encoding="utf-8") as fh:
164
+ json.dump(body, fh)
165
+ os.rename(tmp, os.path.join(spool, f"req-{rid}.json"))
166
+ except OSError:
167
+ out.write(f"{name} refused: the request could not be written to the broker's spool "
168
+ "folder (is it writable from the agent's sandbox?)\n")
169
+ return 3
170
+ req, outp = os.path.join(spool, f"req-{rid}.json"), os.path.join(spool, f"out-{rid}.log")
171
+ done = os.path.join(spool, f"done-{rid}.json")
172
+ try:
173
+ return _follow(name, spool, req, outp, done, out, _wait_seconds(env))
174
+ except KeyboardInterrupt:
175
+ _touch(os.path.join(spool, f"cancel-{rid}"))
176
+ out.write(f"{name} stopped: cancelled\n")
177
+ return 130
178
+ finally:
179
+ for p in (req, outp, done):
180
+ try:
181
+ os.unlink(p)
182
+ except OSError:
183
+ pass
184
+
185
+
186
+ def _touch(path: str) -> None:
187
+ try:
188
+ os.close(os.open(path, os.O_WRONLY | os.O_CREAT | os.O_NOFOLLOW, 0o600))
189
+ except OSError:
190
+ pass
191
+
192
+
193
+ class _NotPlain(Exception):
194
+ pass
195
+
196
+
197
+ def _printable(raw: bytes) -> str:
198
+ """A reply line as the client prints it: control characters (terminal escapes) dropped. The
199
+ spool is the agent's, so a forged reply must not steer the operator's terminal."""
200
+ text = raw.decode("utf-8", "replace")
201
+ return re.sub(r"[\x00-\x08\x0b-\x1f\x7f-\x9f]", "", text)
202
+
203
+
204
+ def _open_plain(path: str):
205
+ """The broker's output or result file, opened without following a link, and only a plain
206
+ file of this user with one name. The spool is in the agent's reach: a link planted there
207
+ would make a client that runs outside the sandbox (the MCP server) read any file."""
208
+ fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC)
209
+ st = os.fstat(fd)
210
+ if not stat.S_ISREG(st.st_mode) or st.st_nlink != 1 or st.st_uid != os.getuid():
211
+ os.close(fd)
212
+ raise _NotPlain(path)
213
+ return os.fdopen(fd, "rb")
214
+
215
+
216
+ def _follow(name: str, spool: str, req: str, outp: str, done: str, out: TextIO,
217
+ wait: float) -> int:
218
+ try:
219
+ return _follow_plain(name, spool, req, outp, done, out, wait)
220
+ except (_NotPlain, OSError):
221
+ out.write(f"{name} refused: the broker's reply in the spool is not a plain file of "
222
+ "yours (something put a link or another file there); nothing is shown\n")
223
+ return 3
224
+
225
+
226
+ def _follow_plain(name: str, spool: str, req: str, outp: str, done: str,
227
+ out: TextIO, wait: float) -> int:
228
+ t0 = time.time()
229
+ pos = 0
230
+ pending = b""
231
+ fh = None
232
+ while True:
233
+ finished = os.path.lexists(done)
234
+ if fh is None and os.path.lexists(outp):
235
+ fh = _open_plain(outp)
236
+ if fh is not None:
237
+ fh.seek(pos)
238
+ chunk = fh.read()
239
+ pos += len(chunk)
240
+ pending += chunk
241
+ *lines, pending = pending.split(b"\n")
242
+ for line in lines:
243
+ out.write(_printable(line) + "\n")
244
+ out.flush()
245
+ if finished:
246
+ if pending:
247
+ out.write(_printable(pending) + "\n")
248
+ try:
249
+ with _open_plain(done) as d:
250
+ code = json.loads(d.read(4096).decode("utf-8")).get("exit")
251
+ return code if isinstance(code, int) else 3
252
+ except (ValueError, AttributeError, UnicodeDecodeError):
253
+ return 3
254
+ if os.path.exists(req) and time.time() - t0 > wait:
255
+ try:
256
+ os.unlink(req) # not claimed: take it back, so it never runs later
257
+ except OSError:
258
+ pass
259
+ else:
260
+ out.write(_no_broker(name, spool))
261
+ return 3
262
+ if not _alive(spool):
263
+ out.write(f"{name} stopped: the broker stopped before the command finished\n")
264
+ return 3
265
+ time.sleep(0.05)
266
+
267
+
268
+ # ---- the broker (outside every sandbox) --------------------------------------------------------
269
+
270
+ class Spool:
271
+ """The spool folder, opened once and used only through its descriptor."""
272
+
273
+ def __init__(self, path: str):
274
+ self.path = path
275
+ parent = os.path.dirname(path)
276
+ try:
277
+ os.makedirs(parent, mode=0o700, exist_ok=True)
278
+ os.mkdir(path, 0o700)
279
+ except FileExistsError:
280
+ pass
281
+ except OSError as exc:
282
+ raise SpoolError(f"the spool folder cannot be made ({type(exc).__name__})")
283
+ try:
284
+ self.fd = os.open(path, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
285
+ except OSError as exc:
286
+ raise SpoolError(f"the spool folder cannot be opened as a plain folder "
287
+ f"({type(exc).__name__})")
288
+ st = os.fstat(self.fd)
289
+ if st.st_uid != os.getuid() or st.st_mode & 0o077:
290
+ os.close(self.fd)
291
+ raise SpoolError("the spool folder must belong to you and have mode 0700")
292
+ self.ident = (st.st_dev, st.st_ino)
293
+
294
+ def still_here(self) -> bool:
295
+ try:
296
+ st = os.lstat(self.path)
297
+ except OSError:
298
+ return False
299
+ return stat.S_ISDIR(st.st_mode) and (st.st_dev, st.st_ino) == self.ident
300
+
301
+ def names(self) -> List[str]:
302
+ return os.listdir(self.fd)
303
+
304
+ def create(self, name: str) -> int:
305
+ """A new file for writing; never an existing file or a link (raises OSError)."""
306
+ return os.open(name, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600,
307
+ dir_fd=self.fd)
308
+
309
+ def write_new(self, name: str, data: bytes) -> None:
310
+ """`name` with `data`, through a new temporary file and a rename."""
311
+ tmp = f".tmp-{secrets.token_hex(16)}"
312
+ fd = self.create(tmp)
313
+ try:
314
+ os.write(fd, data)
315
+ finally:
316
+ os.close(fd)
317
+ os.rename(tmp, name, src_dir_fd=self.fd, dst_dir_fd=self.fd)
318
+
319
+ def unlink(self, name: str) -> None:
320
+ try:
321
+ os.unlink(name, dir_fd=self.fd)
322
+ except OSError:
323
+ pass
324
+
325
+ def exists(self, name: str) -> bool:
326
+ try:
327
+ os.stat(name, dir_fd=self.fd, follow_symlinks=False)
328
+ return True
329
+ except OSError:
330
+ return False
331
+
332
+ def read_request(self, name: str) -> Dict[str, Any]:
333
+ """The request in `name`, checked; raises SpoolError."""
334
+ try:
335
+ fd = os.open(name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=self.fd)
336
+ except OSError:
337
+ raise SpoolError("not a plain file")
338
+ try:
339
+ st = os.fstat(fd)
340
+ if not stat.S_ISREG(st.st_mode) or st.st_nlink != 1 or st.st_uid != os.getuid():
341
+ raise SpoolError("not a plain file with one link")
342
+ if st.st_size > MAX_REQUEST:
343
+ raise SpoolError("too large")
344
+ data = os.read(fd, MAX_REQUEST + 1)
345
+ finally:
346
+ os.close(fd)
347
+ if len(data) > MAX_REQUEST:
348
+ raise SpoolError("too large")
349
+ try:
350
+ body = json.loads(data.decode("utf-8"))
351
+ except (UnicodeDecodeError, ValueError):
352
+ raise SpoolError("not JSON")
353
+ return check_request(body)
354
+
355
+ def heartbeat(self) -> None:
356
+ try:
357
+ # O_NONBLOCK: a FIFO planted under this name fails at once (ENXIO) instead of
358
+ # blocking the main loop
359
+ fd = os.open("broker.alive", os.O_WRONLY | os.O_CREAT | os.O_NOFOLLOW | os.O_NONBLOCK,
360
+ 0o600, dir_fd=self.fd)
361
+ except OSError:
362
+ self.unlink("broker.alive") # not a plain file: remove it; the next beat makes one
363
+ return
364
+ try:
365
+ if stat.S_ISREG(os.fstat(fd).st_mode):
366
+ os.utime(fd)
367
+ finally:
368
+ os.close(fd)
369
+
370
+ def collect_garbage(self) -> None:
371
+ now = time.time()
372
+ for name in self.names():
373
+ if not _LEFTOVER.fullmatch(name) and not _REQ.fullmatch(name):
374
+ continue
375
+ try:
376
+ st = os.stat(name, dir_fd=self.fd, follow_symlinks=False)
377
+ except OSError:
378
+ continue
379
+ if now - st.st_mtime > GC_SECONDS:
380
+ self.unlink(name)
381
+
382
+
383
+ def _plain_words(v: Any, limit: int) -> bool:
384
+ return isinstance(v, list) and len(v) <= limit and all(
385
+ isinstance(w, str) and "\0" not in w for w in v)
386
+
387
+
388
+ def check_request(body: Any) -> Dict[str, Any]:
389
+ if not isinstance(body, dict) or body.get("v") != 1:
390
+ raise SpoolError("unknown request version")
391
+ if body.get("cmd") not in ("run", "copy"):
392
+ raise SpoolError("unknown command")
393
+ if not _plain_words(body.get("args"), MAX_ARGS):
394
+ raise SpoolError("bad arguments")
395
+ cwd = body.get("cwd")
396
+ if not isinstance(cwd, str) or not os.path.isabs(cwd) or "\0" in cwd or \
397
+ not os.path.isdir(cwd):
398
+ raise SpoolError("bad working folder")
399
+ env = body.get("env")
400
+ if not isinstance(env, dict) or not all(
401
+ isinstance(k, str) and isinstance(v, str) and "\0" not in k + v and "=" not in k and k
402
+ for k, v in env.items()):
403
+ raise SpoolError("bad environment")
404
+ return body
405
+
406
+
407
+ def command_env(env: Optional[Dict[str, str]] = None) -> Dict[str, str]:
408
+ """The command's environment: the allow-listed variables of the broker's own environment.
409
+ The request's environment is never used."""
410
+ env = os.environ if env is None else env
411
+ out = {k: v for k, v in env.items() if _ENV_KEEP.fullmatch(k)}
412
+ if "PATH" in out:
413
+ # the same PATH filter as the login service: a relative entry, a folder others can
414
+ # write or one inside a project would let the agent choose the program that runs
415
+ from phiward import service
416
+ out["PATH"] = ":".join(service._safe_path_entries(out["PATH"])) or "/usr/bin:/bin"
417
+ return out
418
+
419
+
420
+ # the most files a listed folder may hold (with its subfolders): a larger one is never approved
421
+ FOLDER_LIMIT = 2000
422
+
423
+
424
+ def program_path() -> str:
425
+ """The PATH on which listed programs are found: the running broker's own (from its status
426
+ file), so `pin` and `doctor` in the operator's terminal see what the broker sees; else this
427
+ process's command PATH."""
428
+ st = read_status()
429
+ p = st.get("path") if isinstance(st, dict) else None
430
+ return p if isinstance(p, str) and p else command_env().get("PATH", "")
431
+
432
+
433
+ def _folder_files(folder: str) -> List[str]:
434
+ """Every file in a listed folder and its subfolders (a script's helpers live there), or a
435
+ marker when it holds more than FOLDER_LIMIT, which is never approved."""
436
+ out: List[str] = []
437
+ for top, dirs, files in os.walk(folder, followlinks=False):
438
+ dirs.sort()
439
+ for n in sorted(files):
440
+ out.append(os.path.join(top, n))
441
+ if len(out) > FOLDER_LIMIT:
442
+ return [f"too-many:{folder}"]
443
+ return out
444
+
445
+
446
+ def listed_programs(cfg: "config.Config", path: str) -> List[str]:
447
+ """The files `[broker] commands` allows, as the broker would find them now: a bare name on
448
+ `path`, a path entry itself, every program in a folder entry."""
449
+ import shutil
450
+ out: List[str] = []
451
+ for lst in cfg.command_lists:
452
+ for entry in lst:
453
+ if entry.startswith("@"):
454
+ names = list(config.COMMAND_PRESETS.get(entry[1:], ()))
455
+ else:
456
+ names = [entry]
457
+ for e in names:
458
+ if "/" in e:
459
+ p = os.path.realpath(os.path.expanduser(e))
460
+ if os.path.isdir(p):
461
+ out += _folder_files(p)
462
+ elif p != os.path.normpath(os.path.expanduser(e)):
463
+ # a path entry with a link on it lists nothing (run.py): say so
464
+ out.append(f"linked:{e}")
465
+ else:
466
+ out.append(p)
467
+ else:
468
+ found = shutil.which(e, path=path)
469
+ out.append(os.path.realpath(found) if found else f"missing:{e}")
470
+ return out
471
+
472
+
473
+ def config_print() -> str:
474
+ """A fingerprint of what decides what the gateway may do and what it masks: the user config,
475
+ the pins, and the names and pack files the config names. Content and resolved place only
476
+ (an editor's same-content save, or `phiward setup` writing the same text, changes nothing).
477
+ The broker takes it at start and refuses requests when it changes, so a change that the
478
+ agent manages to make has no effect until the operator restarts the broker."""
479
+ import hashlib
480
+ from phiward import pin
481
+ h = hashlib.sha256()
482
+ files = [config.user_config_path(), pin.pins_path()]
483
+ cfg = None
484
+ try:
485
+ cfg = config.load("/")
486
+ files += list(cfg.names_from) + [p for p in cfg.packs if "/" in p]
487
+ except Exception: # noqa: BLE001 - a config that cannot be loaded is fingerprinted as text
488
+ pass
489
+ for path in files:
490
+ real = os.path.realpath(os.path.expanduser(path))
491
+ h.update(real.encode("utf-8", "surrogateescape") + b"\0")
492
+ try:
493
+ with open(real, "rb") as fh:
494
+ h.update(hashlib.sha256(fh.read(1 << 24)).digest())
495
+ except OSError as exc:
496
+ h.update(f"missing:{exc.errno}".encode())
497
+ # the programs `[broker] commands` allows, as found on the command PATH, by identity: a
498
+ # program replaced or rewritten (its inode or change time, which no one can set back), or a
499
+ # new one of a listed name earlier on PATH, stops the broker until the operator restarts it
500
+ try:
501
+ progs = listed_programs(cfg, command_env().get("PATH", "")) if cfg else []
502
+ except Exception: # noqa: BLE001
503
+ progs = ["unreadable"]
504
+ for prog in progs:
505
+ h.update(prog.encode("utf-8", "surrogateescape") + b"\0")
506
+ try:
507
+ st = os.stat(prog)
508
+ h.update(f"{st.st_dev}:{st.st_ino}:{st.st_size}:{st.st_ctime_ns}".encode())
509
+ except OSError as exc:
510
+ h.update(f"missing:{exc.errno}".encode())
511
+ return h.hexdigest()
512
+
513
+
514
+ def restart_hint() -> str:
515
+ if sys.platform == "darwin":
516
+ return ("`launchctl kickstart -k gui/$(id -u)/org.phiward.broker` for the login service, "
517
+ "or stop `phiward broker` and start it again")
518
+ return ("`systemctl --user restart phiward-broker.service` for the login service, or stop "
519
+ "`phiward broker` and start it again")
520
+
521
+
522
+ class Broker:
523
+ def __init__(self, spool: Spool, log: TextIO, max_jobs: int = 4):
524
+ self.spool, self.log = spool, log
525
+ self.config_print = config_print()
526
+ # listed programs must match what the operator approved (`phiward pin`): a program
527
+ # replaced while the broker was stopped is not accepted by a restart
528
+ try:
529
+ from phiward import pin
530
+ # on this broker's own PATH (another broker's status file names its own)
531
+ self.unapproved = pin.unapproved(config.load("/"),
532
+ path=command_env().get("PATH", ""))
533
+ except Exception: # noqa: BLE001 - when in doubt, nothing runs
534
+ self.unapproved = ["unreadable"]
535
+ # claimed jobs wait here for one of `max_jobs` workers; the main loop never blocks, so
536
+ # the heartbeat goes on while every worker is busy
537
+ self.jobs: "queue.Queue[Any]" = queue.Queue()
538
+ self.max_jobs = max_jobs
539
+ self.stop = threading.Event()
540
+
541
+ def say(self, msg: str) -> None:
542
+ self.log.write(f"[broker {time.strftime('%H:%M:%S')}] {msg}\n")
543
+ self.log.flush()
544
+
545
+ def serve(self) -> int:
546
+ last_gc = last_beat = last_status = 0.0
547
+ from phiward import run as run_mod
548
+ if run_mod.sweep_scratch(config.cache_dir()):
549
+ self.say("removed scratch folders an earlier broker left")
550
+ status = write_status()
551
+ for _ in range(self.max_jobs):
552
+ threading.Thread(target=self.worker, daemon=True).start()
553
+ if self.unapproved:
554
+ self.say(f"ready, but {len(self.unapproved)} listed program(s) are not approved: "
555
+ "every request is refused until `phiward pin` and a restart")
556
+ else:
557
+ self.say("ready; requests are read from the spool folder")
558
+ while not self.stop.is_set():
559
+ if not self.spool.still_here():
560
+ self.say("the spool folder was moved or replaced; stopping")
561
+ return 3
562
+ if time.time() - last_beat >= HEARTBEAT:
563
+ self.spool.heartbeat()
564
+ last_beat = time.time()
565
+ if time.time() - last_status >= STATUS_SECONDS:
566
+ status = write_status(status)
567
+ last_status = time.time()
568
+ if time.time() - last_gc > 60:
569
+ self.spool.collect_garbage()
570
+ last_gc = time.time()
571
+ for name in self.spool.names():
572
+ m = _REQ.fullmatch(name)
573
+ if m:
574
+ self.claim(m.group(1), name)
575
+ self.stop.wait(0.05)
576
+ return 0
577
+
578
+ def claim(self, rid: str, name: str) -> None:
579
+ """Take a request and queue it. Nothing here reads the request's content or touches its
580
+ folder, so the main loop (and the heartbeat) never waits on it; every
581
+ refusal from here on ends with a done file, so the client never waits forever."""
582
+ job = f"job-{rid}.json"
583
+ if self.spool.exists(job):
584
+ self.spool.unlink(name)
585
+ return
586
+ try:
587
+ os.rename(name, job, src_dir_fd=self.spool.fd, dst_dir_fd=self.spool.fd)
588
+ except OSError:
589
+ return
590
+ try:
591
+ outfd = self.spool.create(f"out-{rid}.log")
592
+ except OSError:
593
+ self.say(f"request {rid[:8]} refused: its output file already exists")
594
+ self.finish(rid, 3)
595
+ return
596
+ if self.jobs.qsize() >= self.max_jobs * QUEUE_PER_JOB:
597
+ self.say(f"request {rid[:8]} refused: busy")
598
+ with os.fdopen(outfd, "w", encoding="utf-8") as out:
599
+ out.write("refused: the broker is busy (too many requests wait); try again "
600
+ "later\n")
601
+ self.finish(rid, 3)
602
+ return
603
+ self.jobs.put((rid, outfd))
604
+
605
+ def worker(self) -> None:
606
+ while True:
607
+ self.run_job(*self.jobs.get())
608
+
609
+ def finish(self, rid: str, code: int) -> None:
610
+ try:
611
+ self.spool.write_new(f"done-{rid}.json", json.dumps({"exit": code}).encode())
612
+ except OSError:
613
+ self.say(f"request {rid[:8]}: the done file could not be written")
614
+ self.spool.unlink(f"job-{rid}.json")
615
+
616
+ def run_job(self, rid: str, outfd: int) -> None:
617
+ code = 3
618
+ try:
619
+ with os.fdopen(outfd, "w", encoding="utf-8", errors="replace") as out:
620
+ try:
621
+ body = self.spool.read_request(f"job-{rid}.json")
622
+ # the request holds the client's words (globs expanded by a shell that is
623
+ # not sandboxed): not left in the spool, which the agent can read
624
+ self.spool.unlink(f"job-{rid}.json")
625
+ except SpoolError as exc:
626
+ self.say(f"request {rid[:8]} refused: {exc}")
627
+ out.write(f"refused: the broker cannot use this request ({exc})\n")
628
+ return
629
+ if self.spool.exists(f"cancel-{rid}"): # cancelled while it waited
630
+ out.write(f"{body['cmd']} stopped: cancelled before it started\n")
631
+ code = 130
632
+ else:
633
+ code = self.execute(rid, body, out)
634
+ except Exception as exc: # noqa: BLE001 - one bad job never stops the broker
635
+ self.say(f"request {rid[:8]} failed ({type(exc).__name__})")
636
+ finally:
637
+ self.finish(rid, code)
638
+
639
+ def execute(self, rid: str, body: Dict[str, Any], out: TextIO) -> int:
640
+ cancelled = lambda: self.spool.exists(f"cancel-{rid}") # noqa: E731
641
+ if self.unapproved:
642
+ self.say(f"request {rid[:8]} refused: {len(self.unapproved)} listed program(s) "
643
+ "not approved")
644
+ out.write(f"refused: {len(self.unapproved)} program(s) of [broker] commands differ "
645
+ "from what the operator approved, or were never approved. The operator "
646
+ "runs `phiward doctor`, which names them, checks them, runs `phiward pin` "
647
+ f"in their own terminal, and restarts the broker: {restart_hint()}.\n")
648
+ return 3
649
+ if config_print() != self.config_print:
650
+ self.say(f"request {rid[:8]} refused: the config changed since the broker started")
651
+ out.write("refused: the phiward config changed since the broker started. The "
652
+ "operator reads the change and restarts the broker: "
653
+ f"{restart_hint()}.\n")
654
+ return 3
655
+ env = command_env()
656
+ self.say(f"request {rid[:8]}: {body['cmd']} ({len(body['args'])} words)")
657
+ from phiward import run
658
+ run.log.fn = lambda msg: self.say(f"request {rid[:8]}: {msg}")
659
+ try:
660
+ if body["cmd"] == "run":
661
+ code = run.execute(body["args"], body["cwd"], env, out, cancelled)
662
+ else:
663
+ from phiward import copying
664
+ code = copying.execute(body["args"], body["cwd"], env, out, cancelled)
665
+ finally:
666
+ run.log.fn = None
667
+ self.say(f"request {rid[:8]}: reported exit {code}")
668
+ return code
669
+
670
+
671
+ def _status() -> Dict[str, Any]:
672
+ cfg = config.load("/")
673
+ import socket as _socket
674
+ from phiward import procinfo
675
+ me = procinfo.info(os.getpid())
676
+ return {"v": 1, "pid": os.getpid(), "start": me[1] if me else None,
677
+ "host": _socket.gethostname(), "config": config.user_config_path(), "cache": config.cache_dir(),
678
+ "spool": config.spool_dir(),
679
+ "roots": cfg.user_roots, "patterns": cfg.user_patterns,
680
+ "path": command_env().get("PATH", "")}
681
+
682
+
683
+ def write_status(last: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
684
+ """Write the status file when it differs from `last`; return what is in force."""
685
+ status = _status()
686
+ if status == last:
687
+ return status
688
+ path = config.status_path()
689
+ try:
690
+ os.makedirs(os.path.dirname(path), mode=0o700, exist_ok=True)
691
+ safeio.write_new(path, json.dumps(status))
692
+ except OSError:
693
+ pass
694
+ return status
695
+
696
+
697
+ def remove_status() -> None:
698
+ try:
699
+ os.unlink(config.status_path())
700
+ except OSError:
701
+ pass
702
+
703
+
704
+ def running(spool: Optional[str] = None) -> str:
705
+ """Whether a broker serves `spool`, for doctor and pin (not the client, whose sandbox may not
706
+ see the broker's process): "here" when the guarded status file names a live process of this
707
+ machine with the start time it wrote, "on HOST" when it names another machine (a shared home
708
+ folder on a cluster: there only the heartbeat can tell), "" otherwise. The heartbeat alone
709
+ is in the spool, which the agent writes, so it is not enough."""
710
+ if not _alive(spool or config.spool_dir()):
711
+ return ""
712
+ st = _read_status_file()
713
+ import socket as _socket
714
+ from phiward import procinfo
715
+ pid, start, host = st.get("pid"), st.get("start"), st.get("host")
716
+ if isinstance(host, str) and host and host != _socket.gethostname():
717
+ return f"on {host}"
718
+ if not isinstance(pid, int) or isinstance(pid, bool):
719
+ return ""
720
+ got = procinfo.info(pid)
721
+ return "here" if got is not None and got[1] == start else ""
722
+
723
+
724
+ def read_status(spool: Optional[str] = None) -> Dict[str, Any]:
725
+ """What the running broker wrote about itself, or {} when no broker runs (`running`). A
726
+ reader may only use it to add protection."""
727
+ if not running(spool):
728
+ return {}
729
+ return _read_status_file()
730
+
731
+
732
+ def other_spool(spool: str) -> Optional[str]:
733
+ """The spool of a broker that runs with another spool than `spool`, or None. Only to say
734
+ where the broker is (`doctor`, a refused request): nothing else is taken from the file."""
735
+ other = _read_status_file().get("spool")
736
+ if other and os.path.abspath(other) != os.path.abspath(spool) and _alive(other):
737
+ return other
738
+ return None
739
+
740
+
741
+ def _read_status_file() -> Dict[str, Any]:
742
+ try:
743
+ fd = os.open(config.status_path(), os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK)
744
+ except OSError:
745
+ return {}
746
+ try:
747
+ st = os.fstat(fd)
748
+ if not stat.S_ISREG(st.st_mode):
749
+ return {}
750
+ private = st.st_uid == os.getuid() and not st.st_mode & 0o077
751
+ data = json.loads(os.read(fd, MAX_REQUEST).decode("utf-8"))
752
+ except (OSError, ValueError, UnicodeDecodeError):
753
+ return {}
754
+ finally:
755
+ os.close(fd)
756
+ if not isinstance(data, dict):
757
+ return {}
758
+ out: Dict[str, Any] = {}
759
+ for key in ("config", "cache", "spool"):
760
+ if isinstance(data.get(key), str) and os.path.isabs(data[key]):
761
+ out[key] = data[key]
762
+ if isinstance(data.get("path"), str) and private:
763
+ # where the broker finds programs (program_path): only folders no one else can write,
764
+ # and only from the broker's own private file
765
+ from phiward import service
766
+ out["path"] = ":".join(service._safe_path_entries(data["path"]))
767
+ for key in ("roots", "patterns"):
768
+ v = data.get(key)
769
+ out[key] = [x for x in v if isinstance(x, str) and os.path.isabs(x)] if isinstance(v, list) else []
770
+ if private: # who wrote it (`running`): only from the broker's own private file
771
+ for key, kind in (("pid", int), ("start", int), ("host", str)):
772
+ v = data.get(key)
773
+ if isinstance(v, kind) and not isinstance(v, bool):
774
+ out[key] = v
775
+ return out
776
+
777
+
778
+ def main(argv: List[str]) -> int:
779
+ max_jobs = 4
780
+ if argv[:1] in (["-h"], ["--help"]):
781
+ sys.stdout.write(HELP)
782
+ return 0
783
+ if argv[:1] == ["--service"]:
784
+ from phiward import service
785
+ return service.main(argv[1:])
786
+ if argv[:1] == ["--max-jobs"] and len(argv) == 2 and argv[1].isdigit() and int(argv[1]) > 0:
787
+ max_jobs = int(argv[1])
788
+ elif argv:
789
+ sys.stderr.write(USAGE)
790
+ return 2
791
+ problem = config.load("/").problem
792
+ if problem:
793
+ sys.stderr.write(f"phiward broker: the user config cannot be used: {problem}\n")
794
+ return 3
795
+ try:
796
+ spool = Spool(config.spool_dir())
797
+ except SpoolError as exc:
798
+ sys.stderr.write(f"phiward broker: {exc}\n")
799
+ return 3
800
+ import fcntl
801
+ try:
802
+ lock = os.open("broker.lock", os.O_WRONLY | os.O_CREAT | os.O_NOFOLLOW | os.O_NONBLOCK,
803
+ 0o600, dir_fd=spool.fd)
804
+ if not stat.S_ISREG(os.fstat(lock).st_mode):
805
+ raise OSError(errno.EINVAL, "not a plain file")
806
+ fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
807
+ except OSError as exc:
808
+ busy = exc.errno in (errno.EWOULDBLOCK, errno.EAGAIN)
809
+ sys.stderr.write("phiward broker: " + ("another broker is running\n" if busy else
810
+ "the lock file cannot be used\n"))
811
+ return 3
812
+ # the login service appends to its log file: keep it from filling the disk
813
+ from phiward import service
814
+ logf = os.path.join(service.log_folder(), "broker.log")
815
+ try:
816
+ if os.path.getsize(logf) > 50 * 1024 * 1024 and not os.path.islink(logf):
817
+ os.truncate(logf, 0)
818
+ except OSError:
819
+ pass
820
+ broker = Broker(spool, sys.stderr, max_jobs)
821
+ for sig in (signal.SIGTERM, signal.SIGHUP):
822
+ signal.signal(sig, lambda *_: broker.stop.set())
823
+ try:
824
+ return broker.serve()
825
+ except KeyboardInterrupt:
826
+ return 0
827
+ finally:
828
+ from phiward import run as run_mod
829
+ run_mod.end_all() # running requests end with the broker, their scratch removed
830
+ spool.unlink("broker.alive")
831
+ remove_status()