phiward 0.2.0b1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- phiward/__init__.py +8 -0
- phiward/__main__.py +5 -0
- phiward/_toml.py +211 -0
- phiward/adapters/__init__.py +105 -0
- phiward/adapters/claude.py +429 -0
- phiward/broker.py +831 -0
- phiward/carryover.py +1613 -0
- phiward/claude_settings.py +1307 -0
- phiward/cli.py +110 -0
- phiward/config.py +936 -0
- phiward/confine.py +476 -0
- phiward/copying.py +533 -0
- phiward/core.py +105 -0
- phiward/data/agents-section.md +43 -0
- phiward/doctor.py +1132 -0
- phiward/fasthook.py +70 -0
- phiward/fsinfo.py +241 -0
- phiward/gateway.py +159 -0
- phiward/githooks.py +248 -0
- phiward/guards/__init__.py +0 -0
- phiward/guards/protected_paths.py +1138 -0
- phiward/hidden.py +281 -0
- phiward/init.py +155 -0
- phiward/install.py +828 -0
- phiward/inventory.py +173 -0
- phiward/mcp.py +186 -0
- phiward/mcp_config.py +1736 -0
- phiward/names.py +5 -0
- phiward/packs/dates.toml +17 -0
- phiward/packs/genomics.toml +51 -0
- phiward/packs/msk.toml +32 -0
- phiward/pathid.py +403 -0
- phiward/pathmatch.py +98 -0
- phiward/paths.py +942 -0
- phiward/pin.py +214 -0
- phiward/procinfo.py +73 -0
- phiward/remote.py +448 -0
- phiward/run.py +788 -0
- phiward/safeio.py +84 -0
- phiward/sanitize.py +691 -0
- phiward/selfcheck.py +1506 -0
- phiward/service.py +321 -0
- phiward/session_note.py +86 -0
- phiward/setup_cmd.py +886 -0
- phiward/shell.py +504 -0
- phiward/wholetool.py +1463 -0
- phiward-0.2.0b1.dist-info/METADATA +147 -0
- phiward-0.2.0b1.dist-info/RECORD +53 -0
- phiward-0.2.0b1.dist-info/WHEEL +5 -0
- phiward-0.2.0b1.dist-info/entry_points.txt +3 -0
- phiward-0.2.0b1.dist-info/licenses/LICENSE +662 -0
- phiward-0.2.0b1.dist-info/licenses/NOTICE +16 -0
- phiward-0.2.0b1.dist-info/top_level.txt +1 -0
phiward/broker.py
ADDED
|
@@ -0,0 +1,831 @@
|
|
|
1
|
+
"""`phiward broker`: the only process that starts gateway commands.
|
|
2
|
+
|
|
3
|
+
The broker runs in the operator's terminal, outside every sandbox. `phiward run ...` and
|
|
4
|
+
`phiward copy ...` are clients only: they run inside the agent's sandbox, leave a request
|
|
5
|
+
in the spool folder, and print the output that the broker writes back. No shell outside the
|
|
6
|
+
sandbox ever reads a gateway command; the broker gets argv.
|
|
7
|
+
|
|
8
|
+
The spool (`$PHIWARD_SPOOL`, else `~/.phiward-spool`) is writable by the agent, so the
|
|
9
|
+
broker treats everything in it as untrusted data:
|
|
10
|
+
- it opens the folder once (no symlink, owned by the user, mode 0700) and works through that
|
|
11
|
+
descriptor; if the path stops naming that folder, it stops;
|
|
12
|
+
- it reads a request only as a regular file with one link, owned by the user, opened without
|
|
13
|
+
following a link and without blocking, and of bounded size;
|
|
14
|
+
- it creates each file it writes with O_CREAT|O_EXCL|O_NOFOLLOW, mode 0600, so a planted link
|
|
15
|
+
or file makes it skip the request, never write through it;
|
|
16
|
+
- it runs argv with no shell, in the gateway profile, in a new session with no terminal. The
|
|
17
|
+
command's environment is not the request's: the broker builds it from a short allow list of
|
|
18
|
+
its own variables (PATH, HOME, USER, LOGNAME, LANG, LC_*, TZ, TERM), so variables such as
|
|
19
|
+
`BASH_ENV`, `RSYNC_RSH` or `LD_PRELOAD` cannot change what an approved command runs.
|
|
20
|
+
|
|
21
|
+
The broker also writes a status file, `~/.phiward/broker.json` (outside the spool; the
|
|
22
|
+
agent may not write it): the user config, cache folder, roots and patterns that it uses. It
|
|
23
|
+
writes it again when they change, and removes it when it stops. `doctor` reports a difference
|
|
24
|
+
from the agent's own view (the spool too). A reader uses it only while the heartbeat is fresh.
|
|
25
|
+
|
|
26
|
+
Files: the client writes `req-<id>.json` (through a temporary name and a rename); the broker
|
|
27
|
+
claims it by renaming it to `job-<id>.json`, writes the sanitized output to `out-<id>.log`, then
|
|
28
|
+
`done-<id>.json` ({"exit": N}). The client writes `cancel-<id>` to stop a command. The broker
|
|
29
|
+
touches `broker.alive` every second; a client that sees no fresh heartbeat says so and exits 3.
|
|
30
|
+
At most `QUEUE_PER_JOB` requests per worker wait; past that a request is refused ("busy"). A
|
|
31
|
+
request cancelled while it waits never starts.
|
|
32
|
+
|
|
33
|
+
Nothing asks the operator at run time: any prompt on the operator's
|
|
34
|
+
screen or terminal is in reach of the agent's code. Grants come from the user config
|
|
35
|
+
(`[broker] write_to`, `[broker] net`, `[copy] remotes`), which the operator writes ahead of time,
|
|
36
|
+
for example with `phiward setup`.
|
|
37
|
+
|
|
38
|
+
usage:
|
|
39
|
+
phiward broker [--max-jobs N]
|
|
40
|
+
phiward broker --service [--write]
|
|
41
|
+
"""
|
|
42
|
+
|
|
43
|
+
import errno
|
|
44
|
+
import json
|
|
45
|
+
import os
|
|
46
|
+
import queue
|
|
47
|
+
import re
|
|
48
|
+
import secrets
|
|
49
|
+
import signal
|
|
50
|
+
import stat
|
|
51
|
+
import sys
|
|
52
|
+
import threading
|
|
53
|
+
import time
|
|
54
|
+
from typing import Any, Dict, List, Optional, TextIO
|
|
55
|
+
|
|
56
|
+
from phiward import config, safeio
|
|
57
|
+
|
|
58
|
+
USAGE = ("usage: phiward broker [--max-jobs N]\n"
|
|
59
|
+
" phiward broker --service [--write]\n")
|
|
60
|
+
HELP = USAGE + """
|
|
61
|
+
Options:
|
|
62
|
+
--max-jobs N run at most N gateway commands at the same time (default 4)
|
|
63
|
+
--service print the login service for the broker, and the command that loads it
|
|
64
|
+
--write with --service: write the service file, then print the commands that load
|
|
65
|
+
it and remove it
|
|
66
|
+
-h, --help show this help
|
|
67
|
+
|
|
68
|
+
The broker runs the gateway commands (`phiward run`, `phiward copy`) outside the agent's
|
|
69
|
+
sandbox, with the grants of your user config. The `phiward run` and `phiward copy` that the
|
|
70
|
+
agent types are only clients: they leave a request in the spool and print the masked output
|
|
71
|
+
that the broker writes back. Without a running broker they fail.
|
|
72
|
+
|
|
73
|
+
Start the broker yourself, outside every agent tool. Either keep it running in a terminal of
|
|
74
|
+
its own (`phiward broker`), or install it once as a login service.
|
|
75
|
+
|
|
76
|
+
The login service:
|
|
77
|
+
- On macOS it is a LaunchAgent (~/Library/LaunchAgents/org.phiward.broker.plist). On Linux it
|
|
78
|
+
is a systemd user unit (~/.config/systemd/user/phiward-broker.service).
|
|
79
|
+
- It starts at login, and starts again if it stops. Its log is in ~/Library/Logs/phiward (macOS)
|
|
80
|
+
or ~/.local/state/phiward (Linux), which no agent session can read. `phiward doctor` shows
|
|
81
|
+
whether the broker runs.
|
|
82
|
+
- `--service` without `--write` prints the file and the command that loads it. It writes
|
|
83
|
+
nothing.
|
|
84
|
+
- `--service --write` writes the file. In your own terminal it then loads the service, or
|
|
85
|
+
restarts it when it runs already (launchctl or systemctl --user); without a terminal it only
|
|
86
|
+
prints those commands.
|
|
87
|
+
- The service gets the PATH of the terminal you install it from (only folders that no one
|
|
88
|
+
else can write). Install it from a terminal where your tools are found, and again after
|
|
89
|
+
your PATH changes.
|
|
90
|
+
- Without a terminal, it refuses when PHIWARD_CONFIG, PHIWARD_SPOOL, XDG_CONFIG_HOME or
|
|
91
|
+
XDG_CACHE_HOME is set.
|
|
92
|
+
|
|
93
|
+
Nothing asks you at run time. The grants come from the user config (`[broker] write_to`,
|
|
94
|
+
`[broker] net`, `[copy] remotes`), which you write ahead of time, for example with
|
|
95
|
+
`phiward setup`.
|
|
96
|
+
"""
|
|
97
|
+
HEARTBEAT = 1.0 # seconds between touches of broker.alive
|
|
98
|
+
STALE = 10.0 # a heartbeat older than this means no broker
|
|
99
|
+
MAX_REQUEST = 256 * 1024 # bytes
|
|
100
|
+
MAX_ARGS = 4096
|
|
101
|
+
QUEUE_PER_JOB = 8 # waiting requests per worker; past this, "busy"
|
|
102
|
+
STATUS_SECONDS = 2.0 # how often the broker checks whether its status changed
|
|
103
|
+
GC_SECONDS = 3600 # leftover files older than this are removed
|
|
104
|
+
_ID = r"[0-9a-f]{16,64}"
|
|
105
|
+
_REQ = re.compile(rf"req-({_ID})\.json")
|
|
106
|
+
_LEFTOVER = re.compile(rf"(?:job|out|done|cancel|\.tmp)-{_ID}(?:\.json|\.log)?")
|
|
107
|
+
_ENV_KEEP = re.compile(r"(?:PATH|HOME|USER|LOGNAME|LANG|LC_[A-Z]+|TZ|TERM)")
|
|
108
|
+
|
|
109
|
+
class SpoolError(Exception):
|
|
110
|
+
pass
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
# ---- shared ------------------------------------------------------------------------------------
|
|
114
|
+
|
|
115
|
+
def _wait_seconds(env: Dict[str, str]) -> float:
|
|
116
|
+
try:
|
|
117
|
+
return max(0.5, float(env.get("PHIWARD_BROKER_WAIT", "10")))
|
|
118
|
+
except ValueError:
|
|
119
|
+
return 10.0
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def _alive(spool: str) -> bool:
|
|
123
|
+
try:
|
|
124
|
+
st = os.lstat(os.path.join(spool, "broker.alive"))
|
|
125
|
+
except OSError:
|
|
126
|
+
return False
|
|
127
|
+
return stat.S_ISREG(st.st_mode) and time.time() - st.st_mtime < STALE
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
# ---- the client (inside the agent's sandbox) ---------------------------------------------------
|
|
131
|
+
|
|
132
|
+
NO_BROKER = ("{name} refused: no phiward broker is running. The operator starts one with "
|
|
133
|
+
"`phiward broker` in a terminal outside the agent (or its login service: "
|
|
134
|
+
"`(cd ~ && phiward broker --service --write)`).\n")
|
|
135
|
+
OTHER_SPOOL = ("{name} refused: the broker runs with the spool {other}, and this shell uses "
|
|
136
|
+
"{spool} (an older phiward used {other}). Ask the operator to restart the broker "
|
|
137
|
+
"(`phiward broker`, or its login service).\n")
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def _no_broker(name: str, spool: str) -> str:
|
|
141
|
+
"""Why a request was not taken: no broker, or one that runs on another spool (an older
|
|
142
|
+
phiward's, before the spool moved out of ~/.phiward)."""
|
|
143
|
+
other = other_spool(spool)
|
|
144
|
+
if other:
|
|
145
|
+
return OTHER_SPOOL.format(name=name, other=other, spool=spool)
|
|
146
|
+
return NO_BROKER.format(name=name)
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def client(name: str, args: List[str], out: TextIO = sys.stdout,
|
|
150
|
+
env: Optional[Dict[str, str]] = None) -> int:
|
|
151
|
+
"""Send `phiward <name> <args>` to the broker and print what it writes back."""
|
|
152
|
+
env = dict(os.environ if env is None else env)
|
|
153
|
+
spool = config.spool_dir(env)
|
|
154
|
+
if not _alive(spool):
|
|
155
|
+
out.write(_no_broker(name, spool))
|
|
156
|
+
return 3
|
|
157
|
+
rid = secrets.token_hex(16)
|
|
158
|
+
# no environment: the broker builds the command's environment itself
|
|
159
|
+
body = {"v": 1, "cmd": name, "args": args, "cwd": os.getcwd(), "env": {}}
|
|
160
|
+
tmp = os.path.join(spool, f".tmp-{rid}")
|
|
161
|
+
try:
|
|
162
|
+
fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
|
|
163
|
+
with os.fdopen(fd, "w", encoding="utf-8") as fh:
|
|
164
|
+
json.dump(body, fh)
|
|
165
|
+
os.rename(tmp, os.path.join(spool, f"req-{rid}.json"))
|
|
166
|
+
except OSError:
|
|
167
|
+
out.write(f"{name} refused: the request could not be written to the broker's spool "
|
|
168
|
+
"folder (is it writable from the agent's sandbox?)\n")
|
|
169
|
+
return 3
|
|
170
|
+
req, outp = os.path.join(spool, f"req-{rid}.json"), os.path.join(spool, f"out-{rid}.log")
|
|
171
|
+
done = os.path.join(spool, f"done-{rid}.json")
|
|
172
|
+
try:
|
|
173
|
+
return _follow(name, spool, req, outp, done, out, _wait_seconds(env))
|
|
174
|
+
except KeyboardInterrupt:
|
|
175
|
+
_touch(os.path.join(spool, f"cancel-{rid}"))
|
|
176
|
+
out.write(f"{name} stopped: cancelled\n")
|
|
177
|
+
return 130
|
|
178
|
+
finally:
|
|
179
|
+
for p in (req, outp, done):
|
|
180
|
+
try:
|
|
181
|
+
os.unlink(p)
|
|
182
|
+
except OSError:
|
|
183
|
+
pass
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def _touch(path: str) -> None:
|
|
187
|
+
try:
|
|
188
|
+
os.close(os.open(path, os.O_WRONLY | os.O_CREAT | os.O_NOFOLLOW, 0o600))
|
|
189
|
+
except OSError:
|
|
190
|
+
pass
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
class _NotPlain(Exception):
|
|
194
|
+
pass
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def _printable(raw: bytes) -> str:
|
|
198
|
+
"""A reply line as the client prints it: control characters (terminal escapes) dropped. The
|
|
199
|
+
spool is the agent's, so a forged reply must not steer the operator's terminal."""
|
|
200
|
+
text = raw.decode("utf-8", "replace")
|
|
201
|
+
return re.sub(r"[\x00-\x08\x0b-\x1f\x7f-\x9f]", "", text)
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def _open_plain(path: str):
|
|
205
|
+
"""The broker's output or result file, opened without following a link, and only a plain
|
|
206
|
+
file of this user with one name. The spool is in the agent's reach: a link planted there
|
|
207
|
+
would make a client that runs outside the sandbox (the MCP server) read any file."""
|
|
208
|
+
fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC)
|
|
209
|
+
st = os.fstat(fd)
|
|
210
|
+
if not stat.S_ISREG(st.st_mode) or st.st_nlink != 1 or st.st_uid != os.getuid():
|
|
211
|
+
os.close(fd)
|
|
212
|
+
raise _NotPlain(path)
|
|
213
|
+
return os.fdopen(fd, "rb")
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def _follow(name: str, spool: str, req: str, outp: str, done: str, out: TextIO,
|
|
217
|
+
wait: float) -> int:
|
|
218
|
+
try:
|
|
219
|
+
return _follow_plain(name, spool, req, outp, done, out, wait)
|
|
220
|
+
except (_NotPlain, OSError):
|
|
221
|
+
out.write(f"{name} refused: the broker's reply in the spool is not a plain file of "
|
|
222
|
+
"yours (something put a link or another file there); nothing is shown\n")
|
|
223
|
+
return 3
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def _follow_plain(name: str, spool: str, req: str, outp: str, done: str,
|
|
227
|
+
out: TextIO, wait: float) -> int:
|
|
228
|
+
t0 = time.time()
|
|
229
|
+
pos = 0
|
|
230
|
+
pending = b""
|
|
231
|
+
fh = None
|
|
232
|
+
while True:
|
|
233
|
+
finished = os.path.lexists(done)
|
|
234
|
+
if fh is None and os.path.lexists(outp):
|
|
235
|
+
fh = _open_plain(outp)
|
|
236
|
+
if fh is not None:
|
|
237
|
+
fh.seek(pos)
|
|
238
|
+
chunk = fh.read()
|
|
239
|
+
pos += len(chunk)
|
|
240
|
+
pending += chunk
|
|
241
|
+
*lines, pending = pending.split(b"\n")
|
|
242
|
+
for line in lines:
|
|
243
|
+
out.write(_printable(line) + "\n")
|
|
244
|
+
out.flush()
|
|
245
|
+
if finished:
|
|
246
|
+
if pending:
|
|
247
|
+
out.write(_printable(pending) + "\n")
|
|
248
|
+
try:
|
|
249
|
+
with _open_plain(done) as d:
|
|
250
|
+
code = json.loads(d.read(4096).decode("utf-8")).get("exit")
|
|
251
|
+
return code if isinstance(code, int) else 3
|
|
252
|
+
except (ValueError, AttributeError, UnicodeDecodeError):
|
|
253
|
+
return 3
|
|
254
|
+
if os.path.exists(req) and time.time() - t0 > wait:
|
|
255
|
+
try:
|
|
256
|
+
os.unlink(req) # not claimed: take it back, so it never runs later
|
|
257
|
+
except OSError:
|
|
258
|
+
pass
|
|
259
|
+
else:
|
|
260
|
+
out.write(_no_broker(name, spool))
|
|
261
|
+
return 3
|
|
262
|
+
if not _alive(spool):
|
|
263
|
+
out.write(f"{name} stopped: the broker stopped before the command finished\n")
|
|
264
|
+
return 3
|
|
265
|
+
time.sleep(0.05)
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
# ---- the broker (outside every sandbox) --------------------------------------------------------
|
|
269
|
+
|
|
270
|
+
class Spool:
|
|
271
|
+
"""The spool folder, opened once and used only through its descriptor."""
|
|
272
|
+
|
|
273
|
+
def __init__(self, path: str):
|
|
274
|
+
self.path = path
|
|
275
|
+
parent = os.path.dirname(path)
|
|
276
|
+
try:
|
|
277
|
+
os.makedirs(parent, mode=0o700, exist_ok=True)
|
|
278
|
+
os.mkdir(path, 0o700)
|
|
279
|
+
except FileExistsError:
|
|
280
|
+
pass
|
|
281
|
+
except OSError as exc:
|
|
282
|
+
raise SpoolError(f"the spool folder cannot be made ({type(exc).__name__})")
|
|
283
|
+
try:
|
|
284
|
+
self.fd = os.open(path, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
|
|
285
|
+
except OSError as exc:
|
|
286
|
+
raise SpoolError(f"the spool folder cannot be opened as a plain folder "
|
|
287
|
+
f"({type(exc).__name__})")
|
|
288
|
+
st = os.fstat(self.fd)
|
|
289
|
+
if st.st_uid != os.getuid() or st.st_mode & 0o077:
|
|
290
|
+
os.close(self.fd)
|
|
291
|
+
raise SpoolError("the spool folder must belong to you and have mode 0700")
|
|
292
|
+
self.ident = (st.st_dev, st.st_ino)
|
|
293
|
+
|
|
294
|
+
def still_here(self) -> bool:
|
|
295
|
+
try:
|
|
296
|
+
st = os.lstat(self.path)
|
|
297
|
+
except OSError:
|
|
298
|
+
return False
|
|
299
|
+
return stat.S_ISDIR(st.st_mode) and (st.st_dev, st.st_ino) == self.ident
|
|
300
|
+
|
|
301
|
+
def names(self) -> List[str]:
|
|
302
|
+
return os.listdir(self.fd)
|
|
303
|
+
|
|
304
|
+
def create(self, name: str) -> int:
|
|
305
|
+
"""A new file for writing; never an existing file or a link (raises OSError)."""
|
|
306
|
+
return os.open(name, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600,
|
|
307
|
+
dir_fd=self.fd)
|
|
308
|
+
|
|
309
|
+
def write_new(self, name: str, data: bytes) -> None:
|
|
310
|
+
"""`name` with `data`, through a new temporary file and a rename."""
|
|
311
|
+
tmp = f".tmp-{secrets.token_hex(16)}"
|
|
312
|
+
fd = self.create(tmp)
|
|
313
|
+
try:
|
|
314
|
+
os.write(fd, data)
|
|
315
|
+
finally:
|
|
316
|
+
os.close(fd)
|
|
317
|
+
os.rename(tmp, name, src_dir_fd=self.fd, dst_dir_fd=self.fd)
|
|
318
|
+
|
|
319
|
+
def unlink(self, name: str) -> None:
|
|
320
|
+
try:
|
|
321
|
+
os.unlink(name, dir_fd=self.fd)
|
|
322
|
+
except OSError:
|
|
323
|
+
pass
|
|
324
|
+
|
|
325
|
+
def exists(self, name: str) -> bool:
|
|
326
|
+
try:
|
|
327
|
+
os.stat(name, dir_fd=self.fd, follow_symlinks=False)
|
|
328
|
+
return True
|
|
329
|
+
except OSError:
|
|
330
|
+
return False
|
|
331
|
+
|
|
332
|
+
def read_request(self, name: str) -> Dict[str, Any]:
|
|
333
|
+
"""The request in `name`, checked; raises SpoolError."""
|
|
334
|
+
try:
|
|
335
|
+
fd = os.open(name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=self.fd)
|
|
336
|
+
except OSError:
|
|
337
|
+
raise SpoolError("not a plain file")
|
|
338
|
+
try:
|
|
339
|
+
st = os.fstat(fd)
|
|
340
|
+
if not stat.S_ISREG(st.st_mode) or st.st_nlink != 1 or st.st_uid != os.getuid():
|
|
341
|
+
raise SpoolError("not a plain file with one link")
|
|
342
|
+
if st.st_size > MAX_REQUEST:
|
|
343
|
+
raise SpoolError("too large")
|
|
344
|
+
data = os.read(fd, MAX_REQUEST + 1)
|
|
345
|
+
finally:
|
|
346
|
+
os.close(fd)
|
|
347
|
+
if len(data) > MAX_REQUEST:
|
|
348
|
+
raise SpoolError("too large")
|
|
349
|
+
try:
|
|
350
|
+
body = json.loads(data.decode("utf-8"))
|
|
351
|
+
except (UnicodeDecodeError, ValueError):
|
|
352
|
+
raise SpoolError("not JSON")
|
|
353
|
+
return check_request(body)
|
|
354
|
+
|
|
355
|
+
def heartbeat(self) -> None:
|
|
356
|
+
try:
|
|
357
|
+
# O_NONBLOCK: a FIFO planted under this name fails at once (ENXIO) instead of
|
|
358
|
+
# blocking the main loop
|
|
359
|
+
fd = os.open("broker.alive", os.O_WRONLY | os.O_CREAT | os.O_NOFOLLOW | os.O_NONBLOCK,
|
|
360
|
+
0o600, dir_fd=self.fd)
|
|
361
|
+
except OSError:
|
|
362
|
+
self.unlink("broker.alive") # not a plain file: remove it; the next beat makes one
|
|
363
|
+
return
|
|
364
|
+
try:
|
|
365
|
+
if stat.S_ISREG(os.fstat(fd).st_mode):
|
|
366
|
+
os.utime(fd)
|
|
367
|
+
finally:
|
|
368
|
+
os.close(fd)
|
|
369
|
+
|
|
370
|
+
def collect_garbage(self) -> None:
|
|
371
|
+
now = time.time()
|
|
372
|
+
for name in self.names():
|
|
373
|
+
if not _LEFTOVER.fullmatch(name) and not _REQ.fullmatch(name):
|
|
374
|
+
continue
|
|
375
|
+
try:
|
|
376
|
+
st = os.stat(name, dir_fd=self.fd, follow_symlinks=False)
|
|
377
|
+
except OSError:
|
|
378
|
+
continue
|
|
379
|
+
if now - st.st_mtime > GC_SECONDS:
|
|
380
|
+
self.unlink(name)
|
|
381
|
+
|
|
382
|
+
|
|
383
|
+
def _plain_words(v: Any, limit: int) -> bool:
|
|
384
|
+
return isinstance(v, list) and len(v) <= limit and all(
|
|
385
|
+
isinstance(w, str) and "\0" not in w for w in v)
|
|
386
|
+
|
|
387
|
+
|
|
388
|
+
def check_request(body: Any) -> Dict[str, Any]:
|
|
389
|
+
if not isinstance(body, dict) or body.get("v") != 1:
|
|
390
|
+
raise SpoolError("unknown request version")
|
|
391
|
+
if body.get("cmd") not in ("run", "copy"):
|
|
392
|
+
raise SpoolError("unknown command")
|
|
393
|
+
if not _plain_words(body.get("args"), MAX_ARGS):
|
|
394
|
+
raise SpoolError("bad arguments")
|
|
395
|
+
cwd = body.get("cwd")
|
|
396
|
+
if not isinstance(cwd, str) or not os.path.isabs(cwd) or "\0" in cwd or \
|
|
397
|
+
not os.path.isdir(cwd):
|
|
398
|
+
raise SpoolError("bad working folder")
|
|
399
|
+
env = body.get("env")
|
|
400
|
+
if not isinstance(env, dict) or not all(
|
|
401
|
+
isinstance(k, str) and isinstance(v, str) and "\0" not in k + v and "=" not in k and k
|
|
402
|
+
for k, v in env.items()):
|
|
403
|
+
raise SpoolError("bad environment")
|
|
404
|
+
return body
|
|
405
|
+
|
|
406
|
+
|
|
407
|
+
def command_env(env: Optional[Dict[str, str]] = None) -> Dict[str, str]:
|
|
408
|
+
"""The command's environment: the allow-listed variables of the broker's own environment.
|
|
409
|
+
The request's environment is never used."""
|
|
410
|
+
env = os.environ if env is None else env
|
|
411
|
+
out = {k: v for k, v in env.items() if _ENV_KEEP.fullmatch(k)}
|
|
412
|
+
if "PATH" in out:
|
|
413
|
+
# the same PATH filter as the login service: a relative entry, a folder others can
|
|
414
|
+
# write or one inside a project would let the agent choose the program that runs
|
|
415
|
+
from phiward import service
|
|
416
|
+
out["PATH"] = ":".join(service._safe_path_entries(out["PATH"])) or "/usr/bin:/bin"
|
|
417
|
+
return out
|
|
418
|
+
|
|
419
|
+
|
|
420
|
+
# the most files a listed folder may hold (with its subfolders): a larger one is never approved
|
|
421
|
+
FOLDER_LIMIT = 2000
|
|
422
|
+
|
|
423
|
+
|
|
424
|
+
def program_path() -> str:
|
|
425
|
+
"""The PATH on which listed programs are found: the running broker's own (from its status
|
|
426
|
+
file), so `pin` and `doctor` in the operator's terminal see what the broker sees; else this
|
|
427
|
+
process's command PATH."""
|
|
428
|
+
st = read_status()
|
|
429
|
+
p = st.get("path") if isinstance(st, dict) else None
|
|
430
|
+
return p if isinstance(p, str) and p else command_env().get("PATH", "")
|
|
431
|
+
|
|
432
|
+
|
|
433
|
+
def _folder_files(folder: str) -> List[str]:
|
|
434
|
+
"""Every file in a listed folder and its subfolders (a script's helpers live there), or a
|
|
435
|
+
marker when it holds more than FOLDER_LIMIT, which is never approved."""
|
|
436
|
+
out: List[str] = []
|
|
437
|
+
for top, dirs, files in os.walk(folder, followlinks=False):
|
|
438
|
+
dirs.sort()
|
|
439
|
+
for n in sorted(files):
|
|
440
|
+
out.append(os.path.join(top, n))
|
|
441
|
+
if len(out) > FOLDER_LIMIT:
|
|
442
|
+
return [f"too-many:{folder}"]
|
|
443
|
+
return out
|
|
444
|
+
|
|
445
|
+
|
|
446
|
+
def listed_programs(cfg: "config.Config", path: str) -> List[str]:
|
|
447
|
+
"""The files `[broker] commands` allows, as the broker would find them now: a bare name on
|
|
448
|
+
`path`, a path entry itself, every program in a folder entry."""
|
|
449
|
+
import shutil
|
|
450
|
+
out: List[str] = []
|
|
451
|
+
for lst in cfg.command_lists:
|
|
452
|
+
for entry in lst:
|
|
453
|
+
if entry.startswith("@"):
|
|
454
|
+
names = list(config.COMMAND_PRESETS.get(entry[1:], ()))
|
|
455
|
+
else:
|
|
456
|
+
names = [entry]
|
|
457
|
+
for e in names:
|
|
458
|
+
if "/" in e:
|
|
459
|
+
p = os.path.realpath(os.path.expanduser(e))
|
|
460
|
+
if os.path.isdir(p):
|
|
461
|
+
out += _folder_files(p)
|
|
462
|
+
elif p != os.path.normpath(os.path.expanduser(e)):
|
|
463
|
+
# a path entry with a link on it lists nothing (run.py): say so
|
|
464
|
+
out.append(f"linked:{e}")
|
|
465
|
+
else:
|
|
466
|
+
out.append(p)
|
|
467
|
+
else:
|
|
468
|
+
found = shutil.which(e, path=path)
|
|
469
|
+
out.append(os.path.realpath(found) if found else f"missing:{e}")
|
|
470
|
+
return out
|
|
471
|
+
|
|
472
|
+
|
|
473
|
+
def config_print() -> str:
|
|
474
|
+
"""A fingerprint of what decides what the gateway may do and what it masks: the user config,
|
|
475
|
+
the pins, and the names and pack files the config names. Content and resolved place only
|
|
476
|
+
(an editor's same-content save, or `phiward setup` writing the same text, changes nothing).
|
|
477
|
+
The broker takes it at start and refuses requests when it changes, so a change that the
|
|
478
|
+
agent manages to make has no effect until the operator restarts the broker."""
|
|
479
|
+
import hashlib
|
|
480
|
+
from phiward import pin
|
|
481
|
+
h = hashlib.sha256()
|
|
482
|
+
files = [config.user_config_path(), pin.pins_path()]
|
|
483
|
+
cfg = None
|
|
484
|
+
try:
|
|
485
|
+
cfg = config.load("/")
|
|
486
|
+
files += list(cfg.names_from) + [p for p in cfg.packs if "/" in p]
|
|
487
|
+
except Exception: # noqa: BLE001 - a config that cannot be loaded is fingerprinted as text
|
|
488
|
+
pass
|
|
489
|
+
for path in files:
|
|
490
|
+
real = os.path.realpath(os.path.expanduser(path))
|
|
491
|
+
h.update(real.encode("utf-8", "surrogateescape") + b"\0")
|
|
492
|
+
try:
|
|
493
|
+
with open(real, "rb") as fh:
|
|
494
|
+
h.update(hashlib.sha256(fh.read(1 << 24)).digest())
|
|
495
|
+
except OSError as exc:
|
|
496
|
+
h.update(f"missing:{exc.errno}".encode())
|
|
497
|
+
# the programs `[broker] commands` allows, as found on the command PATH, by identity: a
|
|
498
|
+
# program replaced or rewritten (its inode or change time, which no one can set back), or a
|
|
499
|
+
# new one of a listed name earlier on PATH, stops the broker until the operator restarts it
|
|
500
|
+
try:
|
|
501
|
+
progs = listed_programs(cfg, command_env().get("PATH", "")) if cfg else []
|
|
502
|
+
except Exception: # noqa: BLE001
|
|
503
|
+
progs = ["unreadable"]
|
|
504
|
+
for prog in progs:
|
|
505
|
+
h.update(prog.encode("utf-8", "surrogateescape") + b"\0")
|
|
506
|
+
try:
|
|
507
|
+
st = os.stat(prog)
|
|
508
|
+
h.update(f"{st.st_dev}:{st.st_ino}:{st.st_size}:{st.st_ctime_ns}".encode())
|
|
509
|
+
except OSError as exc:
|
|
510
|
+
h.update(f"missing:{exc.errno}".encode())
|
|
511
|
+
return h.hexdigest()
|
|
512
|
+
|
|
513
|
+
|
|
514
|
+
def restart_hint() -> str:
|
|
515
|
+
if sys.platform == "darwin":
|
|
516
|
+
return ("`launchctl kickstart -k gui/$(id -u)/org.phiward.broker` for the login service, "
|
|
517
|
+
"or stop `phiward broker` and start it again")
|
|
518
|
+
return ("`systemctl --user restart phiward-broker.service` for the login service, or stop "
|
|
519
|
+
"`phiward broker` and start it again")
|
|
520
|
+
|
|
521
|
+
|
|
522
|
+
class Broker:
|
|
523
|
+
def __init__(self, spool: Spool, log: TextIO, max_jobs: int = 4):
|
|
524
|
+
self.spool, self.log = spool, log
|
|
525
|
+
self.config_print = config_print()
|
|
526
|
+
# listed programs must match what the operator approved (`phiward pin`): a program
|
|
527
|
+
# replaced while the broker was stopped is not accepted by a restart
|
|
528
|
+
try:
|
|
529
|
+
from phiward import pin
|
|
530
|
+
# on this broker's own PATH (another broker's status file names its own)
|
|
531
|
+
self.unapproved = pin.unapproved(config.load("/"),
|
|
532
|
+
path=command_env().get("PATH", ""))
|
|
533
|
+
except Exception: # noqa: BLE001 - when in doubt, nothing runs
|
|
534
|
+
self.unapproved = ["unreadable"]
|
|
535
|
+
# claimed jobs wait here for one of `max_jobs` workers; the main loop never blocks, so
|
|
536
|
+
# the heartbeat goes on while every worker is busy
|
|
537
|
+
self.jobs: "queue.Queue[Any]" = queue.Queue()
|
|
538
|
+
self.max_jobs = max_jobs
|
|
539
|
+
self.stop = threading.Event()
|
|
540
|
+
|
|
541
|
+
def say(self, msg: str) -> None:
|
|
542
|
+
self.log.write(f"[broker {time.strftime('%H:%M:%S')}] {msg}\n")
|
|
543
|
+
self.log.flush()
|
|
544
|
+
|
|
545
|
+
def serve(self) -> int:
|
|
546
|
+
last_gc = last_beat = last_status = 0.0
|
|
547
|
+
from phiward import run as run_mod
|
|
548
|
+
if run_mod.sweep_scratch(config.cache_dir()):
|
|
549
|
+
self.say("removed scratch folders an earlier broker left")
|
|
550
|
+
status = write_status()
|
|
551
|
+
for _ in range(self.max_jobs):
|
|
552
|
+
threading.Thread(target=self.worker, daemon=True).start()
|
|
553
|
+
if self.unapproved:
|
|
554
|
+
self.say(f"ready, but {len(self.unapproved)} listed program(s) are not approved: "
|
|
555
|
+
"every request is refused until `phiward pin` and a restart")
|
|
556
|
+
else:
|
|
557
|
+
self.say("ready; requests are read from the spool folder")
|
|
558
|
+
while not self.stop.is_set():
|
|
559
|
+
if not self.spool.still_here():
|
|
560
|
+
self.say("the spool folder was moved or replaced; stopping")
|
|
561
|
+
return 3
|
|
562
|
+
if time.time() - last_beat >= HEARTBEAT:
|
|
563
|
+
self.spool.heartbeat()
|
|
564
|
+
last_beat = time.time()
|
|
565
|
+
if time.time() - last_status >= STATUS_SECONDS:
|
|
566
|
+
status = write_status(status)
|
|
567
|
+
last_status = time.time()
|
|
568
|
+
if time.time() - last_gc > 60:
|
|
569
|
+
self.spool.collect_garbage()
|
|
570
|
+
last_gc = time.time()
|
|
571
|
+
for name in self.spool.names():
|
|
572
|
+
m = _REQ.fullmatch(name)
|
|
573
|
+
if m:
|
|
574
|
+
self.claim(m.group(1), name)
|
|
575
|
+
self.stop.wait(0.05)
|
|
576
|
+
return 0
|
|
577
|
+
|
|
578
|
+
def claim(self, rid: str, name: str) -> None:
|
|
579
|
+
"""Take a request and queue it. Nothing here reads the request's content or touches its
|
|
580
|
+
folder, so the main loop (and the heartbeat) never waits on it; every
|
|
581
|
+
refusal from here on ends with a done file, so the client never waits forever."""
|
|
582
|
+
job = f"job-{rid}.json"
|
|
583
|
+
if self.spool.exists(job):
|
|
584
|
+
self.spool.unlink(name)
|
|
585
|
+
return
|
|
586
|
+
try:
|
|
587
|
+
os.rename(name, job, src_dir_fd=self.spool.fd, dst_dir_fd=self.spool.fd)
|
|
588
|
+
except OSError:
|
|
589
|
+
return
|
|
590
|
+
try:
|
|
591
|
+
outfd = self.spool.create(f"out-{rid}.log")
|
|
592
|
+
except OSError:
|
|
593
|
+
self.say(f"request {rid[:8]} refused: its output file already exists")
|
|
594
|
+
self.finish(rid, 3)
|
|
595
|
+
return
|
|
596
|
+
if self.jobs.qsize() >= self.max_jobs * QUEUE_PER_JOB:
|
|
597
|
+
self.say(f"request {rid[:8]} refused: busy")
|
|
598
|
+
with os.fdopen(outfd, "w", encoding="utf-8") as out:
|
|
599
|
+
out.write("refused: the broker is busy (too many requests wait); try again "
|
|
600
|
+
"later\n")
|
|
601
|
+
self.finish(rid, 3)
|
|
602
|
+
return
|
|
603
|
+
self.jobs.put((rid, outfd))
|
|
604
|
+
|
|
605
|
+
def worker(self) -> None:
|
|
606
|
+
while True:
|
|
607
|
+
self.run_job(*self.jobs.get())
|
|
608
|
+
|
|
609
|
+
def finish(self, rid: str, code: int) -> None:
|
|
610
|
+
try:
|
|
611
|
+
self.spool.write_new(f"done-{rid}.json", json.dumps({"exit": code}).encode())
|
|
612
|
+
except OSError:
|
|
613
|
+
self.say(f"request {rid[:8]}: the done file could not be written")
|
|
614
|
+
self.spool.unlink(f"job-{rid}.json")
|
|
615
|
+
|
|
616
|
+
def run_job(self, rid: str, outfd: int) -> None:
|
|
617
|
+
code = 3
|
|
618
|
+
try:
|
|
619
|
+
with os.fdopen(outfd, "w", encoding="utf-8", errors="replace") as out:
|
|
620
|
+
try:
|
|
621
|
+
body = self.spool.read_request(f"job-{rid}.json")
|
|
622
|
+
# the request holds the client's words (globs expanded by a shell that is
|
|
623
|
+
# not sandboxed): not left in the spool, which the agent can read
|
|
624
|
+
self.spool.unlink(f"job-{rid}.json")
|
|
625
|
+
except SpoolError as exc:
|
|
626
|
+
self.say(f"request {rid[:8]} refused: {exc}")
|
|
627
|
+
out.write(f"refused: the broker cannot use this request ({exc})\n")
|
|
628
|
+
return
|
|
629
|
+
if self.spool.exists(f"cancel-{rid}"): # cancelled while it waited
|
|
630
|
+
out.write(f"{body['cmd']} stopped: cancelled before it started\n")
|
|
631
|
+
code = 130
|
|
632
|
+
else:
|
|
633
|
+
code = self.execute(rid, body, out)
|
|
634
|
+
except Exception as exc: # noqa: BLE001 - one bad job never stops the broker
|
|
635
|
+
self.say(f"request {rid[:8]} failed ({type(exc).__name__})")
|
|
636
|
+
finally:
|
|
637
|
+
self.finish(rid, code)
|
|
638
|
+
|
|
639
|
+
def execute(self, rid: str, body: Dict[str, Any], out: TextIO) -> int:
|
|
640
|
+
cancelled = lambda: self.spool.exists(f"cancel-{rid}") # noqa: E731
|
|
641
|
+
if self.unapproved:
|
|
642
|
+
self.say(f"request {rid[:8]} refused: {len(self.unapproved)} listed program(s) "
|
|
643
|
+
"not approved")
|
|
644
|
+
out.write(f"refused: {len(self.unapproved)} program(s) of [broker] commands differ "
|
|
645
|
+
"from what the operator approved, or were never approved. The operator "
|
|
646
|
+
"runs `phiward doctor`, which names them, checks them, runs `phiward pin` "
|
|
647
|
+
f"in their own terminal, and restarts the broker: {restart_hint()}.\n")
|
|
648
|
+
return 3
|
|
649
|
+
if config_print() != self.config_print:
|
|
650
|
+
self.say(f"request {rid[:8]} refused: the config changed since the broker started")
|
|
651
|
+
out.write("refused: the phiward config changed since the broker started. The "
|
|
652
|
+
"operator reads the change and restarts the broker: "
|
|
653
|
+
f"{restart_hint()}.\n")
|
|
654
|
+
return 3
|
|
655
|
+
env = command_env()
|
|
656
|
+
self.say(f"request {rid[:8]}: {body['cmd']} ({len(body['args'])} words)")
|
|
657
|
+
from phiward import run
|
|
658
|
+
run.log.fn = lambda msg: self.say(f"request {rid[:8]}: {msg}")
|
|
659
|
+
try:
|
|
660
|
+
if body["cmd"] == "run":
|
|
661
|
+
code = run.execute(body["args"], body["cwd"], env, out, cancelled)
|
|
662
|
+
else:
|
|
663
|
+
from phiward import copying
|
|
664
|
+
code = copying.execute(body["args"], body["cwd"], env, out, cancelled)
|
|
665
|
+
finally:
|
|
666
|
+
run.log.fn = None
|
|
667
|
+
self.say(f"request {rid[:8]}: reported exit {code}")
|
|
668
|
+
return code
|
|
669
|
+
|
|
670
|
+
|
|
671
|
+
def _status() -> Dict[str, Any]:
|
|
672
|
+
cfg = config.load("/")
|
|
673
|
+
import socket as _socket
|
|
674
|
+
from phiward import procinfo
|
|
675
|
+
me = procinfo.info(os.getpid())
|
|
676
|
+
return {"v": 1, "pid": os.getpid(), "start": me[1] if me else None,
|
|
677
|
+
"host": _socket.gethostname(), "config": config.user_config_path(), "cache": config.cache_dir(),
|
|
678
|
+
"spool": config.spool_dir(),
|
|
679
|
+
"roots": cfg.user_roots, "patterns": cfg.user_patterns,
|
|
680
|
+
"path": command_env().get("PATH", "")}
|
|
681
|
+
|
|
682
|
+
|
|
683
|
+
def write_status(last: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
|
|
684
|
+
"""Write the status file when it differs from `last`; return what is in force."""
|
|
685
|
+
status = _status()
|
|
686
|
+
if status == last:
|
|
687
|
+
return status
|
|
688
|
+
path = config.status_path()
|
|
689
|
+
try:
|
|
690
|
+
os.makedirs(os.path.dirname(path), mode=0o700, exist_ok=True)
|
|
691
|
+
safeio.write_new(path, json.dumps(status))
|
|
692
|
+
except OSError:
|
|
693
|
+
pass
|
|
694
|
+
return status
|
|
695
|
+
|
|
696
|
+
|
|
697
|
+
def remove_status() -> None:
|
|
698
|
+
try:
|
|
699
|
+
os.unlink(config.status_path())
|
|
700
|
+
except OSError:
|
|
701
|
+
pass
|
|
702
|
+
|
|
703
|
+
|
|
704
|
+
def running(spool: Optional[str] = None) -> str:
|
|
705
|
+
"""Whether a broker serves `spool`, for doctor and pin (not the client, whose sandbox may not
|
|
706
|
+
see the broker's process): "here" when the guarded status file names a live process of this
|
|
707
|
+
machine with the start time it wrote, "on HOST" when it names another machine (a shared home
|
|
708
|
+
folder on a cluster: there only the heartbeat can tell), "" otherwise. The heartbeat alone
|
|
709
|
+
is in the spool, which the agent writes, so it is not enough."""
|
|
710
|
+
if not _alive(spool or config.spool_dir()):
|
|
711
|
+
return ""
|
|
712
|
+
st = _read_status_file()
|
|
713
|
+
import socket as _socket
|
|
714
|
+
from phiward import procinfo
|
|
715
|
+
pid, start, host = st.get("pid"), st.get("start"), st.get("host")
|
|
716
|
+
if isinstance(host, str) and host and host != _socket.gethostname():
|
|
717
|
+
return f"on {host}"
|
|
718
|
+
if not isinstance(pid, int) or isinstance(pid, bool):
|
|
719
|
+
return ""
|
|
720
|
+
got = procinfo.info(pid)
|
|
721
|
+
return "here" if got is not None and got[1] == start else ""
|
|
722
|
+
|
|
723
|
+
|
|
724
|
+
def read_status(spool: Optional[str] = None) -> Dict[str, Any]:
|
|
725
|
+
"""What the running broker wrote about itself, or {} when no broker runs (`running`). A
|
|
726
|
+
reader may only use it to add protection."""
|
|
727
|
+
if not running(spool):
|
|
728
|
+
return {}
|
|
729
|
+
return _read_status_file()
|
|
730
|
+
|
|
731
|
+
|
|
732
|
+
def other_spool(spool: str) -> Optional[str]:
|
|
733
|
+
"""The spool of a broker that runs with another spool than `spool`, or None. Only to say
|
|
734
|
+
where the broker is (`doctor`, a refused request): nothing else is taken from the file."""
|
|
735
|
+
other = _read_status_file().get("spool")
|
|
736
|
+
if other and os.path.abspath(other) != os.path.abspath(spool) and _alive(other):
|
|
737
|
+
return other
|
|
738
|
+
return None
|
|
739
|
+
|
|
740
|
+
|
|
741
|
+
def _read_status_file() -> Dict[str, Any]:
|
|
742
|
+
try:
|
|
743
|
+
fd = os.open(config.status_path(), os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK)
|
|
744
|
+
except OSError:
|
|
745
|
+
return {}
|
|
746
|
+
try:
|
|
747
|
+
st = os.fstat(fd)
|
|
748
|
+
if not stat.S_ISREG(st.st_mode):
|
|
749
|
+
return {}
|
|
750
|
+
private = st.st_uid == os.getuid() and not st.st_mode & 0o077
|
|
751
|
+
data = json.loads(os.read(fd, MAX_REQUEST).decode("utf-8"))
|
|
752
|
+
except (OSError, ValueError, UnicodeDecodeError):
|
|
753
|
+
return {}
|
|
754
|
+
finally:
|
|
755
|
+
os.close(fd)
|
|
756
|
+
if not isinstance(data, dict):
|
|
757
|
+
return {}
|
|
758
|
+
out: Dict[str, Any] = {}
|
|
759
|
+
for key in ("config", "cache", "spool"):
|
|
760
|
+
if isinstance(data.get(key), str) and os.path.isabs(data[key]):
|
|
761
|
+
out[key] = data[key]
|
|
762
|
+
if isinstance(data.get("path"), str) and private:
|
|
763
|
+
# where the broker finds programs (program_path): only folders no one else can write,
|
|
764
|
+
# and only from the broker's own private file
|
|
765
|
+
from phiward import service
|
|
766
|
+
out["path"] = ":".join(service._safe_path_entries(data["path"]))
|
|
767
|
+
for key in ("roots", "patterns"):
|
|
768
|
+
v = data.get(key)
|
|
769
|
+
out[key] = [x for x in v if isinstance(x, str) and os.path.isabs(x)] if isinstance(v, list) else []
|
|
770
|
+
if private: # who wrote it (`running`): only from the broker's own private file
|
|
771
|
+
for key, kind in (("pid", int), ("start", int), ("host", str)):
|
|
772
|
+
v = data.get(key)
|
|
773
|
+
if isinstance(v, kind) and not isinstance(v, bool):
|
|
774
|
+
out[key] = v
|
|
775
|
+
return out
|
|
776
|
+
|
|
777
|
+
|
|
778
|
+
def main(argv: List[str]) -> int:
|
|
779
|
+
max_jobs = 4
|
|
780
|
+
if argv[:1] in (["-h"], ["--help"]):
|
|
781
|
+
sys.stdout.write(HELP)
|
|
782
|
+
return 0
|
|
783
|
+
if argv[:1] == ["--service"]:
|
|
784
|
+
from phiward import service
|
|
785
|
+
return service.main(argv[1:])
|
|
786
|
+
if argv[:1] == ["--max-jobs"] and len(argv) == 2 and argv[1].isdigit() and int(argv[1]) > 0:
|
|
787
|
+
max_jobs = int(argv[1])
|
|
788
|
+
elif argv:
|
|
789
|
+
sys.stderr.write(USAGE)
|
|
790
|
+
return 2
|
|
791
|
+
problem = config.load("/").problem
|
|
792
|
+
if problem:
|
|
793
|
+
sys.stderr.write(f"phiward broker: the user config cannot be used: {problem}\n")
|
|
794
|
+
return 3
|
|
795
|
+
try:
|
|
796
|
+
spool = Spool(config.spool_dir())
|
|
797
|
+
except SpoolError as exc:
|
|
798
|
+
sys.stderr.write(f"phiward broker: {exc}\n")
|
|
799
|
+
return 3
|
|
800
|
+
import fcntl
|
|
801
|
+
try:
|
|
802
|
+
lock = os.open("broker.lock", os.O_WRONLY | os.O_CREAT | os.O_NOFOLLOW | os.O_NONBLOCK,
|
|
803
|
+
0o600, dir_fd=spool.fd)
|
|
804
|
+
if not stat.S_ISREG(os.fstat(lock).st_mode):
|
|
805
|
+
raise OSError(errno.EINVAL, "not a plain file")
|
|
806
|
+
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
807
|
+
except OSError as exc:
|
|
808
|
+
busy = exc.errno in (errno.EWOULDBLOCK, errno.EAGAIN)
|
|
809
|
+
sys.stderr.write("phiward broker: " + ("another broker is running\n" if busy else
|
|
810
|
+
"the lock file cannot be used\n"))
|
|
811
|
+
return 3
|
|
812
|
+
# the login service appends to its log file: keep it from filling the disk
|
|
813
|
+
from phiward import service
|
|
814
|
+
logf = os.path.join(service.log_folder(), "broker.log")
|
|
815
|
+
try:
|
|
816
|
+
if os.path.getsize(logf) > 50 * 1024 * 1024 and not os.path.islink(logf):
|
|
817
|
+
os.truncate(logf, 0)
|
|
818
|
+
except OSError:
|
|
819
|
+
pass
|
|
820
|
+
broker = Broker(spool, sys.stderr, max_jobs)
|
|
821
|
+
for sig in (signal.SIGTERM, signal.SIGHUP):
|
|
822
|
+
signal.signal(sig, lambda *_: broker.stop.set())
|
|
823
|
+
try:
|
|
824
|
+
return broker.serve()
|
|
825
|
+
except KeyboardInterrupt:
|
|
826
|
+
return 0
|
|
827
|
+
finally:
|
|
828
|
+
from phiward import run as run_mod
|
|
829
|
+
run_mod.end_all() # running requests end with the broker, their scratch removed
|
|
830
|
+
spool.unlink("broker.alive")
|
|
831
|
+
remove_status()
|