nativecluster 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nativecluster/__init__.py +139 -0
- nativecluster/_firings.py +185 -0
- nativecluster/_gate.py +312 -0
- nativecluster/_host.py +337 -0
- nativecluster/_hostsock.py +105 -0
- nativecluster/_launch.py +203 -0
- nativecluster/_models.py +639 -0
- nativecluster/_origin.py +217 -0
- nativecluster/_serve.py +227 -0
- nativecluster/_ticket.py +209 -0
- nativecluster/transport.py +576 -0
- nativecluster-0.1.0.dist-info/METADATA +107 -0
- nativecluster-0.1.0.dist-info/RECORD +16 -0
- nativecluster-0.1.0.dist-info/WHEEL +5 -0
- nativecluster-0.1.0.dist-info/licenses/LICENSE +202 -0
- nativecluster-0.1.0.dist-info/top_level.txt +1 -0
nativecluster/_host.py
ADDED
|
@@ -0,0 +1,337 @@
|
|
|
1
|
+
"""The host's other routes on its socket (spec/hostapi.v1.md): the app's
|
|
2
|
+
own schedules on the host's clock, a firing it took on, its units of work,
|
|
3
|
+
its people's internet items and roles, and its own secrets.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import json
|
|
9
|
+
import urllib.parse
|
|
10
|
+
from datetime import datetime, timezone
|
|
11
|
+
from typing import Any, Dict, List, Optional, Union
|
|
12
|
+
|
|
13
|
+
from ._hostsock import HostError, HostSocket
|
|
14
|
+
|
|
15
|
+
# The host's routes for a running firing: done {firing, outcome, jobs?} and
|
|
16
|
+
# renew {firing} -> {lease_until}; 410 means stop.
|
|
17
|
+
FIRING_DONE_PATH = "/host/v1/triggers/firings/done"
|
|
18
|
+
FIRING_RENEW_PATH = "/host/v1/triggers/firings/renew"
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class StopFiring(Exception):
|
|
22
|
+
"""The host says stop this firing: its person was turned off, or its
|
|
23
|
+
work paused."""
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
# The hand-over (spec/hostapi.v1.md, "Handing the main something to keep"):
|
|
27
|
+
# what an app on a computer hands the same app on its main, for its person.
|
|
28
|
+
HAND_PATH = "/host/v1/main/hand"
|
|
29
|
+
|
|
30
|
+
# HAND_MIN_HOST is the first NativeCluster whose host takes a hand-over: an
|
|
31
|
+
# app that counts on hand_to_main sets its manifest's "min_host" at least
|
|
32
|
+
# this (an older host answers the route 404, a HostError).
|
|
33
|
+
HAND_MIN_HOST = "0.4.13"
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
# An app's own secrets (spec/hostapi.v1.md, "An app's own secrets"): a
|
|
37
|
+
# bot's token, a person's connector token, a sign-in to a service, kept by
|
|
38
|
+
# the host in its own database, never in the app's data folder, each sealed
|
|
39
|
+
# under a key of the app's own derived from the key NativeCluster keeps in
|
|
40
|
+
# the Mac's Keychain. One person's process of an app made for each person
|
|
41
|
+
# never reads another's. A name is a label (1 to 128 letters, digits and
|
|
42
|
+
# . _ : -), stored as it is, so never a secret itself.
|
|
43
|
+
SECRETS_PATH = "/host/v1/secrets"
|
|
44
|
+
|
|
45
|
+
# SECRETS_MIN_HOST is the first NativeCluster whose host keeps an app's
|
|
46
|
+
# secrets: an app that counts on them sets its manifest's "min_host" at
|
|
47
|
+
# least this (an older host answers the route 404, read as no value).
|
|
48
|
+
SECRETS_MIN_HOST = "0.4.15"
|
|
49
|
+
|
|
50
|
+
# The host's caps: one value, and what one store (the app's, or one
|
|
51
|
+
# person's) holds in all.
|
|
52
|
+
MAX_SECRET_VALUE_BYTES = 64 << 10
|
|
53
|
+
MAX_SECRETS_STORE_BYTES = 1 << 20
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
class SecretUnreadable(HostError):
|
|
57
|
+
"""The value was sealed under a key this computer no longer holds: ask
|
|
58
|
+
for it again (the person reconnects) and keep it anew."""
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
class SecretTooLarge(HostError):
|
|
62
|
+
"""The value is over MAX_SECRET_VALUE_BYTES: nothing was kept."""
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
class SecretsFull(HostError):
|
|
66
|
+
"""The app's store has no room for the value: delete some first."""
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _code_and_words(text: str) -> tuple:
|
|
70
|
+
try:
|
|
71
|
+
out = json.loads(text)
|
|
72
|
+
except ValueError:
|
|
73
|
+
return "", ""
|
|
74
|
+
if not isinstance(out, dict):
|
|
75
|
+
return "", ""
|
|
76
|
+
return str(out.get("code") or ""), str(out.get("words") or out.get("error") or "")
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
class Away(HostError):
|
|
80
|
+
"""The main is out of reach now (or this computer is offline by its
|
|
81
|
+
person's choice): nothing was handed over, and the host kept no copy.
|
|
82
|
+
The app keeps what it meant to hand over, and tries again later. words
|
|
83
|
+
is the sentence to show its person."""
|
|
84
|
+
|
|
85
|
+
def __init__(self, method: str, path: str, status: int, text: str) -> None:
|
|
86
|
+
super().__init__(method, path, status, text)
|
|
87
|
+
self.words = _code_and_words(text)[1]
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
class NoMain(HostError):
|
|
91
|
+
"""This computer joined no main (it is one, or on its own): the app
|
|
92
|
+
keeps the work itself, as it always has."""
|
|
93
|
+
|
|
94
|
+
def __init__(self, method: str, path: str, status: int, text: str) -> None:
|
|
95
|
+
super().__init__(method, path, status, text)
|
|
96
|
+
self.words = _code_and_words(text)[1]
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
# --- schedules: a trigger's Spec, as the table takes it ---
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def cron(expr: str) -> dict:
|
|
103
|
+
"""Five fields of digits and `* / , -` (minute hour day month weekday),
|
|
104
|
+
in the main's local time."""
|
|
105
|
+
return {"cron": expr}
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def every(seconds: int) -> dict:
|
|
109
|
+
"""An interval of at least a minute."""
|
|
110
|
+
return {"every": int(seconds)}
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def at(when: Union[int, float, datetime]) -> dict:
|
|
114
|
+
"""Once, at a moment (a reminder): Unix seconds, or a datetime."""
|
|
115
|
+
if isinstance(when, datetime):
|
|
116
|
+
when = when.timestamp()
|
|
117
|
+
return {"at": int(when)}
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def weekly(hour: int, minute: int, weekdays: str = "1111111") -> dict:
|
|
121
|
+
"""At hour:minute, main's local time, on the days weekdays marks
|
|
122
|
+
(seven 0s and 1s, Monday first)."""
|
|
123
|
+
return {"hour": hour, "minute": minute, "weekdays": weekdays}
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def on_event(event: str, min_gap: int = 0) -> dict:
|
|
127
|
+
"""When the app emits event (a short lowercase name), at most once every
|
|
128
|
+
min_gap seconds."""
|
|
129
|
+
out: dict = {"event": event}
|
|
130
|
+
if min_gap:
|
|
131
|
+
out["min_gap"] = int(min_gap)
|
|
132
|
+
return out
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def schedule(id: str, trigger: str, spec: dict, enabled: bool = True) -> dict:
|
|
136
|
+
"""One row of the app's table: id is the trigger, a colon and a key of
|
|
137
|
+
lowercase hex or digits ("reminder:3f9a"), never words; trigger the
|
|
138
|
+
manifest's trigger it is one of. What it does stays in NC_APP_DATA,
|
|
139
|
+
keyed by its id."""
|
|
140
|
+
return {"ID": id, "Trigger": trigger, "Spec": spec, "Enabled": enabled}
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def _rfc3339(when: Union[int, float, datetime]) -> str:
|
|
144
|
+
if not isinstance(when, datetime):
|
|
145
|
+
when = datetime.fromtimestamp(when, tz=timezone.utc)
|
|
146
|
+
if when.tzinfo is None:
|
|
147
|
+
when = when.astimezone()
|
|
148
|
+
return when.isoformat()
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
class HostClient:
|
|
152
|
+
"""An app's way to the host's routes beside its models."""
|
|
153
|
+
|
|
154
|
+
def __init__(self, host: HostSocket) -> None:
|
|
155
|
+
self.host = host
|
|
156
|
+
|
|
157
|
+
def _table(self, op: str, body: dict) -> Any:
|
|
158
|
+
return self.host.call("POST", "/host/v1/triggers/table/" + op, body)
|
|
159
|
+
|
|
160
|
+
# --- the hand-over to the same app on the main (NativeCluster 0.4.13) ---
|
|
161
|
+
|
|
162
|
+
def hand_to_main(self, payload: Any, timeout: float = 90.0) -> Any:
|
|
163
|
+
"""Hands payload (one JSON value, at most 256 KiB) to this app on
|
|
164
|
+
the main, for this process's person, and returns what the app there
|
|
165
|
+
answered (its JSON, None for an empty answer). Raises Away while the
|
|
166
|
+
main is out of reach, NoMain on a computer that joined none (keep the
|
|
167
|
+
work here then), and HostError for any other refusal or the main's
|
|
168
|
+
app's own error status (its status and text). Neither host keeps or
|
|
169
|
+
logs payload."""
|
|
170
|
+
try:
|
|
171
|
+
return self.host.call("POST", HAND_PATH, payload, timeout=timeout)
|
|
172
|
+
except HostError as e:
|
|
173
|
+
code, _ = _code_and_words(e.text)
|
|
174
|
+
if e.status == 503 and code == "away":
|
|
175
|
+
raise Away("POST", HAND_PATH, e.status, e.text) from None
|
|
176
|
+
if e.status == 409 and code == "no_main":
|
|
177
|
+
raise NoMain("POST", HAND_PATH, e.status, e.text) from None
|
|
178
|
+
raise
|
|
179
|
+
|
|
180
|
+
# --- the app's schedules (spec/hostapi.v1.md, "The trigger clock") ---
|
|
181
|
+
|
|
182
|
+
def set_schedules(self, person: str, rows: List[dict]) -> None:
|
|
183
|
+
"""Replaces person's list ("" for the app's own upkeep) with rows."""
|
|
184
|
+
self._table("set", {"person": person, "list": rows})
|
|
185
|
+
|
|
186
|
+
def put_schedule(self, person: str, row: dict) -> None:
|
|
187
|
+
"""Adds or changes one of person's rows, leaving the rest."""
|
|
188
|
+
self._table("put", {"person": person, "trigger": row})
|
|
189
|
+
|
|
190
|
+
def remove_schedule(self, id: str) -> None:
|
|
191
|
+
self._table("remove", {"id": id})
|
|
192
|
+
|
|
193
|
+
def schedules(self, person: str) -> List[dict]:
|
|
194
|
+
return self._table("list", {"person": person}) or []
|
|
195
|
+
|
|
196
|
+
def get_schedule(self, id: str) -> dict:
|
|
197
|
+
return self._table("get", {"id": id})
|
|
198
|
+
|
|
199
|
+
def enable(self, id: str, on: bool) -> None:
|
|
200
|
+
self._table("enable", {"id": id, "on": on})
|
|
201
|
+
|
|
202
|
+
def defer(self, id: str, when: Union[int, float, datetime]) -> None:
|
|
203
|
+
"""Moves a schedule's next run to when."""
|
|
204
|
+
self._table("defer", {"id": id, "at": _rfc3339(when)})
|
|
205
|
+
|
|
206
|
+
def fire(self, id: str) -> Any:
|
|
207
|
+
"""Runs a schedule now (a person's Run it now)."""
|
|
208
|
+
return self._table("fire", {"id": id})
|
|
209
|
+
|
|
210
|
+
def emit(self, event: str, person: str) -> Any:
|
|
211
|
+
"""Says event happened for person: the schedules waiting on it fire."""
|
|
212
|
+
return self._table("emit", {"event": event, "person": person})
|
|
213
|
+
|
|
214
|
+
def pause(self, person: str, until: Optional[Union[int, float, datetime]] = None) -> None:
|
|
215
|
+
"""Pauses person's schedules in this app, as that person."""
|
|
216
|
+
body: dict = {"person": person, "by": "person"}
|
|
217
|
+
if until is not None:
|
|
218
|
+
body["until"] = _rfc3339(until)
|
|
219
|
+
self._table("pause", body)
|
|
220
|
+
|
|
221
|
+
def resume(self, person: str) -> None:
|
|
222
|
+
self._table("resume", {"person": person, "by": "person"})
|
|
223
|
+
|
|
224
|
+
# --- loops ---
|
|
225
|
+
|
|
226
|
+
def start_loop(self, person: str, id: str, trigger: str, *, rounds: int = 0, hours: int = 0, gap_secs: int = 0) -> None:
|
|
227
|
+
"""Starts a loop for person: rounds of the manifest's loop trigger,
|
|
228
|
+
one at a time, until the app answers a round done or failed. rounds
|
|
229
|
+
and hours bound it below the app's caps; gap_secs pauses between
|
|
230
|
+
rounds."""
|
|
231
|
+
loop: dict = {"ID": id, "Trigger": trigger}
|
|
232
|
+
for key, value in (("rounds", rounds), ("hours", hours), ("gap_secs", gap_secs)):
|
|
233
|
+
if value:
|
|
234
|
+
loop[key] = int(value)
|
|
235
|
+
self._table("loops/start", {"person": person, "loop": loop})
|
|
236
|
+
|
|
237
|
+
def stop_loop(self, id: str) -> None:
|
|
238
|
+
self._table("loops/stop", {"id": id})
|
|
239
|
+
|
|
240
|
+
def loops(self) -> List[dict]:
|
|
241
|
+
return self._table("loops/list", {}) or []
|
|
242
|
+
|
|
243
|
+
# --- a firing the app took on ---
|
|
244
|
+
|
|
245
|
+
def firing_done(self, firing: str, outcome: str = "done", jobs: Optional[List[str]] = None) -> None:
|
|
246
|
+
"""Says a firing the app answered running ended: "done", "failed", or
|
|
247
|
+
a loop round's "continue"."""
|
|
248
|
+
body: dict = {"firing": firing, "outcome": outcome}
|
|
249
|
+
if jobs:
|
|
250
|
+
body["jobs"] = jobs
|
|
251
|
+
self._firing(FIRING_DONE_PATH, body)
|
|
252
|
+
|
|
253
|
+
def renew_firing(self, firing: str) -> str:
|
|
254
|
+
"""Gives a firing the app took on its lease again, and says until
|
|
255
|
+
when; StopFiring when the host says stop."""
|
|
256
|
+
out = self._firing(FIRING_RENEW_PATH, {"firing": firing}) or {}
|
|
257
|
+
return out.get("lease_until", "")
|
|
258
|
+
|
|
259
|
+
def _firing(self, path: str, body: dict) -> Any:
|
|
260
|
+
try:
|
|
261
|
+
return self.host.call("POST", path, body)
|
|
262
|
+
except HostError as e:
|
|
263
|
+
if e.status == 410:
|
|
264
|
+
raise StopFiring() from None
|
|
265
|
+
raise
|
|
266
|
+
|
|
267
|
+
def kick_clock(self) -> None:
|
|
268
|
+
"""Asks the host's clock to beat now."""
|
|
269
|
+
self.host.call("POST", "/host/v1/triggers/kick")
|
|
270
|
+
|
|
271
|
+
# --- the rest ---
|
|
272
|
+
|
|
273
|
+
def post_units(self, waiting: bool, pause_ends: Optional[Union[int, float, datetime]] = None,
|
|
274
|
+
ran: Optional[Dict[str, int]] = None) -> None:
|
|
275
|
+
"""Tells the host whether any job waits or runs: while one does, the
|
|
276
|
+
host keeps the app running, and starts it again if it stopped. Say it
|
|
277
|
+
after every change to the app's jobs, at start, and on the way out."""
|
|
278
|
+
body: dict = {"waiting": waiting}
|
|
279
|
+
if pause_ends is not None:
|
|
280
|
+
body["pause_ends"] = _rfc3339(pause_ends)
|
|
281
|
+
if ran:
|
|
282
|
+
body["ran"] = ran
|
|
283
|
+
self.host.call("POST", "/host/v1/units", body)
|
|
284
|
+
|
|
285
|
+
def net_on(self, person: str) -> List[str]:
|
|
286
|
+
"""The app's own net.* items that are on for person now: ask before
|
|
287
|
+
each send, never keep the answer."""
|
|
288
|
+
out = self.host.call("GET", "/host/v1/grants?" + urllib.parse.urlencode({"person": person})) or {}
|
|
289
|
+
return [p for p in out.get("on") or [] if isinstance(p, str)]
|
|
290
|
+
|
|
291
|
+
# --- the app's own secrets (NativeCluster 0.4.15) ---
|
|
292
|
+
|
|
293
|
+
def secret(self, name: str) -> Optional[str]:
|
|
294
|
+
"""The value the app keeps under name, or None for none. Raises
|
|
295
|
+
SecretUnreadable for a value this computer can no longer open."""
|
|
296
|
+
try:
|
|
297
|
+
out = self.host.call("GET", self._secret_path(name)) or {}
|
|
298
|
+
except HostError as e:
|
|
299
|
+
if e.status == 404:
|
|
300
|
+
return None
|
|
301
|
+
raise self._secret_error("GET", name, e) from None
|
|
302
|
+
value = out.get("value")
|
|
303
|
+
return value if isinstance(value, str) else None
|
|
304
|
+
|
|
305
|
+
def set_secret(self, name: str, value: str) -> None:
|
|
306
|
+
"""Keeps value under name, in place of any value there. Raises
|
|
307
|
+
SecretTooLarge or SecretsFull when the host keeps nothing."""
|
|
308
|
+
try:
|
|
309
|
+
self.host.call("PUT", self._secret_path(name), {"value": value})
|
|
310
|
+
except HostError as e:
|
|
311
|
+
raise self._secret_error("PUT", name, e) from None
|
|
312
|
+
|
|
313
|
+
def delete_secret(self, name: str) -> None:
|
|
314
|
+
"""Forgets name's value; one never kept is no error."""
|
|
315
|
+
try:
|
|
316
|
+
self.host.call("DELETE", self._secret_path(name))
|
|
317
|
+
except HostError as e:
|
|
318
|
+
raise self._secret_error("DELETE", name, e) from None
|
|
319
|
+
|
|
320
|
+
@staticmethod
|
|
321
|
+
def _secret_path(name: str) -> str:
|
|
322
|
+
return SECRETS_PATH + "/" + urllib.parse.quote(name, safe="")
|
|
323
|
+
|
|
324
|
+
@staticmethod
|
|
325
|
+
def _secret_error(method: str, name: str, e: HostError) -> HostError:
|
|
326
|
+
code, _ = _code_and_words(e.text)
|
|
327
|
+
path = SECRETS_PATH + "/" + name
|
|
328
|
+
for want, cls in (("unreadable", SecretUnreadable), ("too_large", SecretTooLarge), ("store_full", SecretsFull)):
|
|
329
|
+
if code == want:
|
|
330
|
+
return cls(method, path, e.status, e.text)
|
|
331
|
+
return e
|
|
332
|
+
|
|
333
|
+
def role(self, person: str) -> str:
|
|
334
|
+
"""person's part in the group: "owner", "admin" or "member". Ask each
|
|
335
|
+
time before a change that affects everyone."""
|
|
336
|
+
out = self.host.call("GET", "/host/v1/people/%s/role" % urllib.parse.quote(person, safe="")) or {}
|
|
337
|
+
return out.get("role", "")
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
"""The host's socket (spec/hostapi.v1.md, "The host's socket").
|
|
2
|
+
|
|
3
|
+
Every call to it carries the app's per-launch credential; anything else is
|
|
4
|
+
answered 401. It is a unix socket, so the URL's host is a placeholder with
|
|
5
|
+
no dot ("http://host/..."), never an address of this Mac.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import http.client
|
|
11
|
+
import json
|
|
12
|
+
import socket
|
|
13
|
+
from typing import Any, Callable, Optional, Tuple
|
|
14
|
+
|
|
15
|
+
# BASE_URL is the placeholder every HTTP client of the host's socket names:
|
|
16
|
+
# the socket is the destination, whatever the URL says.
|
|
17
|
+
BASE_URL = "http://host"
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class HostError(Exception):
|
|
21
|
+
"""The host answered a call with an error: its status and its words."""
|
|
22
|
+
|
|
23
|
+
def __init__(self, method: str, path: str, status: int, text: str) -> None:
|
|
24
|
+
super().__init__("NativeCluster's host: %s %s: %d %s" % (method, path, status, text.strip()[:512]))
|
|
25
|
+
self.status = status
|
|
26
|
+
self.text = text
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class _UnixConnection(http.client.HTTPConnection):
|
|
30
|
+
def __init__(self, path: str, timeout: Optional[float]) -> None:
|
|
31
|
+
super().__init__("host", timeout=timeout)
|
|
32
|
+
self._path = path
|
|
33
|
+
|
|
34
|
+
def connect(self) -> None:
|
|
35
|
+
s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
|
36
|
+
try:
|
|
37
|
+
s.settimeout(self.timeout)
|
|
38
|
+
s.connect(self._path)
|
|
39
|
+
except BaseException:
|
|
40
|
+
s.close()
|
|
41
|
+
raise
|
|
42
|
+
self.sock = s
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def close(conn: http.client.HTTPConnection, resp: Optional[http.client.HTTPResponse] = None) -> None:
|
|
46
|
+
"""Closes a call's answer and its connection: the socket closes only
|
|
47
|
+
once both have, and the host reads a wait given up from that."""
|
|
48
|
+
if resp is not None:
|
|
49
|
+
resp.close()
|
|
50
|
+
conn.close()
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class HostSocket:
|
|
54
|
+
"""One app's way to its host: each call on a connection of its own, so
|
|
55
|
+
calls from any number of threads never share one."""
|
|
56
|
+
|
|
57
|
+
def __init__(self, path: str, credential: str) -> None:
|
|
58
|
+
self.path = path
|
|
59
|
+
self._credential = credential
|
|
60
|
+
|
|
61
|
+
def __repr__(self) -> str:
|
|
62
|
+
return "HostSocket(%r)" % self.path
|
|
63
|
+
|
|
64
|
+
@property
|
|
65
|
+
def credential(self) -> str:
|
|
66
|
+
return self._credential
|
|
67
|
+
|
|
68
|
+
def open(self, method: str, path: str, body: Optional[bytes] = None, headers: Optional[dict] = None,
|
|
69
|
+
timeout: Optional[float] = 15.0, sent: Optional[Callable[[socket.socket], None]] = None,
|
|
70
|
+
) -> Tuple[http.client.HTTPConnection, http.client.HTTPResponse]:
|
|
71
|
+
"""Sends one request and returns its connection and its unread
|
|
72
|
+
answer: the caller reads the answer and closes both (close). sent is
|
|
73
|
+
handed the socket once the request is sent, before the answer is
|
|
74
|
+
read (the connection lets go of it to an answer that closes it)."""
|
|
75
|
+
conn = _UnixConnection(self.path, timeout)
|
|
76
|
+
h = {"Authorization": "Bearer " + self._credential}
|
|
77
|
+
if headers:
|
|
78
|
+
h.update(headers)
|
|
79
|
+
try:
|
|
80
|
+
conn.request(method, path, body=body, headers=h)
|
|
81
|
+
if sent is not None and conn.sock is not None:
|
|
82
|
+
sent(conn.sock)
|
|
83
|
+
return conn, conn.getresponse()
|
|
84
|
+
except BaseException:
|
|
85
|
+
conn.close()
|
|
86
|
+
raise
|
|
87
|
+
|
|
88
|
+
def call(self, method: str, path: str, body: Any = None, timeout: Optional[float] = 15.0) -> Any:
|
|
89
|
+
"""One call with a JSON body (None: none), its JSON answer decoded
|
|
90
|
+
(None for an empty one), or HostError."""
|
|
91
|
+
data = None
|
|
92
|
+
headers = {}
|
|
93
|
+
if body is not None:
|
|
94
|
+
data = json.dumps(body).encode("utf-8")
|
|
95
|
+
headers["Content-Type"] = "application/json"
|
|
96
|
+
conn, resp = self.open(method, path, data, headers, timeout)
|
|
97
|
+
try:
|
|
98
|
+
raw = resp.read(4 << 20)
|
|
99
|
+
finally:
|
|
100
|
+
close(conn, resp)
|
|
101
|
+
if resp.status >= 300:
|
|
102
|
+
raise HostError(method, path, resp.status, raw.decode("utf-8", errors="replace"))
|
|
103
|
+
if not raw.strip():
|
|
104
|
+
return None
|
|
105
|
+
return json.loads(raw)
|
nativecluster/_launch.py
ADDED
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
"""Being started (spec/hostapi.v1.md, "Starting an app").
|
|
2
|
+
|
|
3
|
+
The host starts the app's program with its two sockets and its data folder
|
|
4
|
+
in the environment, and its secrets on stdin, never in the environment:
|
|
5
|
+
any process of the same user can read another's launch environment, never
|
|
6
|
+
its stdin.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import json
|
|
12
|
+
import os
|
|
13
|
+
import sys
|
|
14
|
+
import threading
|
|
15
|
+
import time
|
|
16
|
+
from dataclasses import dataclass, field
|
|
17
|
+
from typing import BinaryIO, Callable, Mapping, Optional, Union
|
|
18
|
+
|
|
19
|
+
from ._ticket import TicketKey
|
|
20
|
+
|
|
21
|
+
# What the host sets in the environment of an app it starts.
|
|
22
|
+
APP_SOCKET_ENV = "LB_APP_SOCKET" # the unix socket the app serves on (0600)
|
|
23
|
+
HOST_SOCKET_ENV = "LB_HOST_SOCKET" # the host's socket, for everything the app asks of it
|
|
24
|
+
SECRETS_STDIN_ENV = "LB_SECRETS_STDIN" # "1": the secrets arrive on stdin
|
|
25
|
+
DATA_ENV = "NC_APP_DATA" # the app's own data folder: every file it keeps
|
|
26
|
+
BASE_ENV = "NC_APP_BASE" # the path the app is served at, as the browser sees it
|
|
27
|
+
SOCKETS_ENV = "NC_APP_SOCKETS" # a short folder for the app's own unix sockets (host 0.3.5)
|
|
28
|
+
PERSON_ENV = "NC_APP_PERSON" # an app made for each person: whom this process is for (host 0.3.7)
|
|
29
|
+
EXIT_WITH_PARENT_ENV = "LB_EXIT_WITH_PARENT" # "1": leave when the host does
|
|
30
|
+
|
|
31
|
+
# MANIFEST_ARG is the argument the host starts the program with to ask its
|
|
32
|
+
# manifest: the program prints nc-app.json, byte for byte, and exits.
|
|
33
|
+
MANIFEST_ARG = "manifest"
|
|
34
|
+
|
|
35
|
+
# HEALTH_PATH is the host's readiness check; WHOAMI_PATH the conformance
|
|
36
|
+
# probe `ncdev check` asks with valid and forged tickets. Both stand behind
|
|
37
|
+
# the ticket check like every route.
|
|
38
|
+
HEALTH_PATH = "/healthz"
|
|
39
|
+
WHOAMI_PATH = "/.nc/whoami"
|
|
40
|
+
|
|
41
|
+
_MAX_SECRETS = 64 << 10
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class NotBehindHost(RuntimeError):
|
|
45
|
+
"""This program runs only behind NativeCluster: started any other way it
|
|
46
|
+
would answer nobody (every request needs the host's ticket) and reach no
|
|
47
|
+
model."""
|
|
48
|
+
|
|
49
|
+
def __init__(self) -> None:
|
|
50
|
+
super().__init__(
|
|
51
|
+
"nativecluster: this program runs only behind NativeCluster (no " + APP_SOCKET_ENV
|
|
52
|
+
+ "); try it with `ncdev run`, or install it for your group"
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
@dataclass(frozen=True)
|
|
57
|
+
class Launch:
|
|
58
|
+
"""What the host handed this run of the app."""
|
|
59
|
+
|
|
60
|
+
app_socket: str
|
|
61
|
+
host_socket: str
|
|
62
|
+
# credential is what the app presents on the host's socket (Authorization:
|
|
63
|
+
# Bearer); ticket_key checks the host's ticket on every request. Both are
|
|
64
|
+
# kept in memory only, never logged or written.
|
|
65
|
+
credential: str = field(repr=False)
|
|
66
|
+
ticket_key: TicketKey = field(repr=False)
|
|
67
|
+
data_dir: str = ""
|
|
68
|
+
base: str = ""
|
|
69
|
+
# woken says the host started the app after putting it to sleep itself
|
|
70
|
+
# for being idle.
|
|
71
|
+
woken: bool = False
|
|
72
|
+
# test is set only for a test `ncdev test` runs: the one model the run is
|
|
73
|
+
# for and the ticket of the request it runs under. The host never sends it.
|
|
74
|
+
test: Optional[dict] = None
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def behind_host(environ: Optional[Mapping[str, str]] = None) -> bool:
|
|
78
|
+
"""Whether the host started this process as an app."""
|
|
79
|
+
return bool((environ if environ is not None else os.environ).get(APP_SOCKET_ENV))
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def read_secrets(stream: BinaryIO) -> dict:
|
|
83
|
+
"""One JSON object from stream. It needs no end of file, so a parent may
|
|
84
|
+
keep the pipe open, and never echoes what it read: it is the secrets."""
|
|
85
|
+
buf = b""
|
|
86
|
+
decoder = json.JSONDecoder()
|
|
87
|
+
while len(buf) <= _MAX_SECRETS:
|
|
88
|
+
chunk = stream.read1(4096) if hasattr(stream, "read1") else stream.read(4096) # type: ignore[attr-defined]
|
|
89
|
+
if not chunk:
|
|
90
|
+
break
|
|
91
|
+
buf += chunk
|
|
92
|
+
try:
|
|
93
|
+
obj, _ = decoder.raw_decode(buf.decode("utf-8").lstrip())
|
|
94
|
+
except ValueError:
|
|
95
|
+
continue
|
|
96
|
+
if not isinstance(obj, dict):
|
|
97
|
+
break
|
|
98
|
+
return obj
|
|
99
|
+
if not buf.strip():
|
|
100
|
+
raise RuntimeError("nativecluster: the parent sent no secrets on stdin")
|
|
101
|
+
raise RuntimeError("nativecluster: the secrets on stdin are not a JSON object")
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def read_launch(stdin: Optional[BinaryIO] = None, environ: Optional[Mapping[str, str]] = None) -> Launch:
|
|
105
|
+
"""What the host handed this process: its environment and the secrets on
|
|
106
|
+
its stdin. stdin is read once: use launch() for the process's own."""
|
|
107
|
+
env = environ if environ is not None else os.environ
|
|
108
|
+
if not behind_host(env):
|
|
109
|
+
raise NotBehindHost()
|
|
110
|
+
secrets: dict = {}
|
|
111
|
+
if env.get(SECRETS_STDIN_ENV) == "1":
|
|
112
|
+
secrets = read_secrets(stdin if stdin is not None else sys.stdin.buffer)
|
|
113
|
+
key = str(secrets.get("ticket_key") or "").strip()
|
|
114
|
+
if not key:
|
|
115
|
+
raise RuntimeError("nativecluster: the host sent no ticket key")
|
|
116
|
+
data = env.get(DATA_ENV, "")
|
|
117
|
+
if data:
|
|
118
|
+
os.makedirs(data, mode=0o700, exist_ok=True)
|
|
119
|
+
test = secrets.get("test")
|
|
120
|
+
return Launch(
|
|
121
|
+
app_socket=env[APP_SOCKET_ENV],
|
|
122
|
+
host_socket=env.get(HOST_SOCKET_ENV, ""),
|
|
123
|
+
credential=str(secrets.get("app_credential") or "").strip(),
|
|
124
|
+
ticket_key=TicketKey.from_hex(key),
|
|
125
|
+
data_dir=data,
|
|
126
|
+
base=env.get(BASE_ENV, "").rstrip("/"),
|
|
127
|
+
woken=secrets.get("woken") is True,
|
|
128
|
+
test=test if isinstance(test, dict) else None,
|
|
129
|
+
)
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
_launched: Optional[Launch] = None
|
|
133
|
+
_launch_lock = threading.Lock()
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def launch() -> Launch:
|
|
137
|
+
"""This process's launch, read from its environment and stdin the first
|
|
138
|
+
time it is asked, and kept."""
|
|
139
|
+
global _launched
|
|
140
|
+
with _launch_lock:
|
|
141
|
+
if _launched is None:
|
|
142
|
+
_launched = read_launch()
|
|
143
|
+
return _launched
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def answer_manifest(manifest: Union[bytes, str], argv: Optional[list] = None) -> None:
|
|
147
|
+
"""Answers `<program> manifest`: prints the manifest the program carries,
|
|
148
|
+
byte for byte, and exits (the host holds the package to it). Any other
|
|
149
|
+
run returns at once. manifest is the manifest's bytes, or its path."""
|
|
150
|
+
args = sys.argv if argv is None else argv
|
|
151
|
+
if len(args) < 2 or args[1] != MANIFEST_ARG:
|
|
152
|
+
return
|
|
153
|
+
if isinstance(manifest, str):
|
|
154
|
+
with open(manifest, "rb") as f:
|
|
155
|
+
manifest = f.read()
|
|
156
|
+
sys.stdout.buffer.write(manifest)
|
|
157
|
+
sys.stdout.buffer.flush()
|
|
158
|
+
sys.exit(0)
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def data_dir() -> str:
|
|
162
|
+
"""The app's data folder: every file it keeps goes here, never beside
|
|
163
|
+
its program, whose folder is read-only once installed."""
|
|
164
|
+
return os.environ.get(DATA_ENV, "")
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def base_path() -> str:
|
|
168
|
+
"""The path the app is served at, without a trailing slash: "" at the
|
|
169
|
+
root of its own origin. Links the app writes start with it."""
|
|
170
|
+
return os.environ.get(BASE_ENV, "").rstrip("/")
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def process_person() -> str:
|
|
174
|
+
"""For a process of an app made for each person (its manifest's
|
|
175
|
+
per_person), the id its tickets name its person by: the host runs one
|
|
176
|
+
process for each person, and every request and firing it gets is
|
|
177
|
+
theirs, the waits and schedules it makes name them. "" for any other
|
|
178
|
+
app's process."""
|
|
179
|
+
return os.environ.get(PERSON_ENV, "")
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def sockets_dir() -> str:
|
|
183
|
+
"""A short folder of the app's own for the unix sockets its processes
|
|
184
|
+
use between them (a socket's path holds about 100 bytes, often less
|
|
185
|
+
than the data folder's path leaves): the sandbox lets the app make and
|
|
186
|
+
reach sockets here and in its data folder, nowhere else. Made empty at
|
|
187
|
+
each start. "" from a host before 0.3.5."""
|
|
188
|
+
return os.environ.get(SOCKETS_ENV, "")
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def watch_parent(on_gone: Callable[[], None], every: float = 1.0) -> threading.Thread:
|
|
192
|
+
"""Calls on_gone once the host that started this process is gone (the
|
|
193
|
+
process is then reparented, and its parent id changes)."""
|
|
194
|
+
ppid = os.getppid()
|
|
195
|
+
|
|
196
|
+
def watch() -> None:
|
|
197
|
+
while os.getppid() == ppid:
|
|
198
|
+
time.sleep(every)
|
|
199
|
+
on_gone()
|
|
200
|
+
|
|
201
|
+
t = threading.Thread(target=watch, name="nativecluster-parent", daemon=True)
|
|
202
|
+
t.start()
|
|
203
|
+
return t
|