wasi-python 0.1.0a3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- wasi_python/__init__.py +91 -0
- wasi_python/__main__.py +5 -0
- wasi_python/artifacts/LICENSES/bzip2.txt +44 -0
- wasi_python/artifacts/LICENSES/cpython.txt +1573 -0
- wasi_python/artifacts/LICENSES/llvm-runtimes.txt +1252 -0
- wasi_python/artifacts/LICENSES/numpy.txt +934 -0
- wasi_python/artifacts/LICENSES/pandas.txt +1097 -0
- wasi_python/artifacts/LICENSES/python-dateutil.txt +56 -0
- wasi_python/artifacts/LICENSES/six.txt +20 -0
- wasi_python/artifacts/LICENSES/sqlite.txt +34 -0
- wasi_python/artifacts/LICENSES/tzdata.txt +221 -0
- wasi_python/artifacts/LICENSES/wasi-libc.txt +688 -0
- wasi_python/artifacts/LICENSES/xz.txt +85 -0
- wasi_python/artifacts/LICENSES/zlib.txt +24 -0
- wasi_python/artifacts/MANIFEST +94 -0
- wasi_python/artifacts/SHA256SUMS +16 -0
- wasi_python/artifacts/python-snapshot.wasm +4 -0
- wasi_python/artifacts/python-stdlib.zip +0 -0
- wasi_python/artifacts/python.wasm +0 -0
- wasi_python/cli.py +225 -0
- wasi_python/runner.py +2527 -0
- wasi_python-0.1.0a3.dist-info/METADATA +743 -0
- wasi_python-0.1.0a3.dist-info/RECORD +26 -0
- wasi_python-0.1.0a3.dist-info/WHEEL +4 -0
- wasi_python-0.1.0a3.dist-info/entry_points.txt +2 -0
- wasi_python-0.1.0a3.dist-info/licenses/LICENSE +190 -0
wasi_python/runner.py
ADDED
|
@@ -0,0 +1,2527 @@
|
|
|
1
|
+
"""The runner: one program, one call (spec 001, Decision 3).
|
|
2
|
+
|
|
3
|
+
from wasi_python import run
|
|
4
|
+
|
|
5
|
+
done = run(["-c", "import pandas; print(pandas.__version__)"])
|
|
6
|
+
done.exit_code, done.stdout
|
|
7
|
+
|
|
8
|
+
It does what any embedder does: preopen a directory at `/`, bind the one custom import
|
|
9
|
+
(`host::call`), and call the reactor's exports. wasmtime-py (the `run` extra) is imported only
|
|
10
|
+
here, when a call is made.
|
|
11
|
+
|
|
12
|
+
It is safe by default (spec 006): a call that names no root gets a fresh private one, removed
|
|
13
|
+
afterwards; a call that names no limit still has one for memory, time and output (`DEFAULTS`);
|
|
14
|
+
and nothing a guest frame or a host function does makes `run` raise.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import atexit
|
|
20
|
+
import ctypes
|
|
21
|
+
import errno
|
|
22
|
+
import hashlib
|
|
23
|
+
import itertools
|
|
24
|
+
import math
|
|
25
|
+
import numbers
|
|
26
|
+
import operator
|
|
27
|
+
import os
|
|
28
|
+
import re
|
|
29
|
+
import secrets
|
|
30
|
+
import shutil
|
|
31
|
+
import stat
|
|
32
|
+
import struct
|
|
33
|
+
import sys
|
|
34
|
+
import tempfile
|
|
35
|
+
import threading
|
|
36
|
+
import time
|
|
37
|
+
import warnings
|
|
38
|
+
from collections.abc import Callable, Iterable, Iterator, Mapping
|
|
39
|
+
from contextlib import contextmanager
|
|
40
|
+
from dataclasses import dataclass, field
|
|
41
|
+
from pathlib import Path
|
|
42
|
+
from typing import TYPE_CHECKING, Any, NamedTuple
|
|
43
|
+
|
|
44
|
+
from . import Artifacts
|
|
45
|
+
from . import artifacts as find_artifacts
|
|
46
|
+
|
|
47
|
+
if TYPE_CHECKING:
|
|
48
|
+
import concurrent.futures
|
|
49
|
+
|
|
50
|
+
import wasmtime
|
|
51
|
+
|
|
52
|
+
ARGV0 = "/usr/local/bin/python.wasm"
|
|
53
|
+
NO_HOST = -1
|
|
54
|
+
HOST_FAULT = -2
|
|
55
|
+
CACHE_ENV = "WASI_PYTHON_CACHE"
|
|
56
|
+
TICK_S = 0.01
|
|
57
|
+
NO_DEADLINE = 2**62
|
|
58
|
+
CANCEL_TICKS = 5 # a call with `cancel` is checked every 5 ticks while it computes (spec 015)
|
|
59
|
+
MAX_WORKERS = 1024
|
|
60
|
+
MASK32 = 0xFFFFFFFF
|
|
61
|
+
UNMETERED = 2**63 - 1 # the budget instantiation runs on, before the cap is set (spec 008)
|
|
62
|
+
MAX_RESPONSE = 0x7FFFFFFF # the largest door response: its length must fit a signed i32
|
|
63
|
+
# The guest's environment before the caller's `env`. HOME is added per call: a fresh, empty
|
|
64
|
+
# directory under /tmp that the runner removes afterwards, so nothing a call leaves in its HOME
|
|
65
|
+
# (pdb's .pdbrc, for one) is read by the next (spec 006, Decision 2).
|
|
66
|
+
BASE_ENV = {"TMPDIR": "/tmp", "PYTHONTZPATH": ""}
|
|
67
|
+
|
|
68
|
+
Host = Callable[[bytes], bytes]
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
LIMIT_MAX = 2**63 - 1 # wasmtime reads a memory limit as signed: 2**63 and above mean none
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _shown(value: Any) -> str:
|
|
75
|
+
"""A value for an error message: an integer too long to print is named by its size."""
|
|
76
|
+
if isinstance(value, int) and value.bit_length() > 128:
|
|
77
|
+
return f"an integer of {value.bit_length()} bits"
|
|
78
|
+
return repr(value)
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _integer_limit(name: str, value: Any) -> int | None:
|
|
82
|
+
if value is None:
|
|
83
|
+
return None
|
|
84
|
+
if isinstance(value, bool):
|
|
85
|
+
raise TypeError(f"{name} must be an integer, or None; got {value!r}")
|
|
86
|
+
try:
|
|
87
|
+
number = operator.index(value)
|
|
88
|
+
except TypeError:
|
|
89
|
+
raise TypeError(f"{name} must be an integer, or None; got {value!r}") from None
|
|
90
|
+
if not 0 <= number <= LIMIT_MAX:
|
|
91
|
+
raise ValueError(f"{name} must be from 0 to 2**63 - 1, or None; got {_shown(number)}")
|
|
92
|
+
return number
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _deadline_limit(value: Any) -> float | None:
|
|
96
|
+
if value is None:
|
|
97
|
+
return None
|
|
98
|
+
if isinstance(value, bool) or not isinstance(value, numbers.Real):
|
|
99
|
+
raise TypeError(f"deadline_s must be a number of seconds, or None; got {value!r}")
|
|
100
|
+
rule = "deadline_s must be finite and above 0, and at most about 4.6e16 s, or None"
|
|
101
|
+
try:
|
|
102
|
+
seconds = float(value)
|
|
103
|
+
ticks = math.ceil(seconds / TICK_S) if seconds > 0 else 0
|
|
104
|
+
except (OverflowError, ValueError):
|
|
105
|
+
raise ValueError(f"{rule}; got {_shown(value)}") from None
|
|
106
|
+
if not math.isfinite(seconds) or seconds <= 0 or not 0 < ticks < NO_DEADLINE:
|
|
107
|
+
raise ValueError(f"{rule}; got {_shown(value)}")
|
|
108
|
+
return seconds
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def _check_limits(
|
|
112
|
+
fuel: Any, deadline_s: Any, memory_bytes: Any, output_bytes: Any
|
|
113
|
+
) -> dict[str, Any]:
|
|
114
|
+
"""The four limits as plain values, or TypeError for a wrong type and ValueError for a value
|
|
115
|
+
out of range. A negative fuel or memory cap, or a memory cap of 2**63 or more, would mean no
|
|
116
|
+
limit to wasmtime, and `True` would mean 1, so none of them reaches it (spec 013)."""
|
|
117
|
+
return {
|
|
118
|
+
"fuel": _integer_limit("fuel", fuel),
|
|
119
|
+
"deadline_s": _deadline_limit(deadline_s),
|
|
120
|
+
"memory_bytes": _integer_limit("memory_bytes", memory_bytes),
|
|
121
|
+
"output_bytes": _integer_limit("output_bytes", output_bytes),
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def _text(where: str, value: Any) -> str:
|
|
126
|
+
"""A plain str the guest will see exactly: no NUL, and valid UTF-8, which WASI carries.
|
|
127
|
+
`str.__str__` and `str.encode` are called directly, so a subclass's overrides are ignored.
|
|
128
|
+
Messages name `where`, never the value: environment values are often secrets."""
|
|
129
|
+
if not isinstance(value, str):
|
|
130
|
+
raise TypeError(f"{where} must be a str, not {type(value).__name__}")
|
|
131
|
+
text = str.__str__(value)
|
|
132
|
+
if "\0" in text:
|
|
133
|
+
raise ValueError(f"{where}: embedded null byte")
|
|
134
|
+
try:
|
|
135
|
+
str.encode(text, "utf-8")
|
|
136
|
+
except UnicodeEncodeError:
|
|
137
|
+
raise ValueError(f"{where}: not valid UTF-8 (a lone surrogate)") from None
|
|
138
|
+
return text
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def _check_inputs(argv: Any, env: Any, stdin: Any) -> dict[str, Any]:
|
|
142
|
+
"""argv, env and stdin as plain values, or TypeError for a wrong type and ValueError for a
|
|
143
|
+
value the guest would see differently: a NUL cut an argument or key short, and an = split a
|
|
144
|
+
key, so {"HOME\\0x": ...} set HOME (spec 014)."""
|
|
145
|
+
if isinstance(argv, (str, bytes, bytearray)) or not isinstance(argv, Iterable):
|
|
146
|
+
raise TypeError(f"argv must be an iterable of str, not {type(argv).__name__}")
|
|
147
|
+
args = [_text(f"argv[{i}]", item) for i, item in enumerate(argv)]
|
|
148
|
+
plain_env = None
|
|
149
|
+
if env is not None:
|
|
150
|
+
if not isinstance(env, Mapping):
|
|
151
|
+
raise TypeError(f"env must be a mapping of str to str, not {type(env).__name__}")
|
|
152
|
+
plain_env = {}
|
|
153
|
+
for key, value in env.items():
|
|
154
|
+
name = _text("env key", key)
|
|
155
|
+
if not name or "=" in name:
|
|
156
|
+
raise ValueError("env key: illegal environment variable name") # never quoted
|
|
157
|
+
shown = name if len(name) <= 40 else name[:40] + "…"
|
|
158
|
+
plain_env[name] = _text(f"env value for {shown!r}", value)
|
|
159
|
+
data = None
|
|
160
|
+
if stdin is not None:
|
|
161
|
+
if type(stdin) is bytes:
|
|
162
|
+
data = stdin # not copied again
|
|
163
|
+
else:
|
|
164
|
+
try:
|
|
165
|
+
data = memoryview(stdin).tobytes()
|
|
166
|
+
except TypeError:
|
|
167
|
+
raise TypeError(
|
|
168
|
+
f"stdin must be bytes-like, or None; got {type(stdin).__name__}"
|
|
169
|
+
) from None
|
|
170
|
+
return {"argv": args, "env": plain_env, "stdin": data}
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def _cpus() -> int:
|
|
174
|
+
count = getattr(os, "process_cpu_count", os.cpu_count)() # 3.13 honours CPU affinity
|
|
175
|
+
return count or 1
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def _default_workers() -> int:
|
|
179
|
+
"""At most 8: at the defaults a worker holds up to 1 GiB of guest memory and 32 MiB of output,
|
|
180
|
+
so 8 bound a process at about 8.3 GiB, and past 8 the measured speedup flattens (spec 015)."""
|
|
181
|
+
return min(_cpus(), 8)
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def _check_workers(value: Any) -> int:
|
|
185
|
+
if isinstance(value, bool):
|
|
186
|
+
raise TypeError(f"async_workers must be an integer; got {value!r}")
|
|
187
|
+
try:
|
|
188
|
+
number = operator.index(value)
|
|
189
|
+
except TypeError:
|
|
190
|
+
raise TypeError(f"async_workers must be an integer; got {value!r}") from None
|
|
191
|
+
if not 1 <= number <= MAX_WORKERS:
|
|
192
|
+
raise ValueError(f"async_workers must be from 1 to {MAX_WORKERS}; got {_shown(number)}")
|
|
193
|
+
return number
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
@dataclass(frozen=True)
|
|
197
|
+
class Defaults:
|
|
198
|
+
"""The limits a call has when it names none. `run` reads `DEFAULTS` at call time; the values
|
|
199
|
+
are checked here, by `run`'s rules, so an error from `run` always names what its caller
|
|
200
|
+
passed (spec 013). `async_workers` sizes `run_async`'s default pool (spec 015)."""
|
|
201
|
+
|
|
202
|
+
memory_bytes: int # the guest's linear memory
|
|
203
|
+
deadline_s: float # wall-clock time, by epoch interruption (costs nothing per instruction)
|
|
204
|
+
output_bytes: int # per stream; bytes past it are dropped and the stream named in `truncated`
|
|
205
|
+
async_workers: int = field(default_factory=_default_workers) # guests run_async runs at once
|
|
206
|
+
|
|
207
|
+
def __post_init__(self) -> None:
|
|
208
|
+
_check_limits(None, self.deadline_s, self.memory_bytes, self.output_bytes)
|
|
209
|
+
_check_workers(self.async_workers)
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
DEFAULTS = Defaults(memory_bytes=1 << 30, deadline_s=60.0, output_bytes=16 << 20)
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
class _Unset:
|
|
216
|
+
def __repr__(self) -> str:
|
|
217
|
+
return "<default>"
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
_UNSET: Any = _Unset()
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
@dataclass(frozen=True)
|
|
224
|
+
class Completed:
|
|
225
|
+
"""How one program ended."""
|
|
226
|
+
|
|
227
|
+
exit_code: int | None # run's return value or proc_exit's code; None when it trapped
|
|
228
|
+
stdout: bytes
|
|
229
|
+
stderr: bytes
|
|
230
|
+
trapped: str | None # None, "out of fuel", "deadline", "cancelled", the trap code's plain
|
|
231
|
+
# name ("stack overflow"), or, for a trap without a code, its message's first line
|
|
232
|
+
fuel_used: int | None # instructions spent, when a fuel cap was given and no deadline hit
|
|
233
|
+
truncated: tuple[str, ...] = () # the streams ("stdout", "stderr") whose output was capped
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
@dataclass(frozen=True)
|
|
237
|
+
class Grant:
|
|
238
|
+
"""A host folder the guest sees at `guest`, read-only unless `writable=True` (spec 017).
|
|
239
|
+
|
|
240
|
+
`host` is a str or os.PathLike[str]; `guest` an absolute guest path. A grant may be nested in
|
|
241
|
+
another; `run` checks every grant before any work."""
|
|
242
|
+
|
|
243
|
+
host: str | os.PathLike[str]
|
|
244
|
+
guest: str
|
|
245
|
+
writable: bool = field(default=False, kw_only=True)
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def run(
|
|
249
|
+
argv: Iterable[str],
|
|
250
|
+
*,
|
|
251
|
+
root: str | os.PathLike[str] | bool | None = False,
|
|
252
|
+
grants: Iterable[Grant] = (),
|
|
253
|
+
env: Mapping[str, str] | None = None,
|
|
254
|
+
stdin: bytes | bytearray | memoryview | None = None,
|
|
255
|
+
host: Host | None = None,
|
|
256
|
+
snapshot: bool = True,
|
|
257
|
+
fuel: int | None = None,
|
|
258
|
+
deadline_s: float | None = _UNSET,
|
|
259
|
+
memory_bytes: int | None = _UNSET,
|
|
260
|
+
output_bytes: int | None = _UNSET,
|
|
261
|
+
cache_dir: str | os.PathLike[str] | None | bool = None,
|
|
262
|
+
artifacts: Artifacts | None = None,
|
|
263
|
+
cancel: threading.Event | None = None,
|
|
264
|
+
) -> Completed:
|
|
265
|
+
"""Run one program in a fresh instance of the runtime and return how it ended.
|
|
266
|
+
|
|
267
|
+
`root` is the host directory the guest sees as `/`, writable. By default (`False`, or `None`)
|
|
268
|
+
the guest has no `/` and nothing writable, only its grants (spec 025); `root=True` makes a
|
|
269
|
+
fresh private directory for the call, removed after it. `grants`, each a `Grant`, are host
|
|
270
|
+
folders the guest sees at their guest paths, read-only unless they say otherwise (spec 017).
|
|
271
|
+
The stdlib zip is always granted, read-only, at /usr/local/lib. `memory_bytes`, `deadline_s`
|
|
272
|
+
and `output_bytes` default to `DEFAULTS`; `None` means no limit. The guest's environment
|
|
273
|
+
starts from `BASE_ENV` (`/tmp` for temporary files, `zoneinfo` on the zip's tzdata) and a
|
|
274
|
+
`HOME` made fresh for the call where `/tmp` is writable, with
|
|
275
|
+
`env` laid over it.
|
|
276
|
+
`cache_dir=False` keeps compiled modules in this process only, never on disk.
|
|
277
|
+
`cancel`, a `threading.Event`: once it is set and the guest runs, the guest stops within
|
|
278
|
+
`CANCEL_TICKS` ticks and the call returns `trapped == "cancelled"` (spec 015)."""
|
|
279
|
+
_refuse_forked()
|
|
280
|
+
wt = _wasmtime()
|
|
281
|
+
checked = _checked(argv, env, stdin, fuel, deadline_s, memory_bytes, output_bytes, cancel)
|
|
282
|
+
checked.update(root=_check_root(root), grants=_check_grants(grants))
|
|
283
|
+
return _run_checked(wt, checked, host, snapshot, cache_dir, artifacts)
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
async def run_async(
|
|
287
|
+
argv: Iterable[str],
|
|
288
|
+
*,
|
|
289
|
+
root: str | os.PathLike[str] | bool | None = False,
|
|
290
|
+
grants: Iterable[Grant] = (),
|
|
291
|
+
env: Mapping[str, str] | None = None,
|
|
292
|
+
stdin: bytes | bytearray | memoryview | None = None,
|
|
293
|
+
host: Host | None = None,
|
|
294
|
+
snapshot: bool = True,
|
|
295
|
+
fuel: int | None = None,
|
|
296
|
+
deadline_s: float | None = _UNSET,
|
|
297
|
+
memory_bytes: int | None = _UNSET,
|
|
298
|
+
output_bytes: int | None = _UNSET,
|
|
299
|
+
cache_dir: str | os.PathLike[str] | None | bool = None,
|
|
300
|
+
artifacts: Artifacts | None = None,
|
|
301
|
+
executor: concurrent.futures.Executor | None = None,
|
|
302
|
+
) -> Completed:
|
|
303
|
+
"""`run` from async code, on a worker thread, so the event loop is never blocked (spec 015).
|
|
304
|
+
|
|
305
|
+
The arguments are `run`'s, checked here before anything is queued. The call runs on
|
|
306
|
+
`executor`, which must run callables on threads of this process, or on the runner's default
|
|
307
|
+
pool of `DEFAULTS.async_workers` threads. If the awaiting task is cancelled, the guest is
|
|
308
|
+
stopped and its worker freed before `CancelledError` is raised; a call still queued never
|
|
309
|
+
starts."""
|
|
310
|
+
import asyncio
|
|
311
|
+
import contextvars
|
|
312
|
+
|
|
313
|
+
_refuse_forked()
|
|
314
|
+
checked = _checked(argv, env, stdin, fuel, deadline_s, memory_bytes, output_bytes, None)
|
|
315
|
+
checked.update(root=_check_root(root), grants=_check_grants(grants))
|
|
316
|
+
_check_executor(executor)
|
|
317
|
+
wt = _wasmtime()
|
|
318
|
+
_register_exit_hook()
|
|
319
|
+
event = threading.Event()
|
|
320
|
+
checked["cancel"] = event
|
|
321
|
+
with _live_lock: # the exit hook sets `exiting` before it reads `_live` (the build review, N1)
|
|
322
|
+
if _hook["exiting"]:
|
|
323
|
+
raise RuntimeError("cannot schedule new futures after interpreter shutdown")
|
|
324
|
+
_live.add(event)
|
|
325
|
+
try:
|
|
326
|
+
work = contextvars.copy_context().run # a door host sees the request's context
|
|
327
|
+
args = (_run_checked, wt, checked, host, snapshot, cache_dir, artifacts)
|
|
328
|
+
future = _submit(executor, work, *args)
|
|
329
|
+
try:
|
|
330
|
+
return await asyncio.wrap_future(future)
|
|
331
|
+
except asyncio.CancelledError:
|
|
332
|
+
if future.cancel(): # still queued: it never starts
|
|
333
|
+
raise
|
|
334
|
+
event.set()
|
|
335
|
+
while not future.done(): # wait for the worker, whatever the call's outcome
|
|
336
|
+
try:
|
|
337
|
+
await asyncio.wrap_future(future)
|
|
338
|
+
except BaseException:
|
|
339
|
+
pass
|
|
340
|
+
raise
|
|
341
|
+
finally:
|
|
342
|
+
with _live_lock:
|
|
343
|
+
_live.discard(event)
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
def _check_executor(executor: Any) -> None:
|
|
347
|
+
import concurrent.futures
|
|
348
|
+
|
|
349
|
+
if executor is None:
|
|
350
|
+
return
|
|
351
|
+
interpreters = getattr(concurrent.futures, "InterpreterPoolExecutor", None) # 3.14
|
|
352
|
+
if interpreters is not None and isinstance(executor, interpreters):
|
|
353
|
+
raise TypeError("executor must run on threads of this process, not an interpreter pool")
|
|
354
|
+
if isinstance(executor, concurrent.futures.ProcessPoolExecutor):
|
|
355
|
+
raise TypeError("executor must run on threads of this process, not a process pool")
|
|
356
|
+
if not isinstance(executor, concurrent.futures.Executor):
|
|
357
|
+
raise TypeError(f"executor must be a concurrent.futures.Executor; got {executor!r}")
|
|
358
|
+
|
|
359
|
+
|
|
360
|
+
# run_async's default pool, its live calls and its exit hook (spec 015, Decisions 4 and 6).
|
|
361
|
+
_pool_lock = threading.Lock()
|
|
362
|
+
_pool: dict[str, Any] = {"pool": None, "size": 0}
|
|
363
|
+
_pools: list[concurrent.futures.ThreadPoolExecutor] = [] # every default pool made, for exit
|
|
364
|
+
_live_lock = threading.Lock()
|
|
365
|
+
_live: set[threading.Event] = set()
|
|
366
|
+
_hook = {"registered": False, "exiting": False}
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
def _submit(executor: Any, fn: Callable[..., Any], *args: Any) -> concurrent.futures.Future[Any]:
|
|
370
|
+
"""Submit to the caller's executor, or to the default pool, resized when `DEFAULTS` changed.
|
|
371
|
+
The lookup, any resize and the submit share one lock, so a resize on another thread never
|
|
372
|
+
leaves a submit facing a pool that is shut down."""
|
|
373
|
+
import concurrent.futures
|
|
374
|
+
|
|
375
|
+
if executor is not None:
|
|
376
|
+
return executor.submit(fn, *args)
|
|
377
|
+
with _pool_lock:
|
|
378
|
+
if _hook["exiting"]: # the exit hook has run: never start a pool's call again
|
|
379
|
+
raise RuntimeError("cannot schedule new futures after interpreter shutdown")
|
|
380
|
+
size = DEFAULTS.async_workers
|
|
381
|
+
if _pool["pool"] is None or _pool["size"] != size:
|
|
382
|
+
old = _pool["pool"]
|
|
383
|
+
new = concurrent.futures.ThreadPoolExecutor(size, thread_name_prefix="wasi-python")
|
|
384
|
+
_pools.append(new)
|
|
385
|
+
_pool.update(pool=new, size=size)
|
|
386
|
+
if old is not None:
|
|
387
|
+
old.shutdown(wait=False) # its running calls finish
|
|
388
|
+
return _pool["pool"].submit(fn, *args)
|
|
389
|
+
|
|
390
|
+
|
|
391
|
+
def _register_exit_hook() -> None:
|
|
392
|
+
"""At exit, stop run_async's guests and drop its queued calls. threading runs its exit hooks
|
|
393
|
+
in reverse order of registration, so this one, registered after concurrent.futures.thread is
|
|
394
|
+
imported, runs before that module joins the pools' workers; registered before, exit would
|
|
395
|
+
wait for every queued call (spec 015's confirmation, EXECUTED both ways)."""
|
|
396
|
+
with _pool_lock:
|
|
397
|
+
if _hook["registered"]:
|
|
398
|
+
return
|
|
399
|
+
import concurrent.futures.thread # noqa: F401 (its hook first, so ours runs before it)
|
|
400
|
+
|
|
401
|
+
register = getattr(threading, "_register_atexit", None)
|
|
402
|
+
if register is not None:
|
|
403
|
+
register(_stop_async_calls)
|
|
404
|
+
else:
|
|
405
|
+
atexit.register(_stop_async_calls)
|
|
406
|
+
_hook["registered"] = True
|
|
407
|
+
|
|
408
|
+
|
|
409
|
+
def _stop_async_calls() -> None:
|
|
410
|
+
"""Pools first, so no call can be submitted after its event was missed; then the events of the
|
|
411
|
+
calls already running (spec 015's build review, F3)."""
|
|
412
|
+
with _pool_lock:
|
|
413
|
+
_hook["exiting"] = True
|
|
414
|
+
pools = list(_pools)
|
|
415
|
+
for pool in pools:
|
|
416
|
+
pool.shutdown(wait=False, cancel_futures=True)
|
|
417
|
+
with _live_lock:
|
|
418
|
+
events = list(_live)
|
|
419
|
+
for event in events:
|
|
420
|
+
event.set()
|
|
421
|
+
|
|
422
|
+
|
|
423
|
+
# Private folders and the sweep (spec 016, Decision 2). Each private root and stdin folder has a
|
|
424
|
+
# lock file beside it in the temporary folder, locked for the folder's life and naming the kernel
|
|
425
|
+
# that made it. A process killed mid-call leaves both; the lock dies with it, so the next process
|
|
426
|
+
# on the same kernel can tell its folder is dead. A pid could not: pids repeat across namespaces,
|
|
427
|
+
# are reused, and a killed child is a zombie until reaped.
|
|
428
|
+
_LOCK_NAME = re.compile(r"wasi-python-(root-|zip-)?[0-9a-f]{16}\.lock\Z") # zip-: spec 017
|
|
429
|
+
_held_lock = threading.Lock()
|
|
430
|
+
_held: set[str] = set() # the names this process holds: NFS may treat flock as per-process
|
|
431
|
+
_sweep_state = {"claimed": False, "swept": 0}
|
|
432
|
+
_identity: dict[str, str | None] = {}
|
|
433
|
+
|
|
434
|
+
|
|
435
|
+
def _flags(*names: str) -> int:
|
|
436
|
+
flags = 0
|
|
437
|
+
for name in names:
|
|
438
|
+
flags |= getattr(os, name, 0)
|
|
439
|
+
return flags
|
|
440
|
+
|
|
441
|
+
|
|
442
|
+
def _kernel_identity() -> str | None:
|
|
443
|
+
"""Which kernel this process runs on, or None: a lock taken on one kernel is invisible to
|
|
444
|
+
another that shares the folder (a VM and its host), so a sweep touches only its own."""
|
|
445
|
+
if "value" not in _identity:
|
|
446
|
+
value = None
|
|
447
|
+
try:
|
|
448
|
+
if sys.platform.startswith("linux"):
|
|
449
|
+
value = Path("/proc/sys/kernel/random/boot_id").read_text().strip()
|
|
450
|
+
elif sys.platform == "darwin":
|
|
451
|
+
import ctypes.util
|
|
452
|
+
|
|
453
|
+
libc = ctypes.CDLL(ctypes.util.find_library("c"))
|
|
454
|
+
size_p, void_p = ctypes.POINTER(ctypes.c_size_t), ctypes.c_void_p
|
|
455
|
+
libc.sysctlbyname.argtypes = [
|
|
456
|
+
ctypes.c_char_p,
|
|
457
|
+
void_p,
|
|
458
|
+
size_p,
|
|
459
|
+
void_p,
|
|
460
|
+
ctypes.c_size_t,
|
|
461
|
+
]
|
|
462
|
+
libc.sysctlbyname.restype = ctypes.c_int
|
|
463
|
+
buf, size = ctypes.create_string_buffer(64), ctypes.c_size_t(64)
|
|
464
|
+
name = b"kern.bootsessionuuid"
|
|
465
|
+
if libc.sysctlbyname(name, buf, ctypes.byref(size), None, 0) == 0:
|
|
466
|
+
value = buf.value.decode("ascii")
|
|
467
|
+
elif sys.platform == "win32":
|
|
468
|
+
import winreg
|
|
469
|
+
|
|
470
|
+
access = winreg.KEY_READ | winreg.KEY_WOW64_64KEY
|
|
471
|
+
where = r"SOFTWARE\Microsoft\Cryptography"
|
|
472
|
+
with winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, where, 0, access) as key:
|
|
473
|
+
value = str(winreg.QueryValueEx(key, "MachineGuid")[0])
|
|
474
|
+
except Exception:
|
|
475
|
+
value = None
|
|
476
|
+
_identity["value"] = value if value and "\n" not in value else None
|
|
477
|
+
return _identity["value"]
|
|
478
|
+
|
|
479
|
+
|
|
480
|
+
_BUSY = {errno.EACCES, errno.EDEADLK, getattr(errno, "EDEADLOCK", errno.EDEADLK)}
|
|
481
|
+
|
|
482
|
+
|
|
483
|
+
def _try_lock(fd: int) -> bool | None:
|
|
484
|
+
"""Lock the open file without waiting: True when locked, False when another holder has it,
|
|
485
|
+
None when this filesystem gives no lock (`ENOLCK` on NFS without a lock daemon, say)."""
|
|
486
|
+
try:
|
|
487
|
+
if sys.platform == "win32":
|
|
488
|
+
import msvcrt
|
|
489
|
+
|
|
490
|
+
os.lseek(fd, 0, 0)
|
|
491
|
+
msvcrt.locking(fd, msvcrt.LK_NBLCK, 1)
|
|
492
|
+
else:
|
|
493
|
+
import fcntl
|
|
494
|
+
|
|
495
|
+
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
496
|
+
return True
|
|
497
|
+
except BlockingIOError:
|
|
498
|
+
return False
|
|
499
|
+
except OSError as e:
|
|
500
|
+
return False if sys.platform == "win32" and e.errno in _BUSY else None
|
|
501
|
+
|
|
502
|
+
|
|
503
|
+
def _unlock_close(fd: int) -> None:
|
|
504
|
+
"""Unlock (Windows frees a closed handle's lock only after a delay), then close; quiet."""
|
|
505
|
+
if sys.platform == "win32":
|
|
506
|
+
try:
|
|
507
|
+
import msvcrt
|
|
508
|
+
|
|
509
|
+
os.lseek(fd, 0, 0)
|
|
510
|
+
msvcrt.locking(fd, msvcrt.LK_UNLCK, 1)
|
|
511
|
+
except OSError:
|
|
512
|
+
pass
|
|
513
|
+
try:
|
|
514
|
+
os.close(fd)
|
|
515
|
+
except OSError:
|
|
516
|
+
pass
|
|
517
|
+
|
|
518
|
+
|
|
519
|
+
def _same_file(fd: int, path: str) -> bool:
|
|
520
|
+
"""Whether the open file is still the file at `path`; False when the path is gone."""
|
|
521
|
+
try:
|
|
522
|
+
opened, named = os.fstat(fd), os.lstat(path)
|
|
523
|
+
except OSError:
|
|
524
|
+
return False
|
|
525
|
+
return (opened.st_dev, opened.st_ino) == (named.st_dev, named.st_ino)
|
|
526
|
+
|
|
527
|
+
|
|
528
|
+
class _Taken(Exception):
|
|
529
|
+
pass
|
|
530
|
+
|
|
531
|
+
|
|
532
|
+
class _Private:
|
|
533
|
+
"""A private folder in the temporary folder, with `<name>.lock` beside it, held locked until
|
|
534
|
+
`remove` (spec 016). The name is drawn here, as mkdtemp draws one, because the lock file must
|
|
535
|
+
exist, locked, before the folder does."""
|
|
536
|
+
|
|
537
|
+
def __init__(self, prefix: str) -> None:
|
|
538
|
+
temp = tempfile.gettempdir()
|
|
539
|
+
identity = _kernel_identity()
|
|
540
|
+
create = _flags("O_RDWR", "O_CREAT", "O_EXCL", "O_NOFOLLOW", "O_CLOEXEC", "O_BINARY")
|
|
541
|
+
for _ in range(100):
|
|
542
|
+
name = prefix + secrets.token_hex(8)
|
|
543
|
+
lock_path, path = os.path.join(temp, name + ".lock"), os.path.join(temp, name)
|
|
544
|
+
with _held_lock:
|
|
545
|
+
_held.add(name)
|
|
546
|
+
try:
|
|
547
|
+
fd = os.open(lock_path, create, 0o600)
|
|
548
|
+
except FileExistsError:
|
|
549
|
+
_drop(name)
|
|
550
|
+
continue
|
|
551
|
+
except BaseException:
|
|
552
|
+
_drop(name)
|
|
553
|
+
raise
|
|
554
|
+
owns = False # the file at lock_path is the one this call made
|
|
555
|
+
try:
|
|
556
|
+
locked = _try_lock(fd)
|
|
557
|
+
if locked is False:
|
|
558
|
+
raise _Taken
|
|
559
|
+
owns = _same_file(fd, lock_path) # a sweeper may have removed it, still empty
|
|
560
|
+
if not owns:
|
|
561
|
+
raise _Taken
|
|
562
|
+
if locked and identity is not None: # unlocked: no identity, so never swept
|
|
563
|
+
os.lseek(fd, 0, 0)
|
|
564
|
+
os.write(fd, identity.encode("ascii") + b"\n")
|
|
565
|
+
os.mkdir(path, 0o700)
|
|
566
|
+
except (_Taken, FileExistsError):
|
|
567
|
+
_abandon(fd, lock_path if owns else None, name)
|
|
568
|
+
continue
|
|
569
|
+
except BaseException:
|
|
570
|
+
_abandon(fd, lock_path if owns else None, name)
|
|
571
|
+
raise
|
|
572
|
+
self.name, self.fd, self.lock_path, self.path = name, fd, lock_path, Path(path)
|
|
573
|
+
return
|
|
574
|
+
raise FileExistsError(errno.EEXIST, "no usable temporary folder name found")
|
|
575
|
+
|
|
576
|
+
def remove(self) -> None:
|
|
577
|
+
"""Folder, then lock (close), then the lock file only if the folder is gone, then the
|
|
578
|
+
name; never raises (the 0.1.0a2 security review, F1)."""
|
|
579
|
+
shutil.rmtree(self.path, ignore_errors=True)
|
|
580
|
+
_unlock_close(self.fd) # Windows cannot delete an open file
|
|
581
|
+
if not os.path.lexists(self.path):
|
|
582
|
+
try:
|
|
583
|
+
os.unlink(self.lock_path)
|
|
584
|
+
except OSError:
|
|
585
|
+
pass
|
|
586
|
+
_drop(self.name)
|
|
587
|
+
|
|
588
|
+
|
|
589
|
+
def _drop(name: str) -> None:
|
|
590
|
+
with _held_lock:
|
|
591
|
+
_held.discard(name)
|
|
592
|
+
|
|
593
|
+
|
|
594
|
+
def _abandon(fd: int, lock_path: str | None, name: str) -> None:
|
|
595
|
+
_unlock_close(fd)
|
|
596
|
+
if lock_path is not None: # only a lock file this call made and still has at that name
|
|
597
|
+
try:
|
|
598
|
+
os.unlink(lock_path)
|
|
599
|
+
except OSError:
|
|
600
|
+
pass
|
|
601
|
+
_drop(name)
|
|
602
|
+
|
|
603
|
+
|
|
604
|
+
def _sweep_once() -> None:
|
|
605
|
+
"""At a process's first call: remove the private folders of dead processes on this kernel.
|
|
606
|
+
Claimed under `_lock`, run without it; it never raises into the call."""
|
|
607
|
+
with _lock:
|
|
608
|
+
if _sweep_state["claimed"]:
|
|
609
|
+
return
|
|
610
|
+
_sweep_state["claimed"] = True
|
|
611
|
+
try:
|
|
612
|
+
identity = _kernel_identity()
|
|
613
|
+
if identity is None:
|
|
614
|
+
return
|
|
615
|
+
temp = tempfile.gettempdir()
|
|
616
|
+
for entry in os.listdir(temp):
|
|
617
|
+
try:
|
|
618
|
+
_sweep_entry(temp, entry, identity)
|
|
619
|
+
except Exception:
|
|
620
|
+
continue
|
|
621
|
+
except Exception:
|
|
622
|
+
return
|
|
623
|
+
|
|
624
|
+
|
|
625
|
+
def _owned(st: os.stat_result) -> bool:
|
|
626
|
+
getuid = getattr(os, "getuid", None)
|
|
627
|
+
return getuid is None or st.st_uid == getuid() # Windows: the user's own temporary folder
|
|
628
|
+
|
|
629
|
+
|
|
630
|
+
def _sweep_entry(temp: str, entry: str, identity: str) -> None:
|
|
631
|
+
if not _LOCK_NAME.match(entry):
|
|
632
|
+
return
|
|
633
|
+
name = entry[: -len(".lock")]
|
|
634
|
+
with _held_lock:
|
|
635
|
+
if name in _held:
|
|
636
|
+
return
|
|
637
|
+
lock_path, path = os.path.join(temp, entry), os.path.join(temp, name)
|
|
638
|
+
named = os.lstat(lock_path)
|
|
639
|
+
if not stat.S_ISREG(named.st_mode) or not _owned(named):
|
|
640
|
+
return
|
|
641
|
+
fd = os.open(lock_path, _flags("O_RDWR", "O_NOFOLLOW", "O_NONBLOCK", "O_CLOEXEC", "O_BINARY"))
|
|
642
|
+
unlink = False
|
|
643
|
+
try:
|
|
644
|
+
opened = os.fstat(fd)
|
|
645
|
+
if not stat.S_ISREG(opened.st_mode):
|
|
646
|
+
return
|
|
647
|
+
if (opened.st_dev, opened.st_ino) != (named.st_dev, named.st_ino):
|
|
648
|
+
return
|
|
649
|
+
if not _try_lock(fd):
|
|
650
|
+
return # a live call holds it
|
|
651
|
+
os.lseek(fd, 0, 0)
|
|
652
|
+
first = os.read(fd, 256).split(b"\n", 1)[0].decode("ascii", "replace")
|
|
653
|
+
if not os.path.lexists(path):
|
|
654
|
+
unlink = first in ("", identity) # a lone lock file: nothing of a call's to lose
|
|
655
|
+
return
|
|
656
|
+
if first != identity:
|
|
657
|
+
return # another kernel's, or none
|
|
658
|
+
folder = os.lstat(path)
|
|
659
|
+
isjunction = getattr(os.path, "isjunction", lambda p: False)
|
|
660
|
+
if not stat.S_ISDIR(folder.st_mode) or isjunction(path):
|
|
661
|
+
return
|
|
662
|
+
shutil.rmtree(path, ignore_errors=True)
|
|
663
|
+
if not os.path.lexists(path):
|
|
664
|
+
with _held_lock:
|
|
665
|
+
_sweep_state["swept"] += 1
|
|
666
|
+
unlink = True
|
|
667
|
+
finally:
|
|
668
|
+
# POSIX unlinks while still holding the lock, so a creator that locks the file after
|
|
669
|
+
# this fails its same-file check and draws again; Windows cannot delete an open file.
|
|
670
|
+
if unlink and os.name == "posix":
|
|
671
|
+
_unlink_quietly(lock_path)
|
|
672
|
+
_unlock_close(fd)
|
|
673
|
+
if unlink and os.name != "posix":
|
|
674
|
+
_unlink_quietly(lock_path)
|
|
675
|
+
|
|
676
|
+
|
|
677
|
+
def _unlink_quietly(path: str) -> None:
|
|
678
|
+
try:
|
|
679
|
+
os.unlink(path)
|
|
680
|
+
except OSError:
|
|
681
|
+
pass
|
|
682
|
+
|
|
683
|
+
|
|
684
|
+
# Fork (spec 015, Decision 5). A child of a process that made an engine inherits state from
|
|
685
|
+
# threads that no longer exist, and wasmtime does not support use across a fork (on macOS it
|
|
686
|
+
# aborts), so such a child is refused. A child of a parent that only imported this runs normally.
|
|
687
|
+
_fork = {"used": False, "forked": False}
|
|
688
|
+
|
|
689
|
+
|
|
690
|
+
def _refuse_forked() -> None:
|
|
691
|
+
if _fork["forked"]:
|
|
692
|
+
raise RuntimeError(
|
|
693
|
+
"wasi_python was used before this process was forked; use it in the child only, or "
|
|
694
|
+
"start processes with 'spawn' or 'forkserver'"
|
|
695
|
+
)
|
|
696
|
+
|
|
697
|
+
|
|
698
|
+
def _after_fork_in_child() -> None:
|
|
699
|
+
"""Only assignments, and no lock: a lock may have been held by a thread the fork left out, so
|
|
700
|
+
every lock the runner takes is made anew (spec 015's build review, F8)."""
|
|
701
|
+
global _pool_lock, _live_lock, _lock, _epoch_callback_lock, _outputs_lock, _compiling
|
|
702
|
+
global _held_lock, _zip_lock, _zips
|
|
703
|
+
_pool_lock, _live_lock, _held_lock = threading.Lock(), threading.Lock(), threading.Lock()
|
|
704
|
+
_zip_lock, _zips = threading.Lock(), {} # the parent's zip folders stay the parent's
|
|
705
|
+
_lock, _epoch_callback_lock, _outputs_lock = (
|
|
706
|
+
threading.Lock(),
|
|
707
|
+
threading.Lock(),
|
|
708
|
+
threading.RLock(),
|
|
709
|
+
)
|
|
710
|
+
_compiling = {}
|
|
711
|
+
_pool.update(pool=None, size=0)
|
|
712
|
+
_pools.clear()
|
|
713
|
+
_live.clear()
|
|
714
|
+
_epoch_ticker._thread, _epoch_ticker._active = None, 0
|
|
715
|
+
_epoch_ticker._cond = threading.Condition()
|
|
716
|
+
if _fork["used"]:
|
|
717
|
+
_fork["forked"] = True
|
|
718
|
+
|
|
719
|
+
|
|
720
|
+
if hasattr(os, "register_at_fork"): # not on Windows, which has no fork
|
|
721
|
+
os.register_at_fork(after_in_child=_after_fork_in_child)
|
|
722
|
+
|
|
723
|
+
|
|
724
|
+
def _wasmtime() -> Any:
|
|
725
|
+
try:
|
|
726
|
+
import wasmtime as wt
|
|
727
|
+
except ImportError as e:
|
|
728
|
+
raise ImportError(
|
|
729
|
+
'wasi_python.run needs wasmtime-py: pip install "wasi-python[run]"'
|
|
730
|
+
) from e
|
|
731
|
+
return wt
|
|
732
|
+
|
|
733
|
+
|
|
734
|
+
def _checked(
|
|
735
|
+
argv: Any,
|
|
736
|
+
env: Any,
|
|
737
|
+
stdin: Any,
|
|
738
|
+
fuel: Any,
|
|
739
|
+
deadline_s: Any,
|
|
740
|
+
memory_bytes: Any,
|
|
741
|
+
output_bytes: Any,
|
|
742
|
+
cancel: Any,
|
|
743
|
+
) -> dict[str, Any]:
|
|
744
|
+
"""Every argument a call's guest depends on, checked before any work (specs 013 to 015)."""
|
|
745
|
+
limits = DEFAULTS
|
|
746
|
+
if memory_bytes is _UNSET:
|
|
747
|
+
memory_bytes = limits.memory_bytes
|
|
748
|
+
if deadline_s is _UNSET:
|
|
749
|
+
deadline_s = limits.deadline_s
|
|
750
|
+
if output_bytes is _UNSET:
|
|
751
|
+
output_bytes = limits.output_bytes
|
|
752
|
+
if cancel is not None and not isinstance(cancel, threading.Event):
|
|
753
|
+
raise TypeError(f"cancel must be a threading.Event, or None; got {type(cancel).__name__}")
|
|
754
|
+
checked = _check_limits(fuel, deadline_s, memory_bytes, output_bytes)
|
|
755
|
+
checked.update(_check_inputs(argv, env, stdin))
|
|
756
|
+
checked["cancel"] = cancel
|
|
757
|
+
return checked
|
|
758
|
+
|
|
759
|
+
|
|
760
|
+
# Grants (spec 017). A grant is a host folder the guest sees at a guest path, read-only unless it
|
|
761
|
+
# says `writable=True`. Every host path, `root` included, is walked one component at a time from
|
|
762
|
+
# `/`, judging each link on the way, and every rule compares identities (device and inode), never
|
|
763
|
+
# spellings: macOS folders are case-insensitive, and Windows spells one folder many ways. wasmtime
|
|
764
|
+
# then opens each path itself, so before the guest runs a check module of the runner's own reads
|
|
765
|
+
# back each preopen's identity and compares it with the walk's (Decision 5).
|
|
766
|
+
ZIP_GUEST = "/usr/local/lib"
|
|
767
|
+
MAX_LINKS = 40
|
|
768
|
+
_M64 = (1 << 64) - 1
|
|
769
|
+
|
|
770
|
+
|
|
771
|
+
def _host_path(where: str, value: Any) -> str:
|
|
772
|
+
if isinstance(value, (bytes, bytearray)):
|
|
773
|
+
raise TypeError(f"{where} must be a str or os.PathLike[str], not bytes")
|
|
774
|
+
try:
|
|
775
|
+
path = os.fspath(value)
|
|
776
|
+
except TypeError:
|
|
777
|
+
raise TypeError(
|
|
778
|
+
f"{where} must be a str or os.PathLike[str], not {type(value).__name__}"
|
|
779
|
+
) from None
|
|
780
|
+
if not isinstance(path, str):
|
|
781
|
+
raise TypeError(f"{where} must be a str or os.PathLike[str], not a path of bytes")
|
|
782
|
+
path = _text(where, path)
|
|
783
|
+
if not path:
|
|
784
|
+
raise ValueError(f"{where} is empty")
|
|
785
|
+
return path
|
|
786
|
+
|
|
787
|
+
|
|
788
|
+
def _guest_path(where: str, value: Any) -> str:
|
|
789
|
+
"""An absolute guest path with no empty, `.` or `..` component and no trailing `/`, which
|
|
790
|
+
wasi-libc would otherwise match by text alone (memo §7d); not `/` and not the runtime's."""
|
|
791
|
+
if not isinstance(value, str):
|
|
792
|
+
raise TypeError(f"{where}: the guest path must be a str, not {type(value).__name__}")
|
|
793
|
+
path = _text(f"{where}: the guest path", value)
|
|
794
|
+
if not path.startswith("/"):
|
|
795
|
+
raise ValueError(f"{where}: the guest path must be absolute; got {path!r}")
|
|
796
|
+
if path == "/":
|
|
797
|
+
raise ValueError(f"{where}: the guest path / is the root; pass root= instead")
|
|
798
|
+
if any(part in ("", ".", "..") for part in path.split("/")[1:]):
|
|
799
|
+
raise ValueError(
|
|
800
|
+
f"{where}: the guest path {path!r} has an empty, '.' or '..' part, or ends in '/'"
|
|
801
|
+
)
|
|
802
|
+
if path == ZIP_GUEST or path.startswith(ZIP_GUEST + "/"):
|
|
803
|
+
raise ValueError(f"{where}: {path!r} is the runtime's own ({ZIP_GUEST})")
|
|
804
|
+
return path
|
|
805
|
+
|
|
806
|
+
|
|
807
|
+
def _check_root(root: Any) -> str | bool | None:
|
|
808
|
+
"""The root as the runner keeps it: a host path; None for a private root (`root=True`); or
|
|
809
|
+
False for none, the default (spec 025: nothing writable unless the embedder asks)."""
|
|
810
|
+
if root is None or root is False:
|
|
811
|
+
return False
|
|
812
|
+
if root is True:
|
|
813
|
+
return None
|
|
814
|
+
return _host_path("root", root)
|
|
815
|
+
|
|
816
|
+
|
|
817
|
+
def _check_grants(grants: Any) -> tuple[Grant, ...]:
|
|
818
|
+
"""The grants as a tuple of plain `Grant`s, so `run_async` hands its worker no used iterator."""
|
|
819
|
+
if isinstance(grants, (str, bytes, bytearray, Grant)) or not isinstance(grants, Iterable):
|
|
820
|
+
raise TypeError(f"grants must be an iterable of Grant, not {type(grants).__name__}")
|
|
821
|
+
checked: list[Grant] = []
|
|
822
|
+
seen: set[str] = set()
|
|
823
|
+
for i, grant in enumerate(grants):
|
|
824
|
+
if not isinstance(grant, Grant):
|
|
825
|
+
raise TypeError(f"grants[{i}] must be a Grant, not {type(grant).__name__}")
|
|
826
|
+
guest = _guest_path(f"grants[{i}]", grant.guest)
|
|
827
|
+
where = f"grants[{i}] at {guest}"
|
|
828
|
+
if type(grant.writable) is not bool:
|
|
829
|
+
raise TypeError(f"{where}: writable must be True or False, not {grant.writable!r}")
|
|
830
|
+
if guest in seen:
|
|
831
|
+
raise ValueError(f"{where}: another grant has the same guest path")
|
|
832
|
+
seen.add(guest)
|
|
833
|
+
checked.append(
|
|
834
|
+
Grant(_host_path(f"{where}: host", grant.host), guest, writable=grant.writable)
|
|
835
|
+
)
|
|
836
|
+
return tuple(checked)
|
|
837
|
+
|
|
838
|
+
|
|
839
|
+
Identity = tuple[int, int]
|
|
840
|
+
|
|
841
|
+
|
|
842
|
+
class _Walked:
|
|
843
|
+
"""A host folder walked component by component: the path wasmtime is given, the identities of
|
|
844
|
+
the folder and each ancestor (the folder's first), and what keeps it from changing until the
|
|
845
|
+
identity check: on POSIX the folder's descriptor, on Windows every component's handle. A root
|
|
846
|
+
still to be made has `missing` components, and its chain is its deepest existing ancestor's."""
|
|
847
|
+
|
|
848
|
+
def __init__(self, path: str, chain: tuple[Identity, ...], held: list[int]) -> None:
|
|
849
|
+
self.path, self.chain, self.held = path, chain, held
|
|
850
|
+
self.missing: tuple[str, ...] = ()
|
|
851
|
+
|
|
852
|
+
@property
|
|
853
|
+
def identity(self) -> Identity | None:
|
|
854
|
+
return None if self.missing else self.chain[0]
|
|
855
|
+
|
|
856
|
+
@property
|
|
857
|
+
def fd(self) -> int | None:
|
|
858
|
+
return self.held[-1] if self.held and os.name == "posix" else None
|
|
859
|
+
|
|
860
|
+
def close(self) -> None:
|
|
861
|
+
held, self.held = self.held, []
|
|
862
|
+
for h in held:
|
|
863
|
+
_close_held(h)
|
|
864
|
+
|
|
865
|
+
|
|
866
|
+
def _close_held(h: int) -> None:
|
|
867
|
+
try:
|
|
868
|
+
if os.name == "posix":
|
|
869
|
+
os.close(h)
|
|
870
|
+
else:
|
|
871
|
+
_win().CloseHandle(h)
|
|
872
|
+
except OSError:
|
|
873
|
+
pass
|
|
874
|
+
|
|
875
|
+
|
|
876
|
+
def _walk(where: str, path: str, *, judge: bool, may_make: bool = False) -> _Walked:
|
|
877
|
+
"""Walk `path` from the top. With `judge`, a link in a folder this user can write (POSIX), or
|
|
878
|
+
any link or junction (Windows), raises ValueError; without it (the runtime's own folders),
|
|
879
|
+
every link is followed. With `may_make`, a missing tail is returned as `missing`."""
|
|
880
|
+
try:
|
|
881
|
+
if os.name == "posix":
|
|
882
|
+
return _walk_posix(where, path, judge, may_make)
|
|
883
|
+
return _walk_windows(where, path, judge, may_make)
|
|
884
|
+
except ValueError:
|
|
885
|
+
raise
|
|
886
|
+
except OSError as e:
|
|
887
|
+
if not judge:
|
|
888
|
+
raise
|
|
889
|
+
# A folder that cannot be searched, or vanished mid-walk: the caller's grant is refused.
|
|
890
|
+
raise ValueError(f"{where}: {path!r} cannot be walked: {e.strerror or e}") from None
|
|
891
|
+
|
|
892
|
+
|
|
893
|
+
# Opened for search only where the system allows (Linux O_PATH, macOS O_SEARCH): wasmtime opens a
|
|
894
|
+
# grant below a folder this user may search but not list, so the walk must pass through it too.
|
|
895
|
+
_DIR = (_flags("O_PATH") or _flags("O_SEARCH") or os.O_RDONLY) | _flags(
|
|
896
|
+
"O_DIRECTORY", "O_NOFOLLOW", "O_CLOEXEC"
|
|
897
|
+
)
|
|
898
|
+
|
|
899
|
+
|
|
900
|
+
def _walk_posix(where: str, path: str, judge: bool, may_make: bool) -> _Walked:
|
|
901
|
+
# Joined to the working directory without normalising: `..` is taken on the walked folders,
|
|
902
|
+
# as the kernel takes it, after any link before it is followed.
|
|
903
|
+
todo = list(reversed(os.path.join(os.getcwd(), path).split("/")))
|
|
904
|
+
top = os.open("/", _DIR)
|
|
905
|
+
st = os.fstat(top)
|
|
906
|
+
stack: list[tuple[int, Identity, str]] = [(top, (st.st_dev, st.st_ino), "")]
|
|
907
|
+
links = 0
|
|
908
|
+
try:
|
|
909
|
+
while todo:
|
|
910
|
+
name = todo.pop()
|
|
911
|
+
if name in ("", "."):
|
|
912
|
+
continue
|
|
913
|
+
if name == "..":
|
|
914
|
+
if len(stack) > 1:
|
|
915
|
+
os.close(stack.pop()[0])
|
|
916
|
+
continue
|
|
917
|
+
parent = stack[-1][0]
|
|
918
|
+
try:
|
|
919
|
+
st = os.lstat(name, dir_fd=parent)
|
|
920
|
+
except FileNotFoundError:
|
|
921
|
+
rest = [n for n in reversed(todo) if n not in ("", ".")]
|
|
922
|
+
if not may_make or ".." in rest:
|
|
923
|
+
raise ValueError(f"{where}: {path!r} does not exist") from None
|
|
924
|
+
walked = _walked_posix(stack)
|
|
925
|
+
walked.missing = (name, *rest)
|
|
926
|
+
return walked
|
|
927
|
+
if stat.S_ISLNK(st.st_mode):
|
|
928
|
+
if judge and _writable(parent, stack):
|
|
929
|
+
raise ValueError(
|
|
930
|
+
f"{where}: {path!r} passes through the link {name!r} in a folder this "
|
|
931
|
+
"user can write, which a guest could have made; name its target instead"
|
|
932
|
+
)
|
|
933
|
+
links += 1
|
|
934
|
+
if links > MAX_LINKS:
|
|
935
|
+
raise ValueError(
|
|
936
|
+
f"{where}: {path!r} passes through more than {MAX_LINKS} links"
|
|
937
|
+
)
|
|
938
|
+
target = os.readlink(name, dir_fd=parent)
|
|
939
|
+
if target.startswith("/"):
|
|
940
|
+
while len(stack) > 1:
|
|
941
|
+
os.close(stack.pop()[0])
|
|
942
|
+
todo.extend(reversed(target.split("/")))
|
|
943
|
+
continue
|
|
944
|
+
if not stat.S_ISDIR(st.st_mode):
|
|
945
|
+
raise ValueError(f"{where}: {path!r} is not a folder")
|
|
946
|
+
fd = os.open(name, _DIR, dir_fd=parent)
|
|
947
|
+
opened = os.fstat(fd)
|
|
948
|
+
if (opened.st_dev, opened.st_ino) != (st.st_dev, st.st_ino):
|
|
949
|
+
os.close(fd)
|
|
950
|
+
raise ValueError(f"{where}: {path!r} changed while it was checked")
|
|
951
|
+
stack.append((fd, (opened.st_dev, opened.st_ino), name))
|
|
952
|
+
return _walked_posix(stack)
|
|
953
|
+
except BaseException:
|
|
954
|
+
for fd, _, _ in stack:
|
|
955
|
+
os.close(fd)
|
|
956
|
+
raise
|
|
957
|
+
|
|
958
|
+
|
|
959
|
+
def _walked_posix(stack: list[tuple[int, Identity, str]]) -> _Walked:
|
|
960
|
+
"""Keep the last folder's descriptor, close the rest."""
|
|
961
|
+
for fd, _, _ in stack[:-1]:
|
|
962
|
+
os.close(fd)
|
|
963
|
+
text = "/" + "/".join(name for _, _, name in stack[1:])
|
|
964
|
+
chain = tuple(identity for _, identity, _ in reversed(stack))
|
|
965
|
+
last = stack[-1][0]
|
|
966
|
+
stack[:] = []
|
|
967
|
+
return _Walked(text, chain, [last])
|
|
968
|
+
|
|
969
|
+
|
|
970
|
+
def _writable(fd: int, stack: list[tuple[int, Identity, str]]) -> bool:
|
|
971
|
+
"""Whether this user can write the folder open at `fd`: only a link in such a folder could be
|
|
972
|
+
a guest's, since a guest writes as this user (memo §7a)."""
|
|
973
|
+
effective = os.access in os.supports_effective_ids
|
|
974
|
+
if os.access in os.supports_dir_fd:
|
|
975
|
+
return os.access(".", os.W_OK, dir_fd=fd, effective_ids=effective)
|
|
976
|
+
text = "/" + "/".join(name for _, _, name in stack[1:])
|
|
977
|
+
return os.access(text, os.W_OK, effective_ids=effective)
|
|
978
|
+
|
|
979
|
+
|
|
980
|
+
def _make_missing(where: str, walked: _Walked) -> _Walked:
|
|
981
|
+
"""Make a root's missing components, after the rules were decided on its existing chain, each
|
|
982
|
+
inside the one before through its descriptor; then walk the made root as any other."""
|
|
983
|
+
path = walked.path
|
|
984
|
+
if os.name == "posix":
|
|
985
|
+
fd = os.dup(walked.held[-1])
|
|
986
|
+
try:
|
|
987
|
+
for name in walked.missing:
|
|
988
|
+
try:
|
|
989
|
+
os.mkdir(name, dir_fd=fd)
|
|
990
|
+
except FileExistsError: # a concurrent call made it
|
|
991
|
+
pass
|
|
992
|
+
child = os.open(name, _DIR, dir_fd=fd) # O_NOFOLLOW: a link put there fails
|
|
993
|
+
os.close(fd)
|
|
994
|
+
fd = child
|
|
995
|
+
path = os.path.join(path, name)
|
|
996
|
+
finally:
|
|
997
|
+
os.close(fd)
|
|
998
|
+
else:
|
|
999
|
+
for name in walked.missing:
|
|
1000
|
+
path = os.path.join(path, name)
|
|
1001
|
+
Path(path).mkdir(exist_ok=True)
|
|
1002
|
+
walked.close()
|
|
1003
|
+
return _walk(where, path, judge=True)
|
|
1004
|
+
|
|
1005
|
+
|
|
1006
|
+
# Windows (Decision 5): Python cannot open a folder with os.open, so the walk opens every
|
|
1007
|
+
# component through CreateFileW, refuses a reparse point (a link or a junction), and holds each
|
|
1008
|
+
# handle without FILE_SHARE_DELETE until the identity check, so no walked folder can be renamed,
|
|
1009
|
+
# removed or replaced in between, and no path string is ever compared.
|
|
1010
|
+
# Traverse access, not attributes alone: Windows runs share checks only for data, execute or
|
|
1011
|
+
# delete access (MS-FSA 2.1.5.1.2.2), so an attribute-only handle held without FILE_SHARE_DELETE
|
|
1012
|
+
# stops no rename. FILE_TRAVERSE is FILE_EXECUTE, which is checked, and needs only the permission
|
|
1013
|
+
# to pass through a folder, as POSIX's search does, not to list it.
|
|
1014
|
+
_WALK_ACCESS = 0x20 | 0x80 # FILE_TRAVERSE | FILE_READ_ATTRIBUTES
|
|
1015
|
+
_STAT_ACCESS = 0x80 # FILE_READ_ATTRIBUTES: an identity read that holds nothing
|
|
1016
|
+
_SHARE_READ_WRITE = 0x1 | 0x2
|
|
1017
|
+
_OPEN_EXISTING = 3
|
|
1018
|
+
_BACKUP_SEMANTICS = 0x02000000
|
|
1019
|
+
_OPEN_REPARSE_POINT = 0x00200000
|
|
1020
|
+
_ATTRIBUTE_DIRECTORY = 0x10
|
|
1021
|
+
_ATTRIBUTE_REPARSE_POINT = 0x400
|
|
1022
|
+
_win_api: dict[str, Any] = {}
|
|
1023
|
+
|
|
1024
|
+
|
|
1025
|
+
def _win() -> Any:
|
|
1026
|
+
if "kernel32" not in _win_api:
|
|
1027
|
+
from ctypes import wintypes
|
|
1028
|
+
|
|
1029
|
+
k = ctypes.WinDLL("kernel32", use_last_error=True) # type: ignore[attr-defined]
|
|
1030
|
+
k.CreateFileW.argtypes = [
|
|
1031
|
+
wintypes.LPCWSTR, wintypes.DWORD, wintypes.DWORD, wintypes.LPVOID,
|
|
1032
|
+
wintypes.DWORD, wintypes.DWORD, wintypes.HANDLE,
|
|
1033
|
+
] # fmt: skip
|
|
1034
|
+
k.CreateFileW.restype = wintypes.HANDLE
|
|
1035
|
+
k.GetFileInformationByHandle.argtypes = [wintypes.HANDLE, ctypes.c_void_p]
|
|
1036
|
+
k.GetFileInformationByHandle.restype = wintypes.BOOL
|
|
1037
|
+
k.CloseHandle.argtypes = [wintypes.HANDLE]
|
|
1038
|
+
k.CloseHandle.restype = wintypes.BOOL
|
|
1039
|
+
_win_api["kernel32"] = k
|
|
1040
|
+
return _win_api["kernel32"]
|
|
1041
|
+
|
|
1042
|
+
|
|
1043
|
+
class _ByHandle(ctypes.Structure):
|
|
1044
|
+
_fields_ = [
|
|
1045
|
+
("attributes", ctypes.c_uint32),
|
|
1046
|
+
("times", ctypes.c_uint32 * 6),
|
|
1047
|
+
("volume_serial", ctypes.c_uint32),
|
|
1048
|
+
("size_high", ctypes.c_uint32),
|
|
1049
|
+
("size_low", ctypes.c_uint32),
|
|
1050
|
+
("links", ctypes.c_uint32),
|
|
1051
|
+
("index_high", ctypes.c_uint32),
|
|
1052
|
+
("index_low", ctypes.c_uint32),
|
|
1053
|
+
]
|
|
1054
|
+
|
|
1055
|
+
|
|
1056
|
+
def _win_open(path: str, follow: bool, access: int = _WALK_ACCESS) -> tuple[int, int, Identity]:
|
|
1057
|
+
"""A held handle on `path`, its attributes and its identity, as wasmtime reads it."""
|
|
1058
|
+
k = _win()
|
|
1059
|
+
flags = _BACKUP_SEMANTICS | (0 if follow else _OPEN_REPARSE_POINT)
|
|
1060
|
+
h = k.CreateFileW(path, access, _SHARE_READ_WRITE, None, _OPEN_EXISTING, flags, None)
|
|
1061
|
+
if h is None or h == ctypes.c_void_p(-1).value:
|
|
1062
|
+
code = ctypes.get_last_error() # type: ignore[attr-defined]
|
|
1063
|
+
raise ctypes.WinError(code) # type: ignore[attr-defined]
|
|
1064
|
+
info = _ByHandle()
|
|
1065
|
+
if not k.GetFileInformationByHandle(h, ctypes.byref(info)):
|
|
1066
|
+
code = ctypes.get_last_error() # type: ignore[attr-defined]
|
|
1067
|
+
k.CloseHandle(h)
|
|
1068
|
+
raise ctypes.WinError(code) # type: ignore[attr-defined]
|
|
1069
|
+
index = (info.index_high << 32) | info.index_low
|
|
1070
|
+
return h, info.attributes, (info.volume_serial, index)
|
|
1071
|
+
|
|
1072
|
+
|
|
1073
|
+
def _walk_windows(where: str, path: str, judge: bool, may_make: bool) -> _Walked:
|
|
1074
|
+
full = os.path.abspath(path) if judge else os.path.realpath(path)
|
|
1075
|
+
drive, rest = os.path.splitdrive(full)
|
|
1076
|
+
anchor = drive + "\\"
|
|
1077
|
+
parts = [p for p in re.split(r"[\\/]", rest) if p]
|
|
1078
|
+
held: list[int] = []
|
|
1079
|
+
chain: list[Identity] = []
|
|
1080
|
+
try:
|
|
1081
|
+
here = anchor
|
|
1082
|
+
for i in range(len(parts) + 1):
|
|
1083
|
+
if i:
|
|
1084
|
+
here = os.path.join(here, parts[i - 1])
|
|
1085
|
+
try:
|
|
1086
|
+
h, attributes, identity = _win_open(here, follow=not judge)
|
|
1087
|
+
except FileNotFoundError:
|
|
1088
|
+
if not may_make or i == 0:
|
|
1089
|
+
raise ValueError(f"{where}: {path!r} does not exist") from None
|
|
1090
|
+
walked = _Walked(os.path.dirname(here), tuple(reversed(chain)), held)
|
|
1091
|
+
walked.missing = tuple(parts[i - 1 :])
|
|
1092
|
+
return walked
|
|
1093
|
+
held.append(h)
|
|
1094
|
+
if judge and attributes & _ATTRIBUTE_REPARSE_POINT:
|
|
1095
|
+
raise ValueError(
|
|
1096
|
+
f"{where}: {path!r} passes through a link or junction; on Windows every one is "
|
|
1097
|
+
"refused"
|
|
1098
|
+
)
|
|
1099
|
+
if not attributes & _ATTRIBUTE_DIRECTORY:
|
|
1100
|
+
raise ValueError(f"{where}: {path!r} is not a folder")
|
|
1101
|
+
chain.append(identity)
|
|
1102
|
+
return _Walked(here, tuple(reversed(chain)), held)
|
|
1103
|
+
except BaseException:
|
|
1104
|
+
for h in held:
|
|
1105
|
+
_close_held(h)
|
|
1106
|
+
raise
|
|
1107
|
+
|
|
1108
|
+
|
|
1109
|
+
def _chain(path: Path) -> tuple[Identity | None, tuple[Identity, ...]]:
|
|
1110
|
+
"""A runtime folder's identity (None while it does not exist) and the identities of its
|
|
1111
|
+
deepest existing ancestor and that one's ancestors, every link followed; never refuses."""
|
|
1112
|
+
here = Path(os.path.realpath(path))
|
|
1113
|
+
exists = True
|
|
1114
|
+
while True:
|
|
1115
|
+
try:
|
|
1116
|
+
first = _identity_of(here)
|
|
1117
|
+
break
|
|
1118
|
+
except OSError:
|
|
1119
|
+
exists = False
|
|
1120
|
+
if here.parent == here:
|
|
1121
|
+
return None, ()
|
|
1122
|
+
here = here.parent
|
|
1123
|
+
chain = [first]
|
|
1124
|
+
for parent in here.parents:
|
|
1125
|
+
try:
|
|
1126
|
+
chain.append(_identity_of(parent))
|
|
1127
|
+
except OSError:
|
|
1128
|
+
break
|
|
1129
|
+
return (first if exists else None), tuple(chain)
|
|
1130
|
+
|
|
1131
|
+
|
|
1132
|
+
def _identity_of(path: Path) -> Identity:
|
|
1133
|
+
if os.name == "posix":
|
|
1134
|
+
st = os.stat(path)
|
|
1135
|
+
if not stat.S_ISDIR(st.st_mode):
|
|
1136
|
+
raise NotADirectoryError(str(path))
|
|
1137
|
+
return (st.st_dev, st.st_ino)
|
|
1138
|
+
# Attributes only: a runtime folder's chain must not be cut short by a folder one may not
|
|
1139
|
+
# traverse, or by another handle's share mode (the follow-up review, F1).
|
|
1140
|
+
h, _, identity = _win_open(str(path), follow=True, access=_STAT_ACCESS)
|
|
1141
|
+
_close_held(h)
|
|
1142
|
+
return identity
|
|
1143
|
+
|
|
1144
|
+
|
|
1145
|
+
def _judge(
|
|
1146
|
+
root: _Walked | None,
|
|
1147
|
+
grants: list[tuple[str, Grant, _Walked]],
|
|
1148
|
+
runtime: list,
|
|
1149
|
+
shared: list[tuple[str, tuple[Identity, ...]]] = [], # noqa: B006 (read only)
|
|
1150
|
+
) -> None:
|
|
1151
|
+
"""Decision 4's rules, by identity. `runtime` is (name, identity or None, chain) for each of
|
|
1152
|
+
the runtime's own folders: a writable grant at, inside or above one is refused; a folder not
|
|
1153
|
+
made yet holds nothing, so only a grant above its deepest existing ancestor is. `shared` is
|
|
1154
|
+
(name, chain) for each folder no grant, read-only included, may be at or above."""
|
|
1155
|
+
judged = [("root", None, root)] if root is not None else []
|
|
1156
|
+
for where, _, w in [*judged, *grants]: # root too: the temporary folder may have moved
|
|
1157
|
+
for name, chain in shared:
|
|
1158
|
+
if w.identity is not None and w.identity in chain:
|
|
1159
|
+
raise ValueError(f"{where} holds {name}; grant a folder apart from it")
|
|
1160
|
+
writable = [("root", root)] if root is not None else []
|
|
1161
|
+
writable += [(where, w) for where, g, w in grants if g.writable]
|
|
1162
|
+
for where, w in writable:
|
|
1163
|
+
for name, identity, chain in runtime:
|
|
1164
|
+
inside = identity is not None and identity in w.chain
|
|
1165
|
+
above = w.identity is not None and w.identity in chain
|
|
1166
|
+
if inside or above:
|
|
1167
|
+
how = "is inside" if inside else "holds"
|
|
1168
|
+
raise ValueError(
|
|
1169
|
+
f"{where} is writable and {how} {name}, which later calls trust; grant a "
|
|
1170
|
+
"folder apart from it"
|
|
1171
|
+
)
|
|
1172
|
+
for where, g, r in grants:
|
|
1173
|
+
if g.writable:
|
|
1174
|
+
continue
|
|
1175
|
+
for other, w in writable:
|
|
1176
|
+
if w.identity is not None and w.identity in r.chain:
|
|
1177
|
+
raise ValueError(
|
|
1178
|
+
f"{where} is read-only but inside {other}, which is writable; a guest would "
|
|
1179
|
+
"change it through that grant"
|
|
1180
|
+
)
|
|
1181
|
+
|
|
1182
|
+
|
|
1183
|
+
# The zip (Decision 2): one read-only folder per zip per process, beside a lock file, at
|
|
1184
|
+
# /usr/local/lib on every call. Its own lock, since the epoch ticker takes `_lock` every round.
|
|
1185
|
+
_zip_lock = threading.Lock()
|
|
1186
|
+
_zips: dict[str, list[Any]] = {} # the zip's SHA-256 -> [its _Private folder, the copy's stamp]
|
|
1187
|
+
|
|
1188
|
+
|
|
1189
|
+
def _zip_folder(found: Artifacts) -> Path:
|
|
1190
|
+
want = _digest(found.stdlib_zip)
|
|
1191
|
+
name = f"python{_version_tag(found.manifest)}.zip"
|
|
1192
|
+
with _zip_lock:
|
|
1193
|
+
entry = _zips.get(want)
|
|
1194
|
+
if entry is None:
|
|
1195
|
+
return _zip_folder_locked(found, want, name)
|
|
1196
|
+
private, stamp = entry
|
|
1197
|
+
if not os.path.isdir(private.path) or os.path.islink(private.path):
|
|
1198
|
+
# Removed on the host (a temporary-folder cleaner, say): made again, not trusted.
|
|
1199
|
+
private.remove()
|
|
1200
|
+
del _zips[want]
|
|
1201
|
+
return _zip_folder_locked(found, want, name)
|
|
1202
|
+
dest = private.path / name
|
|
1203
|
+
try:
|
|
1204
|
+
st = os.lstat(dest)
|
|
1205
|
+
except FileNotFoundError:
|
|
1206
|
+
st = None
|
|
1207
|
+
if st is None or not stat.S_ISREG(st.st_mode) or _stamp(st) != stamp:
|
|
1208
|
+
entry[1] = _copy_zip(found, dest, want) # changed on the host: copied again
|
|
1209
|
+
return private.path
|
|
1210
|
+
|
|
1211
|
+
|
|
1212
|
+
def _zip_folder_locked(found: Artifacts, want: str, name: str) -> Path:
|
|
1213
|
+
private = _Private("wasi-python-zip-")
|
|
1214
|
+
try:
|
|
1215
|
+
stamp = _copy_zip(found, private.path / name, want)
|
|
1216
|
+
except BaseException:
|
|
1217
|
+
private.remove()
|
|
1218
|
+
raise
|
|
1219
|
+
_zips[want] = [private, stamp]
|
|
1220
|
+
return private.path
|
|
1221
|
+
|
|
1222
|
+
|
|
1223
|
+
def _zip_known(found: Artifacts) -> Path | None:
|
|
1224
|
+
"""This process's folder for this zip, if it has made one."""
|
|
1225
|
+
with _zip_lock:
|
|
1226
|
+
entry = _zips.get(_digest(found.stdlib_zip))
|
|
1227
|
+
return None if entry is None else entry[0].path
|
|
1228
|
+
|
|
1229
|
+
|
|
1230
|
+
def _copy_zip(found: Artifacts, dest: Path, want: str) -> tuple[int, ...]:
|
|
1231
|
+
fd, tmp = tempfile.mkstemp(prefix=".python-zip-", dir=dest.parent)
|
|
1232
|
+
try:
|
|
1233
|
+
with os.fdopen(fd, "wb") as out, found.stdlib_zip.open("rb") as src:
|
|
1234
|
+
for block in iter(lambda: src.read(1 << 20), b""):
|
|
1235
|
+
out.write(block)
|
|
1236
|
+
# 0644, not read-only: Windows could neither replace nor remove a read-only file, and
|
|
1237
|
+
# the guest's grant is read-only anyway, in a 0700 folder.
|
|
1238
|
+
os.chmod(tmp, 0o644)
|
|
1239
|
+
try:
|
|
1240
|
+
os.replace(tmp, dest)
|
|
1241
|
+
except PermissionError:
|
|
1242
|
+
# Windows cannot replace a file a running guest has open; when the file in place is
|
|
1243
|
+
# already the right zip, the copy is not needed.
|
|
1244
|
+
if os.name != "nt" or not (dest.is_file() and _hash_file(dest) == want):
|
|
1245
|
+
raise
|
|
1246
|
+
Path(tmp).unlink(missing_ok=True)
|
|
1247
|
+
except BaseException:
|
|
1248
|
+
Path(tmp).unlink(missing_ok=True)
|
|
1249
|
+
raise
|
|
1250
|
+
if _hash_file(dest) != want:
|
|
1251
|
+
raise RuntimeError(f"the stdlib zip's copy at {dest} is not the artifact set's")
|
|
1252
|
+
return _stamp(os.lstat(dest))
|
|
1253
|
+
|
|
1254
|
+
|
|
1255
|
+
def _remove_zips() -> None:
|
|
1256
|
+
with _zip_lock:
|
|
1257
|
+
entries = list(_zips.values())
|
|
1258
|
+
_zips.clear()
|
|
1259
|
+
for private, _ in entries:
|
|
1260
|
+
private.remove()
|
|
1261
|
+
|
|
1262
|
+
|
|
1263
|
+
# The identity check (Decision 5): wasmtime reports a preopen's `ino` as SipHash-1-3, zero keys,
|
|
1264
|
+
# over the host folder's two identity numbers as 64-bit words (filesystem.rs:354-371; on Windows
|
|
1265
|
+
# the volume serial and file index, windows.rs:85-91). A wasmtime that hashes otherwise refuses
|
|
1266
|
+
# every grant, loudly; tests/unit/test_runner_grants.py pins it on every OS.
|
|
1267
|
+
_CHECK_WAT = """(module
|
|
1268
|
+
(import "wasi_snapshot_preview1" "fd_prestat_get" (func $pg (param i32 i32) (result i32)))
|
|
1269
|
+
(import "wasi_snapshot_preview1" "fd_prestat_dir_name"
|
|
1270
|
+
(func $pn (param i32 i32 i32) (result i32)))
|
|
1271
|
+
(import "wasi_snapshot_preview1" "fd_filestat_get" (func $st (param i32 i32) (result i32)))
|
|
1272
|
+
(memory (export "memory") 1)
|
|
1273
|
+
(func (export "prestat") (param i32) (result i32) (call $pg (local.get 0) (i32.const 0)))
|
|
1274
|
+
(func (export "name") (param i32 i32) (result i32)
|
|
1275
|
+
(call $pn (local.get 0) (i32.const 1024) (local.get 1)))
|
|
1276
|
+
(func (export "stat") (param i32) (result i32) (call $st (local.get 0) (i32.const 64))))"""
|
|
1277
|
+
_check_modules: dict[bool, Any] = {}
|
|
1278
|
+
|
|
1279
|
+
|
|
1280
|
+
def _rotl(x: int, b: int) -> int:
|
|
1281
|
+
return ((x << b) | (x >> (64 - b))) & _M64
|
|
1282
|
+
|
|
1283
|
+
|
|
1284
|
+
def _siphash13(data: bytes) -> int:
|
|
1285
|
+
v0, v1 = 0x736F6D6570736575, 0x646F72616E646F6D
|
|
1286
|
+
v2, v3 = 0x6C7967656E657261, 0x7465646279746573
|
|
1287
|
+
|
|
1288
|
+
def rounds(n: int) -> None:
|
|
1289
|
+
nonlocal v0, v1, v2, v3
|
|
1290
|
+
for _ in range(n):
|
|
1291
|
+
v0 = (v0 + v1) & _M64
|
|
1292
|
+
v1 = _rotl(v1, 13) ^ v0
|
|
1293
|
+
v0 = _rotl(v0, 32)
|
|
1294
|
+
v2 = (v2 + v3) & _M64
|
|
1295
|
+
v3 = _rotl(v3, 16) ^ v2
|
|
1296
|
+
v0 = (v0 + v3) & _M64
|
|
1297
|
+
v3 = _rotl(v3, 21) ^ v0
|
|
1298
|
+
v2 = (v2 + v1) & _M64
|
|
1299
|
+
v1 = _rotl(v1, 17) ^ v2
|
|
1300
|
+
v2 = _rotl(v2, 32)
|
|
1301
|
+
|
|
1302
|
+
end = len(data) - len(data) % 8
|
|
1303
|
+
for i in range(0, end, 8):
|
|
1304
|
+
m = struct.unpack_from("<Q", data, i)[0]
|
|
1305
|
+
v3 ^= m
|
|
1306
|
+
rounds(1)
|
|
1307
|
+
v0 ^= m
|
|
1308
|
+
last = (len(data) & 0xFF) << 56
|
|
1309
|
+
for i, byte in enumerate(data[end:]):
|
|
1310
|
+
last |= byte << (8 * i)
|
|
1311
|
+
v3 ^= last
|
|
1312
|
+
rounds(1)
|
|
1313
|
+
v0 ^= last
|
|
1314
|
+
v2 ^= 0xFF
|
|
1315
|
+
rounds(3)
|
|
1316
|
+
return v0 ^ v1 ^ v2 ^ v3
|
|
1317
|
+
|
|
1318
|
+
|
|
1319
|
+
def _wasi_ino(identity: Identity) -> int:
|
|
1320
|
+
"""The `ino` wasmtime reports for a folder of this identity. Rust sign-extends a negative
|
|
1321
|
+
dev_t, hence the mask; the words are native-endian, as Rust's Hash writes them."""
|
|
1322
|
+
return _siphash13(struct.pack("=QQ", identity[0] & _M64, identity[1] & _M64))
|
|
1323
|
+
|
|
1324
|
+
|
|
1325
|
+
class _Preopen(NamedTuple):
|
|
1326
|
+
label: str # how an error names it: "grant 2 at /data", "root", "the stdlib folder"
|
|
1327
|
+
walked: _Walked
|
|
1328
|
+
guest: str
|
|
1329
|
+
writable: bool
|
|
1330
|
+
|
|
1331
|
+
|
|
1332
|
+
def _check_module(wt: Any, engine: Any, metered: bool) -> Any:
|
|
1333
|
+
with _lock:
|
|
1334
|
+
if metered not in _check_modules:
|
|
1335
|
+
_check_modules[metered] = wt.Module(engine, _CHECK_WAT)
|
|
1336
|
+
return _check_modules[metered]
|
|
1337
|
+
|
|
1338
|
+
|
|
1339
|
+
def _check_preopens(wt: Any, linker: Any, store: Any, check: Any, preopens: list[_Preopen]) -> None:
|
|
1340
|
+
"""Before the guest: each preopen 3 + i is the folder the walk checked, by guest name and by
|
|
1341
|
+
identity. A mismatch raises before the guest runs; the walk's holds are released either way."""
|
|
1342
|
+
try:
|
|
1343
|
+
exports = linker.instantiate(store, check).exports(store)
|
|
1344
|
+
memory = exports["memory"]
|
|
1345
|
+
for i, p in enumerate(preopens):
|
|
1346
|
+
fd = 3 + i
|
|
1347
|
+
name = p.guest.encode()
|
|
1348
|
+
ok = exports["prestat"](store, fd) == 0
|
|
1349
|
+
if ok:
|
|
1350
|
+
head = bytes(memory.read(store, 0, 8))
|
|
1351
|
+
ok = head[0] == 0 and struct.unpack_from("<I", head, 4)[0] == len(name)
|
|
1352
|
+
ok = ok and exports["name"](store, fd, len(name)) == 0
|
|
1353
|
+
ok = ok and bytes(memory.read(store, 1024, 1024 + len(name))) == name
|
|
1354
|
+
ok = ok and exports["stat"](store, fd) == 0
|
|
1355
|
+
identity = p.walked.identity
|
|
1356
|
+
if ok and identity is not None:
|
|
1357
|
+
reported = struct.unpack("<Q", bytes(memory.read(store, 72, 80)))[0]
|
|
1358
|
+
ok = reported == _wasi_ino(identity)
|
|
1359
|
+
if not ok:
|
|
1360
|
+
raise RuntimeError(f"{p.label} changed between its check and the call")
|
|
1361
|
+
finally:
|
|
1362
|
+
for p in preopens:
|
|
1363
|
+
p.walked.close()
|
|
1364
|
+
|
|
1365
|
+
|
|
1366
|
+
# A seam for the swap check (tests only): runs after every walk and before wasmtime opens anything.
|
|
1367
|
+
_after_walk: Callable[[], None] | None = None
|
|
1368
|
+
|
|
1369
|
+
|
|
1370
|
+
def _walk_grants(
|
|
1371
|
+
root: str | bool | None, grants: tuple[Grant, ...], found: Artifacts, cache: Path | None
|
|
1372
|
+
) -> tuple[_Walked | None, list[tuple[str, Grant, _Walked]]]:
|
|
1373
|
+
"""Every grant and `root`, walked and judged before any work (Decision 4); a missing root is
|
|
1374
|
+
made only once every rule has passed on its existing chain."""
|
|
1375
|
+
walked_root: _Walked | None = None
|
|
1376
|
+
walked: list[tuple[str, Grant, _Walked]] = []
|
|
1377
|
+
runtime = _runtime_folders(found, cache) # first: it makes the module cache if it can
|
|
1378
|
+
try:
|
|
1379
|
+
if isinstance(root, str):
|
|
1380
|
+
walked_root = _walk("root", root, judge=True, may_make=True)
|
|
1381
|
+
for i, grant in enumerate(grants):
|
|
1382
|
+
where = f"grants[{i}] at {grant.guest}"
|
|
1383
|
+
walked.append((where, grant, _walk(where, os.fspath(grant.host), judge=True)))
|
|
1384
|
+
_judge(walked_root, walked, runtime, _shared_folders())
|
|
1385
|
+
if walked_root is not None and walked_root.missing:
|
|
1386
|
+
try:
|
|
1387
|
+
walked_root = _make_missing("root", walked_root)
|
|
1388
|
+
except OSError as e:
|
|
1389
|
+
raise ValueError(f"root: {root!r} cannot be made: {e.strerror or e}") from None
|
|
1390
|
+
return walked_root, walked
|
|
1391
|
+
except BaseException:
|
|
1392
|
+
for w in [walked_root, *(w for _, _, w in walked)]:
|
|
1393
|
+
if w is not None:
|
|
1394
|
+
w.close()
|
|
1395
|
+
raise
|
|
1396
|
+
|
|
1397
|
+
|
|
1398
|
+
def _runtime_folders(found: Artifacts, cache: Path | None) -> list[tuple[str, Any, Any]]:
|
|
1399
|
+
"""The folders later calls and processes trust: this zip's folder (or, before it is made, the
|
|
1400
|
+
temporary folder that will hold it), the artifact set's files, the module cache, and the
|
|
1401
|
+
runtime's own code (this package and wasmtime-py's, whose native library a guest must never
|
|
1402
|
+
replace). The module cache is made first, if it can be, so it is judged as it will be: a root
|
|
1403
|
+
still to be made above a cache still to be made would otherwise come to hold it (the build
|
|
1404
|
+
review, finding 1)."""
|
|
1405
|
+
import wasmtime
|
|
1406
|
+
|
|
1407
|
+
if cache is not None:
|
|
1408
|
+
try:
|
|
1409
|
+
cache.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
1410
|
+
except OSError:
|
|
1411
|
+
pass # unusable: `_module` then keeps compiled modules in memory
|
|
1412
|
+
zip_dir = _zip_known(found)
|
|
1413
|
+
if zip_dir is not None:
|
|
1414
|
+
folders = [("the runtime's stdlib folder", *_chain(zip_dir))]
|
|
1415
|
+
else:
|
|
1416
|
+
temp = Path(tempfile.gettempdir())
|
|
1417
|
+
folders = [
|
|
1418
|
+
("the temporary folder, where the runtime's stdlib folder goes", None, _chain(temp)[1])
|
|
1419
|
+
]
|
|
1420
|
+
parts = {found.manifest.parent, found.snapshot.parent, found.plain.parent}
|
|
1421
|
+
parts.add(found.stdlib_zip.parent)
|
|
1422
|
+
folders += [("the artifact set", *_chain(d)) for d in sorted(parts)]
|
|
1423
|
+
if cache is not None:
|
|
1424
|
+
folders.append(("the module cache", *_chain(cache)))
|
|
1425
|
+
folders.append(("wasi_python's own code", *_chain(Path(__file__).resolve().parent)))
|
|
1426
|
+
folders.append(("wasmtime-py's own code", *_chain(Path(wasmtime.__file__).resolve().parent)))
|
|
1427
|
+
# The host's own interpreter, which the next host process runs (spec 024, S2): its stdlib,
|
|
1428
|
+
# its site-packages and scripts, and the user's site-packages it imports at start-up.
|
|
1429
|
+
import site
|
|
1430
|
+
import sysconfig
|
|
1431
|
+
|
|
1432
|
+
paths = sysconfig.get_paths()
|
|
1433
|
+
keys = ("stdlib", "platstdlib", "purelib", "platlib", "scripts")
|
|
1434
|
+
own = {Path(paths[k]).resolve() for k in keys if k in paths}
|
|
1435
|
+
if site.ENABLE_USER_SITE:
|
|
1436
|
+
own.add(Path(site.getusersitepackages()).resolve())
|
|
1437
|
+
folders += [("the host Python's own code", *_chain(d)) for d in sorted(own) if d.is_dir()]
|
|
1438
|
+
return folders
|
|
1439
|
+
|
|
1440
|
+
|
|
1441
|
+
def _shared_folders() -> list[tuple[str, tuple[Identity, ...]]]:
|
|
1442
|
+
"""Folders no grant may be at or above, read-only included: the temporary folder holds every
|
|
1443
|
+
call's private root and stdin (the build review, finding 2)."""
|
|
1444
|
+
temp = Path(tempfile.gettempdir())
|
|
1445
|
+
return [
|
|
1446
|
+
(
|
|
1447
|
+
"the temporary folder, which holds every call's private folders and stdin",
|
|
1448
|
+
_chain(temp)[1],
|
|
1449
|
+
)
|
|
1450
|
+
]
|
|
1451
|
+
|
|
1452
|
+
|
|
1453
|
+
def _run_checked(
|
|
1454
|
+
wt: Any,
|
|
1455
|
+
c: dict[str, Any],
|
|
1456
|
+
host: Host | None,
|
|
1457
|
+
snapshot: bool,
|
|
1458
|
+
cache_dir: str | os.PathLike[str] | None | bool,
|
|
1459
|
+
artifacts: Artifacts | None,
|
|
1460
|
+
) -> Completed:
|
|
1461
|
+
found = artifacts if artifacts is not None else find_artifacts()
|
|
1462
|
+
cache = _cache_dir(cache_dir)
|
|
1463
|
+
root = c["root"]
|
|
1464
|
+
walked_root, walked = _walk_grants(root, c["grants"], found, cache)
|
|
1465
|
+
preopens: list[_Preopen] = []
|
|
1466
|
+
private = None
|
|
1467
|
+
try:
|
|
1468
|
+
if _after_walk is not None:
|
|
1469
|
+
_after_walk()
|
|
1470
|
+
binary = found.snapshot if snapshot else found.plain
|
|
1471
|
+
engine = _engine(c["fuel"] is not None)
|
|
1472
|
+
module = _module(engine, c["fuel"] is not None, binary, cache)
|
|
1473
|
+
_sweep_once()
|
|
1474
|
+
zip_dir = _zip_folder(found)
|
|
1475
|
+
preopens.append(
|
|
1476
|
+
_Preopen("the stdlib folder", _walk("", str(zip_dir), judge=False), ZIP_GUEST, False)
|
|
1477
|
+
)
|
|
1478
|
+
root_path = None
|
|
1479
|
+
if root is None:
|
|
1480
|
+
private = _Private("wasi-python-root-") # 0700, this user's own, locked (spec 016)
|
|
1481
|
+
(private.path / "usr" / "local" / "lib").mkdir(parents=True)
|
|
1482
|
+
root_path = private.path
|
|
1483
|
+
preopens.append(
|
|
1484
|
+
_Preopen("the private root", _walk("", str(root_path), judge=False), "/", True)
|
|
1485
|
+
)
|
|
1486
|
+
elif walked_root is not None:
|
|
1487
|
+
root_path = Path(walked_root.path)
|
|
1488
|
+
# Real folders, never a guest's link: without them /usr would vanish from a fresh
|
|
1489
|
+
# root's listing, and the zip's path could resolve into a file a guest wrote. Made
|
|
1490
|
+
# through the walked folder, never its path, which another guest may have swapped
|
|
1491
|
+
# since the walk (spec 024).
|
|
1492
|
+
if walked_root.fd is not None and _dir_fd_ok():
|
|
1493
|
+
os.close(_real_dir_at(walked_root.fd, ("usr", "local", "lib")))
|
|
1494
|
+
else:
|
|
1495
|
+
_real_dir(root_path, ("usr", "local", "lib"))
|
|
1496
|
+
preopens.append(_Preopen("root", walked_root, "/", True))
|
|
1497
|
+
for where, grant, w in walked:
|
|
1498
|
+
preopens.append(_Preopen(where, w, grant.guest, grant.writable))
|
|
1499
|
+
return _call(wt, engine, module, root_path, preopens, c, host)
|
|
1500
|
+
finally:
|
|
1501
|
+
# Windows holds every walked folder until here; a private root can be removed only after.
|
|
1502
|
+
for w in [walked_root, *(w for _, _, w in walked), *(p.walked for p in preopens)]:
|
|
1503
|
+
if w is not None:
|
|
1504
|
+
w.close()
|
|
1505
|
+
if private is not None:
|
|
1506
|
+
private.remove()
|
|
1507
|
+
|
|
1508
|
+
|
|
1509
|
+
def _call(
|
|
1510
|
+
wt: Any,
|
|
1511
|
+
engine: wasmtime.Engine,
|
|
1512
|
+
module: wasmtime.Module,
|
|
1513
|
+
root: Path | None,
|
|
1514
|
+
preopens: list[_Preopen],
|
|
1515
|
+
c: dict[str, Any],
|
|
1516
|
+
host: Host | None,
|
|
1517
|
+
) -> Completed:
|
|
1518
|
+
"""One instance, one program. The store is closed before this returns, trapped or not, so the
|
|
1519
|
+
guest's memory and open files are released at once (a trapped store is otherwise kept alive by
|
|
1520
|
+
a reference cycle until Python's collector runs) and a private root can be removed afterwards
|
|
1521
|
+
(Windows refuses to remove an open directory). Ctrl-C is deferred while the guest runs."""
|
|
1522
|
+
with _deferred_ctrl_c():
|
|
1523
|
+
return _call_deferred(wt, engine, module, root, preopens, c, host)
|
|
1524
|
+
|
|
1525
|
+
|
|
1526
|
+
def _call_deferred(
|
|
1527
|
+
wt: Any,
|
|
1528
|
+
engine: wasmtime.Engine,
|
|
1529
|
+
module: wasmtime.Module,
|
|
1530
|
+
root: Path | None,
|
|
1531
|
+
preopens: list[_Preopen],
|
|
1532
|
+
c: dict[str, Any],
|
|
1533
|
+
host: Host | None,
|
|
1534
|
+
) -> Completed:
|
|
1535
|
+
tmp, tmp_fd = _home_folder(root, preopens)
|
|
1536
|
+
home = None
|
|
1537
|
+
if tmp is not None:
|
|
1538
|
+
# The fresh HOME is made, and removed, through a handle on /tmp opened without following a
|
|
1539
|
+
# link, so a guest running concurrently on the same folder cannot turn /tmp into a link
|
|
1540
|
+
# between the check and the mkdir (Windows has no dir_fd, and its guests cannot make links).
|
|
1541
|
+
name = f".home-{secrets.token_hex(16)}"
|
|
1542
|
+
try:
|
|
1543
|
+
if tmp_fd is not None:
|
|
1544
|
+
os.mkdir(name, dir_fd=tmp_fd)
|
|
1545
|
+
else:
|
|
1546
|
+
(tmp / name).mkdir()
|
|
1547
|
+
except BaseException:
|
|
1548
|
+
_close(tmp_fd)
|
|
1549
|
+
raise
|
|
1550
|
+
home = (name, f"/tmp/{name}")
|
|
1551
|
+
store = wt.Store(engine)
|
|
1552
|
+
try:
|
|
1553
|
+
guest_home = None if home is None else home[1]
|
|
1554
|
+
return _instance(wt, engine, module, store, preopens, guest_home, c, host)
|
|
1555
|
+
finally:
|
|
1556
|
+
store.close()
|
|
1557
|
+
if home is not None and tmp_fd is not None:
|
|
1558
|
+
shutil.rmtree(home[0], ignore_errors=True, dir_fd=tmp_fd)
|
|
1559
|
+
elif home is not None and tmp is not None:
|
|
1560
|
+
shutil.rmtree(tmp / home[0], ignore_errors=True)
|
|
1561
|
+
_close(tmp_fd)
|
|
1562
|
+
|
|
1563
|
+
|
|
1564
|
+
def _home_folder(root: Path | None, preopens: list[_Preopen]) -> tuple[Path | None, int | None]:
|
|
1565
|
+
"""Where HOME is made (Decision 3): a writable grant at /tmp, else the root's tmp, else
|
|
1566
|
+
nowhere. A read-only grant at /tmp hides the root's tmp, so then there is no HOME either."""
|
|
1567
|
+
at_tmp = [p for p in preopens if p.guest == "/tmp"]
|
|
1568
|
+
if at_tmp:
|
|
1569
|
+
p = at_tmp[0]
|
|
1570
|
+
if not p.writable:
|
|
1571
|
+
return None, None
|
|
1572
|
+
fd = p.walked.fd
|
|
1573
|
+
return Path(p.walked.path), (os.dup(fd) if fd is not None and _dir_fd_ok() else None)
|
|
1574
|
+
if root is None:
|
|
1575
|
+
return None, None
|
|
1576
|
+
# TMPDIR, a real folder (a guest's link there is removed), made through the root's walked
|
|
1577
|
+
# folder, never its path (spec 024).
|
|
1578
|
+
fd = next(p.walked.fd for p in preopens if p.guest == "/")
|
|
1579
|
+
if fd is not None and _dir_fd_ok():
|
|
1580
|
+
return root / "tmp", _real_dir_at(fd, ("tmp",))
|
|
1581
|
+
return _real_dir(root, ("tmp",)), None
|
|
1582
|
+
|
|
1583
|
+
|
|
1584
|
+
def _dir_fd_ok() -> bool:
|
|
1585
|
+
return os.mkdir in os.supports_dir_fd and shutil.rmtree.avoids_symlink_attacks is not False
|
|
1586
|
+
|
|
1587
|
+
|
|
1588
|
+
def _open_dir(path: Path) -> int | None:
|
|
1589
|
+
"""A handle on a directory, opened without following a link; None where the platform has no
|
|
1590
|
+
dir_fd support (Windows)."""
|
|
1591
|
+
if not _dir_fd_ok():
|
|
1592
|
+
return None
|
|
1593
|
+
flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0)
|
|
1594
|
+
return os.open(path, flags)
|
|
1595
|
+
|
|
1596
|
+
|
|
1597
|
+
def _close(fd: int | None) -> None:
|
|
1598
|
+
if fd is not None:
|
|
1599
|
+
os.close(fd)
|
|
1600
|
+
|
|
1601
|
+
|
|
1602
|
+
def _instance(
|
|
1603
|
+
wt: Any,
|
|
1604
|
+
engine: wasmtime.Engine,
|
|
1605
|
+
module: wasmtime.Module,
|
|
1606
|
+
store: wasmtime.Store,
|
|
1607
|
+
preopens: list[_Preopen],
|
|
1608
|
+
home: str | None,
|
|
1609
|
+
c: dict[str, Any],
|
|
1610
|
+
host: Host | None,
|
|
1611
|
+
) -> Completed:
|
|
1612
|
+
argv, env, stdin, cancel = c["argv"], c["env"], c["stdin"], c["cancel"]
|
|
1613
|
+
fuel, deadline_s = c["fuel"], c["deadline_s"]
|
|
1614
|
+
memory_bytes, output_bytes = c["memory_bytes"], c["output_bytes"]
|
|
1615
|
+
if fuel is not None:
|
|
1616
|
+
# Fuel is metered from the end of instantiation (spec 008): instantiation runs only
|
|
1617
|
+
# wasmtime's own memory fill, which it charges per byte when it cannot map the image
|
|
1618
|
+
# (macOS when compiled in the process, Windows always); the guest's work starts after.
|
|
1619
|
+
store.set_fuel(UNMETERED)
|
|
1620
|
+
if memory_bytes is not None:
|
|
1621
|
+
store.set_limits(memory_size=memory_bytes)
|
|
1622
|
+
ticks = NO_DEADLINE if deadline_s is None else max(1, math.ceil(deadline_s / TICK_S))
|
|
1623
|
+
# A call with `cancel` is first checked within CANCEL_TICKS ticks, whatever its deadline; its
|
|
1624
|
+
# callback then enforces the deadline too (spec 015).
|
|
1625
|
+
store.set_epoch_deadline(ticks if cancel is None else min(CANCEL_TICKS, ticks))
|
|
1626
|
+
# The ticker's count at which this store's deadline has passed, read after the deadline is set
|
|
1627
|
+
# (spec 010): a wait in poll_oneoff stops there, so it never outlives the deadline.
|
|
1628
|
+
stop_at = None if deadline_s is None else _stop_at(_epoch_ticker.count(), ticks)
|
|
1629
|
+
checks = [] if stop_at is None else [lambda: _epoch_ticker.count() >= stop_at]
|
|
1630
|
+
if cancel is not None:
|
|
1631
|
+
checks.append(cancel.is_set) # a wait ends at a cancel, deadline or not
|
|
1632
|
+
if _ctrl_c_watched():
|
|
1633
|
+
checks.append(lambda: _ctrl_c["pending"]) # a wait on the main thread ends at Ctrl-C
|
|
1634
|
+
waits = _Waits(
|
|
1635
|
+
lambda: time.perf_counter(),
|
|
1636
|
+
time.sleep,
|
|
1637
|
+
(lambda: any(check() for check in checks)) if checks else None,
|
|
1638
|
+
)
|
|
1639
|
+
|
|
1640
|
+
out, err = _Stream("stdout", output_bytes), _Stream("stderr", output_bytes)
|
|
1641
|
+
wasi = wt.WasiConfig()
|
|
1642
|
+
origin_ns = time.perf_counter_ns() # wasmtime's monotonic clock counts from here (spec 010)
|
|
1643
|
+
wasi.argv = [ARGV0, *argv]
|
|
1644
|
+
fresh = {} if home is None else {"HOME": home} # no HOME where there is no /tmp to make it
|
|
1645
|
+
wasi.env = sorted({**BASE_ENV, **fresh, **(env or {})}.items())
|
|
1646
|
+
for p in preopens: # the stdlib folder first; wasi-libc picks by the longest guest prefix
|
|
1647
|
+
wasi.preopen_dir(p.walked.path, p.guest, p.writable)
|
|
1648
|
+
check = _check_module(wt, engine, fuel is not None)
|
|
1649
|
+
keys = _send_outputs(wasi, out, err)
|
|
1650
|
+
scratch = None
|
|
1651
|
+
try:
|
|
1652
|
+
scratch = _Private("wasi-python-") # 0700, this user's own, locked (spec 016)
|
|
1653
|
+
stdin_path = scratch.path / "stdin"
|
|
1654
|
+
stdin_path.write_bytes(stdin or b"")
|
|
1655
|
+
wasi.stdin_file = str(stdin_path)
|
|
1656
|
+
store.set_wasi(wasi)
|
|
1657
|
+
linker = _linker(wt, engine) # held for the whole call: clearing the cache cannot free it
|
|
1658
|
+
call = _Call(store, host, waits, origin_ns, cancel, stop_at)
|
|
1659
|
+
if cancel is not None:
|
|
1660
|
+
_watch(store)
|
|
1661
|
+
with _epoch_ticker(engine, deadline_s is not None or cancel is not None):
|
|
1662
|
+
done = _drive(wt, linker, store, module, call, fuel, preopens, check)
|
|
1663
|
+
fuel_used = _fuel_used(fuel, done, call, store.get_fuel)
|
|
1664
|
+
finally:
|
|
1665
|
+
# The store holds every descriptor the guest opened; a guest can open them until the
|
|
1666
|
+
# host refuses. Close it first, so the clean-up below has descriptors to work with, and
|
|
1667
|
+
# make that clean-up unable to raise (the 0.1.0a2 security review, F1).
|
|
1668
|
+
store.close()
|
|
1669
|
+
with _outputs_lock: # the guest has ended; wasmtime frees the keys later, on its own thread
|
|
1670
|
+
for key in keys:
|
|
1671
|
+
_outputs.pop(key, None)
|
|
1672
|
+
if scratch is not None:
|
|
1673
|
+
scratch.remove()
|
|
1674
|
+
truncated = tuple(s.name for s in (out, err) if s.dropped)
|
|
1675
|
+
return Completed(
|
|
1676
|
+
done.exit_code, bytes(out.data), bytes(err.data), done.trapped, fuel_used, truncated
|
|
1677
|
+
)
|
|
1678
|
+
|
|
1679
|
+
|
|
1680
|
+
def _fuel_used(
|
|
1681
|
+
fuel: int | None, done: _Driven, call: _Call, remaining: Callable[[], int]
|
|
1682
|
+
) -> int | None:
|
|
1683
|
+
"""The instructions the guest ran, or None when the count is meaningless: after an epoch
|
|
1684
|
+
interrupt wasmtime's fuel counter is not brought up to date (spec 001 landing notes), and a
|
|
1685
|
+
stopped call ended on one (spec 011)."""
|
|
1686
|
+
if fuel is None or done.trapped == "deadline":
|
|
1687
|
+
return None
|
|
1688
|
+
if call.stopped is not None or call.ended is not None:
|
|
1689
|
+
return None
|
|
1690
|
+
return fuel - remaining() if done.metered else 0
|
|
1691
|
+
|
|
1692
|
+
|
|
1693
|
+
class _Driven(NamedTuple):
|
|
1694
|
+
exit_code: int | None
|
|
1695
|
+
trapped: str | None
|
|
1696
|
+
metered: bool # the cap is set only once instantiation has succeeded
|
|
1697
|
+
instance: Any
|
|
1698
|
+
|
|
1699
|
+
|
|
1700
|
+
def _drive(
|
|
1701
|
+
wt: Any,
|
|
1702
|
+
linker: Any,
|
|
1703
|
+
store: Any,
|
|
1704
|
+
module: Any,
|
|
1705
|
+
call: _Call,
|
|
1706
|
+
fuel: int | None,
|
|
1707
|
+
preopens: list[_Preopen] = (), # type: ignore[assignment]
|
|
1708
|
+
check: Any = None,
|
|
1709
|
+
) -> _Driven:
|
|
1710
|
+
"""Instantiate and run the guest with its call on this thread's stack. What a host function
|
|
1711
|
+
caught and recorded comes first: an exit or a trap is the call's result, and any other
|
|
1712
|
+
exception (Ctrl-C, a host's cancellation) is raised here, on the thread that made the call
|
|
1713
|
+
(spec 011)."""
|
|
1714
|
+
exit_code: int | None = None
|
|
1715
|
+
trapped: str | None = None
|
|
1716
|
+
metered = False
|
|
1717
|
+
instance = None
|
|
1718
|
+
stack = _stack()
|
|
1719
|
+
stack.append(call)
|
|
1720
|
+
_calls.lost = False
|
|
1721
|
+
try:
|
|
1722
|
+
if call.cancel is not None and call.cancel.is_set(): # cancelled before the guest ran
|
|
1723
|
+
call.ended = "cancelled"
|
|
1724
|
+
return _Driven(None, "cancelled", False, None)
|
|
1725
|
+
if preopens:
|
|
1726
|
+
_check_preopens(wt, linker, store, check, list(preopens)) # spec 017, Decision 5
|
|
1727
|
+
instance = linker.instantiate(store, module)
|
|
1728
|
+
if fuel is not None:
|
|
1729
|
+
store.set_fuel(fuel)
|
|
1730
|
+
metered = True
|
|
1731
|
+
exports = instance.exports(store)
|
|
1732
|
+
if "_initialize" in exports:
|
|
1733
|
+
exports["_initialize"](store)
|
|
1734
|
+
exit_code = exports["run"](store)
|
|
1735
|
+
except wt.ExitTrap as e:
|
|
1736
|
+
exit_code = e.code
|
|
1737
|
+
except wt.Trap as e:
|
|
1738
|
+
trapped = _trap_name(e)
|
|
1739
|
+
except wt.WasmtimeError as e:
|
|
1740
|
+
trapped = _first_line(str(e))
|
|
1741
|
+
finally:
|
|
1742
|
+
stack.pop()
|
|
1743
|
+
lost, _calls.lost = getattr(_calls, "lost", False), False
|
|
1744
|
+
if lost:
|
|
1745
|
+
raise RuntimeError("wasi_python lost track of a call")
|
|
1746
|
+
cause = call.stopped
|
|
1747
|
+
if cause is not None:
|
|
1748
|
+
exit_code, trapped = None, None
|
|
1749
|
+
if isinstance(cause, wt.ExitTrap):
|
|
1750
|
+
exit_code = cause.code
|
|
1751
|
+
elif isinstance(cause, wt.Trap):
|
|
1752
|
+
trapped = _trap_name(cause)
|
|
1753
|
+
elif isinstance(cause, wt.WasmtimeError):
|
|
1754
|
+
trapped = _first_line(str(cause))
|
|
1755
|
+
else:
|
|
1756
|
+
raise cause
|
|
1757
|
+
elif call.ended is not None: # the epoch callback stopped it (spec 015)
|
|
1758
|
+
exit_code, trapped = None, call.ended
|
|
1759
|
+
return _Driven(exit_code, trapped, metered, instance)
|
|
1760
|
+
|
|
1761
|
+
|
|
1762
|
+
class _Call:
|
|
1763
|
+
"""One call's state, which the shared host functions find on the calling thread's stack: its
|
|
1764
|
+
door, its poll_oneoff, its store's context address, and the first exception they caught; for
|
|
1765
|
+
a call with `cancel`, the event, the ticker count its deadline passes at, and the outcome the
|
|
1766
|
+
epoch callback recorded (spec 015)."""
|
|
1767
|
+
|
|
1768
|
+
def __init__(
|
|
1769
|
+
self,
|
|
1770
|
+
store: Any,
|
|
1771
|
+
host: Host | None,
|
|
1772
|
+
waits: Any,
|
|
1773
|
+
origin_ns: int,
|
|
1774
|
+
cancel: threading.Event | None = None,
|
|
1775
|
+
stop_at: int | None = None,
|
|
1776
|
+
) -> None:
|
|
1777
|
+
self.context = _context_address(store)
|
|
1778
|
+
self.door = _door(host)
|
|
1779
|
+
self.poll = _poll_oneoff(origin_ns, waits)
|
|
1780
|
+
self.stopped: BaseException | None = None
|
|
1781
|
+
self.cancel = cancel
|
|
1782
|
+
self.stop_at = stop_at
|
|
1783
|
+
self.ended: str | None = None # "cancelled" or "deadline", from the epoch callback
|
|
1784
|
+
|
|
1785
|
+
def stop(self, caller: Any, cause: BaseException) -> None:
|
|
1786
|
+
"""Record the cause and stop the guest: an epoch deadline of 0, set through the caller's
|
|
1787
|
+
context, traps it at its next function entry, with nothing raised through wasmtime-py."""
|
|
1788
|
+
if self.stopped is None:
|
|
1789
|
+
self.stopped = cause
|
|
1790
|
+
from wasmtime import _ffi
|
|
1791
|
+
|
|
1792
|
+
_ffi.wasmtime_context_set_epoch_deadline(caller._context(), 0)
|
|
1793
|
+
|
|
1794
|
+
|
|
1795
|
+
_calls = threading.local()
|
|
1796
|
+
|
|
1797
|
+
|
|
1798
|
+
def _stack() -> list[_Call]:
|
|
1799
|
+
stack = getattr(_calls, "stack", None)
|
|
1800
|
+
if stack is None:
|
|
1801
|
+
stack = _calls.stack = []
|
|
1802
|
+
return stack
|
|
1803
|
+
|
|
1804
|
+
|
|
1805
|
+
def _context_address(storelike: Any) -> int:
|
|
1806
|
+
return ctypes.addressof(storelike._context().contents)
|
|
1807
|
+
|
|
1808
|
+
|
|
1809
|
+
def _shared(kind: str, refused: int, stopped: int) -> Callable[..., int]:
|
|
1810
|
+
"""A host function defined once per engine. It serves the call on top of this thread's
|
|
1811
|
+
stack, and only when the caller is that call's store; it never raises into wasmtime-py,
|
|
1812
|
+
whose exceptions travel by a process-wide slot (spec 011)."""
|
|
1813
|
+
|
|
1814
|
+
def function(caller: Any, *args: int) -> int:
|
|
1815
|
+
call = None
|
|
1816
|
+
try:
|
|
1817
|
+
stack = _stack()
|
|
1818
|
+
call = stack[-1] if stack else None
|
|
1819
|
+
if call is None or _context_address(caller) != call.context:
|
|
1820
|
+
return refused
|
|
1821
|
+
if call.stopped is None and _ctrl_c_watched() and _ctrl_c["pending"]:
|
|
1822
|
+
_ctrl_c["pending"] = False
|
|
1823
|
+
raise KeyboardInterrupt
|
|
1824
|
+
if call.stopped is not None:
|
|
1825
|
+
return stopped
|
|
1826
|
+
answer = getattr(call, kind)(caller, *args)
|
|
1827
|
+
if _ctrl_c_watched() and _ctrl_c["pending"]: # Ctrl-C during a wait
|
|
1828
|
+
_ctrl_c["pending"] = False
|
|
1829
|
+
raise KeyboardInterrupt
|
|
1830
|
+
if call.cancel is not None and call.cancel.is_set():
|
|
1831
|
+
_epoch_now(caller) # the epoch callback names it at the guest's next entry
|
|
1832
|
+
return answer
|
|
1833
|
+
except BaseException as cause:
|
|
1834
|
+
try:
|
|
1835
|
+
if call is not None:
|
|
1836
|
+
call.stop(caller, cause)
|
|
1837
|
+
except BaseException:
|
|
1838
|
+
pass
|
|
1839
|
+
return stopped
|
|
1840
|
+
|
|
1841
|
+
return function
|
|
1842
|
+
|
|
1843
|
+
|
|
1844
|
+
def _epoch_now(caller: Any) -> None:
|
|
1845
|
+
"""An epoch deadline of 0 for the caller's store: its guest stops, or meets its epoch
|
|
1846
|
+
callback, at its next function entry."""
|
|
1847
|
+
from wasmtime import _ffi
|
|
1848
|
+
|
|
1849
|
+
_ffi.wasmtime_context_set_epoch_deadline(caller._context(), 0)
|
|
1850
|
+
|
|
1851
|
+
|
|
1852
|
+
def _epoch_answer(context: Any, delta: Any) -> int:
|
|
1853
|
+
"""The epoch callback's decision for the call on top of this thread's stack, in order: a
|
|
1854
|
+
recorded stop holds, a deferred Ctrl-C becomes a stop, then a cancel, then the deadline;
|
|
1855
|
+
otherwise continue for up to CANCEL_TICKS ticks. 0 continues; an error (an int address) stops
|
|
1856
|
+
the guest (spec 015)."""
|
|
1857
|
+
call = None
|
|
1858
|
+
try:
|
|
1859
|
+
try:
|
|
1860
|
+
stack = _stack()
|
|
1861
|
+
call = stack[-1] if stack else None
|
|
1862
|
+
if call is not None and ctypes.addressof(context.contents) != call.context:
|
|
1863
|
+
call = None
|
|
1864
|
+
except BaseException:
|
|
1865
|
+
call = None
|
|
1866
|
+
if call is None:
|
|
1867
|
+
_calls.lost = True
|
|
1868
|
+
return _stop_error(delta)
|
|
1869
|
+
if call.stopped is not None:
|
|
1870
|
+
return _stop_error(delta)
|
|
1871
|
+
if _ctrl_c_watched() and _ctrl_c["pending"]:
|
|
1872
|
+
_ctrl_c["pending"] = False
|
|
1873
|
+
call.stopped = KeyboardInterrupt()
|
|
1874
|
+
return _stop_error(delta)
|
|
1875
|
+
if call.cancel is not None and call.cancel.is_set():
|
|
1876
|
+
call.ended = "cancelled"
|
|
1877
|
+
return _stop_error(delta)
|
|
1878
|
+
count = _epoch_ticker.count()
|
|
1879
|
+
if call.stop_at is not None and count >= call.stop_at:
|
|
1880
|
+
call.ended = "deadline"
|
|
1881
|
+
return _stop_error(delta)
|
|
1882
|
+
left = CANCEL_TICKS if call.stop_at is None else call.stop_at - count
|
|
1883
|
+
delta[0] = max(1, min(CANCEL_TICKS, left))
|
|
1884
|
+
return 0
|
|
1885
|
+
except BaseException as cause:
|
|
1886
|
+
try:
|
|
1887
|
+
if call is None:
|
|
1888
|
+
_calls.lost = True
|
|
1889
|
+
elif call.stopped is None:
|
|
1890
|
+
call.stopped = cause
|
|
1891
|
+
except BaseException:
|
|
1892
|
+
pass
|
|
1893
|
+
return _stop_error(delta)
|
|
1894
|
+
|
|
1895
|
+
|
|
1896
|
+
def _stop_error(delta: Any) -> int:
|
|
1897
|
+
"""A new wasmtime error, which wasmtime owns once returned. If none can be made, continue for
|
|
1898
|
+
one tick, never 0, so the next callback tries again (spec 015's build review, F2)."""
|
|
1899
|
+
try:
|
|
1900
|
+
from wasmtime import _ffi
|
|
1901
|
+
|
|
1902
|
+
error = ctypes.cast(_ffi.wasmtime_error_new(b"stopped by wasi_python"), ctypes.c_void_p)
|
|
1903
|
+
if error.value:
|
|
1904
|
+
return error.value
|
|
1905
|
+
except BaseException:
|
|
1906
|
+
pass
|
|
1907
|
+
try:
|
|
1908
|
+
delta[0] = 1
|
|
1909
|
+
except BaseException:
|
|
1910
|
+
pass
|
|
1911
|
+
return 0
|
|
1912
|
+
|
|
1913
|
+
|
|
1914
|
+
def _on_epoch(context: Any, data: Any, delta: Any, kind: Any) -> int:
|
|
1915
|
+
return _epoch_answer(context, delta) # never writes `kind`: wasmtime panics on a bad value
|
|
1916
|
+
|
|
1917
|
+
|
|
1918
|
+
_epoch_callback: Any = None # the one C callback, made on first use
|
|
1919
|
+
_epoch_callback_lock = threading.Lock()
|
|
1920
|
+
|
|
1921
|
+
|
|
1922
|
+
def _watch(store: Any) -> None:
|
|
1923
|
+
"""Give a store the epoch callback, with a NULL finalizer: no per-call ctypes object exists,
|
|
1924
|
+
and the callback finds its call through the thread's stack (spec 015)."""
|
|
1925
|
+
global _epoch_callback
|
|
1926
|
+
from wasmtime import _bindings, _ffi
|
|
1927
|
+
|
|
1928
|
+
argtypes = _bindings._wasmtime_store_epoch_deadline_callback.argtypes
|
|
1929
|
+
with _epoch_callback_lock: # one object for the process: a store must never outlive its own
|
|
1930
|
+
if _epoch_callback is None:
|
|
1931
|
+
_epoch_callback = argtypes[1](_on_epoch)
|
|
1932
|
+
callback = _epoch_callback
|
|
1933
|
+
_ffi.wasmtime_store_epoch_deadline_callback(store.ptr(), callback, None, argtypes[3]())
|
|
1934
|
+
|
|
1935
|
+
|
|
1936
|
+
_linkers: dict[Any, Any] = {}
|
|
1937
|
+
|
|
1938
|
+
|
|
1939
|
+
def _linker(wt: Any, engine: wasmtime.Engine) -> Any:
|
|
1940
|
+
"""The engine's one linker: WASI, the shadowed poll_oneoff and the door, defined once, so a
|
|
1941
|
+
call adds nothing to wasmtime-py's process-wide table of host functions (spec 011)."""
|
|
1942
|
+
with _lock:
|
|
1943
|
+
linker = _linkers.get(engine)
|
|
1944
|
+
if linker is None:
|
|
1945
|
+
linker = wt.Linker(engine)
|
|
1946
|
+
linker.define_wasi()
|
|
1947
|
+
linker.allow_shadowing = True
|
|
1948
|
+
i32 = wt.ValType.i32()
|
|
1949
|
+
ty = wt.FuncType([i32] * 4, [i32])
|
|
1950
|
+
poll = _shared("poll", EINVAL, EINVAL)
|
|
1951
|
+
linker.define_func("wasi_snapshot_preview1", "poll_oneoff", ty, poll, True)
|
|
1952
|
+
linker.define_func("host", "call", ty, _shared("door", NO_HOST, HOST_FAULT), True)
|
|
1953
|
+
_linkers[engine] = linker
|
|
1954
|
+
return linker
|
|
1955
|
+
|
|
1956
|
+
|
|
1957
|
+
# The guest's stdout and stderr, by a key that is never reused. wasmtime calls the two functions
|
|
1958
|
+
# below on its own threads, and frees a key when it drops the stream; wasmtime-py's own table for
|
|
1959
|
+
# custom outputs is not used, since it has no lock (spec 011).
|
|
1960
|
+
_outputs: dict[int, _Stream] = {}
|
|
1961
|
+
_outputs_lock = threading.RLock() # a finalizer the collector runs inside a locked block
|
|
1962
|
+
_output_keys = itertools.count(1)
|
|
1963
|
+
|
|
1964
|
+
|
|
1965
|
+
@ctypes.CFUNCTYPE(
|
|
1966
|
+
ctypes.c_ssize_t, ctypes.c_void_p, ctypes.POINTER(ctypes.c_ubyte), ctypes.c_size_t
|
|
1967
|
+
)
|
|
1968
|
+
def _output_write(key: int | None, data: Any, size: int) -> int:
|
|
1969
|
+
try:
|
|
1970
|
+
with _outputs_lock:
|
|
1971
|
+
stream = _outputs.get(key or 0)
|
|
1972
|
+
return size if stream is None else stream.write(ctypes.string_at(data, size))
|
|
1973
|
+
except BaseException:
|
|
1974
|
+
return size # never raises, never prints: the bytes are dropped and the write succeeds
|
|
1975
|
+
|
|
1976
|
+
|
|
1977
|
+
@ctypes.CFUNCTYPE(None, ctypes.c_void_p)
|
|
1978
|
+
def _output_free(key: int | None) -> None:
|
|
1979
|
+
with _outputs_lock:
|
|
1980
|
+
_outputs.pop(key or 0, None)
|
|
1981
|
+
|
|
1982
|
+
|
|
1983
|
+
def _send_outputs(wasi: Any, out: _Stream, err: _Stream) -> tuple[int, int]:
|
|
1984
|
+
from wasmtime import _ffi
|
|
1985
|
+
|
|
1986
|
+
with _outputs_lock:
|
|
1987
|
+
keys = next(_output_keys), next(_output_keys)
|
|
1988
|
+
_outputs[keys[0]], _outputs[keys[1]] = out, err
|
|
1989
|
+
_ffi.wasi_config_set_stdout_custom(wasi.ptr(), _output_write, keys[0], _output_free)
|
|
1990
|
+
_ffi.wasi_config_set_stderr_custom(wasi.ptr(), _output_write, keys[1], _output_free)
|
|
1991
|
+
return keys
|
|
1992
|
+
|
|
1993
|
+
|
|
1994
|
+
# Ctrl-C while a guest runs on the main thread. Python's default handler raises KeyboardInterrupt
|
|
1995
|
+
# wherever the main thread next runs Python, and when that is the first line of wasmtime-py's
|
|
1996
|
+
# trampoline the exception cannot be caught and the process aborts (spec 011's review, EXECUTED:
|
|
1997
|
+
# exit 134). So while run() holds the main thread under the default handler, Ctrl-C only sets a
|
|
1998
|
+
# flag: the shared host functions turn it into a recorded stop, a wait ends at it, and a flag no
|
|
1999
|
+
# host function saw is raised once the call has ended.
|
|
2000
|
+
_ctrl_c = {"pending": False}
|
|
2001
|
+
|
|
2002
|
+
|
|
2003
|
+
def _note_ctrl_c(signum: int, frame: Any) -> None:
|
|
2004
|
+
_ctrl_c["pending"] = True
|
|
2005
|
+
|
|
2006
|
+
|
|
2007
|
+
def _ctrl_c_watched() -> bool:
|
|
2008
|
+
import signal
|
|
2009
|
+
|
|
2010
|
+
main = threading.current_thread() is threading.main_thread()
|
|
2011
|
+
return main and signal.getsignal(signal.SIGINT) is _note_ctrl_c
|
|
2012
|
+
|
|
2013
|
+
|
|
2014
|
+
@contextmanager
|
|
2015
|
+
def _deferred_ctrl_c() -> Iterator[None]:
|
|
2016
|
+
import signal
|
|
2017
|
+
|
|
2018
|
+
main = threading.current_thread() is threading.main_thread()
|
|
2019
|
+
if not main or signal.getsignal(signal.SIGINT) is not signal.default_int_handler:
|
|
2020
|
+
yield # another thread, an embedder's own handler, or a nested call: nothing to defer
|
|
2021
|
+
return
|
|
2022
|
+
_ctrl_c["pending"] = False
|
|
2023
|
+
signal.signal(signal.SIGINT, _note_ctrl_c)
|
|
2024
|
+
try:
|
|
2025
|
+
yield
|
|
2026
|
+
finally:
|
|
2027
|
+
if signal.getsignal(signal.SIGINT) is _note_ctrl_c: # a host may have set its own
|
|
2028
|
+
signal.signal(signal.SIGINT, signal.default_int_handler)
|
|
2029
|
+
pending, _ctrl_c["pending"] = _ctrl_c["pending"], False
|
|
2030
|
+
if pending:
|
|
2031
|
+
raise KeyboardInterrupt
|
|
2032
|
+
|
|
2033
|
+
|
|
2034
|
+
EINVAL, ENOMEM, EBADF, EFAULT = 28, 48, 8, 21 # WASI preview1 errno values
|
|
2035
|
+
MAX_SUBSCRIPTIONS = 4096 # one poll_oneoff's host work is bounded (spec 010)
|
|
2036
|
+
SUBSCRIPTION = (
|
|
2037
|
+
"<QB7xIxxxxQQH6x" # userdata, tag, clock id or fd, timeout, precision, flags: 48 bytes
|
|
2038
|
+
)
|
|
2039
|
+
EVENT = "<QHB5xQH6x" # userdata, error, type, nbytes, flags: 32 bytes
|
|
2040
|
+
CLOCK, FD_READ, FD_WRITE = 0, 1, 2
|
|
2041
|
+
REALTIME, MONOTONIC = 0, 1
|
|
2042
|
+
ABSTIME = 1
|
|
2043
|
+
|
|
2044
|
+
|
|
2045
|
+
@dataclass(frozen=True)
|
|
2046
|
+
class _Plan:
|
|
2047
|
+
"""What one poll_oneoff answers: an errno, or the events to write after waiting wait_ns."""
|
|
2048
|
+
|
|
2049
|
+
errno: int | None
|
|
2050
|
+
events: tuple[tuple[int, int, int, int, int], ...] = ()
|
|
2051
|
+
wait_ns: int = 0
|
|
2052
|
+
|
|
2053
|
+
|
|
2054
|
+
def _stop_at(count: int, ticks: int) -> int:
|
|
2055
|
+
"""The ticker's count at which a deadline of `ticks`, set when the count was `count`, has
|
|
2056
|
+
passed. The engine's epoch can be one round ahead of the count (the ticker advances the engines
|
|
2057
|
+
first), hence the + 1: a wait never ends before the epoch reaches the deadline (spec 010)."""
|
|
2058
|
+
return count + ticks + 1
|
|
2059
|
+
|
|
2060
|
+
|
|
2061
|
+
def _poll_plan(raw: bytes, n: int, realtime_ns: int, monotonic_ns: int) -> _Plan:
|
|
2062
|
+
"""poll_oneoff as wasmtime 49 answers it (crates/wasi/src/p1.rs), for the guest's files and
|
|
2063
|
+
stdio: a descriptor subscription is ready at once (stdin for writing, stdout and stderr for
|
|
2064
|
+
reading and the preopened root answer EBADF); otherwise the shortest clock wins, and every
|
|
2065
|
+
clock due by then is reported, with nbytes 0 as wasmtime writes it (spec 010)."""
|
|
2066
|
+
if n == 0:
|
|
2067
|
+
return _Plan(EINVAL)
|
|
2068
|
+
if n > MAX_SUBSCRIPTIONS:
|
|
2069
|
+
return _Plan(ENOMEM)
|
|
2070
|
+
ready, clocks = [], []
|
|
2071
|
+
for userdata, tag, which, timeout, _precision, flags in struct.iter_unpack(SUBSCRIPTION, raw):
|
|
2072
|
+
if tag == CLOCK:
|
|
2073
|
+
if which not in (REALTIME, MONOTONIC):
|
|
2074
|
+
return _Plan(EINVAL)
|
|
2075
|
+
if flags & ABSTIME:
|
|
2076
|
+
timeout -= realtime_ns if which == REALTIME else monotonic_ns
|
|
2077
|
+
clocks.append((max(0, timeout), userdata))
|
|
2078
|
+
elif tag in (FD_READ, FD_WRITE):
|
|
2079
|
+
if (
|
|
2080
|
+
which == 3
|
|
2081
|
+
or (tag == FD_READ and which in (1, 2))
|
|
2082
|
+
or (tag == FD_WRITE and which == 0)
|
|
2083
|
+
):
|
|
2084
|
+
return _Plan(EBADF)
|
|
2085
|
+
ready.append((userdata, 0, tag, 1, 0))
|
|
2086
|
+
else:
|
|
2087
|
+
return _Plan(EINVAL)
|
|
2088
|
+
if ready:
|
|
2089
|
+
return _Plan(None, tuple(ready))
|
|
2090
|
+
wait_ns = min(timeout for timeout, _ in clocks)
|
|
2091
|
+
due = tuple((userdata, 0, CLOCK, 0, 0) for timeout, userdata in clocks if timeout <= wait_ns)
|
|
2092
|
+
return _Plan(None, due, wait_ns)
|
|
2093
|
+
|
|
2094
|
+
|
|
2095
|
+
class _Waits:
|
|
2096
|
+
"""A wait that the call's deadline ends: in slices of at most TICK_S while a deadline is set,
|
|
2097
|
+
one sleep otherwise. `now`, `sleep` and `deadline_passed` are injected so the judge counts the
|
|
2098
|
+
slices without measuring time (spec 010)."""
|
|
2099
|
+
|
|
2100
|
+
def __init__(
|
|
2101
|
+
self,
|
|
2102
|
+
now: Callable[[], float],
|
|
2103
|
+
sleep: Callable[[float], None],
|
|
2104
|
+
deadline_passed: Callable[[], bool] | None,
|
|
2105
|
+
) -> None:
|
|
2106
|
+
self.now, self.sleep, self.deadline_passed = now, sleep, deadline_passed
|
|
2107
|
+
|
|
2108
|
+
def wait(self, seconds: float) -> None:
|
|
2109
|
+
if self.deadline_passed is None:
|
|
2110
|
+
if seconds > 0:
|
|
2111
|
+
self.sleep(seconds)
|
|
2112
|
+
return
|
|
2113
|
+
until = self.now() + seconds
|
|
2114
|
+
while (left := until - self.now()) > 0 and not self.deadline_passed():
|
|
2115
|
+
self.sleep(min(left, TICK_S))
|
|
2116
|
+
|
|
2117
|
+
|
|
2118
|
+
def _poll_oneoff(
|
|
2119
|
+
origin_ns: int,
|
|
2120
|
+
waits: _Waits,
|
|
2121
|
+
realtime_ns: Callable[[], int] = time.time_ns,
|
|
2122
|
+
monotonic_ns: Callable[[], int] = time.perf_counter_ns,
|
|
2123
|
+
) -> Callable[..., int]:
|
|
2124
|
+
"""The shadowed poll_oneoff. It never raises into wasmtime-py (an exception there travels by a
|
|
2125
|
+
process-wide slot another call can take): a wait ended by the deadline answers normally, and
|
|
2126
|
+
the guest traps at its next epoch check. A BaseException goes to the shared function, which
|
|
2127
|
+
records it and stops the guest (spec 011). The clocks are injectable for the judge."""
|
|
2128
|
+
|
|
2129
|
+
def call(caller, in_ptr: int, out_ptr: int, n: int, nevents_ptr: int) -> int:
|
|
2130
|
+
try:
|
|
2131
|
+
memory = caller.get("memory")
|
|
2132
|
+
in_ptr, out_ptr, n, nevents_ptr = (
|
|
2133
|
+
v & MASK32 for v in (in_ptr, out_ptr, n, nevents_ptr)
|
|
2134
|
+
)
|
|
2135
|
+
if n == 0 or n > MAX_SUBSCRIPTIONS:
|
|
2136
|
+
return EINVAL if n == 0 else ENOMEM
|
|
2137
|
+
size = memory.data_len(caller)
|
|
2138
|
+
if in_ptr + 48 * n > size or out_ptr + 32 * n > size or nevents_ptr + 4 > size:
|
|
2139
|
+
return EFAULT
|
|
2140
|
+
raw = bytes(memory.read(caller, in_ptr, in_ptr + 48 * n))
|
|
2141
|
+
plan = _poll_plan(raw, n, realtime_ns(), monotonic_ns() - origin_ns)
|
|
2142
|
+
if plan.errno is not None:
|
|
2143
|
+
return plan.errno
|
|
2144
|
+
if plan.wait_ns:
|
|
2145
|
+
waits.wait(plan.wait_ns / 1e9)
|
|
2146
|
+
events = b"".join(struct.pack(EVENT, *e) for e in plan.events)
|
|
2147
|
+
memory.write(caller, events, out_ptr)
|
|
2148
|
+
memory.write(caller, struct.pack("<I", len(plan.events)), nevents_ptr)
|
|
2149
|
+
return 0
|
|
2150
|
+
except Exception:
|
|
2151
|
+
return EINVAL # anything else is recorded by the shared function (spec 011)
|
|
2152
|
+
|
|
2153
|
+
return call
|
|
2154
|
+
|
|
2155
|
+
|
|
2156
|
+
class _Stream:
|
|
2157
|
+
"""A guest's stdout or stderr, kept up to a cap. Bytes past it are dropped, never buffered,
|
|
2158
|
+
and still reported as written, so the guest's write succeeds and is not retried (wasmtime-py
|
|
2159
|
+
takes the callback's return value as the count written)."""
|
|
2160
|
+
|
|
2161
|
+
def __init__(self, name: str, cap: int | None) -> None:
|
|
2162
|
+
self.name, self.cap, self.data, self.dropped = name, cap, bytearray(), False
|
|
2163
|
+
|
|
2164
|
+
def write(self, data: bytes) -> int:
|
|
2165
|
+
room = len(data) if self.cap is None else max(0, self.cap - len(self.data))
|
|
2166
|
+
if room < len(data):
|
|
2167
|
+
self.dropped = True
|
|
2168
|
+
self.data.extend(data[:room])
|
|
2169
|
+
return len(data)
|
|
2170
|
+
|
|
2171
|
+
|
|
2172
|
+
def _first_line(text: str) -> str:
|
|
2173
|
+
lines = text.strip().splitlines()
|
|
2174
|
+
return lines[0] if lines else "trap"
|
|
2175
|
+
|
|
2176
|
+
|
|
2177
|
+
def _trap_name(e: wasmtime.Trap) -> str:
|
|
2178
|
+
import wasmtime as wt
|
|
2179
|
+
|
|
2180
|
+
if e.trap_code == wt.TrapCode.OUT_OF_FUEL:
|
|
2181
|
+
return "out of fuel"
|
|
2182
|
+
if e.trap_code == wt.TrapCode.INTERRUPT:
|
|
2183
|
+
return "deadline"
|
|
2184
|
+
if e.trap_code is not None: # by its kind, not wasmtime's heading (spec 016, Decision 3)
|
|
2185
|
+
return _trap_code_name(e.trap_code)
|
|
2186
|
+
return _first_line(e.message)
|
|
2187
|
+
|
|
2188
|
+
|
|
2189
|
+
def _trap_code_name(code: Any) -> str:
|
|
2190
|
+
"""`TrapCode.STACK_OVERFLOW` is "stack overflow"."""
|
|
2191
|
+
return code.name.lower().replace("_", " ")
|
|
2192
|
+
|
|
2193
|
+
|
|
2194
|
+
def _door(host: Host | None) -> Callable[..., int]:
|
|
2195
|
+
"""host::call(ptr, len, out_ptr_ptr, out_len_ptr) -> 0, -1 or -2 (spec 003, Decision 5)."""
|
|
2196
|
+
|
|
2197
|
+
import wasmtime as wt
|
|
2198
|
+
|
|
2199
|
+
def call(caller, ptr: int, length: int, out_ptr_ptr: int, out_len_ptr: int) -> int:
|
|
2200
|
+
if host is None:
|
|
2201
|
+
return NO_HOST
|
|
2202
|
+
try:
|
|
2203
|
+
return frame(caller, ptr, length, out_ptr_ptr, out_len_ptr)
|
|
2204
|
+
except (wt.Trap, wt.WasmtimeError):
|
|
2205
|
+
raise # a trap or exit inside host_alloc: the shared function records it (spec 011)
|
|
2206
|
+
except Exception:
|
|
2207
|
+
return HOST_FAULT # nothing a frame or a host does escapes run() (spec 006)
|
|
2208
|
+
|
|
2209
|
+
def frame(caller, ptr: int, length: int, out_ptr_ptr: int, out_len_ptr: int) -> int:
|
|
2210
|
+
memory, alloc = caller.get("memory"), caller.get("host_alloc")
|
|
2211
|
+
if memory is None or alloc is None:
|
|
2212
|
+
return HOST_FAULT
|
|
2213
|
+
ptr, length = ptr & MASK32, length & MASK32 # i32 values: addresses are unsigned
|
|
2214
|
+
out_ptr_ptr, out_len_ptr = out_ptr_ptr & MASK32, out_len_ptr & MASK32
|
|
2215
|
+
size = memory.data_len(caller)
|
|
2216
|
+
if ptr + length > size or out_ptr_ptr + 4 > size or out_len_ptr + 4 > size:
|
|
2217
|
+
return HOST_FAULT # memory.read would clamp a frame that runs past memory
|
|
2218
|
+
response = host(bytes(memory.read(caller, ptr, ptr + length)))
|
|
2219
|
+
if not isinstance(response, (bytes, bytearray, memoryview)):
|
|
2220
|
+
return HOST_FAULT
|
|
2221
|
+
if memoryview(response).nbytes > MAX_RESPONSE:
|
|
2222
|
+
return HOST_FAULT # checked in bytes, before the copy
|
|
2223
|
+
response = bytes(response)
|
|
2224
|
+
p = alloc(caller, len(response)) & MASK32
|
|
2225
|
+
if not p or p + len(response) > memory.data_len(caller):
|
|
2226
|
+
return HOST_FAULT
|
|
2227
|
+
memory.write(caller, response, p)
|
|
2228
|
+
memory.write(caller, struct.pack("<I", p), out_ptr_ptr)
|
|
2229
|
+
memory.write(caller, struct.pack("<I", len(response)), out_len_ptr)
|
|
2230
|
+
return 0
|
|
2231
|
+
|
|
2232
|
+
return call
|
|
2233
|
+
|
|
2234
|
+
|
|
2235
|
+
def _stamp(st: os.stat_result) -> tuple[int, ...]:
|
|
2236
|
+
return (st.st_dev, st.st_ino, st.st_size, st.st_mtime_ns, st.st_ctime_ns)
|
|
2237
|
+
|
|
2238
|
+
|
|
2239
|
+
def _real_dir_at(fd: int, parts: tuple[str, ...]) -> int:
|
|
2240
|
+
"""A descriptor on fd/parts…, each a real directory, each step taken from the folder before:
|
|
2241
|
+
a link or file a guest left in the way is removed, never followed (spec 024). The caller
|
|
2242
|
+
closes the result; `fd` stays open."""
|
|
2243
|
+
here = os.dup(fd)
|
|
2244
|
+
try:
|
|
2245
|
+
for part in parts:
|
|
2246
|
+
try:
|
|
2247
|
+
st = os.stat(part, dir_fd=here, follow_symlinks=False)
|
|
2248
|
+
except FileNotFoundError:
|
|
2249
|
+
st = None
|
|
2250
|
+
if st is not None and not stat.S_ISDIR(st.st_mode):
|
|
2251
|
+
os.unlink(part, dir_fd=here) # a symbolic link or a file: removed, never followed
|
|
2252
|
+
st = None
|
|
2253
|
+
if st is None:
|
|
2254
|
+
try:
|
|
2255
|
+
os.mkdir(part, dir_fd=here)
|
|
2256
|
+
except FileExistsError: # a concurrent call made it
|
|
2257
|
+
pass
|
|
2258
|
+
child = os.open(part, _DIR, dir_fd=here) # O_NOFOLLOW: a link put there fails
|
|
2259
|
+
os.close(here)
|
|
2260
|
+
here = child
|
|
2261
|
+
except BaseException:
|
|
2262
|
+
os.close(here)
|
|
2263
|
+
raise
|
|
2264
|
+
return here
|
|
2265
|
+
|
|
2266
|
+
|
|
2267
|
+
def _real_dir(root: Path, parts: tuple[str, ...]) -> Path:
|
|
2268
|
+
"""root/parts…, each a real directory: a link or file a guest left in the way is removed.
|
|
2269
|
+
Windows only, where the walk holds every folder of the root's path (spec 017, Decision 5) and
|
|
2270
|
+
guests cannot make links; elsewhere `_real_dir_at`. The root is never made again here."""
|
|
2271
|
+
here = root
|
|
2272
|
+
for part in parts:
|
|
2273
|
+
here = here / part
|
|
2274
|
+
try:
|
|
2275
|
+
st = os.lstat(here)
|
|
2276
|
+
except FileNotFoundError:
|
|
2277
|
+
st = None
|
|
2278
|
+
if st is not None and not stat.S_ISDIR(st.st_mode):
|
|
2279
|
+
here.unlink() # a symbolic link or a file: removed, never followed
|
|
2280
|
+
st = None
|
|
2281
|
+
if st is None:
|
|
2282
|
+
try:
|
|
2283
|
+
here.mkdir()
|
|
2284
|
+
except FileExistsError: # a concurrent call made it
|
|
2285
|
+
if not stat.S_ISDIR(os.lstat(here).st_mode):
|
|
2286
|
+
raise
|
|
2287
|
+
return here
|
|
2288
|
+
|
|
2289
|
+
|
|
2290
|
+
def _hash_file(path: Path) -> str:
|
|
2291
|
+
flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_BINARY", 0)
|
|
2292
|
+
h = hashlib.sha256()
|
|
2293
|
+
with os.fdopen(os.open(path, flags), "rb") as f:
|
|
2294
|
+
for block in iter(lambda: f.read(1 << 20), b""):
|
|
2295
|
+
h.update(block)
|
|
2296
|
+
return h.hexdigest()
|
|
2297
|
+
|
|
2298
|
+
|
|
2299
|
+
_versions: dict[Path, str] = {}
|
|
2300
|
+
|
|
2301
|
+
|
|
2302
|
+
def _version_tag(manifest: Path) -> str:
|
|
2303
|
+
"""'315' from MANIFEST's `python 3.15.0` line: the stdlib zip's guest name is python315.zip."""
|
|
2304
|
+
if manifest not in _versions:
|
|
2305
|
+
for line in manifest.read_text().splitlines():
|
|
2306
|
+
if line.startswith("python "):
|
|
2307
|
+
major, minor = line.split()[1].split(".")[:2]
|
|
2308
|
+
_versions[manifest] = f"{major}{minor}"
|
|
2309
|
+
break
|
|
2310
|
+
else:
|
|
2311
|
+
raise ValueError(f"{manifest} has no python line")
|
|
2312
|
+
return _versions[manifest]
|
|
2313
|
+
|
|
2314
|
+
|
|
2315
|
+
def _cache_dir(given: str | os.PathLike[str] | None | bool) -> Path | None:
|
|
2316
|
+
if given is False:
|
|
2317
|
+
return None
|
|
2318
|
+
if given is not None and given is not True:
|
|
2319
|
+
return Path(given)
|
|
2320
|
+
if os.environ.get(CACHE_ENV):
|
|
2321
|
+
return Path(os.environ[CACHE_ENV])
|
|
2322
|
+
home = Path.home()
|
|
2323
|
+
if sys.platform == "darwin":
|
|
2324
|
+
return home / "Library" / "Caches" / "wasi_python"
|
|
2325
|
+
if sys.platform == "win32":
|
|
2326
|
+
base = os.environ.get("LOCALAPPDATA")
|
|
2327
|
+
return (Path(base) if base else home / "AppData" / "Local") / "wasi_python" / "cache"
|
|
2328
|
+
base = os.environ.get("XDG_CACHE_HOME")
|
|
2329
|
+
return (Path(base) if base else home / ".cache") / "wasi_python"
|
|
2330
|
+
|
|
2331
|
+
|
|
2332
|
+
_lock = threading.Lock()
|
|
2333
|
+
_engines: dict[bool, wasmtime.Engine] = {}
|
|
2334
|
+
_modules: dict[tuple[bool, str], wasmtime.Module] = {}
|
|
2335
|
+
_digests: dict[tuple[str, int, int], str] = {}
|
|
2336
|
+
_counts = {"compiled": 0, "loaded": 0, "written": 0}
|
|
2337
|
+
|
|
2338
|
+
|
|
2339
|
+
def stats() -> dict[str, int]:
|
|
2340
|
+
"""How this process got its compiled modules: compiled from the binary, loaded from a .cwasm,
|
|
2341
|
+
and .cwasm files written. A diagnostic; the judge's cache check reads it."""
|
|
2342
|
+
with _lock:
|
|
2343
|
+
return dict(_counts)
|
|
2344
|
+
|
|
2345
|
+
|
|
2346
|
+
@atexit.register
|
|
2347
|
+
def _release() -> None:
|
|
2348
|
+
"""Remove this process's zip folders, and free the cached linkers, modules and engines while
|
|
2349
|
+
wasmtime's bindings still exist."""
|
|
2350
|
+
_remove_zips()
|
|
2351
|
+
_check_modules.clear()
|
|
2352
|
+
_linkers.clear()
|
|
2353
|
+
_modules.clear()
|
|
2354
|
+
_engines.clear()
|
|
2355
|
+
|
|
2356
|
+
|
|
2357
|
+
def _engine(metered: bool) -> wasmtime.Engine:
|
|
2358
|
+
"""Two engines per process: fuel metering changes the compiled code."""
|
|
2359
|
+
import wasmtime as wt
|
|
2360
|
+
|
|
2361
|
+
with _lock:
|
|
2362
|
+
if metered not in _engines:
|
|
2363
|
+
_fork["used"] = True # before the engine exists: a fork from here on is refused
|
|
2364
|
+
cfg = wt.Config()
|
|
2365
|
+
cfg.consume_fuel = metered
|
|
2366
|
+
cfg.epoch_interruption = True
|
|
2367
|
+
_engines[metered] = wt.Engine(cfg)
|
|
2368
|
+
return _engines[metered]
|
|
2369
|
+
|
|
2370
|
+
|
|
2371
|
+
def _digest(path: Path) -> str:
|
|
2372
|
+
st = path.stat()
|
|
2373
|
+
key = (str(path.resolve()), st.st_size, st.st_mtime_ns)
|
|
2374
|
+
if key not in _digests:
|
|
2375
|
+
h = hashlib.sha256()
|
|
2376
|
+
with path.open("rb") as f:
|
|
2377
|
+
for block in iter(lambda: f.read(1 << 20), b""):
|
|
2378
|
+
h.update(block)
|
|
2379
|
+
_digests[key] = h.hexdigest()
|
|
2380
|
+
return _digests[key]
|
|
2381
|
+
|
|
2382
|
+
|
|
2383
|
+
def _module(
|
|
2384
|
+
engine: wasmtime.Engine, metered: bool, binary: Path, cache: Path | None
|
|
2385
|
+
) -> wasmtime.Module:
|
|
2386
|
+
"""The compiled module: from this process, else the .cwasm cache, else compiled and cached.
|
|
2387
|
+
|
|
2388
|
+
A .cwasm is native code that wasmtime runs as it finds it, so the cache is used only where no
|
|
2389
|
+
one else can write: on POSIX, a directory and files owned by this user and writable by no one
|
|
2390
|
+
else (the directory is made 0700, the files 0600). Anything else is ignored, with a warning,
|
|
2391
|
+
and the module is compiled in this process."""
|
|
2392
|
+
import wasmtime as wt
|
|
2393
|
+
|
|
2394
|
+
digest = _digest(binary)
|
|
2395
|
+
with _lock:
|
|
2396
|
+
mod = _modules.get((metered, digest))
|
|
2397
|
+
guard = _compiling.setdefault((metered, digest), threading.Lock())
|
|
2398
|
+
if mod is not None:
|
|
2399
|
+
return mod
|
|
2400
|
+
# One thread compiles or loads; the others wait for it (spec 011). Only this lock is held
|
|
2401
|
+
# while compiling: the epoch ticker takes _lock every round.
|
|
2402
|
+
with guard:
|
|
2403
|
+
with _lock:
|
|
2404
|
+
mod = _modules.get((metered, digest))
|
|
2405
|
+
if mod is not None:
|
|
2406
|
+
return mod
|
|
2407
|
+
return _module_miss(wt, engine, metered, binary, cache, digest)
|
|
2408
|
+
|
|
2409
|
+
|
|
2410
|
+
_compiling: dict[tuple[bool, str], threading.Lock] = {}
|
|
2411
|
+
|
|
2412
|
+
|
|
2413
|
+
def _module_miss(
|
|
2414
|
+
wt: Any, engine: wasmtime.Engine, metered: bool, binary: Path, cache: Path | None, digest: str
|
|
2415
|
+
) -> wasmtime.Module:
|
|
2416
|
+
flavor = "fuel" if metered else "plain"
|
|
2417
|
+
mod = None
|
|
2418
|
+
cwasm = None
|
|
2419
|
+
if cache is not None and _private(cache, is_dir=True):
|
|
2420
|
+
cwasm = cache / f"{digest[:32]}-wasmtime{_wasmtime_version()}-{flavor}.cwasm"
|
|
2421
|
+
if cwasm is not None and cwasm.exists() and _private(cwasm, is_dir=False):
|
|
2422
|
+
try:
|
|
2423
|
+
mod = wt.Module.deserialize_file(engine, str(cwasm))
|
|
2424
|
+
with _lock:
|
|
2425
|
+
_counts["loaded"] += 1
|
|
2426
|
+
except wt.WasmtimeError:
|
|
2427
|
+
mod = None # stale or foreign: compiled again below and replaced
|
|
2428
|
+
cwasm.unlink(missing_ok=True)
|
|
2429
|
+
if mod is None:
|
|
2430
|
+
mod = wt.Module.from_file(engine, str(binary))
|
|
2431
|
+
with _lock:
|
|
2432
|
+
_counts["compiled"] += 1
|
|
2433
|
+
if cwasm is not None and not cwasm.exists():
|
|
2434
|
+
fd, tmp = tempfile.mkstemp(prefix=".cwasm-", dir=cache) # created 0600
|
|
2435
|
+
try:
|
|
2436
|
+
with os.fdopen(fd, "wb") as f:
|
|
2437
|
+
f.write(mod.serialize())
|
|
2438
|
+
os.replace(tmp, cwasm)
|
|
2439
|
+
with _lock:
|
|
2440
|
+
_counts["written"] += 1
|
|
2441
|
+
except BaseException:
|
|
2442
|
+
Path(tmp).unlink(missing_ok=True)
|
|
2443
|
+
raise
|
|
2444
|
+
with _lock:
|
|
2445
|
+
_modules[(metered, digest)] = mod
|
|
2446
|
+
return mod
|
|
2447
|
+
|
|
2448
|
+
|
|
2449
|
+
def _private(path: Path, *, is_dir: bool) -> bool:
|
|
2450
|
+
"""Whether a cache path is this user's alone (POSIX); a directory is made if absent."""
|
|
2451
|
+
if is_dir:
|
|
2452
|
+
path.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
2453
|
+
if os.name != "posix":
|
|
2454
|
+
return True # the default cache is under the user's own profile
|
|
2455
|
+
st = os.lstat(path)
|
|
2456
|
+
kind_ok = stat.S_ISDIR(st.st_mode) if is_dir else stat.S_ISREG(st.st_mode)
|
|
2457
|
+
if kind_ok and st.st_uid == os.getuid() and not st.st_mode & 0o022:
|
|
2458
|
+
return True
|
|
2459
|
+
warnings.warn(
|
|
2460
|
+
f"wasi_python: not using the module cache at {path}: it must be a "
|
|
2461
|
+
f"{'directory' if is_dir else 'file'} owned by this user and writable by no one else",
|
|
2462
|
+
RuntimeWarning,
|
|
2463
|
+
stacklevel=5,
|
|
2464
|
+
)
|
|
2465
|
+
return False
|
|
2466
|
+
|
|
2467
|
+
|
|
2468
|
+
def _wasmtime_version() -> str:
|
|
2469
|
+
from importlib.metadata import version
|
|
2470
|
+
|
|
2471
|
+
return version("wasmtime")
|
|
2472
|
+
|
|
2473
|
+
|
|
2474
|
+
class _epoch_ticker:
|
|
2475
|
+
"""While at least one call with a deadline runs, one thread per process advances every
|
|
2476
|
+
engine's epoch once per TICK_S of wall time elapsed, catching up when a sleep runs long, so a
|
|
2477
|
+
deadline of n ticks ends close to n * TICK_S."""
|
|
2478
|
+
|
|
2479
|
+
_active = 0
|
|
2480
|
+
_count = 0 # epochs advanced so far, updated after the engines are (spec 010)
|
|
2481
|
+
_thread: threading.Thread | None = None
|
|
2482
|
+
_cond = threading.Condition()
|
|
2483
|
+
|
|
2484
|
+
def __init__(self, engine: wasmtime.Engine, wanted: bool) -> None:
|
|
2485
|
+
self.wanted = wanted
|
|
2486
|
+
|
|
2487
|
+
@classmethod
|
|
2488
|
+
def count(cls) -> int:
|
|
2489
|
+
return cls._count
|
|
2490
|
+
|
|
2491
|
+
def __enter__(self) -> None:
|
|
2492
|
+
if not self.wanted:
|
|
2493
|
+
return
|
|
2494
|
+
cls = type(self)
|
|
2495
|
+
with cls._cond:
|
|
2496
|
+
cls._active += 1
|
|
2497
|
+
if cls._thread is None:
|
|
2498
|
+
cls._thread = threading.Thread(target=cls._tick, daemon=True)
|
|
2499
|
+
cls._thread.start()
|
|
2500
|
+
cls._cond.notify_all()
|
|
2501
|
+
|
|
2502
|
+
def __exit__(self, *exc: object) -> None:
|
|
2503
|
+
if not self.wanted:
|
|
2504
|
+
return
|
|
2505
|
+
with type(self)._cond:
|
|
2506
|
+
type(self)._active -= 1
|
|
2507
|
+
|
|
2508
|
+
@classmethod
|
|
2509
|
+
def _tick(cls) -> None:
|
|
2510
|
+
while True:
|
|
2511
|
+
with cls._cond:
|
|
2512
|
+
while cls._active == 0:
|
|
2513
|
+
cls._cond.wait()
|
|
2514
|
+
start, done = time.monotonic(), 0
|
|
2515
|
+
while True:
|
|
2516
|
+
with cls._cond:
|
|
2517
|
+
if cls._active == 0:
|
|
2518
|
+
break
|
|
2519
|
+
due = int((time.monotonic() - start) / TICK_S)
|
|
2520
|
+
with _lock:
|
|
2521
|
+
engines = list(_engines.values())
|
|
2522
|
+
for _ in range(due - done):
|
|
2523
|
+
for engine in engines:
|
|
2524
|
+
engine.increment_epoch()
|
|
2525
|
+
cls._count += 1
|
|
2526
|
+
done = due
|
|
2527
|
+
time.sleep(TICK_S / 4)
|