wasi-python 0.1.0a3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
wasi_python/runner.py ADDED
@@ -0,0 +1,2527 @@
1
+ """The runner: one program, one call (spec 001, Decision 3).
2
+
3
+ from wasi_python import run
4
+
5
+ done = run(["-c", "import pandas; print(pandas.__version__)"])
6
+ done.exit_code, done.stdout
7
+
8
+ It does what any embedder does: preopen a directory at `/`, bind the one custom import
9
+ (`host::call`), and call the reactor's exports. wasmtime-py (the `run` extra) is imported only
10
+ here, when a call is made.
11
+
12
+ It is safe by default (spec 006): a call that names no root gets a fresh private one, removed
13
+ afterwards; a call that names no limit still has one for memory, time and output (`DEFAULTS`);
14
+ and nothing a guest frame or a host function does makes `run` raise.
15
+ """
16
+
17
+ from __future__ import annotations
18
+
19
+ import atexit
20
+ import ctypes
21
+ import errno
22
+ import hashlib
23
+ import itertools
24
+ import math
25
+ import numbers
26
+ import operator
27
+ import os
28
+ import re
29
+ import secrets
30
+ import shutil
31
+ import stat
32
+ import struct
33
+ import sys
34
+ import tempfile
35
+ import threading
36
+ import time
37
+ import warnings
38
+ from collections.abc import Callable, Iterable, Iterator, Mapping
39
+ from contextlib import contextmanager
40
+ from dataclasses import dataclass, field
41
+ from pathlib import Path
42
+ from typing import TYPE_CHECKING, Any, NamedTuple
43
+
44
+ from . import Artifacts
45
+ from . import artifacts as find_artifacts
46
+
47
+ if TYPE_CHECKING:
48
+ import concurrent.futures
49
+
50
+ import wasmtime
51
+
52
+ ARGV0 = "/usr/local/bin/python.wasm"
53
+ NO_HOST = -1
54
+ HOST_FAULT = -2
55
+ CACHE_ENV = "WASI_PYTHON_CACHE"
56
+ TICK_S = 0.01
57
+ NO_DEADLINE = 2**62
58
+ CANCEL_TICKS = 5 # a call with `cancel` is checked every 5 ticks while it computes (spec 015)
59
+ MAX_WORKERS = 1024
60
+ MASK32 = 0xFFFFFFFF
61
+ UNMETERED = 2**63 - 1 # the budget instantiation runs on, before the cap is set (spec 008)
62
+ MAX_RESPONSE = 0x7FFFFFFF # the largest door response: its length must fit a signed i32
63
+ # The guest's environment before the caller's `env`. HOME is added per call: a fresh, empty
64
+ # directory under /tmp that the runner removes afterwards, so nothing a call leaves in its HOME
65
+ # (pdb's .pdbrc, for one) is read by the next (spec 006, Decision 2).
66
+ BASE_ENV = {"TMPDIR": "/tmp", "PYTHONTZPATH": ""}
67
+
68
+ Host = Callable[[bytes], bytes]
69
+
70
+
71
+ LIMIT_MAX = 2**63 - 1 # wasmtime reads a memory limit as signed: 2**63 and above mean none
72
+
73
+
74
+ def _shown(value: Any) -> str:
75
+ """A value for an error message: an integer too long to print is named by its size."""
76
+ if isinstance(value, int) and value.bit_length() > 128:
77
+ return f"an integer of {value.bit_length()} bits"
78
+ return repr(value)
79
+
80
+
81
+ def _integer_limit(name: str, value: Any) -> int | None:
82
+ if value is None:
83
+ return None
84
+ if isinstance(value, bool):
85
+ raise TypeError(f"{name} must be an integer, or None; got {value!r}")
86
+ try:
87
+ number = operator.index(value)
88
+ except TypeError:
89
+ raise TypeError(f"{name} must be an integer, or None; got {value!r}") from None
90
+ if not 0 <= number <= LIMIT_MAX:
91
+ raise ValueError(f"{name} must be from 0 to 2**63 - 1, or None; got {_shown(number)}")
92
+ return number
93
+
94
+
95
+ def _deadline_limit(value: Any) -> float | None:
96
+ if value is None:
97
+ return None
98
+ if isinstance(value, bool) or not isinstance(value, numbers.Real):
99
+ raise TypeError(f"deadline_s must be a number of seconds, or None; got {value!r}")
100
+ rule = "deadline_s must be finite and above 0, and at most about 4.6e16 s, or None"
101
+ try:
102
+ seconds = float(value)
103
+ ticks = math.ceil(seconds / TICK_S) if seconds > 0 else 0
104
+ except (OverflowError, ValueError):
105
+ raise ValueError(f"{rule}; got {_shown(value)}") from None
106
+ if not math.isfinite(seconds) or seconds <= 0 or not 0 < ticks < NO_DEADLINE:
107
+ raise ValueError(f"{rule}; got {_shown(value)}")
108
+ return seconds
109
+
110
+
111
+ def _check_limits(
112
+ fuel: Any, deadline_s: Any, memory_bytes: Any, output_bytes: Any
113
+ ) -> dict[str, Any]:
114
+ """The four limits as plain values, or TypeError for a wrong type and ValueError for a value
115
+ out of range. A negative fuel or memory cap, or a memory cap of 2**63 or more, would mean no
116
+ limit to wasmtime, and `True` would mean 1, so none of them reaches it (spec 013)."""
117
+ return {
118
+ "fuel": _integer_limit("fuel", fuel),
119
+ "deadline_s": _deadline_limit(deadline_s),
120
+ "memory_bytes": _integer_limit("memory_bytes", memory_bytes),
121
+ "output_bytes": _integer_limit("output_bytes", output_bytes),
122
+ }
123
+
124
+
125
+ def _text(where: str, value: Any) -> str:
126
+ """A plain str the guest will see exactly: no NUL, and valid UTF-8, which WASI carries.
127
+ `str.__str__` and `str.encode` are called directly, so a subclass's overrides are ignored.
128
+ Messages name `where`, never the value: environment values are often secrets."""
129
+ if not isinstance(value, str):
130
+ raise TypeError(f"{where} must be a str, not {type(value).__name__}")
131
+ text = str.__str__(value)
132
+ if "\0" in text:
133
+ raise ValueError(f"{where}: embedded null byte")
134
+ try:
135
+ str.encode(text, "utf-8")
136
+ except UnicodeEncodeError:
137
+ raise ValueError(f"{where}: not valid UTF-8 (a lone surrogate)") from None
138
+ return text
139
+
140
+
141
+ def _check_inputs(argv: Any, env: Any, stdin: Any) -> dict[str, Any]:
142
+ """argv, env and stdin as plain values, or TypeError for a wrong type and ValueError for a
143
+ value the guest would see differently: a NUL cut an argument or key short, and an = split a
144
+ key, so {"HOME\\0x": ...} set HOME (spec 014)."""
145
+ if isinstance(argv, (str, bytes, bytearray)) or not isinstance(argv, Iterable):
146
+ raise TypeError(f"argv must be an iterable of str, not {type(argv).__name__}")
147
+ args = [_text(f"argv[{i}]", item) for i, item in enumerate(argv)]
148
+ plain_env = None
149
+ if env is not None:
150
+ if not isinstance(env, Mapping):
151
+ raise TypeError(f"env must be a mapping of str to str, not {type(env).__name__}")
152
+ plain_env = {}
153
+ for key, value in env.items():
154
+ name = _text("env key", key)
155
+ if not name or "=" in name:
156
+ raise ValueError("env key: illegal environment variable name") # never quoted
157
+ shown = name if len(name) <= 40 else name[:40] + "…"
158
+ plain_env[name] = _text(f"env value for {shown!r}", value)
159
+ data = None
160
+ if stdin is not None:
161
+ if type(stdin) is bytes:
162
+ data = stdin # not copied again
163
+ else:
164
+ try:
165
+ data = memoryview(stdin).tobytes()
166
+ except TypeError:
167
+ raise TypeError(
168
+ f"stdin must be bytes-like, or None; got {type(stdin).__name__}"
169
+ ) from None
170
+ return {"argv": args, "env": plain_env, "stdin": data}
171
+
172
+
173
+ def _cpus() -> int:
174
+ count = getattr(os, "process_cpu_count", os.cpu_count)() # 3.13 honours CPU affinity
175
+ return count or 1
176
+
177
+
178
+ def _default_workers() -> int:
179
+ """At most 8: at the defaults a worker holds up to 1 GiB of guest memory and 32 MiB of output,
180
+ so 8 bound a process at about 8.3 GiB, and past 8 the measured speedup flattens (spec 015)."""
181
+ return min(_cpus(), 8)
182
+
183
+
184
+ def _check_workers(value: Any) -> int:
185
+ if isinstance(value, bool):
186
+ raise TypeError(f"async_workers must be an integer; got {value!r}")
187
+ try:
188
+ number = operator.index(value)
189
+ except TypeError:
190
+ raise TypeError(f"async_workers must be an integer; got {value!r}") from None
191
+ if not 1 <= number <= MAX_WORKERS:
192
+ raise ValueError(f"async_workers must be from 1 to {MAX_WORKERS}; got {_shown(number)}")
193
+ return number
194
+
195
+
196
+ @dataclass(frozen=True)
197
+ class Defaults:
198
+ """The limits a call has when it names none. `run` reads `DEFAULTS` at call time; the values
199
+ are checked here, by `run`'s rules, so an error from `run` always names what its caller
200
+ passed (spec 013). `async_workers` sizes `run_async`'s default pool (spec 015)."""
201
+
202
+ memory_bytes: int # the guest's linear memory
203
+ deadline_s: float # wall-clock time, by epoch interruption (costs nothing per instruction)
204
+ output_bytes: int # per stream; bytes past it are dropped and the stream named in `truncated`
205
+ async_workers: int = field(default_factory=_default_workers) # guests run_async runs at once
206
+
207
+ def __post_init__(self) -> None:
208
+ _check_limits(None, self.deadline_s, self.memory_bytes, self.output_bytes)
209
+ _check_workers(self.async_workers)
210
+
211
+
212
+ DEFAULTS = Defaults(memory_bytes=1 << 30, deadline_s=60.0, output_bytes=16 << 20)
213
+
214
+
215
+ class _Unset:
216
+ def __repr__(self) -> str:
217
+ return "<default>"
218
+
219
+
220
+ _UNSET: Any = _Unset()
221
+
222
+
223
+ @dataclass(frozen=True)
224
+ class Completed:
225
+ """How one program ended."""
226
+
227
+ exit_code: int | None # run's return value or proc_exit's code; None when it trapped
228
+ stdout: bytes
229
+ stderr: bytes
230
+ trapped: str | None # None, "out of fuel", "deadline", "cancelled", the trap code's plain
231
+ # name ("stack overflow"), or, for a trap without a code, its message's first line
232
+ fuel_used: int | None # instructions spent, when a fuel cap was given and no deadline hit
233
+ truncated: tuple[str, ...] = () # the streams ("stdout", "stderr") whose output was capped
234
+
235
+
236
+ @dataclass(frozen=True)
237
+ class Grant:
238
+ """A host folder the guest sees at `guest`, read-only unless `writable=True` (spec 017).
239
+
240
+ `host` is a str or os.PathLike[str]; `guest` an absolute guest path. A grant may be nested in
241
+ another; `run` checks every grant before any work."""
242
+
243
+ host: str | os.PathLike[str]
244
+ guest: str
245
+ writable: bool = field(default=False, kw_only=True)
246
+
247
+
248
+ def run(
249
+ argv: Iterable[str],
250
+ *,
251
+ root: str | os.PathLike[str] | bool | None = False,
252
+ grants: Iterable[Grant] = (),
253
+ env: Mapping[str, str] | None = None,
254
+ stdin: bytes | bytearray | memoryview | None = None,
255
+ host: Host | None = None,
256
+ snapshot: bool = True,
257
+ fuel: int | None = None,
258
+ deadline_s: float | None = _UNSET,
259
+ memory_bytes: int | None = _UNSET,
260
+ output_bytes: int | None = _UNSET,
261
+ cache_dir: str | os.PathLike[str] | None | bool = None,
262
+ artifacts: Artifacts | None = None,
263
+ cancel: threading.Event | None = None,
264
+ ) -> Completed:
265
+ """Run one program in a fresh instance of the runtime and return how it ended.
266
+
267
+ `root` is the host directory the guest sees as `/`, writable. By default (`False`, or `None`)
268
+ the guest has no `/` and nothing writable, only its grants (spec 025); `root=True` makes a
269
+ fresh private directory for the call, removed after it. `grants`, each a `Grant`, are host
270
+ folders the guest sees at their guest paths, read-only unless they say otherwise (spec 017).
271
+ The stdlib zip is always granted, read-only, at /usr/local/lib. `memory_bytes`, `deadline_s`
272
+ and `output_bytes` default to `DEFAULTS`; `None` means no limit. The guest's environment
273
+ starts from `BASE_ENV` (`/tmp` for temporary files, `zoneinfo` on the zip's tzdata) and a
274
+ `HOME` made fresh for the call where `/tmp` is writable, with
275
+ `env` laid over it.
276
+ `cache_dir=False` keeps compiled modules in this process only, never on disk.
277
+ `cancel`, a `threading.Event`: once it is set and the guest runs, the guest stops within
278
+ `CANCEL_TICKS` ticks and the call returns `trapped == "cancelled"` (spec 015)."""
279
+ _refuse_forked()
280
+ wt = _wasmtime()
281
+ checked = _checked(argv, env, stdin, fuel, deadline_s, memory_bytes, output_bytes, cancel)
282
+ checked.update(root=_check_root(root), grants=_check_grants(grants))
283
+ return _run_checked(wt, checked, host, snapshot, cache_dir, artifacts)
284
+
285
+
286
+ async def run_async(
287
+ argv: Iterable[str],
288
+ *,
289
+ root: str | os.PathLike[str] | bool | None = False,
290
+ grants: Iterable[Grant] = (),
291
+ env: Mapping[str, str] | None = None,
292
+ stdin: bytes | bytearray | memoryview | None = None,
293
+ host: Host | None = None,
294
+ snapshot: bool = True,
295
+ fuel: int | None = None,
296
+ deadline_s: float | None = _UNSET,
297
+ memory_bytes: int | None = _UNSET,
298
+ output_bytes: int | None = _UNSET,
299
+ cache_dir: str | os.PathLike[str] | None | bool = None,
300
+ artifacts: Artifacts | None = None,
301
+ executor: concurrent.futures.Executor | None = None,
302
+ ) -> Completed:
303
+ """`run` from async code, on a worker thread, so the event loop is never blocked (spec 015).
304
+
305
+ The arguments are `run`'s, checked here before anything is queued. The call runs on
306
+ `executor`, which must run callables on threads of this process, or on the runner's default
307
+ pool of `DEFAULTS.async_workers` threads. If the awaiting task is cancelled, the guest is
308
+ stopped and its worker freed before `CancelledError` is raised; a call still queued never
309
+ starts."""
310
+ import asyncio
311
+ import contextvars
312
+
313
+ _refuse_forked()
314
+ checked = _checked(argv, env, stdin, fuel, deadline_s, memory_bytes, output_bytes, None)
315
+ checked.update(root=_check_root(root), grants=_check_grants(grants))
316
+ _check_executor(executor)
317
+ wt = _wasmtime()
318
+ _register_exit_hook()
319
+ event = threading.Event()
320
+ checked["cancel"] = event
321
+ with _live_lock: # the exit hook sets `exiting` before it reads `_live` (the build review, N1)
322
+ if _hook["exiting"]:
323
+ raise RuntimeError("cannot schedule new futures after interpreter shutdown")
324
+ _live.add(event)
325
+ try:
326
+ work = contextvars.copy_context().run # a door host sees the request's context
327
+ args = (_run_checked, wt, checked, host, snapshot, cache_dir, artifacts)
328
+ future = _submit(executor, work, *args)
329
+ try:
330
+ return await asyncio.wrap_future(future)
331
+ except asyncio.CancelledError:
332
+ if future.cancel(): # still queued: it never starts
333
+ raise
334
+ event.set()
335
+ while not future.done(): # wait for the worker, whatever the call's outcome
336
+ try:
337
+ await asyncio.wrap_future(future)
338
+ except BaseException:
339
+ pass
340
+ raise
341
+ finally:
342
+ with _live_lock:
343
+ _live.discard(event)
344
+
345
+
346
+ def _check_executor(executor: Any) -> None:
347
+ import concurrent.futures
348
+
349
+ if executor is None:
350
+ return
351
+ interpreters = getattr(concurrent.futures, "InterpreterPoolExecutor", None) # 3.14
352
+ if interpreters is not None and isinstance(executor, interpreters):
353
+ raise TypeError("executor must run on threads of this process, not an interpreter pool")
354
+ if isinstance(executor, concurrent.futures.ProcessPoolExecutor):
355
+ raise TypeError("executor must run on threads of this process, not a process pool")
356
+ if not isinstance(executor, concurrent.futures.Executor):
357
+ raise TypeError(f"executor must be a concurrent.futures.Executor; got {executor!r}")
358
+
359
+
360
+ # run_async's default pool, its live calls and its exit hook (spec 015, Decisions 4 and 6).
361
+ _pool_lock = threading.Lock()
362
+ _pool: dict[str, Any] = {"pool": None, "size": 0}
363
+ _pools: list[concurrent.futures.ThreadPoolExecutor] = [] # every default pool made, for exit
364
+ _live_lock = threading.Lock()
365
+ _live: set[threading.Event] = set()
366
+ _hook = {"registered": False, "exiting": False}
367
+
368
+
369
+ def _submit(executor: Any, fn: Callable[..., Any], *args: Any) -> concurrent.futures.Future[Any]:
370
+ """Submit to the caller's executor, or to the default pool, resized when `DEFAULTS` changed.
371
+ The lookup, any resize and the submit share one lock, so a resize on another thread never
372
+ leaves a submit facing a pool that is shut down."""
373
+ import concurrent.futures
374
+
375
+ if executor is not None:
376
+ return executor.submit(fn, *args)
377
+ with _pool_lock:
378
+ if _hook["exiting"]: # the exit hook has run: never start a pool's call again
379
+ raise RuntimeError("cannot schedule new futures after interpreter shutdown")
380
+ size = DEFAULTS.async_workers
381
+ if _pool["pool"] is None or _pool["size"] != size:
382
+ old = _pool["pool"]
383
+ new = concurrent.futures.ThreadPoolExecutor(size, thread_name_prefix="wasi-python")
384
+ _pools.append(new)
385
+ _pool.update(pool=new, size=size)
386
+ if old is not None:
387
+ old.shutdown(wait=False) # its running calls finish
388
+ return _pool["pool"].submit(fn, *args)
389
+
390
+
391
+ def _register_exit_hook() -> None:
392
+ """At exit, stop run_async's guests and drop its queued calls. threading runs its exit hooks
393
+ in reverse order of registration, so this one, registered after concurrent.futures.thread is
394
+ imported, runs before that module joins the pools' workers; registered before, exit would
395
+ wait for every queued call (spec 015's confirmation, EXECUTED both ways)."""
396
+ with _pool_lock:
397
+ if _hook["registered"]:
398
+ return
399
+ import concurrent.futures.thread # noqa: F401 (its hook first, so ours runs before it)
400
+
401
+ register = getattr(threading, "_register_atexit", None)
402
+ if register is not None:
403
+ register(_stop_async_calls)
404
+ else:
405
+ atexit.register(_stop_async_calls)
406
+ _hook["registered"] = True
407
+
408
+
409
+ def _stop_async_calls() -> None:
410
+ """Pools first, so no call can be submitted after its event was missed; then the events of the
411
+ calls already running (spec 015's build review, F3)."""
412
+ with _pool_lock:
413
+ _hook["exiting"] = True
414
+ pools = list(_pools)
415
+ for pool in pools:
416
+ pool.shutdown(wait=False, cancel_futures=True)
417
+ with _live_lock:
418
+ events = list(_live)
419
+ for event in events:
420
+ event.set()
421
+
422
+
423
+ # Private folders and the sweep (spec 016, Decision 2). Each private root and stdin folder has a
424
+ # lock file beside it in the temporary folder, locked for the folder's life and naming the kernel
425
+ # that made it. A process killed mid-call leaves both; the lock dies with it, so the next process
426
+ # on the same kernel can tell its folder is dead. A pid could not: pids repeat across namespaces,
427
+ # are reused, and a killed child is a zombie until reaped.
428
+ _LOCK_NAME = re.compile(r"wasi-python-(root-|zip-)?[0-9a-f]{16}\.lock\Z") # zip-: spec 017
429
+ _held_lock = threading.Lock()
430
+ _held: set[str] = set() # the names this process holds: NFS may treat flock as per-process
431
+ _sweep_state = {"claimed": False, "swept": 0}
432
+ _identity: dict[str, str | None] = {}
433
+
434
+
435
+ def _flags(*names: str) -> int:
436
+ flags = 0
437
+ for name in names:
438
+ flags |= getattr(os, name, 0)
439
+ return flags
440
+
441
+
442
+ def _kernel_identity() -> str | None:
443
+ """Which kernel this process runs on, or None: a lock taken on one kernel is invisible to
444
+ another that shares the folder (a VM and its host), so a sweep touches only its own."""
445
+ if "value" not in _identity:
446
+ value = None
447
+ try:
448
+ if sys.platform.startswith("linux"):
449
+ value = Path("/proc/sys/kernel/random/boot_id").read_text().strip()
450
+ elif sys.platform == "darwin":
451
+ import ctypes.util
452
+
453
+ libc = ctypes.CDLL(ctypes.util.find_library("c"))
454
+ size_p, void_p = ctypes.POINTER(ctypes.c_size_t), ctypes.c_void_p
455
+ libc.sysctlbyname.argtypes = [
456
+ ctypes.c_char_p,
457
+ void_p,
458
+ size_p,
459
+ void_p,
460
+ ctypes.c_size_t,
461
+ ]
462
+ libc.sysctlbyname.restype = ctypes.c_int
463
+ buf, size = ctypes.create_string_buffer(64), ctypes.c_size_t(64)
464
+ name = b"kern.bootsessionuuid"
465
+ if libc.sysctlbyname(name, buf, ctypes.byref(size), None, 0) == 0:
466
+ value = buf.value.decode("ascii")
467
+ elif sys.platform == "win32":
468
+ import winreg
469
+
470
+ access = winreg.KEY_READ | winreg.KEY_WOW64_64KEY
471
+ where = r"SOFTWARE\Microsoft\Cryptography"
472
+ with winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, where, 0, access) as key:
473
+ value = str(winreg.QueryValueEx(key, "MachineGuid")[0])
474
+ except Exception:
475
+ value = None
476
+ _identity["value"] = value if value and "\n" not in value else None
477
+ return _identity["value"]
478
+
479
+
480
+ _BUSY = {errno.EACCES, errno.EDEADLK, getattr(errno, "EDEADLOCK", errno.EDEADLK)}
481
+
482
+
483
+ def _try_lock(fd: int) -> bool | None:
484
+ """Lock the open file without waiting: True when locked, False when another holder has it,
485
+ None when this filesystem gives no lock (`ENOLCK` on NFS without a lock daemon, say)."""
486
+ try:
487
+ if sys.platform == "win32":
488
+ import msvcrt
489
+
490
+ os.lseek(fd, 0, 0)
491
+ msvcrt.locking(fd, msvcrt.LK_NBLCK, 1)
492
+ else:
493
+ import fcntl
494
+
495
+ fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
496
+ return True
497
+ except BlockingIOError:
498
+ return False
499
+ except OSError as e:
500
+ return False if sys.platform == "win32" and e.errno in _BUSY else None
501
+
502
+
503
+ def _unlock_close(fd: int) -> None:
504
+ """Unlock (Windows frees a closed handle's lock only after a delay), then close; quiet."""
505
+ if sys.platform == "win32":
506
+ try:
507
+ import msvcrt
508
+
509
+ os.lseek(fd, 0, 0)
510
+ msvcrt.locking(fd, msvcrt.LK_UNLCK, 1)
511
+ except OSError:
512
+ pass
513
+ try:
514
+ os.close(fd)
515
+ except OSError:
516
+ pass
517
+
518
+
519
+ def _same_file(fd: int, path: str) -> bool:
520
+ """Whether the open file is still the file at `path`; False when the path is gone."""
521
+ try:
522
+ opened, named = os.fstat(fd), os.lstat(path)
523
+ except OSError:
524
+ return False
525
+ return (opened.st_dev, opened.st_ino) == (named.st_dev, named.st_ino)
526
+
527
+
528
+ class _Taken(Exception):
529
+ pass
530
+
531
+
532
+ class _Private:
533
+ """A private folder in the temporary folder, with `<name>.lock` beside it, held locked until
534
+ `remove` (spec 016). The name is drawn here, as mkdtemp draws one, because the lock file must
535
+ exist, locked, before the folder does."""
536
+
537
+ def __init__(self, prefix: str) -> None:
538
+ temp = tempfile.gettempdir()
539
+ identity = _kernel_identity()
540
+ create = _flags("O_RDWR", "O_CREAT", "O_EXCL", "O_NOFOLLOW", "O_CLOEXEC", "O_BINARY")
541
+ for _ in range(100):
542
+ name = prefix + secrets.token_hex(8)
543
+ lock_path, path = os.path.join(temp, name + ".lock"), os.path.join(temp, name)
544
+ with _held_lock:
545
+ _held.add(name)
546
+ try:
547
+ fd = os.open(lock_path, create, 0o600)
548
+ except FileExistsError:
549
+ _drop(name)
550
+ continue
551
+ except BaseException:
552
+ _drop(name)
553
+ raise
554
+ owns = False # the file at lock_path is the one this call made
555
+ try:
556
+ locked = _try_lock(fd)
557
+ if locked is False:
558
+ raise _Taken
559
+ owns = _same_file(fd, lock_path) # a sweeper may have removed it, still empty
560
+ if not owns:
561
+ raise _Taken
562
+ if locked and identity is not None: # unlocked: no identity, so never swept
563
+ os.lseek(fd, 0, 0)
564
+ os.write(fd, identity.encode("ascii") + b"\n")
565
+ os.mkdir(path, 0o700)
566
+ except (_Taken, FileExistsError):
567
+ _abandon(fd, lock_path if owns else None, name)
568
+ continue
569
+ except BaseException:
570
+ _abandon(fd, lock_path if owns else None, name)
571
+ raise
572
+ self.name, self.fd, self.lock_path, self.path = name, fd, lock_path, Path(path)
573
+ return
574
+ raise FileExistsError(errno.EEXIST, "no usable temporary folder name found")
575
+
576
+ def remove(self) -> None:
577
+ """Folder, then lock (close), then the lock file only if the folder is gone, then the
578
+ name; never raises (the 0.1.0a2 security review, F1)."""
579
+ shutil.rmtree(self.path, ignore_errors=True)
580
+ _unlock_close(self.fd) # Windows cannot delete an open file
581
+ if not os.path.lexists(self.path):
582
+ try:
583
+ os.unlink(self.lock_path)
584
+ except OSError:
585
+ pass
586
+ _drop(self.name)
587
+
588
+
589
+ def _drop(name: str) -> None:
590
+ with _held_lock:
591
+ _held.discard(name)
592
+
593
+
594
+ def _abandon(fd: int, lock_path: str | None, name: str) -> None:
595
+ _unlock_close(fd)
596
+ if lock_path is not None: # only a lock file this call made and still has at that name
597
+ try:
598
+ os.unlink(lock_path)
599
+ except OSError:
600
+ pass
601
+ _drop(name)
602
+
603
+
604
+ def _sweep_once() -> None:
605
+ """At a process's first call: remove the private folders of dead processes on this kernel.
606
+ Claimed under `_lock`, run without it; it never raises into the call."""
607
+ with _lock:
608
+ if _sweep_state["claimed"]:
609
+ return
610
+ _sweep_state["claimed"] = True
611
+ try:
612
+ identity = _kernel_identity()
613
+ if identity is None:
614
+ return
615
+ temp = tempfile.gettempdir()
616
+ for entry in os.listdir(temp):
617
+ try:
618
+ _sweep_entry(temp, entry, identity)
619
+ except Exception:
620
+ continue
621
+ except Exception:
622
+ return
623
+
624
+
625
+ def _owned(st: os.stat_result) -> bool:
626
+ getuid = getattr(os, "getuid", None)
627
+ return getuid is None or st.st_uid == getuid() # Windows: the user's own temporary folder
628
+
629
+
630
+ def _sweep_entry(temp: str, entry: str, identity: str) -> None:
631
+ if not _LOCK_NAME.match(entry):
632
+ return
633
+ name = entry[: -len(".lock")]
634
+ with _held_lock:
635
+ if name in _held:
636
+ return
637
+ lock_path, path = os.path.join(temp, entry), os.path.join(temp, name)
638
+ named = os.lstat(lock_path)
639
+ if not stat.S_ISREG(named.st_mode) or not _owned(named):
640
+ return
641
+ fd = os.open(lock_path, _flags("O_RDWR", "O_NOFOLLOW", "O_NONBLOCK", "O_CLOEXEC", "O_BINARY"))
642
+ unlink = False
643
+ try:
644
+ opened = os.fstat(fd)
645
+ if not stat.S_ISREG(opened.st_mode):
646
+ return
647
+ if (opened.st_dev, opened.st_ino) != (named.st_dev, named.st_ino):
648
+ return
649
+ if not _try_lock(fd):
650
+ return # a live call holds it
651
+ os.lseek(fd, 0, 0)
652
+ first = os.read(fd, 256).split(b"\n", 1)[0].decode("ascii", "replace")
653
+ if not os.path.lexists(path):
654
+ unlink = first in ("", identity) # a lone lock file: nothing of a call's to lose
655
+ return
656
+ if first != identity:
657
+ return # another kernel's, or none
658
+ folder = os.lstat(path)
659
+ isjunction = getattr(os.path, "isjunction", lambda p: False)
660
+ if not stat.S_ISDIR(folder.st_mode) or isjunction(path):
661
+ return
662
+ shutil.rmtree(path, ignore_errors=True)
663
+ if not os.path.lexists(path):
664
+ with _held_lock:
665
+ _sweep_state["swept"] += 1
666
+ unlink = True
667
+ finally:
668
+ # POSIX unlinks while still holding the lock, so a creator that locks the file after
669
+ # this fails its same-file check and draws again; Windows cannot delete an open file.
670
+ if unlink and os.name == "posix":
671
+ _unlink_quietly(lock_path)
672
+ _unlock_close(fd)
673
+ if unlink and os.name != "posix":
674
+ _unlink_quietly(lock_path)
675
+
676
+
677
+ def _unlink_quietly(path: str) -> None:
678
+ try:
679
+ os.unlink(path)
680
+ except OSError:
681
+ pass
682
+
683
+
684
+ # Fork (spec 015, Decision 5). A child of a process that made an engine inherits state from
685
+ # threads that no longer exist, and wasmtime does not support use across a fork (on macOS it
686
+ # aborts), so such a child is refused. A child of a parent that only imported this runs normally.
687
+ _fork = {"used": False, "forked": False}
688
+
689
+
690
+ def _refuse_forked() -> None:
691
+ if _fork["forked"]:
692
+ raise RuntimeError(
693
+ "wasi_python was used before this process was forked; use it in the child only, or "
694
+ "start processes with 'spawn' or 'forkserver'"
695
+ )
696
+
697
+
698
+ def _after_fork_in_child() -> None:
699
+ """Only assignments, and no lock: a lock may have been held by a thread the fork left out, so
700
+ every lock the runner takes is made anew (spec 015's build review, F8)."""
701
+ global _pool_lock, _live_lock, _lock, _epoch_callback_lock, _outputs_lock, _compiling
702
+ global _held_lock, _zip_lock, _zips
703
+ _pool_lock, _live_lock, _held_lock = threading.Lock(), threading.Lock(), threading.Lock()
704
+ _zip_lock, _zips = threading.Lock(), {} # the parent's zip folders stay the parent's
705
+ _lock, _epoch_callback_lock, _outputs_lock = (
706
+ threading.Lock(),
707
+ threading.Lock(),
708
+ threading.RLock(),
709
+ )
710
+ _compiling = {}
711
+ _pool.update(pool=None, size=0)
712
+ _pools.clear()
713
+ _live.clear()
714
+ _epoch_ticker._thread, _epoch_ticker._active = None, 0
715
+ _epoch_ticker._cond = threading.Condition()
716
+ if _fork["used"]:
717
+ _fork["forked"] = True
718
+
719
+
720
+ if hasattr(os, "register_at_fork"): # not on Windows, which has no fork
721
+ os.register_at_fork(after_in_child=_after_fork_in_child)
722
+
723
+
724
+ def _wasmtime() -> Any:
725
+ try:
726
+ import wasmtime as wt
727
+ except ImportError as e:
728
+ raise ImportError(
729
+ 'wasi_python.run needs wasmtime-py: pip install "wasi-python[run]"'
730
+ ) from e
731
+ return wt
732
+
733
+
734
+ def _checked(
735
+ argv: Any,
736
+ env: Any,
737
+ stdin: Any,
738
+ fuel: Any,
739
+ deadline_s: Any,
740
+ memory_bytes: Any,
741
+ output_bytes: Any,
742
+ cancel: Any,
743
+ ) -> dict[str, Any]:
744
+ """Every argument a call's guest depends on, checked before any work (specs 013 to 015)."""
745
+ limits = DEFAULTS
746
+ if memory_bytes is _UNSET:
747
+ memory_bytes = limits.memory_bytes
748
+ if deadline_s is _UNSET:
749
+ deadline_s = limits.deadline_s
750
+ if output_bytes is _UNSET:
751
+ output_bytes = limits.output_bytes
752
+ if cancel is not None and not isinstance(cancel, threading.Event):
753
+ raise TypeError(f"cancel must be a threading.Event, or None; got {type(cancel).__name__}")
754
+ checked = _check_limits(fuel, deadline_s, memory_bytes, output_bytes)
755
+ checked.update(_check_inputs(argv, env, stdin))
756
+ checked["cancel"] = cancel
757
+ return checked
758
+
759
+
760
+ # Grants (spec 017). A grant is a host folder the guest sees at a guest path, read-only unless it
761
+ # says `writable=True`. Every host path, `root` included, is walked one component at a time from
762
+ # `/`, judging each link on the way, and every rule compares identities (device and inode), never
763
+ # spellings: macOS folders are case-insensitive, and Windows spells one folder many ways. wasmtime
764
+ # then opens each path itself, so before the guest runs a check module of the runner's own reads
765
+ # back each preopen's identity and compares it with the walk's (Decision 5).
766
+ ZIP_GUEST = "/usr/local/lib"
767
+ MAX_LINKS = 40
768
+ _M64 = (1 << 64) - 1
769
+
770
+
771
+ def _host_path(where: str, value: Any) -> str:
772
+ if isinstance(value, (bytes, bytearray)):
773
+ raise TypeError(f"{where} must be a str or os.PathLike[str], not bytes")
774
+ try:
775
+ path = os.fspath(value)
776
+ except TypeError:
777
+ raise TypeError(
778
+ f"{where} must be a str or os.PathLike[str], not {type(value).__name__}"
779
+ ) from None
780
+ if not isinstance(path, str):
781
+ raise TypeError(f"{where} must be a str or os.PathLike[str], not a path of bytes")
782
+ path = _text(where, path)
783
+ if not path:
784
+ raise ValueError(f"{where} is empty")
785
+ return path
786
+
787
+
788
+ def _guest_path(where: str, value: Any) -> str:
789
+ """An absolute guest path with no empty, `.` or `..` component and no trailing `/`, which
790
+ wasi-libc would otherwise match by text alone (memo §7d); not `/` and not the runtime's."""
791
+ if not isinstance(value, str):
792
+ raise TypeError(f"{where}: the guest path must be a str, not {type(value).__name__}")
793
+ path = _text(f"{where}: the guest path", value)
794
+ if not path.startswith("/"):
795
+ raise ValueError(f"{where}: the guest path must be absolute; got {path!r}")
796
+ if path == "/":
797
+ raise ValueError(f"{where}: the guest path / is the root; pass root= instead")
798
+ if any(part in ("", ".", "..") for part in path.split("/")[1:]):
799
+ raise ValueError(
800
+ f"{where}: the guest path {path!r} has an empty, '.' or '..' part, or ends in '/'"
801
+ )
802
+ if path == ZIP_GUEST or path.startswith(ZIP_GUEST + "/"):
803
+ raise ValueError(f"{where}: {path!r} is the runtime's own ({ZIP_GUEST})")
804
+ return path
805
+
806
+
807
+ def _check_root(root: Any) -> str | bool | None:
808
+ """The root as the runner keeps it: a host path; None for a private root (`root=True`); or
809
+ False for none, the default (spec 025: nothing writable unless the embedder asks)."""
810
+ if root is None or root is False:
811
+ return False
812
+ if root is True:
813
+ return None
814
+ return _host_path("root", root)
815
+
816
+
817
+ def _check_grants(grants: Any) -> tuple[Grant, ...]:
818
+ """The grants as a tuple of plain `Grant`s, so `run_async` hands its worker no used iterator."""
819
+ if isinstance(grants, (str, bytes, bytearray, Grant)) or not isinstance(grants, Iterable):
820
+ raise TypeError(f"grants must be an iterable of Grant, not {type(grants).__name__}")
821
+ checked: list[Grant] = []
822
+ seen: set[str] = set()
823
+ for i, grant in enumerate(grants):
824
+ if not isinstance(grant, Grant):
825
+ raise TypeError(f"grants[{i}] must be a Grant, not {type(grant).__name__}")
826
+ guest = _guest_path(f"grants[{i}]", grant.guest)
827
+ where = f"grants[{i}] at {guest}"
828
+ if type(grant.writable) is not bool:
829
+ raise TypeError(f"{where}: writable must be True or False, not {grant.writable!r}")
830
+ if guest in seen:
831
+ raise ValueError(f"{where}: another grant has the same guest path")
832
+ seen.add(guest)
833
+ checked.append(
834
+ Grant(_host_path(f"{where}: host", grant.host), guest, writable=grant.writable)
835
+ )
836
+ return tuple(checked)
837
+
838
+
839
+ Identity = tuple[int, int]
840
+
841
+
842
+ class _Walked:
843
+ """A host folder walked component by component: the path wasmtime is given, the identities of
844
+ the folder and each ancestor (the folder's first), and what keeps it from changing until the
845
+ identity check: on POSIX the folder's descriptor, on Windows every component's handle. A root
846
+ still to be made has `missing` components, and its chain is its deepest existing ancestor's."""
847
+
848
+ def __init__(self, path: str, chain: tuple[Identity, ...], held: list[int]) -> None:
849
+ self.path, self.chain, self.held = path, chain, held
850
+ self.missing: tuple[str, ...] = ()
851
+
852
+ @property
853
+ def identity(self) -> Identity | None:
854
+ return None if self.missing else self.chain[0]
855
+
856
+ @property
857
+ def fd(self) -> int | None:
858
+ return self.held[-1] if self.held and os.name == "posix" else None
859
+
860
+ def close(self) -> None:
861
+ held, self.held = self.held, []
862
+ for h in held:
863
+ _close_held(h)
864
+
865
+
866
+ def _close_held(h: int) -> None:
867
+ try:
868
+ if os.name == "posix":
869
+ os.close(h)
870
+ else:
871
+ _win().CloseHandle(h)
872
+ except OSError:
873
+ pass
874
+
875
+
876
+ def _walk(where: str, path: str, *, judge: bool, may_make: bool = False) -> _Walked:
877
+ """Walk `path` from the top. With `judge`, a link in a folder this user can write (POSIX), or
878
+ any link or junction (Windows), raises ValueError; without it (the runtime's own folders),
879
+ every link is followed. With `may_make`, a missing tail is returned as `missing`."""
880
+ try:
881
+ if os.name == "posix":
882
+ return _walk_posix(where, path, judge, may_make)
883
+ return _walk_windows(where, path, judge, may_make)
884
+ except ValueError:
885
+ raise
886
+ except OSError as e:
887
+ if not judge:
888
+ raise
889
+ # A folder that cannot be searched, or vanished mid-walk: the caller's grant is refused.
890
+ raise ValueError(f"{where}: {path!r} cannot be walked: {e.strerror or e}") from None
891
+
892
+
893
+ # Opened for search only where the system allows (Linux O_PATH, macOS O_SEARCH): wasmtime opens a
894
+ # grant below a folder this user may search but not list, so the walk must pass through it too.
895
+ _DIR = (_flags("O_PATH") or _flags("O_SEARCH") or os.O_RDONLY) | _flags(
896
+ "O_DIRECTORY", "O_NOFOLLOW", "O_CLOEXEC"
897
+ )
898
+
899
+
900
+ def _walk_posix(where: str, path: str, judge: bool, may_make: bool) -> _Walked:
901
+ # Joined to the working directory without normalising: `..` is taken on the walked folders,
902
+ # as the kernel takes it, after any link before it is followed.
903
+ todo = list(reversed(os.path.join(os.getcwd(), path).split("/")))
904
+ top = os.open("/", _DIR)
905
+ st = os.fstat(top)
906
+ stack: list[tuple[int, Identity, str]] = [(top, (st.st_dev, st.st_ino), "")]
907
+ links = 0
908
+ try:
909
+ while todo:
910
+ name = todo.pop()
911
+ if name in ("", "."):
912
+ continue
913
+ if name == "..":
914
+ if len(stack) > 1:
915
+ os.close(stack.pop()[0])
916
+ continue
917
+ parent = stack[-1][0]
918
+ try:
919
+ st = os.lstat(name, dir_fd=parent)
920
+ except FileNotFoundError:
921
+ rest = [n for n in reversed(todo) if n not in ("", ".")]
922
+ if not may_make or ".." in rest:
923
+ raise ValueError(f"{where}: {path!r} does not exist") from None
924
+ walked = _walked_posix(stack)
925
+ walked.missing = (name, *rest)
926
+ return walked
927
+ if stat.S_ISLNK(st.st_mode):
928
+ if judge and _writable(parent, stack):
929
+ raise ValueError(
930
+ f"{where}: {path!r} passes through the link {name!r} in a folder this "
931
+ "user can write, which a guest could have made; name its target instead"
932
+ )
933
+ links += 1
934
+ if links > MAX_LINKS:
935
+ raise ValueError(
936
+ f"{where}: {path!r} passes through more than {MAX_LINKS} links"
937
+ )
938
+ target = os.readlink(name, dir_fd=parent)
939
+ if target.startswith("/"):
940
+ while len(stack) > 1:
941
+ os.close(stack.pop()[0])
942
+ todo.extend(reversed(target.split("/")))
943
+ continue
944
+ if not stat.S_ISDIR(st.st_mode):
945
+ raise ValueError(f"{where}: {path!r} is not a folder")
946
+ fd = os.open(name, _DIR, dir_fd=parent)
947
+ opened = os.fstat(fd)
948
+ if (opened.st_dev, opened.st_ino) != (st.st_dev, st.st_ino):
949
+ os.close(fd)
950
+ raise ValueError(f"{where}: {path!r} changed while it was checked")
951
+ stack.append((fd, (opened.st_dev, opened.st_ino), name))
952
+ return _walked_posix(stack)
953
+ except BaseException:
954
+ for fd, _, _ in stack:
955
+ os.close(fd)
956
+ raise
957
+
958
+
959
+ def _walked_posix(stack: list[tuple[int, Identity, str]]) -> _Walked:
960
+ """Keep the last folder's descriptor, close the rest."""
961
+ for fd, _, _ in stack[:-1]:
962
+ os.close(fd)
963
+ text = "/" + "/".join(name for _, _, name in stack[1:])
964
+ chain = tuple(identity for _, identity, _ in reversed(stack))
965
+ last = stack[-1][0]
966
+ stack[:] = []
967
+ return _Walked(text, chain, [last])
968
+
969
+
970
+ def _writable(fd: int, stack: list[tuple[int, Identity, str]]) -> bool:
971
+ """Whether this user can write the folder open at `fd`: only a link in such a folder could be
972
+ a guest's, since a guest writes as this user (memo §7a)."""
973
+ effective = os.access in os.supports_effective_ids
974
+ if os.access in os.supports_dir_fd:
975
+ return os.access(".", os.W_OK, dir_fd=fd, effective_ids=effective)
976
+ text = "/" + "/".join(name for _, _, name in stack[1:])
977
+ return os.access(text, os.W_OK, effective_ids=effective)
978
+
979
+
980
+ def _make_missing(where: str, walked: _Walked) -> _Walked:
981
+ """Make a root's missing components, after the rules were decided on its existing chain, each
982
+ inside the one before through its descriptor; then walk the made root as any other."""
983
+ path = walked.path
984
+ if os.name == "posix":
985
+ fd = os.dup(walked.held[-1])
986
+ try:
987
+ for name in walked.missing:
988
+ try:
989
+ os.mkdir(name, dir_fd=fd)
990
+ except FileExistsError: # a concurrent call made it
991
+ pass
992
+ child = os.open(name, _DIR, dir_fd=fd) # O_NOFOLLOW: a link put there fails
993
+ os.close(fd)
994
+ fd = child
995
+ path = os.path.join(path, name)
996
+ finally:
997
+ os.close(fd)
998
+ else:
999
+ for name in walked.missing:
1000
+ path = os.path.join(path, name)
1001
+ Path(path).mkdir(exist_ok=True)
1002
+ walked.close()
1003
+ return _walk(where, path, judge=True)
1004
+
1005
+
1006
+ # Windows (Decision 5): Python cannot open a folder with os.open, so the walk opens every
1007
+ # component through CreateFileW, refuses a reparse point (a link or a junction), and holds each
1008
+ # handle without FILE_SHARE_DELETE until the identity check, so no walked folder can be renamed,
1009
+ # removed or replaced in between, and no path string is ever compared.
1010
+ # Traverse access, not attributes alone: Windows runs share checks only for data, execute or
1011
+ # delete access (MS-FSA 2.1.5.1.2.2), so an attribute-only handle held without FILE_SHARE_DELETE
1012
+ # stops no rename. FILE_TRAVERSE is FILE_EXECUTE, which is checked, and needs only the permission
1013
+ # to pass through a folder, as POSIX's search does, not to list it.
1014
+ _WALK_ACCESS = 0x20 | 0x80 # FILE_TRAVERSE | FILE_READ_ATTRIBUTES
1015
+ _STAT_ACCESS = 0x80 # FILE_READ_ATTRIBUTES: an identity read that holds nothing
1016
+ _SHARE_READ_WRITE = 0x1 | 0x2
1017
+ _OPEN_EXISTING = 3
1018
+ _BACKUP_SEMANTICS = 0x02000000
1019
+ _OPEN_REPARSE_POINT = 0x00200000
1020
+ _ATTRIBUTE_DIRECTORY = 0x10
1021
+ _ATTRIBUTE_REPARSE_POINT = 0x400
1022
+ _win_api: dict[str, Any] = {}
1023
+
1024
+
1025
+ def _win() -> Any:
1026
+ if "kernel32" not in _win_api:
1027
+ from ctypes import wintypes
1028
+
1029
+ k = ctypes.WinDLL("kernel32", use_last_error=True) # type: ignore[attr-defined]
1030
+ k.CreateFileW.argtypes = [
1031
+ wintypes.LPCWSTR, wintypes.DWORD, wintypes.DWORD, wintypes.LPVOID,
1032
+ wintypes.DWORD, wintypes.DWORD, wintypes.HANDLE,
1033
+ ] # fmt: skip
1034
+ k.CreateFileW.restype = wintypes.HANDLE
1035
+ k.GetFileInformationByHandle.argtypes = [wintypes.HANDLE, ctypes.c_void_p]
1036
+ k.GetFileInformationByHandle.restype = wintypes.BOOL
1037
+ k.CloseHandle.argtypes = [wintypes.HANDLE]
1038
+ k.CloseHandle.restype = wintypes.BOOL
1039
+ _win_api["kernel32"] = k
1040
+ return _win_api["kernel32"]
1041
+
1042
+
1043
+ class _ByHandle(ctypes.Structure):
1044
+ _fields_ = [
1045
+ ("attributes", ctypes.c_uint32),
1046
+ ("times", ctypes.c_uint32 * 6),
1047
+ ("volume_serial", ctypes.c_uint32),
1048
+ ("size_high", ctypes.c_uint32),
1049
+ ("size_low", ctypes.c_uint32),
1050
+ ("links", ctypes.c_uint32),
1051
+ ("index_high", ctypes.c_uint32),
1052
+ ("index_low", ctypes.c_uint32),
1053
+ ]
1054
+
1055
+
1056
+ def _win_open(path: str, follow: bool, access: int = _WALK_ACCESS) -> tuple[int, int, Identity]:
1057
+ """A held handle on `path`, its attributes and its identity, as wasmtime reads it."""
1058
+ k = _win()
1059
+ flags = _BACKUP_SEMANTICS | (0 if follow else _OPEN_REPARSE_POINT)
1060
+ h = k.CreateFileW(path, access, _SHARE_READ_WRITE, None, _OPEN_EXISTING, flags, None)
1061
+ if h is None or h == ctypes.c_void_p(-1).value:
1062
+ code = ctypes.get_last_error() # type: ignore[attr-defined]
1063
+ raise ctypes.WinError(code) # type: ignore[attr-defined]
1064
+ info = _ByHandle()
1065
+ if not k.GetFileInformationByHandle(h, ctypes.byref(info)):
1066
+ code = ctypes.get_last_error() # type: ignore[attr-defined]
1067
+ k.CloseHandle(h)
1068
+ raise ctypes.WinError(code) # type: ignore[attr-defined]
1069
+ index = (info.index_high << 32) | info.index_low
1070
+ return h, info.attributes, (info.volume_serial, index)
1071
+
1072
+
1073
+ def _walk_windows(where: str, path: str, judge: bool, may_make: bool) -> _Walked:
1074
+ full = os.path.abspath(path) if judge else os.path.realpath(path)
1075
+ drive, rest = os.path.splitdrive(full)
1076
+ anchor = drive + "\\"
1077
+ parts = [p for p in re.split(r"[\\/]", rest) if p]
1078
+ held: list[int] = []
1079
+ chain: list[Identity] = []
1080
+ try:
1081
+ here = anchor
1082
+ for i in range(len(parts) + 1):
1083
+ if i:
1084
+ here = os.path.join(here, parts[i - 1])
1085
+ try:
1086
+ h, attributes, identity = _win_open(here, follow=not judge)
1087
+ except FileNotFoundError:
1088
+ if not may_make or i == 0:
1089
+ raise ValueError(f"{where}: {path!r} does not exist") from None
1090
+ walked = _Walked(os.path.dirname(here), tuple(reversed(chain)), held)
1091
+ walked.missing = tuple(parts[i - 1 :])
1092
+ return walked
1093
+ held.append(h)
1094
+ if judge and attributes & _ATTRIBUTE_REPARSE_POINT:
1095
+ raise ValueError(
1096
+ f"{where}: {path!r} passes through a link or junction; on Windows every one is "
1097
+ "refused"
1098
+ )
1099
+ if not attributes & _ATTRIBUTE_DIRECTORY:
1100
+ raise ValueError(f"{where}: {path!r} is not a folder")
1101
+ chain.append(identity)
1102
+ return _Walked(here, tuple(reversed(chain)), held)
1103
+ except BaseException:
1104
+ for h in held:
1105
+ _close_held(h)
1106
+ raise
1107
+
1108
+
1109
+ def _chain(path: Path) -> tuple[Identity | None, tuple[Identity, ...]]:
1110
+ """A runtime folder's identity (None while it does not exist) and the identities of its
1111
+ deepest existing ancestor and that one's ancestors, every link followed; never refuses."""
1112
+ here = Path(os.path.realpath(path))
1113
+ exists = True
1114
+ while True:
1115
+ try:
1116
+ first = _identity_of(here)
1117
+ break
1118
+ except OSError:
1119
+ exists = False
1120
+ if here.parent == here:
1121
+ return None, ()
1122
+ here = here.parent
1123
+ chain = [first]
1124
+ for parent in here.parents:
1125
+ try:
1126
+ chain.append(_identity_of(parent))
1127
+ except OSError:
1128
+ break
1129
+ return (first if exists else None), tuple(chain)
1130
+
1131
+
1132
+ def _identity_of(path: Path) -> Identity:
1133
+ if os.name == "posix":
1134
+ st = os.stat(path)
1135
+ if not stat.S_ISDIR(st.st_mode):
1136
+ raise NotADirectoryError(str(path))
1137
+ return (st.st_dev, st.st_ino)
1138
+ # Attributes only: a runtime folder's chain must not be cut short by a folder one may not
1139
+ # traverse, or by another handle's share mode (the follow-up review, F1).
1140
+ h, _, identity = _win_open(str(path), follow=True, access=_STAT_ACCESS)
1141
+ _close_held(h)
1142
+ return identity
1143
+
1144
+
1145
+ def _judge(
1146
+ root: _Walked | None,
1147
+ grants: list[tuple[str, Grant, _Walked]],
1148
+ runtime: list,
1149
+ shared: list[tuple[str, tuple[Identity, ...]]] = [], # noqa: B006 (read only)
1150
+ ) -> None:
1151
+ """Decision 4's rules, by identity. `runtime` is (name, identity or None, chain) for each of
1152
+ the runtime's own folders: a writable grant at, inside or above one is refused; a folder not
1153
+ made yet holds nothing, so only a grant above its deepest existing ancestor is. `shared` is
1154
+ (name, chain) for each folder no grant, read-only included, may be at or above."""
1155
+ judged = [("root", None, root)] if root is not None else []
1156
+ for where, _, w in [*judged, *grants]: # root too: the temporary folder may have moved
1157
+ for name, chain in shared:
1158
+ if w.identity is not None and w.identity in chain:
1159
+ raise ValueError(f"{where} holds {name}; grant a folder apart from it")
1160
+ writable = [("root", root)] if root is not None else []
1161
+ writable += [(where, w) for where, g, w in grants if g.writable]
1162
+ for where, w in writable:
1163
+ for name, identity, chain in runtime:
1164
+ inside = identity is not None and identity in w.chain
1165
+ above = w.identity is not None and w.identity in chain
1166
+ if inside or above:
1167
+ how = "is inside" if inside else "holds"
1168
+ raise ValueError(
1169
+ f"{where} is writable and {how} {name}, which later calls trust; grant a "
1170
+ "folder apart from it"
1171
+ )
1172
+ for where, g, r in grants:
1173
+ if g.writable:
1174
+ continue
1175
+ for other, w in writable:
1176
+ if w.identity is not None and w.identity in r.chain:
1177
+ raise ValueError(
1178
+ f"{where} is read-only but inside {other}, which is writable; a guest would "
1179
+ "change it through that grant"
1180
+ )
1181
+
1182
+
1183
+ # The zip (Decision 2): one read-only folder per zip per process, beside a lock file, at
1184
+ # /usr/local/lib on every call. Its own lock, since the epoch ticker takes `_lock` every round.
1185
+ _zip_lock = threading.Lock()
1186
+ _zips: dict[str, list[Any]] = {} # the zip's SHA-256 -> [its _Private folder, the copy's stamp]
1187
+
1188
+
1189
+ def _zip_folder(found: Artifacts) -> Path:
1190
+ want = _digest(found.stdlib_zip)
1191
+ name = f"python{_version_tag(found.manifest)}.zip"
1192
+ with _zip_lock:
1193
+ entry = _zips.get(want)
1194
+ if entry is None:
1195
+ return _zip_folder_locked(found, want, name)
1196
+ private, stamp = entry
1197
+ if not os.path.isdir(private.path) or os.path.islink(private.path):
1198
+ # Removed on the host (a temporary-folder cleaner, say): made again, not trusted.
1199
+ private.remove()
1200
+ del _zips[want]
1201
+ return _zip_folder_locked(found, want, name)
1202
+ dest = private.path / name
1203
+ try:
1204
+ st = os.lstat(dest)
1205
+ except FileNotFoundError:
1206
+ st = None
1207
+ if st is None or not stat.S_ISREG(st.st_mode) or _stamp(st) != stamp:
1208
+ entry[1] = _copy_zip(found, dest, want) # changed on the host: copied again
1209
+ return private.path
1210
+
1211
+
1212
+ def _zip_folder_locked(found: Artifacts, want: str, name: str) -> Path:
1213
+ private = _Private("wasi-python-zip-")
1214
+ try:
1215
+ stamp = _copy_zip(found, private.path / name, want)
1216
+ except BaseException:
1217
+ private.remove()
1218
+ raise
1219
+ _zips[want] = [private, stamp]
1220
+ return private.path
1221
+
1222
+
1223
+ def _zip_known(found: Artifacts) -> Path | None:
1224
+ """This process's folder for this zip, if it has made one."""
1225
+ with _zip_lock:
1226
+ entry = _zips.get(_digest(found.stdlib_zip))
1227
+ return None if entry is None else entry[0].path
1228
+
1229
+
1230
+ def _copy_zip(found: Artifacts, dest: Path, want: str) -> tuple[int, ...]:
1231
+ fd, tmp = tempfile.mkstemp(prefix=".python-zip-", dir=dest.parent)
1232
+ try:
1233
+ with os.fdopen(fd, "wb") as out, found.stdlib_zip.open("rb") as src:
1234
+ for block in iter(lambda: src.read(1 << 20), b""):
1235
+ out.write(block)
1236
+ # 0644, not read-only: Windows could neither replace nor remove a read-only file, and
1237
+ # the guest's grant is read-only anyway, in a 0700 folder.
1238
+ os.chmod(tmp, 0o644)
1239
+ try:
1240
+ os.replace(tmp, dest)
1241
+ except PermissionError:
1242
+ # Windows cannot replace a file a running guest has open; when the file in place is
1243
+ # already the right zip, the copy is not needed.
1244
+ if os.name != "nt" or not (dest.is_file() and _hash_file(dest) == want):
1245
+ raise
1246
+ Path(tmp).unlink(missing_ok=True)
1247
+ except BaseException:
1248
+ Path(tmp).unlink(missing_ok=True)
1249
+ raise
1250
+ if _hash_file(dest) != want:
1251
+ raise RuntimeError(f"the stdlib zip's copy at {dest} is not the artifact set's")
1252
+ return _stamp(os.lstat(dest))
1253
+
1254
+
1255
+ def _remove_zips() -> None:
1256
+ with _zip_lock:
1257
+ entries = list(_zips.values())
1258
+ _zips.clear()
1259
+ for private, _ in entries:
1260
+ private.remove()
1261
+
1262
+
1263
+ # The identity check (Decision 5): wasmtime reports a preopen's `ino` as SipHash-1-3, zero keys,
1264
+ # over the host folder's two identity numbers as 64-bit words (filesystem.rs:354-371; on Windows
1265
+ # the volume serial and file index, windows.rs:85-91). A wasmtime that hashes otherwise refuses
1266
+ # every grant, loudly; tests/unit/test_runner_grants.py pins it on every OS.
1267
+ _CHECK_WAT = """(module
1268
+ (import "wasi_snapshot_preview1" "fd_prestat_get" (func $pg (param i32 i32) (result i32)))
1269
+ (import "wasi_snapshot_preview1" "fd_prestat_dir_name"
1270
+ (func $pn (param i32 i32 i32) (result i32)))
1271
+ (import "wasi_snapshot_preview1" "fd_filestat_get" (func $st (param i32 i32) (result i32)))
1272
+ (memory (export "memory") 1)
1273
+ (func (export "prestat") (param i32) (result i32) (call $pg (local.get 0) (i32.const 0)))
1274
+ (func (export "name") (param i32 i32) (result i32)
1275
+ (call $pn (local.get 0) (i32.const 1024) (local.get 1)))
1276
+ (func (export "stat") (param i32) (result i32) (call $st (local.get 0) (i32.const 64))))"""
1277
+ _check_modules: dict[bool, Any] = {}
1278
+
1279
+
1280
+ def _rotl(x: int, b: int) -> int:
1281
+ return ((x << b) | (x >> (64 - b))) & _M64
1282
+
1283
+
1284
+ def _siphash13(data: bytes) -> int:
1285
+ v0, v1 = 0x736F6D6570736575, 0x646F72616E646F6D
1286
+ v2, v3 = 0x6C7967656E657261, 0x7465646279746573
1287
+
1288
+ def rounds(n: int) -> None:
1289
+ nonlocal v0, v1, v2, v3
1290
+ for _ in range(n):
1291
+ v0 = (v0 + v1) & _M64
1292
+ v1 = _rotl(v1, 13) ^ v0
1293
+ v0 = _rotl(v0, 32)
1294
+ v2 = (v2 + v3) & _M64
1295
+ v3 = _rotl(v3, 16) ^ v2
1296
+ v0 = (v0 + v3) & _M64
1297
+ v3 = _rotl(v3, 21) ^ v0
1298
+ v2 = (v2 + v1) & _M64
1299
+ v1 = _rotl(v1, 17) ^ v2
1300
+ v2 = _rotl(v2, 32)
1301
+
1302
+ end = len(data) - len(data) % 8
1303
+ for i in range(0, end, 8):
1304
+ m = struct.unpack_from("<Q", data, i)[0]
1305
+ v3 ^= m
1306
+ rounds(1)
1307
+ v0 ^= m
1308
+ last = (len(data) & 0xFF) << 56
1309
+ for i, byte in enumerate(data[end:]):
1310
+ last |= byte << (8 * i)
1311
+ v3 ^= last
1312
+ rounds(1)
1313
+ v0 ^= last
1314
+ v2 ^= 0xFF
1315
+ rounds(3)
1316
+ return v0 ^ v1 ^ v2 ^ v3
1317
+
1318
+
1319
+ def _wasi_ino(identity: Identity) -> int:
1320
+ """The `ino` wasmtime reports for a folder of this identity. Rust sign-extends a negative
1321
+ dev_t, hence the mask; the words are native-endian, as Rust's Hash writes them."""
1322
+ return _siphash13(struct.pack("=QQ", identity[0] & _M64, identity[1] & _M64))
1323
+
1324
+
1325
+ class _Preopen(NamedTuple):
1326
+ label: str # how an error names it: "grant 2 at /data", "root", "the stdlib folder"
1327
+ walked: _Walked
1328
+ guest: str
1329
+ writable: bool
1330
+
1331
+
1332
+ def _check_module(wt: Any, engine: Any, metered: bool) -> Any:
1333
+ with _lock:
1334
+ if metered not in _check_modules:
1335
+ _check_modules[metered] = wt.Module(engine, _CHECK_WAT)
1336
+ return _check_modules[metered]
1337
+
1338
+
1339
+ def _check_preopens(wt: Any, linker: Any, store: Any, check: Any, preopens: list[_Preopen]) -> None:
1340
+ """Before the guest: each preopen 3 + i is the folder the walk checked, by guest name and by
1341
+ identity. A mismatch raises before the guest runs; the walk's holds are released either way."""
1342
+ try:
1343
+ exports = linker.instantiate(store, check).exports(store)
1344
+ memory = exports["memory"]
1345
+ for i, p in enumerate(preopens):
1346
+ fd = 3 + i
1347
+ name = p.guest.encode()
1348
+ ok = exports["prestat"](store, fd) == 0
1349
+ if ok:
1350
+ head = bytes(memory.read(store, 0, 8))
1351
+ ok = head[0] == 0 and struct.unpack_from("<I", head, 4)[0] == len(name)
1352
+ ok = ok and exports["name"](store, fd, len(name)) == 0
1353
+ ok = ok and bytes(memory.read(store, 1024, 1024 + len(name))) == name
1354
+ ok = ok and exports["stat"](store, fd) == 0
1355
+ identity = p.walked.identity
1356
+ if ok and identity is not None:
1357
+ reported = struct.unpack("<Q", bytes(memory.read(store, 72, 80)))[0]
1358
+ ok = reported == _wasi_ino(identity)
1359
+ if not ok:
1360
+ raise RuntimeError(f"{p.label} changed between its check and the call")
1361
+ finally:
1362
+ for p in preopens:
1363
+ p.walked.close()
1364
+
1365
+
1366
+ # A seam for the swap check (tests only): runs after every walk and before wasmtime opens anything.
1367
+ _after_walk: Callable[[], None] | None = None
1368
+
1369
+
1370
+ def _walk_grants(
1371
+ root: str | bool | None, grants: tuple[Grant, ...], found: Artifacts, cache: Path | None
1372
+ ) -> tuple[_Walked | None, list[tuple[str, Grant, _Walked]]]:
1373
+ """Every grant and `root`, walked and judged before any work (Decision 4); a missing root is
1374
+ made only once every rule has passed on its existing chain."""
1375
+ walked_root: _Walked | None = None
1376
+ walked: list[tuple[str, Grant, _Walked]] = []
1377
+ runtime = _runtime_folders(found, cache) # first: it makes the module cache if it can
1378
+ try:
1379
+ if isinstance(root, str):
1380
+ walked_root = _walk("root", root, judge=True, may_make=True)
1381
+ for i, grant in enumerate(grants):
1382
+ where = f"grants[{i}] at {grant.guest}"
1383
+ walked.append((where, grant, _walk(where, os.fspath(grant.host), judge=True)))
1384
+ _judge(walked_root, walked, runtime, _shared_folders())
1385
+ if walked_root is not None and walked_root.missing:
1386
+ try:
1387
+ walked_root = _make_missing("root", walked_root)
1388
+ except OSError as e:
1389
+ raise ValueError(f"root: {root!r} cannot be made: {e.strerror or e}") from None
1390
+ return walked_root, walked
1391
+ except BaseException:
1392
+ for w in [walked_root, *(w for _, _, w in walked)]:
1393
+ if w is not None:
1394
+ w.close()
1395
+ raise
1396
+
1397
+
1398
+ def _runtime_folders(found: Artifacts, cache: Path | None) -> list[tuple[str, Any, Any]]:
1399
+ """The folders later calls and processes trust: this zip's folder (or, before it is made, the
1400
+ temporary folder that will hold it), the artifact set's files, the module cache, and the
1401
+ runtime's own code (this package and wasmtime-py's, whose native library a guest must never
1402
+ replace). The module cache is made first, if it can be, so it is judged as it will be: a root
1403
+ still to be made above a cache still to be made would otherwise come to hold it (the build
1404
+ review, finding 1)."""
1405
+ import wasmtime
1406
+
1407
+ if cache is not None:
1408
+ try:
1409
+ cache.mkdir(mode=0o700, parents=True, exist_ok=True)
1410
+ except OSError:
1411
+ pass # unusable: `_module` then keeps compiled modules in memory
1412
+ zip_dir = _zip_known(found)
1413
+ if zip_dir is not None:
1414
+ folders = [("the runtime's stdlib folder", *_chain(zip_dir))]
1415
+ else:
1416
+ temp = Path(tempfile.gettempdir())
1417
+ folders = [
1418
+ ("the temporary folder, where the runtime's stdlib folder goes", None, _chain(temp)[1])
1419
+ ]
1420
+ parts = {found.manifest.parent, found.snapshot.parent, found.plain.parent}
1421
+ parts.add(found.stdlib_zip.parent)
1422
+ folders += [("the artifact set", *_chain(d)) for d in sorted(parts)]
1423
+ if cache is not None:
1424
+ folders.append(("the module cache", *_chain(cache)))
1425
+ folders.append(("wasi_python's own code", *_chain(Path(__file__).resolve().parent)))
1426
+ folders.append(("wasmtime-py's own code", *_chain(Path(wasmtime.__file__).resolve().parent)))
1427
+ # The host's own interpreter, which the next host process runs (spec 024, S2): its stdlib,
1428
+ # its site-packages and scripts, and the user's site-packages it imports at start-up.
1429
+ import site
1430
+ import sysconfig
1431
+
1432
+ paths = sysconfig.get_paths()
1433
+ keys = ("stdlib", "platstdlib", "purelib", "platlib", "scripts")
1434
+ own = {Path(paths[k]).resolve() for k in keys if k in paths}
1435
+ if site.ENABLE_USER_SITE:
1436
+ own.add(Path(site.getusersitepackages()).resolve())
1437
+ folders += [("the host Python's own code", *_chain(d)) for d in sorted(own) if d.is_dir()]
1438
+ return folders
1439
+
1440
+
1441
+ def _shared_folders() -> list[tuple[str, tuple[Identity, ...]]]:
1442
+ """Folders no grant may be at or above, read-only included: the temporary folder holds every
1443
+ call's private root and stdin (the build review, finding 2)."""
1444
+ temp = Path(tempfile.gettempdir())
1445
+ return [
1446
+ (
1447
+ "the temporary folder, which holds every call's private folders and stdin",
1448
+ _chain(temp)[1],
1449
+ )
1450
+ ]
1451
+
1452
+
1453
+ def _run_checked(
1454
+ wt: Any,
1455
+ c: dict[str, Any],
1456
+ host: Host | None,
1457
+ snapshot: bool,
1458
+ cache_dir: str | os.PathLike[str] | None | bool,
1459
+ artifacts: Artifacts | None,
1460
+ ) -> Completed:
1461
+ found = artifacts if artifacts is not None else find_artifacts()
1462
+ cache = _cache_dir(cache_dir)
1463
+ root = c["root"]
1464
+ walked_root, walked = _walk_grants(root, c["grants"], found, cache)
1465
+ preopens: list[_Preopen] = []
1466
+ private = None
1467
+ try:
1468
+ if _after_walk is not None:
1469
+ _after_walk()
1470
+ binary = found.snapshot if snapshot else found.plain
1471
+ engine = _engine(c["fuel"] is not None)
1472
+ module = _module(engine, c["fuel"] is not None, binary, cache)
1473
+ _sweep_once()
1474
+ zip_dir = _zip_folder(found)
1475
+ preopens.append(
1476
+ _Preopen("the stdlib folder", _walk("", str(zip_dir), judge=False), ZIP_GUEST, False)
1477
+ )
1478
+ root_path = None
1479
+ if root is None:
1480
+ private = _Private("wasi-python-root-") # 0700, this user's own, locked (spec 016)
1481
+ (private.path / "usr" / "local" / "lib").mkdir(parents=True)
1482
+ root_path = private.path
1483
+ preopens.append(
1484
+ _Preopen("the private root", _walk("", str(root_path), judge=False), "/", True)
1485
+ )
1486
+ elif walked_root is not None:
1487
+ root_path = Path(walked_root.path)
1488
+ # Real folders, never a guest's link: without them /usr would vanish from a fresh
1489
+ # root's listing, and the zip's path could resolve into a file a guest wrote. Made
1490
+ # through the walked folder, never its path, which another guest may have swapped
1491
+ # since the walk (spec 024).
1492
+ if walked_root.fd is not None and _dir_fd_ok():
1493
+ os.close(_real_dir_at(walked_root.fd, ("usr", "local", "lib")))
1494
+ else:
1495
+ _real_dir(root_path, ("usr", "local", "lib"))
1496
+ preopens.append(_Preopen("root", walked_root, "/", True))
1497
+ for where, grant, w in walked:
1498
+ preopens.append(_Preopen(where, w, grant.guest, grant.writable))
1499
+ return _call(wt, engine, module, root_path, preopens, c, host)
1500
+ finally:
1501
+ # Windows holds every walked folder until here; a private root can be removed only after.
1502
+ for w in [walked_root, *(w for _, _, w in walked), *(p.walked for p in preopens)]:
1503
+ if w is not None:
1504
+ w.close()
1505
+ if private is not None:
1506
+ private.remove()
1507
+
1508
+
1509
+ def _call(
1510
+ wt: Any,
1511
+ engine: wasmtime.Engine,
1512
+ module: wasmtime.Module,
1513
+ root: Path | None,
1514
+ preopens: list[_Preopen],
1515
+ c: dict[str, Any],
1516
+ host: Host | None,
1517
+ ) -> Completed:
1518
+ """One instance, one program. The store is closed before this returns, trapped or not, so the
1519
+ guest's memory and open files are released at once (a trapped store is otherwise kept alive by
1520
+ a reference cycle until Python's collector runs) and a private root can be removed afterwards
1521
+ (Windows refuses to remove an open directory). Ctrl-C is deferred while the guest runs."""
1522
+ with _deferred_ctrl_c():
1523
+ return _call_deferred(wt, engine, module, root, preopens, c, host)
1524
+
1525
+
1526
+ def _call_deferred(
1527
+ wt: Any,
1528
+ engine: wasmtime.Engine,
1529
+ module: wasmtime.Module,
1530
+ root: Path | None,
1531
+ preopens: list[_Preopen],
1532
+ c: dict[str, Any],
1533
+ host: Host | None,
1534
+ ) -> Completed:
1535
+ tmp, tmp_fd = _home_folder(root, preopens)
1536
+ home = None
1537
+ if tmp is not None:
1538
+ # The fresh HOME is made, and removed, through a handle on /tmp opened without following a
1539
+ # link, so a guest running concurrently on the same folder cannot turn /tmp into a link
1540
+ # between the check and the mkdir (Windows has no dir_fd, and its guests cannot make links).
1541
+ name = f".home-{secrets.token_hex(16)}"
1542
+ try:
1543
+ if tmp_fd is not None:
1544
+ os.mkdir(name, dir_fd=tmp_fd)
1545
+ else:
1546
+ (tmp / name).mkdir()
1547
+ except BaseException:
1548
+ _close(tmp_fd)
1549
+ raise
1550
+ home = (name, f"/tmp/{name}")
1551
+ store = wt.Store(engine)
1552
+ try:
1553
+ guest_home = None if home is None else home[1]
1554
+ return _instance(wt, engine, module, store, preopens, guest_home, c, host)
1555
+ finally:
1556
+ store.close()
1557
+ if home is not None and tmp_fd is not None:
1558
+ shutil.rmtree(home[0], ignore_errors=True, dir_fd=tmp_fd)
1559
+ elif home is not None and tmp is not None:
1560
+ shutil.rmtree(tmp / home[0], ignore_errors=True)
1561
+ _close(tmp_fd)
1562
+
1563
+
1564
+ def _home_folder(root: Path | None, preopens: list[_Preopen]) -> tuple[Path | None, int | None]:
1565
+ """Where HOME is made (Decision 3): a writable grant at /tmp, else the root's tmp, else
1566
+ nowhere. A read-only grant at /tmp hides the root's tmp, so then there is no HOME either."""
1567
+ at_tmp = [p for p in preopens if p.guest == "/tmp"]
1568
+ if at_tmp:
1569
+ p = at_tmp[0]
1570
+ if not p.writable:
1571
+ return None, None
1572
+ fd = p.walked.fd
1573
+ return Path(p.walked.path), (os.dup(fd) if fd is not None and _dir_fd_ok() else None)
1574
+ if root is None:
1575
+ return None, None
1576
+ # TMPDIR, a real folder (a guest's link there is removed), made through the root's walked
1577
+ # folder, never its path (spec 024).
1578
+ fd = next(p.walked.fd for p in preopens if p.guest == "/")
1579
+ if fd is not None and _dir_fd_ok():
1580
+ return root / "tmp", _real_dir_at(fd, ("tmp",))
1581
+ return _real_dir(root, ("tmp",)), None
1582
+
1583
+
1584
+ def _dir_fd_ok() -> bool:
1585
+ return os.mkdir in os.supports_dir_fd and shutil.rmtree.avoids_symlink_attacks is not False
1586
+
1587
+
1588
+ def _open_dir(path: Path) -> int | None:
1589
+ """A handle on a directory, opened without following a link; None where the platform has no
1590
+ dir_fd support (Windows)."""
1591
+ if not _dir_fd_ok():
1592
+ return None
1593
+ flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0)
1594
+ return os.open(path, flags)
1595
+
1596
+
1597
+ def _close(fd: int | None) -> None:
1598
+ if fd is not None:
1599
+ os.close(fd)
1600
+
1601
+
1602
+ def _instance(
1603
+ wt: Any,
1604
+ engine: wasmtime.Engine,
1605
+ module: wasmtime.Module,
1606
+ store: wasmtime.Store,
1607
+ preopens: list[_Preopen],
1608
+ home: str | None,
1609
+ c: dict[str, Any],
1610
+ host: Host | None,
1611
+ ) -> Completed:
1612
+ argv, env, stdin, cancel = c["argv"], c["env"], c["stdin"], c["cancel"]
1613
+ fuel, deadline_s = c["fuel"], c["deadline_s"]
1614
+ memory_bytes, output_bytes = c["memory_bytes"], c["output_bytes"]
1615
+ if fuel is not None:
1616
+ # Fuel is metered from the end of instantiation (spec 008): instantiation runs only
1617
+ # wasmtime's own memory fill, which it charges per byte when it cannot map the image
1618
+ # (macOS when compiled in the process, Windows always); the guest's work starts after.
1619
+ store.set_fuel(UNMETERED)
1620
+ if memory_bytes is not None:
1621
+ store.set_limits(memory_size=memory_bytes)
1622
+ ticks = NO_DEADLINE if deadline_s is None else max(1, math.ceil(deadline_s / TICK_S))
1623
+ # A call with `cancel` is first checked within CANCEL_TICKS ticks, whatever its deadline; its
1624
+ # callback then enforces the deadline too (spec 015).
1625
+ store.set_epoch_deadline(ticks if cancel is None else min(CANCEL_TICKS, ticks))
1626
+ # The ticker's count at which this store's deadline has passed, read after the deadline is set
1627
+ # (spec 010): a wait in poll_oneoff stops there, so it never outlives the deadline.
1628
+ stop_at = None if deadline_s is None else _stop_at(_epoch_ticker.count(), ticks)
1629
+ checks = [] if stop_at is None else [lambda: _epoch_ticker.count() >= stop_at]
1630
+ if cancel is not None:
1631
+ checks.append(cancel.is_set) # a wait ends at a cancel, deadline or not
1632
+ if _ctrl_c_watched():
1633
+ checks.append(lambda: _ctrl_c["pending"]) # a wait on the main thread ends at Ctrl-C
1634
+ waits = _Waits(
1635
+ lambda: time.perf_counter(),
1636
+ time.sleep,
1637
+ (lambda: any(check() for check in checks)) if checks else None,
1638
+ )
1639
+
1640
+ out, err = _Stream("stdout", output_bytes), _Stream("stderr", output_bytes)
1641
+ wasi = wt.WasiConfig()
1642
+ origin_ns = time.perf_counter_ns() # wasmtime's monotonic clock counts from here (spec 010)
1643
+ wasi.argv = [ARGV0, *argv]
1644
+ fresh = {} if home is None else {"HOME": home} # no HOME where there is no /tmp to make it
1645
+ wasi.env = sorted({**BASE_ENV, **fresh, **(env or {})}.items())
1646
+ for p in preopens: # the stdlib folder first; wasi-libc picks by the longest guest prefix
1647
+ wasi.preopen_dir(p.walked.path, p.guest, p.writable)
1648
+ check = _check_module(wt, engine, fuel is not None)
1649
+ keys = _send_outputs(wasi, out, err)
1650
+ scratch = None
1651
+ try:
1652
+ scratch = _Private("wasi-python-") # 0700, this user's own, locked (spec 016)
1653
+ stdin_path = scratch.path / "stdin"
1654
+ stdin_path.write_bytes(stdin or b"")
1655
+ wasi.stdin_file = str(stdin_path)
1656
+ store.set_wasi(wasi)
1657
+ linker = _linker(wt, engine) # held for the whole call: clearing the cache cannot free it
1658
+ call = _Call(store, host, waits, origin_ns, cancel, stop_at)
1659
+ if cancel is not None:
1660
+ _watch(store)
1661
+ with _epoch_ticker(engine, deadline_s is not None or cancel is not None):
1662
+ done = _drive(wt, linker, store, module, call, fuel, preopens, check)
1663
+ fuel_used = _fuel_used(fuel, done, call, store.get_fuel)
1664
+ finally:
1665
+ # The store holds every descriptor the guest opened; a guest can open them until the
1666
+ # host refuses. Close it first, so the clean-up below has descriptors to work with, and
1667
+ # make that clean-up unable to raise (the 0.1.0a2 security review, F1).
1668
+ store.close()
1669
+ with _outputs_lock: # the guest has ended; wasmtime frees the keys later, on its own thread
1670
+ for key in keys:
1671
+ _outputs.pop(key, None)
1672
+ if scratch is not None:
1673
+ scratch.remove()
1674
+ truncated = tuple(s.name for s in (out, err) if s.dropped)
1675
+ return Completed(
1676
+ done.exit_code, bytes(out.data), bytes(err.data), done.trapped, fuel_used, truncated
1677
+ )
1678
+
1679
+
1680
+ def _fuel_used(
1681
+ fuel: int | None, done: _Driven, call: _Call, remaining: Callable[[], int]
1682
+ ) -> int | None:
1683
+ """The instructions the guest ran, or None when the count is meaningless: after an epoch
1684
+ interrupt wasmtime's fuel counter is not brought up to date (spec 001 landing notes), and a
1685
+ stopped call ended on one (spec 011)."""
1686
+ if fuel is None or done.trapped == "deadline":
1687
+ return None
1688
+ if call.stopped is not None or call.ended is not None:
1689
+ return None
1690
+ return fuel - remaining() if done.metered else 0
1691
+
1692
+
1693
+ class _Driven(NamedTuple):
1694
+ exit_code: int | None
1695
+ trapped: str | None
1696
+ metered: bool # the cap is set only once instantiation has succeeded
1697
+ instance: Any
1698
+
1699
+
1700
+ def _drive(
1701
+ wt: Any,
1702
+ linker: Any,
1703
+ store: Any,
1704
+ module: Any,
1705
+ call: _Call,
1706
+ fuel: int | None,
1707
+ preopens: list[_Preopen] = (), # type: ignore[assignment]
1708
+ check: Any = None,
1709
+ ) -> _Driven:
1710
+ """Instantiate and run the guest with its call on this thread's stack. What a host function
1711
+ caught and recorded comes first: an exit or a trap is the call's result, and any other
1712
+ exception (Ctrl-C, a host's cancellation) is raised here, on the thread that made the call
1713
+ (spec 011)."""
1714
+ exit_code: int | None = None
1715
+ trapped: str | None = None
1716
+ metered = False
1717
+ instance = None
1718
+ stack = _stack()
1719
+ stack.append(call)
1720
+ _calls.lost = False
1721
+ try:
1722
+ if call.cancel is not None and call.cancel.is_set(): # cancelled before the guest ran
1723
+ call.ended = "cancelled"
1724
+ return _Driven(None, "cancelled", False, None)
1725
+ if preopens:
1726
+ _check_preopens(wt, linker, store, check, list(preopens)) # spec 017, Decision 5
1727
+ instance = linker.instantiate(store, module)
1728
+ if fuel is not None:
1729
+ store.set_fuel(fuel)
1730
+ metered = True
1731
+ exports = instance.exports(store)
1732
+ if "_initialize" in exports:
1733
+ exports["_initialize"](store)
1734
+ exit_code = exports["run"](store)
1735
+ except wt.ExitTrap as e:
1736
+ exit_code = e.code
1737
+ except wt.Trap as e:
1738
+ trapped = _trap_name(e)
1739
+ except wt.WasmtimeError as e:
1740
+ trapped = _first_line(str(e))
1741
+ finally:
1742
+ stack.pop()
1743
+ lost, _calls.lost = getattr(_calls, "lost", False), False
1744
+ if lost:
1745
+ raise RuntimeError("wasi_python lost track of a call")
1746
+ cause = call.stopped
1747
+ if cause is not None:
1748
+ exit_code, trapped = None, None
1749
+ if isinstance(cause, wt.ExitTrap):
1750
+ exit_code = cause.code
1751
+ elif isinstance(cause, wt.Trap):
1752
+ trapped = _trap_name(cause)
1753
+ elif isinstance(cause, wt.WasmtimeError):
1754
+ trapped = _first_line(str(cause))
1755
+ else:
1756
+ raise cause
1757
+ elif call.ended is not None: # the epoch callback stopped it (spec 015)
1758
+ exit_code, trapped = None, call.ended
1759
+ return _Driven(exit_code, trapped, metered, instance)
1760
+
1761
+
1762
+ class _Call:
1763
+ """One call's state, which the shared host functions find on the calling thread's stack: its
1764
+ door, its poll_oneoff, its store's context address, and the first exception they caught; for
1765
+ a call with `cancel`, the event, the ticker count its deadline passes at, and the outcome the
1766
+ epoch callback recorded (spec 015)."""
1767
+
1768
+ def __init__(
1769
+ self,
1770
+ store: Any,
1771
+ host: Host | None,
1772
+ waits: Any,
1773
+ origin_ns: int,
1774
+ cancel: threading.Event | None = None,
1775
+ stop_at: int | None = None,
1776
+ ) -> None:
1777
+ self.context = _context_address(store)
1778
+ self.door = _door(host)
1779
+ self.poll = _poll_oneoff(origin_ns, waits)
1780
+ self.stopped: BaseException | None = None
1781
+ self.cancel = cancel
1782
+ self.stop_at = stop_at
1783
+ self.ended: str | None = None # "cancelled" or "deadline", from the epoch callback
1784
+
1785
+ def stop(self, caller: Any, cause: BaseException) -> None:
1786
+ """Record the cause and stop the guest: an epoch deadline of 0, set through the caller's
1787
+ context, traps it at its next function entry, with nothing raised through wasmtime-py."""
1788
+ if self.stopped is None:
1789
+ self.stopped = cause
1790
+ from wasmtime import _ffi
1791
+
1792
+ _ffi.wasmtime_context_set_epoch_deadline(caller._context(), 0)
1793
+
1794
+
1795
+ _calls = threading.local()
1796
+
1797
+
1798
+ def _stack() -> list[_Call]:
1799
+ stack = getattr(_calls, "stack", None)
1800
+ if stack is None:
1801
+ stack = _calls.stack = []
1802
+ return stack
1803
+
1804
+
1805
+ def _context_address(storelike: Any) -> int:
1806
+ return ctypes.addressof(storelike._context().contents)
1807
+
1808
+
1809
+ def _shared(kind: str, refused: int, stopped: int) -> Callable[..., int]:
1810
+ """A host function defined once per engine. It serves the call on top of this thread's
1811
+ stack, and only when the caller is that call's store; it never raises into wasmtime-py,
1812
+ whose exceptions travel by a process-wide slot (spec 011)."""
1813
+
1814
+ def function(caller: Any, *args: int) -> int:
1815
+ call = None
1816
+ try:
1817
+ stack = _stack()
1818
+ call = stack[-1] if stack else None
1819
+ if call is None or _context_address(caller) != call.context:
1820
+ return refused
1821
+ if call.stopped is None and _ctrl_c_watched() and _ctrl_c["pending"]:
1822
+ _ctrl_c["pending"] = False
1823
+ raise KeyboardInterrupt
1824
+ if call.stopped is not None:
1825
+ return stopped
1826
+ answer = getattr(call, kind)(caller, *args)
1827
+ if _ctrl_c_watched() and _ctrl_c["pending"]: # Ctrl-C during a wait
1828
+ _ctrl_c["pending"] = False
1829
+ raise KeyboardInterrupt
1830
+ if call.cancel is not None and call.cancel.is_set():
1831
+ _epoch_now(caller) # the epoch callback names it at the guest's next entry
1832
+ return answer
1833
+ except BaseException as cause:
1834
+ try:
1835
+ if call is not None:
1836
+ call.stop(caller, cause)
1837
+ except BaseException:
1838
+ pass
1839
+ return stopped
1840
+
1841
+ return function
1842
+
1843
+
1844
+ def _epoch_now(caller: Any) -> None:
1845
+ """An epoch deadline of 0 for the caller's store: its guest stops, or meets its epoch
1846
+ callback, at its next function entry."""
1847
+ from wasmtime import _ffi
1848
+
1849
+ _ffi.wasmtime_context_set_epoch_deadline(caller._context(), 0)
1850
+
1851
+
1852
+ def _epoch_answer(context: Any, delta: Any) -> int:
1853
+ """The epoch callback's decision for the call on top of this thread's stack, in order: a
1854
+ recorded stop holds, a deferred Ctrl-C becomes a stop, then a cancel, then the deadline;
1855
+ otherwise continue for up to CANCEL_TICKS ticks. 0 continues; an error (an int address) stops
1856
+ the guest (spec 015)."""
1857
+ call = None
1858
+ try:
1859
+ try:
1860
+ stack = _stack()
1861
+ call = stack[-1] if stack else None
1862
+ if call is not None and ctypes.addressof(context.contents) != call.context:
1863
+ call = None
1864
+ except BaseException:
1865
+ call = None
1866
+ if call is None:
1867
+ _calls.lost = True
1868
+ return _stop_error(delta)
1869
+ if call.stopped is not None:
1870
+ return _stop_error(delta)
1871
+ if _ctrl_c_watched() and _ctrl_c["pending"]:
1872
+ _ctrl_c["pending"] = False
1873
+ call.stopped = KeyboardInterrupt()
1874
+ return _stop_error(delta)
1875
+ if call.cancel is not None and call.cancel.is_set():
1876
+ call.ended = "cancelled"
1877
+ return _stop_error(delta)
1878
+ count = _epoch_ticker.count()
1879
+ if call.stop_at is not None and count >= call.stop_at:
1880
+ call.ended = "deadline"
1881
+ return _stop_error(delta)
1882
+ left = CANCEL_TICKS if call.stop_at is None else call.stop_at - count
1883
+ delta[0] = max(1, min(CANCEL_TICKS, left))
1884
+ return 0
1885
+ except BaseException as cause:
1886
+ try:
1887
+ if call is None:
1888
+ _calls.lost = True
1889
+ elif call.stopped is None:
1890
+ call.stopped = cause
1891
+ except BaseException:
1892
+ pass
1893
+ return _stop_error(delta)
1894
+
1895
+
1896
+ def _stop_error(delta: Any) -> int:
1897
+ """A new wasmtime error, which wasmtime owns once returned. If none can be made, continue for
1898
+ one tick, never 0, so the next callback tries again (spec 015's build review, F2)."""
1899
+ try:
1900
+ from wasmtime import _ffi
1901
+
1902
+ error = ctypes.cast(_ffi.wasmtime_error_new(b"stopped by wasi_python"), ctypes.c_void_p)
1903
+ if error.value:
1904
+ return error.value
1905
+ except BaseException:
1906
+ pass
1907
+ try:
1908
+ delta[0] = 1
1909
+ except BaseException:
1910
+ pass
1911
+ return 0
1912
+
1913
+
1914
+ def _on_epoch(context: Any, data: Any, delta: Any, kind: Any) -> int:
1915
+ return _epoch_answer(context, delta) # never writes `kind`: wasmtime panics on a bad value
1916
+
1917
+
1918
+ _epoch_callback: Any = None # the one C callback, made on first use
1919
+ _epoch_callback_lock = threading.Lock()
1920
+
1921
+
1922
+ def _watch(store: Any) -> None:
1923
+ """Give a store the epoch callback, with a NULL finalizer: no per-call ctypes object exists,
1924
+ and the callback finds its call through the thread's stack (spec 015)."""
1925
+ global _epoch_callback
1926
+ from wasmtime import _bindings, _ffi
1927
+
1928
+ argtypes = _bindings._wasmtime_store_epoch_deadline_callback.argtypes
1929
+ with _epoch_callback_lock: # one object for the process: a store must never outlive its own
1930
+ if _epoch_callback is None:
1931
+ _epoch_callback = argtypes[1](_on_epoch)
1932
+ callback = _epoch_callback
1933
+ _ffi.wasmtime_store_epoch_deadline_callback(store.ptr(), callback, None, argtypes[3]())
1934
+
1935
+
1936
+ _linkers: dict[Any, Any] = {}
1937
+
1938
+
1939
+ def _linker(wt: Any, engine: wasmtime.Engine) -> Any:
1940
+ """The engine's one linker: WASI, the shadowed poll_oneoff and the door, defined once, so a
1941
+ call adds nothing to wasmtime-py's process-wide table of host functions (spec 011)."""
1942
+ with _lock:
1943
+ linker = _linkers.get(engine)
1944
+ if linker is None:
1945
+ linker = wt.Linker(engine)
1946
+ linker.define_wasi()
1947
+ linker.allow_shadowing = True
1948
+ i32 = wt.ValType.i32()
1949
+ ty = wt.FuncType([i32] * 4, [i32])
1950
+ poll = _shared("poll", EINVAL, EINVAL)
1951
+ linker.define_func("wasi_snapshot_preview1", "poll_oneoff", ty, poll, True)
1952
+ linker.define_func("host", "call", ty, _shared("door", NO_HOST, HOST_FAULT), True)
1953
+ _linkers[engine] = linker
1954
+ return linker
1955
+
1956
+
1957
+ # The guest's stdout and stderr, by a key that is never reused. wasmtime calls the two functions
1958
+ # below on its own threads, and frees a key when it drops the stream; wasmtime-py's own table for
1959
+ # custom outputs is not used, since it has no lock (spec 011).
1960
+ _outputs: dict[int, _Stream] = {}
1961
+ _outputs_lock = threading.RLock() # a finalizer the collector runs inside a locked block
1962
+ _output_keys = itertools.count(1)
1963
+
1964
+
1965
+ @ctypes.CFUNCTYPE(
1966
+ ctypes.c_ssize_t, ctypes.c_void_p, ctypes.POINTER(ctypes.c_ubyte), ctypes.c_size_t
1967
+ )
1968
+ def _output_write(key: int | None, data: Any, size: int) -> int:
1969
+ try:
1970
+ with _outputs_lock:
1971
+ stream = _outputs.get(key or 0)
1972
+ return size if stream is None else stream.write(ctypes.string_at(data, size))
1973
+ except BaseException:
1974
+ return size # never raises, never prints: the bytes are dropped and the write succeeds
1975
+
1976
+
1977
+ @ctypes.CFUNCTYPE(None, ctypes.c_void_p)
1978
+ def _output_free(key: int | None) -> None:
1979
+ with _outputs_lock:
1980
+ _outputs.pop(key or 0, None)
1981
+
1982
+
1983
+ def _send_outputs(wasi: Any, out: _Stream, err: _Stream) -> tuple[int, int]:
1984
+ from wasmtime import _ffi
1985
+
1986
+ with _outputs_lock:
1987
+ keys = next(_output_keys), next(_output_keys)
1988
+ _outputs[keys[0]], _outputs[keys[1]] = out, err
1989
+ _ffi.wasi_config_set_stdout_custom(wasi.ptr(), _output_write, keys[0], _output_free)
1990
+ _ffi.wasi_config_set_stderr_custom(wasi.ptr(), _output_write, keys[1], _output_free)
1991
+ return keys
1992
+
1993
+
1994
+ # Ctrl-C while a guest runs on the main thread. Python's default handler raises KeyboardInterrupt
1995
+ # wherever the main thread next runs Python, and when that is the first line of wasmtime-py's
1996
+ # trampoline the exception cannot be caught and the process aborts (spec 011's review, EXECUTED:
1997
+ # exit 134). So while run() holds the main thread under the default handler, Ctrl-C only sets a
1998
+ # flag: the shared host functions turn it into a recorded stop, a wait ends at it, and a flag no
1999
+ # host function saw is raised once the call has ended.
2000
+ _ctrl_c = {"pending": False}
2001
+
2002
+
2003
+ def _note_ctrl_c(signum: int, frame: Any) -> None:
2004
+ _ctrl_c["pending"] = True
2005
+
2006
+
2007
+ def _ctrl_c_watched() -> bool:
2008
+ import signal
2009
+
2010
+ main = threading.current_thread() is threading.main_thread()
2011
+ return main and signal.getsignal(signal.SIGINT) is _note_ctrl_c
2012
+
2013
+
2014
+ @contextmanager
2015
+ def _deferred_ctrl_c() -> Iterator[None]:
2016
+ import signal
2017
+
2018
+ main = threading.current_thread() is threading.main_thread()
2019
+ if not main or signal.getsignal(signal.SIGINT) is not signal.default_int_handler:
2020
+ yield # another thread, an embedder's own handler, or a nested call: nothing to defer
2021
+ return
2022
+ _ctrl_c["pending"] = False
2023
+ signal.signal(signal.SIGINT, _note_ctrl_c)
2024
+ try:
2025
+ yield
2026
+ finally:
2027
+ if signal.getsignal(signal.SIGINT) is _note_ctrl_c: # a host may have set its own
2028
+ signal.signal(signal.SIGINT, signal.default_int_handler)
2029
+ pending, _ctrl_c["pending"] = _ctrl_c["pending"], False
2030
+ if pending:
2031
+ raise KeyboardInterrupt
2032
+
2033
+
2034
+ EINVAL, ENOMEM, EBADF, EFAULT = 28, 48, 8, 21 # WASI preview1 errno values
2035
+ MAX_SUBSCRIPTIONS = 4096 # one poll_oneoff's host work is bounded (spec 010)
2036
+ SUBSCRIPTION = (
2037
+ "<QB7xIxxxxQQH6x" # userdata, tag, clock id or fd, timeout, precision, flags: 48 bytes
2038
+ )
2039
+ EVENT = "<QHB5xQH6x" # userdata, error, type, nbytes, flags: 32 bytes
2040
+ CLOCK, FD_READ, FD_WRITE = 0, 1, 2
2041
+ REALTIME, MONOTONIC = 0, 1
2042
+ ABSTIME = 1
2043
+
2044
+
2045
+ @dataclass(frozen=True)
2046
+ class _Plan:
2047
+ """What one poll_oneoff answers: an errno, or the events to write after waiting wait_ns."""
2048
+
2049
+ errno: int | None
2050
+ events: tuple[tuple[int, int, int, int, int], ...] = ()
2051
+ wait_ns: int = 0
2052
+
2053
+
2054
+ def _stop_at(count: int, ticks: int) -> int:
2055
+ """The ticker's count at which a deadline of `ticks`, set when the count was `count`, has
2056
+ passed. The engine's epoch can be one round ahead of the count (the ticker advances the engines
2057
+ first), hence the + 1: a wait never ends before the epoch reaches the deadline (spec 010)."""
2058
+ return count + ticks + 1
2059
+
2060
+
2061
+ def _poll_plan(raw: bytes, n: int, realtime_ns: int, monotonic_ns: int) -> _Plan:
2062
+ """poll_oneoff as wasmtime 49 answers it (crates/wasi/src/p1.rs), for the guest's files and
2063
+ stdio: a descriptor subscription is ready at once (stdin for writing, stdout and stderr for
2064
+ reading and the preopened root answer EBADF); otherwise the shortest clock wins, and every
2065
+ clock due by then is reported, with nbytes 0 as wasmtime writes it (spec 010)."""
2066
+ if n == 0:
2067
+ return _Plan(EINVAL)
2068
+ if n > MAX_SUBSCRIPTIONS:
2069
+ return _Plan(ENOMEM)
2070
+ ready, clocks = [], []
2071
+ for userdata, tag, which, timeout, _precision, flags in struct.iter_unpack(SUBSCRIPTION, raw):
2072
+ if tag == CLOCK:
2073
+ if which not in (REALTIME, MONOTONIC):
2074
+ return _Plan(EINVAL)
2075
+ if flags & ABSTIME:
2076
+ timeout -= realtime_ns if which == REALTIME else monotonic_ns
2077
+ clocks.append((max(0, timeout), userdata))
2078
+ elif tag in (FD_READ, FD_WRITE):
2079
+ if (
2080
+ which == 3
2081
+ or (tag == FD_READ and which in (1, 2))
2082
+ or (tag == FD_WRITE and which == 0)
2083
+ ):
2084
+ return _Plan(EBADF)
2085
+ ready.append((userdata, 0, tag, 1, 0))
2086
+ else:
2087
+ return _Plan(EINVAL)
2088
+ if ready:
2089
+ return _Plan(None, tuple(ready))
2090
+ wait_ns = min(timeout for timeout, _ in clocks)
2091
+ due = tuple((userdata, 0, CLOCK, 0, 0) for timeout, userdata in clocks if timeout <= wait_ns)
2092
+ return _Plan(None, due, wait_ns)
2093
+
2094
+
2095
+ class _Waits:
2096
+ """A wait that the call's deadline ends: in slices of at most TICK_S while a deadline is set,
2097
+ one sleep otherwise. `now`, `sleep` and `deadline_passed` are injected so the judge counts the
2098
+ slices without measuring time (spec 010)."""
2099
+
2100
+ def __init__(
2101
+ self,
2102
+ now: Callable[[], float],
2103
+ sleep: Callable[[float], None],
2104
+ deadline_passed: Callable[[], bool] | None,
2105
+ ) -> None:
2106
+ self.now, self.sleep, self.deadline_passed = now, sleep, deadline_passed
2107
+
2108
+ def wait(self, seconds: float) -> None:
2109
+ if self.deadline_passed is None:
2110
+ if seconds > 0:
2111
+ self.sleep(seconds)
2112
+ return
2113
+ until = self.now() + seconds
2114
+ while (left := until - self.now()) > 0 and not self.deadline_passed():
2115
+ self.sleep(min(left, TICK_S))
2116
+
2117
+
2118
+ def _poll_oneoff(
2119
+ origin_ns: int,
2120
+ waits: _Waits,
2121
+ realtime_ns: Callable[[], int] = time.time_ns,
2122
+ monotonic_ns: Callable[[], int] = time.perf_counter_ns,
2123
+ ) -> Callable[..., int]:
2124
+ """The shadowed poll_oneoff. It never raises into wasmtime-py (an exception there travels by a
2125
+ process-wide slot another call can take): a wait ended by the deadline answers normally, and
2126
+ the guest traps at its next epoch check. A BaseException goes to the shared function, which
2127
+ records it and stops the guest (spec 011). The clocks are injectable for the judge."""
2128
+
2129
+ def call(caller, in_ptr: int, out_ptr: int, n: int, nevents_ptr: int) -> int:
2130
+ try:
2131
+ memory = caller.get("memory")
2132
+ in_ptr, out_ptr, n, nevents_ptr = (
2133
+ v & MASK32 for v in (in_ptr, out_ptr, n, nevents_ptr)
2134
+ )
2135
+ if n == 0 or n > MAX_SUBSCRIPTIONS:
2136
+ return EINVAL if n == 0 else ENOMEM
2137
+ size = memory.data_len(caller)
2138
+ if in_ptr + 48 * n > size or out_ptr + 32 * n > size or nevents_ptr + 4 > size:
2139
+ return EFAULT
2140
+ raw = bytes(memory.read(caller, in_ptr, in_ptr + 48 * n))
2141
+ plan = _poll_plan(raw, n, realtime_ns(), monotonic_ns() - origin_ns)
2142
+ if plan.errno is not None:
2143
+ return plan.errno
2144
+ if plan.wait_ns:
2145
+ waits.wait(plan.wait_ns / 1e9)
2146
+ events = b"".join(struct.pack(EVENT, *e) for e in plan.events)
2147
+ memory.write(caller, events, out_ptr)
2148
+ memory.write(caller, struct.pack("<I", len(plan.events)), nevents_ptr)
2149
+ return 0
2150
+ except Exception:
2151
+ return EINVAL # anything else is recorded by the shared function (spec 011)
2152
+
2153
+ return call
2154
+
2155
+
2156
+ class _Stream:
2157
+ """A guest's stdout or stderr, kept up to a cap. Bytes past it are dropped, never buffered,
2158
+ and still reported as written, so the guest's write succeeds and is not retried (wasmtime-py
2159
+ takes the callback's return value as the count written)."""
2160
+
2161
+ def __init__(self, name: str, cap: int | None) -> None:
2162
+ self.name, self.cap, self.data, self.dropped = name, cap, bytearray(), False
2163
+
2164
+ def write(self, data: bytes) -> int:
2165
+ room = len(data) if self.cap is None else max(0, self.cap - len(self.data))
2166
+ if room < len(data):
2167
+ self.dropped = True
2168
+ self.data.extend(data[:room])
2169
+ return len(data)
2170
+
2171
+
2172
+ def _first_line(text: str) -> str:
2173
+ lines = text.strip().splitlines()
2174
+ return lines[0] if lines else "trap"
2175
+
2176
+
2177
+ def _trap_name(e: wasmtime.Trap) -> str:
2178
+ import wasmtime as wt
2179
+
2180
+ if e.trap_code == wt.TrapCode.OUT_OF_FUEL:
2181
+ return "out of fuel"
2182
+ if e.trap_code == wt.TrapCode.INTERRUPT:
2183
+ return "deadline"
2184
+ if e.trap_code is not None: # by its kind, not wasmtime's heading (spec 016, Decision 3)
2185
+ return _trap_code_name(e.trap_code)
2186
+ return _first_line(e.message)
2187
+
2188
+
2189
+ def _trap_code_name(code: Any) -> str:
2190
+ """`TrapCode.STACK_OVERFLOW` is "stack overflow"."""
2191
+ return code.name.lower().replace("_", " ")
2192
+
2193
+
2194
+ def _door(host: Host | None) -> Callable[..., int]:
2195
+ """host::call(ptr, len, out_ptr_ptr, out_len_ptr) -> 0, -1 or -2 (spec 003, Decision 5)."""
2196
+
2197
+ import wasmtime as wt
2198
+
2199
+ def call(caller, ptr: int, length: int, out_ptr_ptr: int, out_len_ptr: int) -> int:
2200
+ if host is None:
2201
+ return NO_HOST
2202
+ try:
2203
+ return frame(caller, ptr, length, out_ptr_ptr, out_len_ptr)
2204
+ except (wt.Trap, wt.WasmtimeError):
2205
+ raise # a trap or exit inside host_alloc: the shared function records it (spec 011)
2206
+ except Exception:
2207
+ return HOST_FAULT # nothing a frame or a host does escapes run() (spec 006)
2208
+
2209
+ def frame(caller, ptr: int, length: int, out_ptr_ptr: int, out_len_ptr: int) -> int:
2210
+ memory, alloc = caller.get("memory"), caller.get("host_alloc")
2211
+ if memory is None or alloc is None:
2212
+ return HOST_FAULT
2213
+ ptr, length = ptr & MASK32, length & MASK32 # i32 values: addresses are unsigned
2214
+ out_ptr_ptr, out_len_ptr = out_ptr_ptr & MASK32, out_len_ptr & MASK32
2215
+ size = memory.data_len(caller)
2216
+ if ptr + length > size or out_ptr_ptr + 4 > size or out_len_ptr + 4 > size:
2217
+ return HOST_FAULT # memory.read would clamp a frame that runs past memory
2218
+ response = host(bytes(memory.read(caller, ptr, ptr + length)))
2219
+ if not isinstance(response, (bytes, bytearray, memoryview)):
2220
+ return HOST_FAULT
2221
+ if memoryview(response).nbytes > MAX_RESPONSE:
2222
+ return HOST_FAULT # checked in bytes, before the copy
2223
+ response = bytes(response)
2224
+ p = alloc(caller, len(response)) & MASK32
2225
+ if not p or p + len(response) > memory.data_len(caller):
2226
+ return HOST_FAULT
2227
+ memory.write(caller, response, p)
2228
+ memory.write(caller, struct.pack("<I", p), out_ptr_ptr)
2229
+ memory.write(caller, struct.pack("<I", len(response)), out_len_ptr)
2230
+ return 0
2231
+
2232
+ return call
2233
+
2234
+
2235
+ def _stamp(st: os.stat_result) -> tuple[int, ...]:
2236
+ return (st.st_dev, st.st_ino, st.st_size, st.st_mtime_ns, st.st_ctime_ns)
2237
+
2238
+
2239
+ def _real_dir_at(fd: int, parts: tuple[str, ...]) -> int:
2240
+ """A descriptor on fd/parts…, each a real directory, each step taken from the folder before:
2241
+ a link or file a guest left in the way is removed, never followed (spec 024). The caller
2242
+ closes the result; `fd` stays open."""
2243
+ here = os.dup(fd)
2244
+ try:
2245
+ for part in parts:
2246
+ try:
2247
+ st = os.stat(part, dir_fd=here, follow_symlinks=False)
2248
+ except FileNotFoundError:
2249
+ st = None
2250
+ if st is not None and not stat.S_ISDIR(st.st_mode):
2251
+ os.unlink(part, dir_fd=here) # a symbolic link or a file: removed, never followed
2252
+ st = None
2253
+ if st is None:
2254
+ try:
2255
+ os.mkdir(part, dir_fd=here)
2256
+ except FileExistsError: # a concurrent call made it
2257
+ pass
2258
+ child = os.open(part, _DIR, dir_fd=here) # O_NOFOLLOW: a link put there fails
2259
+ os.close(here)
2260
+ here = child
2261
+ except BaseException:
2262
+ os.close(here)
2263
+ raise
2264
+ return here
2265
+
2266
+
2267
+ def _real_dir(root: Path, parts: tuple[str, ...]) -> Path:
2268
+ """root/parts…, each a real directory: a link or file a guest left in the way is removed.
2269
+ Windows only, where the walk holds every folder of the root's path (spec 017, Decision 5) and
2270
+ guests cannot make links; elsewhere `_real_dir_at`. The root is never made again here."""
2271
+ here = root
2272
+ for part in parts:
2273
+ here = here / part
2274
+ try:
2275
+ st = os.lstat(here)
2276
+ except FileNotFoundError:
2277
+ st = None
2278
+ if st is not None and not stat.S_ISDIR(st.st_mode):
2279
+ here.unlink() # a symbolic link or a file: removed, never followed
2280
+ st = None
2281
+ if st is None:
2282
+ try:
2283
+ here.mkdir()
2284
+ except FileExistsError: # a concurrent call made it
2285
+ if not stat.S_ISDIR(os.lstat(here).st_mode):
2286
+ raise
2287
+ return here
2288
+
2289
+
2290
+ def _hash_file(path: Path) -> str:
2291
+ flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_BINARY", 0)
2292
+ h = hashlib.sha256()
2293
+ with os.fdopen(os.open(path, flags), "rb") as f:
2294
+ for block in iter(lambda: f.read(1 << 20), b""):
2295
+ h.update(block)
2296
+ return h.hexdigest()
2297
+
2298
+
2299
+ _versions: dict[Path, str] = {}
2300
+
2301
+
2302
+ def _version_tag(manifest: Path) -> str:
2303
+ """'315' from MANIFEST's `python 3.15.0` line: the stdlib zip's guest name is python315.zip."""
2304
+ if manifest not in _versions:
2305
+ for line in manifest.read_text().splitlines():
2306
+ if line.startswith("python "):
2307
+ major, minor = line.split()[1].split(".")[:2]
2308
+ _versions[manifest] = f"{major}{minor}"
2309
+ break
2310
+ else:
2311
+ raise ValueError(f"{manifest} has no python line")
2312
+ return _versions[manifest]
2313
+
2314
+
2315
+ def _cache_dir(given: str | os.PathLike[str] | None | bool) -> Path | None:
2316
+ if given is False:
2317
+ return None
2318
+ if given is not None and given is not True:
2319
+ return Path(given)
2320
+ if os.environ.get(CACHE_ENV):
2321
+ return Path(os.environ[CACHE_ENV])
2322
+ home = Path.home()
2323
+ if sys.platform == "darwin":
2324
+ return home / "Library" / "Caches" / "wasi_python"
2325
+ if sys.platform == "win32":
2326
+ base = os.environ.get("LOCALAPPDATA")
2327
+ return (Path(base) if base else home / "AppData" / "Local") / "wasi_python" / "cache"
2328
+ base = os.environ.get("XDG_CACHE_HOME")
2329
+ return (Path(base) if base else home / ".cache") / "wasi_python"
2330
+
2331
+
2332
+ _lock = threading.Lock()
2333
+ _engines: dict[bool, wasmtime.Engine] = {}
2334
+ _modules: dict[tuple[bool, str], wasmtime.Module] = {}
2335
+ _digests: dict[tuple[str, int, int], str] = {}
2336
+ _counts = {"compiled": 0, "loaded": 0, "written": 0}
2337
+
2338
+
2339
+ def stats() -> dict[str, int]:
2340
+ """How this process got its compiled modules: compiled from the binary, loaded from a .cwasm,
2341
+ and .cwasm files written. A diagnostic; the judge's cache check reads it."""
2342
+ with _lock:
2343
+ return dict(_counts)
2344
+
2345
+
2346
+ @atexit.register
2347
+ def _release() -> None:
2348
+ """Remove this process's zip folders, and free the cached linkers, modules and engines while
2349
+ wasmtime's bindings still exist."""
2350
+ _remove_zips()
2351
+ _check_modules.clear()
2352
+ _linkers.clear()
2353
+ _modules.clear()
2354
+ _engines.clear()
2355
+
2356
+
2357
+ def _engine(metered: bool) -> wasmtime.Engine:
2358
+ """Two engines per process: fuel metering changes the compiled code."""
2359
+ import wasmtime as wt
2360
+
2361
+ with _lock:
2362
+ if metered not in _engines:
2363
+ _fork["used"] = True # before the engine exists: a fork from here on is refused
2364
+ cfg = wt.Config()
2365
+ cfg.consume_fuel = metered
2366
+ cfg.epoch_interruption = True
2367
+ _engines[metered] = wt.Engine(cfg)
2368
+ return _engines[metered]
2369
+
2370
+
2371
+ def _digest(path: Path) -> str:
2372
+ st = path.stat()
2373
+ key = (str(path.resolve()), st.st_size, st.st_mtime_ns)
2374
+ if key not in _digests:
2375
+ h = hashlib.sha256()
2376
+ with path.open("rb") as f:
2377
+ for block in iter(lambda: f.read(1 << 20), b""):
2378
+ h.update(block)
2379
+ _digests[key] = h.hexdigest()
2380
+ return _digests[key]
2381
+
2382
+
2383
+ def _module(
2384
+ engine: wasmtime.Engine, metered: bool, binary: Path, cache: Path | None
2385
+ ) -> wasmtime.Module:
2386
+ """The compiled module: from this process, else the .cwasm cache, else compiled and cached.
2387
+
2388
+ A .cwasm is native code that wasmtime runs as it finds it, so the cache is used only where no
2389
+ one else can write: on POSIX, a directory and files owned by this user and writable by no one
2390
+ else (the directory is made 0700, the files 0600). Anything else is ignored, with a warning,
2391
+ and the module is compiled in this process."""
2392
+ import wasmtime as wt
2393
+
2394
+ digest = _digest(binary)
2395
+ with _lock:
2396
+ mod = _modules.get((metered, digest))
2397
+ guard = _compiling.setdefault((metered, digest), threading.Lock())
2398
+ if mod is not None:
2399
+ return mod
2400
+ # One thread compiles or loads; the others wait for it (spec 011). Only this lock is held
2401
+ # while compiling: the epoch ticker takes _lock every round.
2402
+ with guard:
2403
+ with _lock:
2404
+ mod = _modules.get((metered, digest))
2405
+ if mod is not None:
2406
+ return mod
2407
+ return _module_miss(wt, engine, metered, binary, cache, digest)
2408
+
2409
+
2410
+ _compiling: dict[tuple[bool, str], threading.Lock] = {}
2411
+
2412
+
2413
+ def _module_miss(
2414
+ wt: Any, engine: wasmtime.Engine, metered: bool, binary: Path, cache: Path | None, digest: str
2415
+ ) -> wasmtime.Module:
2416
+ flavor = "fuel" if metered else "plain"
2417
+ mod = None
2418
+ cwasm = None
2419
+ if cache is not None and _private(cache, is_dir=True):
2420
+ cwasm = cache / f"{digest[:32]}-wasmtime{_wasmtime_version()}-{flavor}.cwasm"
2421
+ if cwasm is not None and cwasm.exists() and _private(cwasm, is_dir=False):
2422
+ try:
2423
+ mod = wt.Module.deserialize_file(engine, str(cwasm))
2424
+ with _lock:
2425
+ _counts["loaded"] += 1
2426
+ except wt.WasmtimeError:
2427
+ mod = None # stale or foreign: compiled again below and replaced
2428
+ cwasm.unlink(missing_ok=True)
2429
+ if mod is None:
2430
+ mod = wt.Module.from_file(engine, str(binary))
2431
+ with _lock:
2432
+ _counts["compiled"] += 1
2433
+ if cwasm is not None and not cwasm.exists():
2434
+ fd, tmp = tempfile.mkstemp(prefix=".cwasm-", dir=cache) # created 0600
2435
+ try:
2436
+ with os.fdopen(fd, "wb") as f:
2437
+ f.write(mod.serialize())
2438
+ os.replace(tmp, cwasm)
2439
+ with _lock:
2440
+ _counts["written"] += 1
2441
+ except BaseException:
2442
+ Path(tmp).unlink(missing_ok=True)
2443
+ raise
2444
+ with _lock:
2445
+ _modules[(metered, digest)] = mod
2446
+ return mod
2447
+
2448
+
2449
+ def _private(path: Path, *, is_dir: bool) -> bool:
2450
+ """Whether a cache path is this user's alone (POSIX); a directory is made if absent."""
2451
+ if is_dir:
2452
+ path.mkdir(mode=0o700, parents=True, exist_ok=True)
2453
+ if os.name != "posix":
2454
+ return True # the default cache is under the user's own profile
2455
+ st = os.lstat(path)
2456
+ kind_ok = stat.S_ISDIR(st.st_mode) if is_dir else stat.S_ISREG(st.st_mode)
2457
+ if kind_ok and st.st_uid == os.getuid() and not st.st_mode & 0o022:
2458
+ return True
2459
+ warnings.warn(
2460
+ f"wasi_python: not using the module cache at {path}: it must be a "
2461
+ f"{'directory' if is_dir else 'file'} owned by this user and writable by no one else",
2462
+ RuntimeWarning,
2463
+ stacklevel=5,
2464
+ )
2465
+ return False
2466
+
2467
+
2468
+ def _wasmtime_version() -> str:
2469
+ from importlib.metadata import version
2470
+
2471
+ return version("wasmtime")
2472
+
2473
+
2474
+ class _epoch_ticker:
2475
+ """While at least one call with a deadline runs, one thread per process advances every
2476
+ engine's epoch once per TICK_S of wall time elapsed, catching up when a sleep runs long, so a
2477
+ deadline of n ticks ends close to n * TICK_S."""
2478
+
2479
+ _active = 0
2480
+ _count = 0 # epochs advanced so far, updated after the engines are (spec 010)
2481
+ _thread: threading.Thread | None = None
2482
+ _cond = threading.Condition()
2483
+
2484
+ def __init__(self, engine: wasmtime.Engine, wanted: bool) -> None:
2485
+ self.wanted = wanted
2486
+
2487
+ @classmethod
2488
+ def count(cls) -> int:
2489
+ return cls._count
2490
+
2491
+ def __enter__(self) -> None:
2492
+ if not self.wanted:
2493
+ return
2494
+ cls = type(self)
2495
+ with cls._cond:
2496
+ cls._active += 1
2497
+ if cls._thread is None:
2498
+ cls._thread = threading.Thread(target=cls._tick, daemon=True)
2499
+ cls._thread.start()
2500
+ cls._cond.notify_all()
2501
+
2502
+ def __exit__(self, *exc: object) -> None:
2503
+ if not self.wanted:
2504
+ return
2505
+ with type(self)._cond:
2506
+ type(self)._active -= 1
2507
+
2508
+ @classmethod
2509
+ def _tick(cls) -> None:
2510
+ while True:
2511
+ with cls._cond:
2512
+ while cls._active == 0:
2513
+ cls._cond.wait()
2514
+ start, done = time.monotonic(), 0
2515
+ while True:
2516
+ with cls._cond:
2517
+ if cls._active == 0:
2518
+ break
2519
+ due = int((time.monotonic() - start) / TICK_S)
2520
+ with _lock:
2521
+ engines = list(_engines.values())
2522
+ for _ in range(due - done):
2523
+ for engine in engines:
2524
+ engine.increment_epoch()
2525
+ cls._count += 1
2526
+ done = due
2527
+ time.sleep(TICK_S / 4)