testflows.machine 0.1.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- testflows/machine/v1/__init__.py +280 -0
- testflows/machine/v1/_args.py +528 -0
- testflows/machine/v1/_requires.py +113 -0
- testflows/machine/v1/_service.py +125 -0
- testflows/machine/v1/_version.py +16 -0
- testflows/machine/v1/_warn.py +38 -0
- testflows/machine/v1/_when.py +55 -0
- testflows/machine/v1/client.py +2077 -0
- testflows/machine/v1/disks.py +669 -0
- testflows/machine/v1/errors.py +256 -0
- testflows/machine/v1/login.py +107 -0
- testflows/machine/v1/ping.py +70 -0
- testflows/machine/v1/results.py +3231 -0
- testflows/machine/v1/runs.py +7648 -0
- testflows/machine/v1/sessions.py +875 -0
- testflows_machine-0.1.4.dist-info/METADATA +125 -0
- testflows_machine-0.1.4.dist-info/RECORD +21 -0
- testflows_machine-0.1.4.dist-info/WHEEL +5 -0
- testflows_machine-0.1.4.dist-info/licenses/LICENSE +203 -0
- testflows_machine-0.1.4.dist-info/licenses/NOTICE +7 -0
- testflows_machine-0.1.4.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,2077 @@
|
|
|
1
|
+
# Copyright 2026 Katteli Inc.
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
"""The client: where the SDK starts, and the one object a program holds.
|
|
15
|
+
|
|
16
|
+
The SDK mirrors the CLI. A top-level command is a method of `Client`, a
|
|
17
|
+
command group is an attribute, and a command that takes a run is a method of
|
|
18
|
+
`Run`: `machine create` is `client.create()`, `machine sessions use` is
|
|
19
|
+
`client.sessions.use()`, and `machine start app` is `run.start()`.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
import builtins
|
|
25
|
+
import contextlib
|
|
26
|
+
import dataclasses
|
|
27
|
+
from datetime import datetime, timezone
|
|
28
|
+
import time
|
|
29
|
+
import os
|
|
30
|
+
from pathlib import Path
|
|
31
|
+
from typing import Any, Callable, Generator, Literal, Optional, Sequence, overload
|
|
32
|
+
|
|
33
|
+
from testflows.machine._core import api
|
|
34
|
+
from testflows.machine._core import branches as _branches
|
|
35
|
+
from testflows.machine._core import cloud_run_source as _cloud_run_source
|
|
36
|
+
from testflows.machine._core import ipc as _ipc
|
|
37
|
+
from testflows.machine._core import lifecycle as _lifecycle
|
|
38
|
+
from testflows.machine._core import machines as _machines
|
|
39
|
+
from testflows.machine._core import parsers as _parsers
|
|
40
|
+
from testflows.machine._core import ping as _ping
|
|
41
|
+
from testflows.machine._core import storage_ops as _storage_ops
|
|
42
|
+
from testflows.machine._core import run_source as _run_source
|
|
43
|
+
from testflows.machine._core import sessions as _sessions
|
|
44
|
+
from testflows.machine._core import sorting as _sorting
|
|
45
|
+
from testflows.machine._core import wait as _wait
|
|
46
|
+
from testflows.machine._core import wire as _wire
|
|
47
|
+
|
|
48
|
+
from . import _args
|
|
49
|
+
from . import _service
|
|
50
|
+
from . import _warn
|
|
51
|
+
from . import _when
|
|
52
|
+
from . import results
|
|
53
|
+
from .errors import ApiError, InvalidArgument, ServiceUnavailable, _translated
|
|
54
|
+
from .disks import Disk, Disks
|
|
55
|
+
from .ping import Ping, _public
|
|
56
|
+
from .runs import Run
|
|
57
|
+
from .sessions import Session, SessionClosing, Sessions
|
|
58
|
+
|
|
59
|
+
#: Set to 1, a client given no session works with the service on this
|
|
60
|
+
#: machine.
|
|
61
|
+
_LOCAL_ENV = "TESTFLOWS_MACHINE_LOCAL"
|
|
62
|
+
|
|
63
|
+
#: The values `create` accepts for its enumerated options.
|
|
64
|
+
_MEM_TYPES = ("auto", "hugepages")
|
|
65
|
+
_MEM_MODES = ("auto", "map", "lazy")
|
|
66
|
+
_CONSOLES = ("serial", "virtio")
|
|
67
|
+
_FRAMEBUFFERS = ("vnc", "gtk", "sdl", "none")
|
|
68
|
+
|
|
69
|
+
#: A notice's deadline as the client keys it, and as `Session.closing_at`
|
|
70
|
+
#: holds it when the service cannot be read: ISO 8601 in UTC.
|
|
71
|
+
_NOTICE_TIME = "%Y-%m-%dT%H:%M:%S"
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _deadline(value: str) -> datetime:
|
|
75
|
+
"""A deadline as the service or the record writes it, in UTC. One with no
|
|
76
|
+
offset is UTC already."""
|
|
77
|
+
at = datetime.fromisoformat(value)
|
|
78
|
+
if at.tzinfo is None:
|
|
79
|
+
return at.replace(tzinfo=timezone.utc)
|
|
80
|
+
return at.astimezone(timezone.utc)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _deadline_key(at: datetime) -> str:
|
|
84
|
+
"""How the client remembers a deadline it told."""
|
|
85
|
+
return at.astimezone(timezone.utc).strftime(_NOTICE_TIME)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
#: `runs --parents`: where a run reads its data parents from.
|
|
89
|
+
_PARENTS = ("local", "remote")
|
|
90
|
+
|
|
91
|
+
#: What `create` accepts, the bounds the CLI holds the same flags to. A vCPU
|
|
92
|
+
#: count and a memory size the machine cannot serve are refused here rather
|
|
93
|
+
#: than at the spawn.
|
|
94
|
+
_MAX_CPUS = _wire.MAX_VCPUS
|
|
95
|
+
_MAX_MEM_MB = 1048576
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
class Client:
|
|
99
|
+
"""The one object a program holds, as the `machine` command is.
|
|
100
|
+
|
|
101
|
+
A top-level CLI command is a method here, and a command group is an
|
|
102
|
+
attribute: `machine create` is `create`, `machine runs` is `runs`,
|
|
103
|
+
`machine sessions use` is `sessions.use`. A command that takes a run is
|
|
104
|
+
a method of the `Run` these return.
|
|
105
|
+
|
|
106
|
+
A run lives in a session: a machine the account rents, with its own
|
|
107
|
+
CPUs, memory and storage. The client has a current session, as the CLI
|
|
108
|
+
does: `session=` sets it, as `-s` does, and `sessions.use` or
|
|
109
|
+
`sessions.connect` change it for this client only, never for the CLI.
|
|
110
|
+
Commands that need a machine go to the current session at the time of
|
|
111
|
+
the call. With no session, reading the account's runs still works:
|
|
112
|
+
`runs`, `branches`, `marks`, and a run's `describe`, `lineage` and
|
|
113
|
+
`checkpoints` read the account's catalog.
|
|
114
|
+
|
|
115
|
+
Nothing is contacted until a call needs it. Every call signs in with
|
|
116
|
+
TESTFLOWS_MACHINE_API_KEY when it is set, otherwise the login stored by
|
|
117
|
+
`machine login` or `login.verify`.
|
|
118
|
+
|
|
119
|
+
Example:
|
|
120
|
+
>>> import testflows.machine.v1 as machine
|
|
121
|
+
>>> client = machine.Client()
|
|
122
|
+
>>> with client.sessions.create("ci-1", cpus=4) as session:
|
|
123
|
+
... client.sessions.use(session)
|
|
124
|
+
... run = client.create("app", disk="app")
|
|
125
|
+
... run.run(until="tasks")
|
|
126
|
+
... run.focus.add("/app")
|
|
127
|
+
... run.run(until="/app:exit")
|
|
128
|
+
... run.checkpoint("done")
|
|
129
|
+
|
|
130
|
+
A session can be scheduled to stop for maintenance of its host. The
|
|
131
|
+
client reads the notice at the start of a call into the session, at most
|
|
132
|
+
once every ten minutes from the service, and tells the program once for
|
|
133
|
+
each notice: through `on_session_closing` when it is given, or with a
|
|
134
|
+
`SessionClosing` warning. The session's `closing_at` and
|
|
135
|
+
`closing_reason` say when and why. Use `Session.migrate` before then to
|
|
136
|
+
keep its machines running. A call that is already running when the
|
|
137
|
+
deadline passes is not told.
|
|
138
|
+
|
|
139
|
+
Example:
|
|
140
|
+
>>> def closing(session):
|
|
141
|
+
... session.migrate()
|
|
142
|
+
>>> client = machine.Client(on_session_closing=closing)
|
|
143
|
+
|
|
144
|
+
Args:
|
|
145
|
+
session (str, optional): The current session, by name, id or unique
|
|
146
|
+
id prefix, as `-s` names it.
|
|
147
|
+
api_url (str, optional): The service's URL, as `--api-url` gives it;
|
|
148
|
+
defaults to TESTFLOWS_MACHINE_API_URL, else the public service.
|
|
149
|
+
on_session_closing (callable, optional): Called with the `Session`
|
|
150
|
+
once for each maintenance notice, before the call that found it
|
|
151
|
+
goes on. A call it makes goes on in the session it leaves
|
|
152
|
+
current, so a callback that migrates moves the call with it. An
|
|
153
|
+
exception it raises is raised from that call, and the notice is
|
|
154
|
+
not told again.
|
|
155
|
+
"""
|
|
156
|
+
|
|
157
|
+
def __init__(
|
|
158
|
+
self,
|
|
159
|
+
*,
|
|
160
|
+
session: Optional[str] = None,
|
|
161
|
+
api_url: Optional[str] = None,
|
|
162
|
+
on_session_closing: Optional[Callable[[Session], None]] = None,
|
|
163
|
+
) -> None:
|
|
164
|
+
local = os.environ.get(_LOCAL_ENV) == "1"
|
|
165
|
+
if local and session is not None:
|
|
166
|
+
raise InvalidArgument(
|
|
167
|
+
f"{_LOCAL_ENV} is set, so a client cannot take a session."
|
|
168
|
+
f" Unset it to work with session {session!r}."
|
|
169
|
+
)
|
|
170
|
+
self._local = local
|
|
171
|
+
self._session_ref = session
|
|
172
|
+
self._session: Optional[Session] = None
|
|
173
|
+
#: The session each moved one went to, `sessions migrate`, so every
|
|
174
|
+
#: Session object of a moved session names where it went.
|
|
175
|
+
self._moved_ids: dict[str, str] = {}
|
|
176
|
+
self._api_url = api_url
|
|
177
|
+
self._api: Optional[tuple[api.client.Client, dict[str, Any]]] = None
|
|
178
|
+
self._services: dict[str, _service._Service] = {}
|
|
179
|
+
self._on_session_closing = on_session_closing
|
|
180
|
+
#: The deadlines each session's notice was told for, so a notice is
|
|
181
|
+
#: told once, and again only when it changes. Both the record's and
|
|
182
|
+
#: the session's as read again, which can differ while the record lags.
|
|
183
|
+
self._closing_told: dict[str, set[str]] = {}
|
|
184
|
+
|
|
185
|
+
@classmethod
|
|
186
|
+
def _signed_in(
|
|
187
|
+
cls, client: api.client.Client, creds: dict[str, Any], api_url: Optional[str]
|
|
188
|
+
) -> "Client":
|
|
189
|
+
"""A client with no session, over a login already made."""
|
|
190
|
+
made = cls(api_url=api_url)
|
|
191
|
+
made._api = (client, creds)
|
|
192
|
+
return made
|
|
193
|
+
|
|
194
|
+
# -- the session -----------------------------------------------------------
|
|
195
|
+
|
|
196
|
+
@property
|
|
197
|
+
def session(self) -> Optional[Session]:
|
|
198
|
+
"""Session: The current session, read on first use; None when the
|
|
199
|
+
client has none.
|
|
200
|
+
|
|
201
|
+
Raises:
|
|
202
|
+
NotFound: No session matches the name the client was given.
|
|
203
|
+
InvalidArgument: The name matches more than one session.
|
|
204
|
+
"""
|
|
205
|
+
if self._session_ref is None:
|
|
206
|
+
return None
|
|
207
|
+
if self._session is None:
|
|
208
|
+
self._session = self.sessions.show(self._session_ref)
|
|
209
|
+
return self._session
|
|
210
|
+
|
|
211
|
+
@property
|
|
212
|
+
def sessions(self) -> Sessions:
|
|
213
|
+
"""Sessions: The account's sessions, as `machine sessions`: create,
|
|
214
|
+
list, use and delete them.
|
|
215
|
+
|
|
216
|
+
Raises:
|
|
217
|
+
NotSignedIn: There are no credentials.
|
|
218
|
+
"""
|
|
219
|
+
return Sessions(self)
|
|
220
|
+
|
|
221
|
+
@property
|
|
222
|
+
def disks(self) -> Disks:
|
|
223
|
+
"""Disks: The account's disks, as `machine disks`: push, list and
|
|
224
|
+
delete them.
|
|
225
|
+
|
|
226
|
+
Raises:
|
|
227
|
+
NotSignedIn: There are no credentials.
|
|
228
|
+
"""
|
|
229
|
+
return Disks(*self._account())
|
|
230
|
+
|
|
231
|
+
@property
|
|
232
|
+
def storage(self) -> "Storage":
|
|
233
|
+
"""Storage: What the account stores, as `machine storage`."""
|
|
234
|
+
return Storage(self)
|
|
235
|
+
|
|
236
|
+
@property
|
|
237
|
+
def preload(self) -> "Preload":
|
|
238
|
+
"""Preload: Copies onto the current session's disk, as `machine
|
|
239
|
+
preload`."""
|
|
240
|
+
return Preload(self)
|
|
241
|
+
|
|
242
|
+
@property
|
|
243
|
+
def offload(self) -> "Offload":
|
|
244
|
+
"""Offload: Copies dropped from the current session's disk, as
|
|
245
|
+
`machine offload`."""
|
|
246
|
+
return Offload(self)
|
|
247
|
+
|
|
248
|
+
@property
|
|
249
|
+
def marks(self) -> "Marks":
|
|
250
|
+
"""Marks: The marks of every tree, as `machine marks`."""
|
|
251
|
+
return Marks(self)
|
|
252
|
+
|
|
253
|
+
@property
|
|
254
|
+
def delete(self) -> "Delete":
|
|
255
|
+
"""Delete: Deleting every run of the session, or those booting a
|
|
256
|
+
disk, as `machine delete -a`; `cleanup` resumes interrupted ones."""
|
|
257
|
+
return Delete(self)
|
|
258
|
+
|
|
259
|
+
@property
|
|
260
|
+
def runs(self) -> "Runs":
|
|
261
|
+
"""Runs: The account's runs, as `machine runs`: call it to list them,
|
|
262
|
+
and index it to get one."""
|
|
263
|
+
return Runs(self)
|
|
264
|
+
|
|
265
|
+
# -- runs ------------------------------------------------------------------
|
|
266
|
+
|
|
267
|
+
def create(
|
|
268
|
+
self,
|
|
269
|
+
name: Optional[str] = None,
|
|
270
|
+
*,
|
|
271
|
+
disk: "Disk | str",
|
|
272
|
+
like: "Run | str | None" = None,
|
|
273
|
+
labels: Optional[dict[str, str]] = None,
|
|
274
|
+
keep_output: "Sequence[str] | str | None" = None,
|
|
275
|
+
drop_output: "Sequence[str] | str | None" = None,
|
|
276
|
+
cpus: Optional[int] = None,
|
|
277
|
+
mem: Optional[int] = None,
|
|
278
|
+
mem_type: Optional[str] = None,
|
|
279
|
+
console: Optional[str] = None,
|
|
280
|
+
mem_mode: Optional[str] = None,
|
|
281
|
+
fs_sync: Optional[bool] = None,
|
|
282
|
+
fs_dirsync: Optional[bool] = None,
|
|
283
|
+
fb: Optional[str] = None,
|
|
284
|
+
rng: Optional[bool] = None,
|
|
285
|
+
rng_seed: Optional[int] = None,
|
|
286
|
+
daemon: Optional[bool] = None,
|
|
287
|
+
single_step: Optional[bool] = None,
|
|
288
|
+
limit: Optional[int] = None,
|
|
289
|
+
backstop_limit: Optional[int] = None,
|
|
290
|
+
pin_cpu: Optional[int] = None,
|
|
291
|
+
vtime_base: Optional[int] = None,
|
|
292
|
+
vtime_rate: Optional[str] = None,
|
|
293
|
+
vtime_io_cost: Optional[int] = None,
|
|
294
|
+
vtime_rdtsc_cost: Optional[int] = None,
|
|
295
|
+
vtime_pause_cost: Optional[int] = None,
|
|
296
|
+
auto_checkpoints: Optional[int] = None,
|
|
297
|
+
side_dump: Optional[bool] = None,
|
|
298
|
+
side_trace: Optional[bool] = None,
|
|
299
|
+
mem_hash: Optional[bool] = None,
|
|
300
|
+
disk_hash: Optional[bool] = None,
|
|
301
|
+
repeatable: Optional[bool] = None,
|
|
302
|
+
limit_margin: Optional[int] = None,
|
|
303
|
+
timer_slack: Optional[str] = None,
|
|
304
|
+
instr_margin: Optional[int] = None,
|
|
305
|
+
dispatch_bound: Optional[str] = None,
|
|
306
|
+
fast_backstop: Optional[int] = None,
|
|
307
|
+
irq_wait: Optional[int] = None,
|
|
308
|
+
wait: bool = True,
|
|
309
|
+
timeout: Optional[float] = None,
|
|
310
|
+
) -> Run:
|
|
311
|
+
"""Create a run on a disk and bring its machine up, as
|
|
312
|
+
`machine create` does, in the current session.
|
|
313
|
+
|
|
314
|
+
The machine boots nothing on its own: it waits for commands, and
|
|
315
|
+
`Run.run` drives it. Every option left None takes its value from
|
|
316
|
+
`like`, a run whose options are copied, else the default: 1 vCPU,
|
|
317
|
+
256 MB, an RNG device, fast execution with no limit. Used in a
|
|
318
|
+
`with` block, the run is deleted when the block exits, unless the
|
|
319
|
+
block calls `Run.keep`.
|
|
320
|
+
|
|
321
|
+
Example:
|
|
322
|
+
>>> run = client.create("app", disk="app", cpus=2,
|
|
323
|
+
... rng_seed=7, labels={"suite": "nightly"})
|
|
324
|
+
>>> twin = client.create(disk="app", like=run, cpus=4)
|
|
325
|
+
>>> with client.create(disk="app") as scratch:
|
|
326
|
+
... scratch.run(until="tasks")
|
|
327
|
+
|
|
328
|
+
Args:
|
|
329
|
+
name (str, optional): The run's name; one is generated when
|
|
330
|
+
omitted.
|
|
331
|
+
disk (str): The disk to boot, by name; `Client.disks` lists
|
|
332
|
+
them and `Disks.push` adds one.
|
|
333
|
+
like (Run or str, optional): A run whose options to copy.
|
|
334
|
+
labels (dict[str, str], optional): Labels to attach, for
|
|
335
|
+
`runs(labels=...)`; an empty value removes one `like`
|
|
336
|
+
carried.
|
|
337
|
+
keep_output (str or list[str], optional): Sources whose output
|
|
338
|
+
the run stores though it is dropped: an output file or a
|
|
339
|
+
directory of them by its path, such as
|
|
340
|
+
`/output/drop/app/debug.log`, or a pattern such as
|
|
341
|
+
`/output/drop/*/*.log`. The run keeps them for its whole
|
|
342
|
+
life, and a branch of it has them unless it names its own.
|
|
343
|
+
drop_output (str or list[str], optional): Sources whose output
|
|
344
|
+
the run does not store: a path, a device such as `ttyS1`,
|
|
345
|
+
or a pattern. `Run.fork` describes the patterns and which
|
|
346
|
+
of a kept and a dropped name decides.
|
|
347
|
+
cpus (int, optional): vCPUs.
|
|
348
|
+
mem (int, optional): Guest memory in MB.
|
|
349
|
+
mem_type (str, optional): Guest memory backing: `auto` or
|
|
350
|
+
`hugepages`.
|
|
351
|
+
mem_mode (str, optional): How a cold restore maps memory:
|
|
352
|
+
`auto`, `map` or `lazy`.
|
|
353
|
+
console (str, optional): The device the run's console is:
|
|
354
|
+
`serial` for the first serial port, `ttyS0`, or `virtio`
|
|
355
|
+
for the virtio console, `hvc0`. Defaults to `serial`.
|
|
356
|
+
fs_sync (bool, optional): Mount the root filesystem `sync`, so
|
|
357
|
+
writes reach the disk at once.
|
|
358
|
+
fs_dirsync (bool, optional): Mount it `dirsync`.
|
|
359
|
+
fb (str, optional): Open a framebuffer to watch the guest:
|
|
360
|
+
`vnc`, `gtk`, `sdl` or `none`.
|
|
361
|
+
rng (bool, optional): Give the guest an RNG device, whose seed
|
|
362
|
+
a branch can change; on by default, False leaves it out.
|
|
363
|
+
rng_seed (int, optional): The RNG's seed; the same seed gives
|
|
364
|
+
the same random bytes.
|
|
365
|
+
daemon (bool, optional): Wait for commands; False runs the
|
|
366
|
+
machine to completion. The run keeps it, and every later
|
|
367
|
+
start and fork repeats it. Defaults to True.
|
|
368
|
+
single_step (bool, optional): Run in step mode, for runs that
|
|
369
|
+
do not wait for commands. Kept the same way. Defaults to
|
|
370
|
+
False.
|
|
371
|
+
limit (int, optional): Conditional branches a fast dispatch may
|
|
372
|
+
run, steps in step mode; 0 for no limit.
|
|
373
|
+
backstop_limit (int, optional): Step modes only: instructions a
|
|
374
|
+
dispatch may run without stepping.
|
|
375
|
+
pin_cpu (int, optional): Pin this machine to a host CPU; the
|
|
376
|
+
default lets it pick the least-loaded. A CPU belongs to one
|
|
377
|
+
machine, so the run does not keep it.
|
|
378
|
+
vtime_base (int, optional): The guest's clock at boot, in Unix
|
|
379
|
+
seconds; defaults to 2026-03-01.
|
|
380
|
+
vtime_rate (str, optional): Virtual nanoseconds per instruction,
|
|
381
|
+
`n` or `n/d`.
|
|
382
|
+
vtime_io_cost (int, optional): Virtual nanoseconds each I/O exit
|
|
383
|
+
costs.
|
|
384
|
+
vtime_rdtsc_cost (int, optional): Virtual nanoseconds each RDTSC
|
|
385
|
+
exit costs.
|
|
386
|
+
vtime_pause_cost (int, optional): Virtual nanoseconds each PAUSE
|
|
387
|
+
costs.
|
|
388
|
+
auto_checkpoints (int, optional): Checkpoint every this many
|
|
389
|
+
entries; 0 for never.
|
|
390
|
+
side_dump (bool, optional): Record the machine's state with every
|
|
391
|
+
entry, which `Run.dump(entry=...)` reads back; costs disk.
|
|
392
|
+
side_trace (bool, optional): Record an execution trace with
|
|
393
|
+
every entry.
|
|
394
|
+
mem_hash (bool, optional): Hash the machine's memory at every
|
|
395
|
+
entry, so that a replay compares memory too. Off by default.
|
|
396
|
+
disk_hash (bool, optional): Hash the machine's disk at every
|
|
397
|
+
entry, so that a replay compares the disk too. Off by default.
|
|
398
|
+
repeatable (bool, optional): Record so that the same run,
|
|
399
|
+
recorded again from the same start with the same commands,
|
|
400
|
+
records the same entries. Fast execution stops exactly at
|
|
401
|
+
its limit and at timer deadlines, which records more
|
|
402
|
+
slowly. The run keeps it, and every start and fork of it
|
|
403
|
+
records the same way. None: the session's default. A
|
|
404
|
+
run's describe shows the value it was given.
|
|
405
|
+
limit_margin (int, optional): How many branches short of each
|
|
406
|
+
stop the branch counter interrupts. Fewer is faster, too
|
|
407
|
+
few misses stops. Defaults to the skid of the host's
|
|
408
|
+
branch counter.
|
|
409
|
+
timer_slack (str, optional): How late a timer may be in code
|
|
410
|
+
that takes no exit, a duration such as `20ms`, or `0` for
|
|
411
|
+
no later than its deadline. More is faster. Defaults to
|
|
412
|
+
`20ms`.
|
|
413
|
+
instr_margin (int, optional): How many instructions short of a
|
|
414
|
+
stop placed by an instruction count the instruction counter
|
|
415
|
+
interrupts. A replay that passes such a stop needs more.
|
|
416
|
+
Defaults to the host's: 50,000 on AMD, 3,630 on Intel.
|
|
417
|
+
dispatch_bound (str, optional): The host time a fast dispatch
|
|
418
|
+
may run before it stops, a duration such as `500ms`. A stop
|
|
419
|
+
it places does not repeat in another recording. Defaults to
|
|
420
|
+
`1s`.
|
|
421
|
+
fast_backstop (int, optional): How many instructions a fast
|
|
422
|
+
dispatch may run with no other stop, in code that retires no
|
|
423
|
+
branch. Defaults to 100,000,000.
|
|
424
|
+
irq_wait (int, optional): How many branches a dispatch runs
|
|
425
|
+
while an interrupt waits for the guest to take it. Defaults
|
|
426
|
+
to 200.
|
|
427
|
+
wait (bool, optional): Return once the machine accepts commands.
|
|
428
|
+
Defaults to True.
|
|
429
|
+
timeout (float, optional): Seconds to wait; None waits without a
|
|
430
|
+
limit.
|
|
431
|
+
|
|
432
|
+
Returns:
|
|
433
|
+
Run: The new run.
|
|
434
|
+
|
|
435
|
+
Raises:
|
|
436
|
+
InvalidArgument: An option or label is not valid, or the client
|
|
437
|
+
has no session.
|
|
438
|
+
AlreadyExists: A run has this name.
|
|
439
|
+
NotFound: The disk or the `like` run does not exist.
|
|
440
|
+
Timeout: The machine was not ready within `timeout`.
|
|
441
|
+
ApiError: The machine failed to start; the message says why.
|
|
442
|
+
"""
|
|
443
|
+
# The same bounds the CLI holds these to, so a number the machine would
|
|
444
|
+
# refuse is refused here, with the range in the message.
|
|
445
|
+
_args.bounded("cpus", cpus, 1, _MAX_CPUS)
|
|
446
|
+
_args.bounded("mem", mem, 1, _MAX_MEM_MB)
|
|
447
|
+
_args.bounded("limit", limit, 0)
|
|
448
|
+
_args.bounded("backstop_limit", backstop_limit, 0)
|
|
449
|
+
_args.bounded("auto_checkpoints", auto_checkpoints, 0)
|
|
450
|
+
_args.bounded("pin_cpu", pin_cpu, _args.MIN_PIN_CPU, _args.MAX_PIN_CPU)
|
|
451
|
+
explicit: dict[str, Any] = {
|
|
452
|
+
key: value
|
|
453
|
+
for key, value in (
|
|
454
|
+
("cpus", cpus),
|
|
455
|
+
("memory", mem),
|
|
456
|
+
("mem_type", mem_type),
|
|
457
|
+
("mem_mode", mem_mode),
|
|
458
|
+
("console", console),
|
|
459
|
+
("fs_sync", fs_sync),
|
|
460
|
+
("fs_dirsync", fs_dirsync),
|
|
461
|
+
("fb", fb),
|
|
462
|
+
("rng", rng),
|
|
463
|
+
("rng_seed", rng_seed),
|
|
464
|
+
("daemon", daemon),
|
|
465
|
+
("single_step", single_step),
|
|
466
|
+
("limit", limit),
|
|
467
|
+
("backstop_limit", backstop_limit),
|
|
468
|
+
("pin_cpu", pin_cpu),
|
|
469
|
+
("vtime_base", vtime_base),
|
|
470
|
+
("vtime_io_cost", vtime_io_cost),
|
|
471
|
+
("vtime_rdtsc_cost", vtime_rdtsc_cost),
|
|
472
|
+
("vtime_pause_cost", vtime_pause_cost),
|
|
473
|
+
("auto_checkpoint_interval", auto_checkpoints),
|
|
474
|
+
("side_dump", side_dump),
|
|
475
|
+
("side_trace", side_trace),
|
|
476
|
+
("mem_hash", mem_hash),
|
|
477
|
+
("disk_hash", disk_hash),
|
|
478
|
+
("repeatable", repeatable),
|
|
479
|
+
("limit_margin", _args.limit_margin(limit_margin)),
|
|
480
|
+
("timer_slack", _args.timer_slack(timer_slack)),
|
|
481
|
+
("instr_margin", _args.stop_count("instr_margin", instr_margin)),
|
|
482
|
+
("dispatch_bound", _args.dispatch_bound(dispatch_bound)),
|
|
483
|
+
("fast_backstop", _args.stop_count("fast_backstop", fast_backstop)),
|
|
484
|
+
("irq_wait", _args.irq_wait(irq_wait)),
|
|
485
|
+
)
|
|
486
|
+
if value is not None
|
|
487
|
+
}
|
|
488
|
+
_args.check_choice("mem_type", mem_type, _MEM_TYPES)
|
|
489
|
+
_args.check_choice("mem_mode", mem_mode, _MEM_MODES)
|
|
490
|
+
_args.check_choice("console", console, _CONSOLES)
|
|
491
|
+
_args.check_choice("fb", fb, _FRAMEBUFFERS)
|
|
492
|
+
with _translated():
|
|
493
|
+
for label in labels or {}:
|
|
494
|
+
_machines.validate_label_name(label)
|
|
495
|
+
if vtime_rate is not None:
|
|
496
|
+
num, den = _parsers.vtime_rate_type(vtime_rate)
|
|
497
|
+
explicit.update(vtime_rate_num=num, vtime_rate_den=den)
|
|
498
|
+
service = self._service()
|
|
499
|
+
resolved = service.disk(_args.disk_ref(disk))
|
|
500
|
+
with service.client() as client, _translated():
|
|
501
|
+
kwargs = dict(_machines.RUN_DEFAULTS)
|
|
502
|
+
if like is not None:
|
|
503
|
+
like_id = like.id if isinstance(like, Run) else like
|
|
504
|
+
kwargs.update(
|
|
505
|
+
_machines.describe_to_run_kwargs(client.describe(like_id))
|
|
506
|
+
)
|
|
507
|
+
kwargs.update(explicit)
|
|
508
|
+
# `like` first, then the disk's own labels, then this call's: a
|
|
509
|
+
# disk labelled `version=1.2` is not relabelled 1.0 by copying
|
|
510
|
+
# the options of a run that booted an older one.
|
|
511
|
+
merged = {
|
|
512
|
+
**(kwargs.pop("labels", None) or {}),
|
|
513
|
+
**resolved.labels,
|
|
514
|
+
**(labels or {}),
|
|
515
|
+
}
|
|
516
|
+
merged = {key: value for key, value in merged.items() if value}
|
|
517
|
+
if merged:
|
|
518
|
+
kwargs["labels"] = merged
|
|
519
|
+
# What `like` copied stands for each list this call does not
|
|
520
|
+
# give.
|
|
521
|
+
kwargs["output"] = _args.stored(keep_output, drop_output).over(
|
|
522
|
+
kwargs.get("output") or _ipc.STORED_BY_DIRECTORY
|
|
523
|
+
)
|
|
524
|
+
made = _machines.run_machine(
|
|
525
|
+
client, name=name or "", disk=resolved.id, **kwargs
|
|
526
|
+
)
|
|
527
|
+
if wait:
|
|
528
|
+
_lifecycle.wait_ready(client, made, timeout_s=timeout)
|
|
529
|
+
run_name = _machines.display_name(client, made.run_id)
|
|
530
|
+
return Run(self, made.run_id, run_name)
|
|
531
|
+
|
|
532
|
+
@overload
|
|
533
|
+
def ops(
|
|
534
|
+
self,
|
|
535
|
+
*,
|
|
536
|
+
op_id: Optional[int] = ...,
|
|
537
|
+
states: Optional[builtins.list[str]] = ...,
|
|
538
|
+
types: Optional[builtins.list[str]] = ...,
|
|
539
|
+
since: "_when.When | None" = ...,
|
|
540
|
+
until: "_when.When | None" = ...,
|
|
541
|
+
limit: Optional[int] = ...,
|
|
542
|
+
offset: int = ...,
|
|
543
|
+
watch: Literal[False] = ...,
|
|
544
|
+
timeout: Optional[float] = ...,
|
|
545
|
+
) -> builtins.list[results.Op]: ...
|
|
546
|
+
|
|
547
|
+
@overload
|
|
548
|
+
def ops(
|
|
549
|
+
self,
|
|
550
|
+
*,
|
|
551
|
+
op_id: Optional[int] = ...,
|
|
552
|
+
states: Optional[builtins.list[str]] = ...,
|
|
553
|
+
types: Optional[builtins.list[str]] = ...,
|
|
554
|
+
since: "_when.When | None" = ...,
|
|
555
|
+
until: "_when.When | None" = ...,
|
|
556
|
+
limit: Optional[int] = ...,
|
|
557
|
+
offset: int = ...,
|
|
558
|
+
watch: Literal[True],
|
|
559
|
+
timeout: Optional[float] = ...,
|
|
560
|
+
) -> Generator[builtins.list[results.Op], None, None]: ...
|
|
561
|
+
|
|
562
|
+
def ops(
|
|
563
|
+
self,
|
|
564
|
+
*,
|
|
565
|
+
op_id: Optional[int] = None,
|
|
566
|
+
states: Optional[builtins.list[str]] = None,
|
|
567
|
+
types: Optional[builtins.list[str]] = None,
|
|
568
|
+
since: "_when.When | None" = None,
|
|
569
|
+
until: "_when.When | None" = None,
|
|
570
|
+
limit: Optional[int] = None,
|
|
571
|
+
offset: int = 0,
|
|
572
|
+
watch: bool = False,
|
|
573
|
+
timeout: Optional[float] = None,
|
|
574
|
+
) -> "builtins.list[results.Op] | Generator[builtins.list[results.Op], None, None]":
|
|
575
|
+
"""List the current session's operations on runs, newest first, as
|
|
576
|
+
`machine ops` does.
|
|
577
|
+
|
|
578
|
+
An operation is a checkpoint, branch, switch or rewind the session
|
|
579
|
+
carried out. The list is every operation in every state, which is
|
|
580
|
+
where to look when one of those calls returned an error; `states`
|
|
581
|
+
keeps some. Every matching operation is returned unless `limit` is
|
|
582
|
+
given. `Run.ops` lists one run's.
|
|
583
|
+
|
|
584
|
+
With `watch`, it yields the list at once, then again each time it
|
|
585
|
+
changes, such as a checkpoint finishing. Leave the loop, or pass
|
|
586
|
+
`timeout`, to stop watching.
|
|
587
|
+
|
|
588
|
+
Example:
|
|
589
|
+
>>> client.ops(states=["failed"], since="1h")
|
|
590
|
+
[{'op_id': '17897...', 'op_type': 'checkpoint', 'state': 'failed', ...}]
|
|
591
|
+
>>> for running in client.ops(states=["running"], watch=True, timeout=300):
|
|
592
|
+
... if not running:
|
|
593
|
+
... break
|
|
594
|
+
|
|
595
|
+
Args:
|
|
596
|
+
op_id (int, optional): Return this operation alone, whatever its
|
|
597
|
+
state or type.
|
|
598
|
+
states (list[str], optional): Keep these states: `running`,
|
|
599
|
+
`unfinished`, `done`, `failed`; `^state` leaves one out.
|
|
600
|
+
Defaults to every state.
|
|
601
|
+
types (list[str], optional): Keep these types: `checkpoint`,
|
|
602
|
+
`branch`, `switch`, `rewind`, `start`, `delete`; `^type`
|
|
603
|
+
leaves one out. Defaults to every type.
|
|
604
|
+
since (datetime or int or str, optional): Keep those started at
|
|
605
|
+
or after this time: a datetime, Unix seconds, a date or an
|
|
606
|
+
age such as `1h`.
|
|
607
|
+
until (datetime or int or str, optional): Keep those started at
|
|
608
|
+
or before this time.
|
|
609
|
+
limit (int, optional): Return at most this many.
|
|
610
|
+
offset (int, optional): Skip this many first. Defaults to 0.
|
|
611
|
+
watch (bool, optional): Yield the list each time it changes.
|
|
612
|
+
Defaults to False.
|
|
613
|
+
timeout (float, optional): With `watch`, end the generator after
|
|
614
|
+
this many seconds, without raising; None watches until the
|
|
615
|
+
caller leaves the loop.
|
|
616
|
+
|
|
617
|
+
Returns:
|
|
618
|
+
list[Op]: One record per operation: its id,
|
|
619
|
+
type, state, run, entry, start time and error; with `watch`, an
|
|
620
|
+
iterator of such lists.
|
|
621
|
+
|
|
622
|
+
Raises:
|
|
623
|
+
InvalidArgument: A state, type or time is not valid, or the
|
|
624
|
+
client has no session.
|
|
625
|
+
"""
|
|
626
|
+
return self._ops(
|
|
627
|
+
None,
|
|
628
|
+
op_id=op_id,
|
|
629
|
+
states=states,
|
|
630
|
+
types=types,
|
|
631
|
+
since=since,
|
|
632
|
+
until=until,
|
|
633
|
+
limit=limit,
|
|
634
|
+
offset=offset,
|
|
635
|
+
watch=watch,
|
|
636
|
+
timeout=timeout,
|
|
637
|
+
)
|
|
638
|
+
|
|
639
|
+
@overload
|
|
640
|
+
def branches(
|
|
641
|
+
self,
|
|
642
|
+
*,
|
|
643
|
+
running: bool = ...,
|
|
644
|
+
depth: Optional[int] = ...,
|
|
645
|
+
since: "_when.When | None" = ...,
|
|
646
|
+
until: "_when.When | None" = ...,
|
|
647
|
+
sort: str = ...,
|
|
648
|
+
reverse: bool = ...,
|
|
649
|
+
limit: Optional[int] = ...,
|
|
650
|
+
offset: int = ...,
|
|
651
|
+
watch: Literal[False] = ...,
|
|
652
|
+
timeout: Optional[float] = ...,
|
|
653
|
+
) -> builtins.list[Run]: ...
|
|
654
|
+
|
|
655
|
+
@overload
|
|
656
|
+
def branches(
|
|
657
|
+
self,
|
|
658
|
+
*,
|
|
659
|
+
running: bool = ...,
|
|
660
|
+
depth: Optional[int] = ...,
|
|
661
|
+
since: "_when.When | None" = ...,
|
|
662
|
+
until: "_when.When | None" = ...,
|
|
663
|
+
sort: str = ...,
|
|
664
|
+
reverse: bool = ...,
|
|
665
|
+
limit: Optional[int] = ...,
|
|
666
|
+
offset: int = ...,
|
|
667
|
+
watch: Literal[True],
|
|
668
|
+
timeout: Optional[float] = ...,
|
|
669
|
+
) -> Generator[builtins.list[Run], None, None]: ...
|
|
670
|
+
|
|
671
|
+
def branches(
|
|
672
|
+
self,
|
|
673
|
+
*,
|
|
674
|
+
running: bool = False,
|
|
675
|
+
depth: Optional[int] = None,
|
|
676
|
+
since: "_when.When | None" = None,
|
|
677
|
+
until: "_when.When | None" = None,
|
|
678
|
+
sort: str = "created",
|
|
679
|
+
reverse: bool = False,
|
|
680
|
+
limit: Optional[int] = None,
|
|
681
|
+
offset: int = 0,
|
|
682
|
+
watch: bool = False,
|
|
683
|
+
timeout: Optional[float] = None,
|
|
684
|
+
) -> "builtins.list[Run] | Generator[builtins.list[Run], None, None]":
|
|
685
|
+
"""List the runs of every tree, newest first, as `machine branches`
|
|
686
|
+
does.
|
|
687
|
+
|
|
688
|
+
Every matching run is returned unless `limit` is given. Answers with
|
|
689
|
+
no session, from the account's catalog. `Run.branches` lists one
|
|
690
|
+
run's tree.
|
|
691
|
+
|
|
692
|
+
With `watch`, it yields the list at once, then again each time it
|
|
693
|
+
changes: a branch forked or deleted, or one starting or stopping.
|
|
694
|
+
Leave the loop, or pass `timeout`, to stop watching.
|
|
695
|
+
|
|
696
|
+
Example:
|
|
697
|
+
>>> [b.name for b in client.branches(depth=0)]
|
|
698
|
+
['app', 'db']
|
|
699
|
+
>>> for running in client.branches(running=True, watch=True, timeout=600):
|
|
700
|
+
... if not running:
|
|
701
|
+
... break
|
|
702
|
+
|
|
703
|
+
Args:
|
|
704
|
+
running (bool, optional): Keep those with a machine. Defaults to
|
|
705
|
+
False.
|
|
706
|
+
depth (int, optional): Keep those this many forks from their
|
|
707
|
+
root; 0 is the roots.
|
|
708
|
+
since (datetime or int or str, optional): Keep those created at
|
|
709
|
+
or after this time: a datetime, Unix seconds, a date or an
|
|
710
|
+
age such as `1h`.
|
|
711
|
+
until (datetime or int or str, optional): Keep those created at
|
|
712
|
+
or before this time.
|
|
713
|
+
sort (str, optional): `created`, newest first, or `name`, A to
|
|
714
|
+
Z. Defaults to `created`.
|
|
715
|
+
reverse (bool, optional): Turn the order round. Defaults to
|
|
716
|
+
False.
|
|
717
|
+
limit (int, optional): Return at most this many.
|
|
718
|
+
offset (int, optional): Skip this many first. Defaults to 0.
|
|
719
|
+
watch (bool, optional): Yield the list each time it changes.
|
|
720
|
+
Defaults to False.
|
|
721
|
+
timeout (float, optional): With `watch`, end the generator after
|
|
722
|
+
this many seconds, without raising; None watches until the
|
|
723
|
+
caller leaves the loop.
|
|
724
|
+
|
|
725
|
+
Returns:
|
|
726
|
+
list[Run]: The runs; with `watch`, an iterator of such lists.
|
|
727
|
+
|
|
728
|
+
Raises:
|
|
729
|
+
InvalidArgument: `sort` or a time is not valid.
|
|
730
|
+
"""
|
|
731
|
+
return self._branches(
|
|
732
|
+
None,
|
|
733
|
+
running=running,
|
|
734
|
+
depth=depth,
|
|
735
|
+
since=since,
|
|
736
|
+
until=until,
|
|
737
|
+
sort=sort,
|
|
738
|
+
reverse=reverse,
|
|
739
|
+
limit=limit,
|
|
740
|
+
offset=offset,
|
|
741
|
+
watch=watch,
|
|
742
|
+
timeout=timeout,
|
|
743
|
+
)
|
|
744
|
+
|
|
745
|
+
def cleanup(self, *, force: bool = False) -> results.Cleanup:
|
|
746
|
+
"""Remove what machines that ended uncleanly left behind in the
|
|
747
|
+
current session, as `machine cleanup` does.
|
|
748
|
+
|
|
749
|
+
A machine that crashed or was killed from outside leaves stale
|
|
750
|
+
entries the service clears here. Runs themselves are not deleted.
|
|
751
|
+
With `force`, machines that no longer answer are killed and their
|
|
752
|
+
entries removed too.
|
|
753
|
+
|
|
754
|
+
Args:
|
|
755
|
+
force (bool, optional): Also kill orphaned machines and remove
|
|
756
|
+
entries whose machine is already gone. Defaults to False.
|
|
757
|
+
|
|
758
|
+
Returns:
|
|
759
|
+
Cleanup: What was cleaned, and the runs it touched.
|
|
760
|
+
|
|
761
|
+
Raises:
|
|
762
|
+
InvalidArgument: The client has no session.
|
|
763
|
+
"""
|
|
764
|
+
with self._connection() as client, _translated():
|
|
765
|
+
done = _machines.clean(client, force=force)
|
|
766
|
+
return results.cleanup(dataclasses.asdict(done))
|
|
767
|
+
|
|
768
|
+
def ping(self, *, count: int = 3) -> Ping:
|
|
769
|
+
"""Measure whether the current session's service answers, and how
|
|
770
|
+
fast, as `machine ping` does.
|
|
771
|
+
|
|
772
|
+
For a client with no session it measures the account's service
|
|
773
|
+
instead. A service that cannot be reached is a result, not an
|
|
774
|
+
exception: `Ping.ok` is False and `Ping.error` says why.
|
|
775
|
+
|
|
776
|
+
Example:
|
|
777
|
+
>>> ping = client.ping()
|
|
778
|
+
>>> ping.ok, round(ping.rtt_ms, 1)
|
|
779
|
+
(True, 1.8)
|
|
780
|
+
|
|
781
|
+
Args:
|
|
782
|
+
count (int, optional): Round trips to time, at least 1.
|
|
783
|
+
Defaults to 3.
|
|
784
|
+
|
|
785
|
+
Returns:
|
|
786
|
+
Ping: Whether it answered, and its round trips.
|
|
787
|
+
|
|
788
|
+
Raises:
|
|
789
|
+
NotRunning: The current session is not running.
|
|
790
|
+
"""
|
|
791
|
+
count = max(1, count)
|
|
792
|
+
if self._local:
|
|
793
|
+
return _public(_ping.probe_socket(None, count))
|
|
794
|
+
session = self.session
|
|
795
|
+
client, creds = self._account()
|
|
796
|
+
if session is None:
|
|
797
|
+
base_url = _ping.api_base_url(creds, self._api_url)
|
|
798
|
+
return _public(_ping.probe_api(creds, base_url, count))
|
|
799
|
+
try:
|
|
800
|
+
with _translated():
|
|
801
|
+
self._service()
|
|
802
|
+
except ServiceUnavailable as unreachable:
|
|
803
|
+
return _public(_ping.Probe(False, error=str(unreachable)))
|
|
804
|
+
return _public(
|
|
805
|
+
_ping.probe_session(client, creds, session.id, count, self._api_url)
|
|
806
|
+
)
|
|
807
|
+
|
|
808
|
+
def stop(
|
|
809
|
+
self,
|
|
810
|
+
*,
|
|
811
|
+
all: bool,
|
|
812
|
+
dry_run: bool = False,
|
|
813
|
+
wait: bool = True,
|
|
814
|
+
timeout: Optional[float] = None,
|
|
815
|
+
) -> builtins.list[Run]:
|
|
816
|
+
"""Stop every running machine in the current session, as `machine
|
|
817
|
+
stop -a` does.
|
|
818
|
+
|
|
819
|
+
Each machine checkpoints its tip first, so its run starts again
|
|
820
|
+
where it stopped. `Run.stop` stops one.
|
|
821
|
+
|
|
822
|
+
Example:
|
|
823
|
+
>>> [r.name for r in client.stop(all=True, dry_run=True)]
|
|
824
|
+
['app', 'app/try-1']
|
|
825
|
+
>>> client.stop(all=True)
|
|
826
|
+
|
|
827
|
+
Args:
|
|
828
|
+
all (bool): Stop every running machine; required, as `-a` is.
|
|
829
|
+
dry_run (bool, optional): List what would be stopped, stopping
|
|
830
|
+
nothing. Defaults to False.
|
|
831
|
+
wait (bool, optional): Return once every machine is gone.
|
|
832
|
+
Defaults to True.
|
|
833
|
+
timeout (float, optional): Seconds to wait; None waits without a
|
|
834
|
+
limit.
|
|
835
|
+
|
|
836
|
+
Returns:
|
|
837
|
+
list[Run]: The runs stopped, or with `dry_run` those that would
|
|
838
|
+
be.
|
|
839
|
+
|
|
840
|
+
Raises:
|
|
841
|
+
InvalidArgument: `all` is False, or the client has no session.
|
|
842
|
+
ApiError: Some machines could not be stopped; the message names
|
|
843
|
+
the first.
|
|
844
|
+
"""
|
|
845
|
+
_require_all(all, "Stop one run with run.stop().")
|
|
846
|
+
with self._connection() as client, _translated():
|
|
847
|
+
names = _names(client)
|
|
848
|
+
done, errors = _lifecycle.stop_all(
|
|
849
|
+
client, dry_run=dry_run, until_stopped=wait, timeout_s=timeout
|
|
850
|
+
)
|
|
851
|
+
if isinstance(done, _machines.StopDryRunResult):
|
|
852
|
+
return [Run(self, run_id, name) for name, run_id in done.would_stop]
|
|
853
|
+
_raise_first(errors, f"Stopped {done.stopped}")
|
|
854
|
+
return [Run(self, run_id, names.get(run_id, "")) for run_id in done.run_ids]
|
|
855
|
+
|
|
856
|
+
def wait(
|
|
857
|
+
self,
|
|
858
|
+
*runs: "Run | str",
|
|
859
|
+
for_: str,
|
|
860
|
+
all: bool = False,
|
|
861
|
+
interval: float = _wait.DEFAULT_INTERVAL_S,
|
|
862
|
+
timeout: Optional[float] = None,
|
|
863
|
+
) -> None:
|
|
864
|
+
"""Wait until runs satisfy a predicate, as `machine wait` does.
|
|
865
|
+
|
|
866
|
+
It watches without driving: other programs or threads run the
|
|
867
|
+
machines. With several runs, it returns once every one satisfies
|
|
868
|
+
it. `Run.wait` waits for one.
|
|
869
|
+
|
|
870
|
+
Example:
|
|
871
|
+
>>> client.wait(a, b, for_="/app:exit", timeout=600)
|
|
872
|
+
>>> client.wait(for_="stopped", all=True)
|
|
873
|
+
|
|
874
|
+
Args:
|
|
875
|
+
*runs (Run or str): The runs, as `Run` objects or names.
|
|
876
|
+
for_ (str): The predicate, as for `Run.wait`. The trailing
|
|
877
|
+
underscore avoids the Python keyword.
|
|
878
|
+
all (bool, optional): Wait for every running run in the session
|
|
879
|
+
instead of naming them. Defaults to False.
|
|
880
|
+
interval (float, optional): Seconds between checks. Defaults
|
|
881
|
+
to 1.
|
|
882
|
+
timeout (float, optional): Seconds to wait; None waits without a
|
|
883
|
+
limit, 0 checks once.
|
|
884
|
+
|
|
885
|
+
Raises:
|
|
886
|
+
InvalidArgument: `for_` is not a predicate, runs are named with
|
|
887
|
+
`all`, or none are named without it.
|
|
888
|
+
Timeout: It did not hold for every run within `timeout`.
|
|
889
|
+
NotRunning: A run stopped, so the predicate can no longer hold.
|
|
890
|
+
"""
|
|
891
|
+
if all and runs:
|
|
892
|
+
raise InvalidArgument("Name runs or pass all=True, not both.")
|
|
893
|
+
if not all and not runs:
|
|
894
|
+
raise InvalidArgument("Run name required.")
|
|
895
|
+
_args.positive("interval", interval)
|
|
896
|
+
predicate = _args.predicate(for_)
|
|
897
|
+
with self._connection() as client, _translated():
|
|
898
|
+
if all:
|
|
899
|
+
live = _machines.list_machines(client, enrich=False)
|
|
900
|
+
ids = [m.run_id for m in live]
|
|
901
|
+
else:
|
|
902
|
+
ids = [
|
|
903
|
+
r.id if isinstance(r, Run) else client.resolve_run_ref(r)
|
|
904
|
+
for r in runs
|
|
905
|
+
]
|
|
906
|
+
_wait.poll_all(
|
|
907
|
+
client, ids, predicate, timeout_s=timeout, interval_s=interval
|
|
908
|
+
)
|
|
909
|
+
|
|
910
|
+
def __repr__(self) -> str:
|
|
911
|
+
if self._session_ref is not None:
|
|
912
|
+
return f"Client(session={self._session_ref!r})"
|
|
913
|
+
return "Client()"
|
|
914
|
+
|
|
915
|
+
# -- internals -------------------------------------------------------------
|
|
916
|
+
|
|
917
|
+
def _use(self, session: Session) -> None:
|
|
918
|
+
"""Make `session` current for this client."""
|
|
919
|
+
self._session_ref = session.id
|
|
920
|
+
self._session = session
|
|
921
|
+
|
|
922
|
+
def _moved(self, old_id: str, session: Session) -> None:
|
|
923
|
+
"""`old_id` moved to `session`: every Session object of `old_id` names it
|
|
924
|
+
now, and the client uses it in its place when it was current."""
|
|
925
|
+
self._moved_ids[old_id] = session.id
|
|
926
|
+
self._services.pop(old_id, None)
|
|
927
|
+
if self._session is not None and self._session.id == session.id:
|
|
928
|
+
self._use(session)
|
|
929
|
+
|
|
930
|
+
def _forget(self, session_id: str) -> None:
|
|
931
|
+
"""Drop `session_id` as current, when it is: it was deleted."""
|
|
932
|
+
self._services.pop(session_id, None)
|
|
933
|
+
if self._session is not None and self._session.id == session_id:
|
|
934
|
+
self._session_ref = None
|
|
935
|
+
self._session = None
|
|
936
|
+
|
|
937
|
+
def _account(self) -> tuple[api.client.Client, dict[str, Any]]:
|
|
938
|
+
"""The account's service and the login to it, signed in once."""
|
|
939
|
+
if self._api is None:
|
|
940
|
+
with _translated():
|
|
941
|
+
self._api = api.auth.signed_in(self._api_url)
|
|
942
|
+
return self._api
|
|
943
|
+
|
|
944
|
+
def _service(self) -> _service._Service:
|
|
945
|
+
"""The current session's service."""
|
|
946
|
+
if self._local:
|
|
947
|
+
return self._services.setdefault("", _service.local())
|
|
948
|
+
session = self.session
|
|
949
|
+
if session is None:
|
|
950
|
+
raise InvalidArgument(
|
|
951
|
+
"No session. Use one with client.sessions.use(name),"
|
|
952
|
+
" or create one with client.sessions.create()."
|
|
953
|
+
)
|
|
954
|
+
if session.id not in self._services:
|
|
955
|
+
client, creds = self._account()
|
|
956
|
+
session_id = session.id
|
|
957
|
+
self._services[session_id] = _service.of_session(
|
|
958
|
+
client,
|
|
959
|
+
creds,
|
|
960
|
+
session_id,
|
|
961
|
+
session.name,
|
|
962
|
+
follow=lambda: self._follow_from(session_id),
|
|
963
|
+
)
|
|
964
|
+
return self._services[session.id]
|
|
965
|
+
|
|
966
|
+
@contextlib.contextmanager
|
|
967
|
+
def _connection(self) -> Generator[_ipc.Client, None, None]:
|
|
968
|
+
"""A connection to the current session's service.
|
|
969
|
+
|
|
970
|
+
A call that fails because the session moved, `sessions migrate`, makes
|
|
971
|
+
the client follow it: the next call goes to where the session went. The
|
|
972
|
+
failed call is raised as it failed, since it may have done its work.
|
|
973
|
+
|
|
974
|
+
A session's maintenance notice is told first, so a callback that
|
|
975
|
+
migrates sends the call to the new session."""
|
|
976
|
+
if not self._local:
|
|
977
|
+
self._tell_closing()
|
|
978
|
+
try:
|
|
979
|
+
with self._service().client() as client:
|
|
980
|
+
yield client
|
|
981
|
+
except Exception as err:
|
|
982
|
+
if not self._local and _sessions.moved_away(err):
|
|
983
|
+
self._follow()
|
|
984
|
+
raise
|
|
985
|
+
|
|
986
|
+
def _tell_closing(self) -> None:
|
|
987
|
+
"""Tell the program of the current session's maintenance notice, once
|
|
988
|
+
for each deadline: `on_session_closing`, or a `SessionClosing`
|
|
989
|
+
warning. Marked told before the callback runs, so the calls it makes
|
|
990
|
+
do not tell it again."""
|
|
991
|
+
session = self.session
|
|
992
|
+
if session is None:
|
|
993
|
+
return
|
|
994
|
+
notice = _sessions.notice(session.id, self._account)
|
|
995
|
+
if notice is None:
|
|
996
|
+
if self._closing_told.pop(session.id, None):
|
|
997
|
+
session._closing(None, None)
|
|
998
|
+
return
|
|
999
|
+
by = _deadline_key(notice.by)
|
|
1000
|
+
told = self._closing_told.setdefault(session.id, set())
|
|
1001
|
+
if by in told:
|
|
1002
|
+
return
|
|
1003
|
+
told.add(by)
|
|
1004
|
+
if not self._closing_still(session, by, notice.reason):
|
|
1005
|
+
return
|
|
1006
|
+
deadline = _deadline(str(session.closing_at))
|
|
1007
|
+
told.add(_deadline_key(deadline))
|
|
1008
|
+
if self._on_session_closing is not None:
|
|
1009
|
+
self._on_session_closing(session)
|
|
1010
|
+
return
|
|
1011
|
+
reason = session.closing_reason or notice.reason
|
|
1012
|
+
_warn.warn(
|
|
1013
|
+
f"Session {session.name} stops at {deadline:%Y-%m-%d %H:%M} UTC"
|
|
1014
|
+
f" because of {reason}. Move it before then with Session.migrate.",
|
|
1015
|
+
SessionClosing,
|
|
1016
|
+
)
|
|
1017
|
+
|
|
1018
|
+
def _closing_still(self, session: Session, by: str, reason: str) -> bool:
|
|
1019
|
+
"""Read `session` again before a notice is told: the notice is read
|
|
1020
|
+
from a record up to ten minutes old, and a session that moved or ended
|
|
1021
|
+
since keeps its notice, which nothing can act on. When the service
|
|
1022
|
+
does not answer, the notice is told as recorded: a notice never fails
|
|
1023
|
+
the call it rides on."""
|
|
1024
|
+
try:
|
|
1025
|
+
session.refresh()
|
|
1026
|
+
except Exception:
|
|
1027
|
+
session._closing(by, reason)
|
|
1028
|
+
return True
|
|
1029
|
+
live = session.state in _sessions.NOTICE_STATES
|
|
1030
|
+
moved = bool(session._row.get("successor_id"))
|
|
1031
|
+
return live and not moved and session.closing_at is not None
|
|
1032
|
+
|
|
1033
|
+
def _follow(self) -> None:
|
|
1034
|
+
"""Use the session the current one moved to, when it moved."""
|
|
1035
|
+
if self._session is not None:
|
|
1036
|
+
self._follow_from(self._session.id)
|
|
1037
|
+
|
|
1038
|
+
def _follow_from(self, old_id: str) -> Optional[Path]:
|
|
1039
|
+
"""Follow session `old_id` to where it moved, and give its socket; None
|
|
1040
|
+
when it did not move. Bound to each session's service, so a session that
|
|
1041
|
+
is not the current one follows its own move, not the current one's."""
|
|
1042
|
+
client, creds = self._account()
|
|
1043
|
+
try:
|
|
1044
|
+
info = _sessions.follow(client, creds, old_id)
|
|
1045
|
+
except Exception: # the error that led here is the one to report
|
|
1046
|
+
return None
|
|
1047
|
+
if info is None:
|
|
1048
|
+
return None
|
|
1049
|
+
moved = Session(self, info)
|
|
1050
|
+
self._moved(old_id, moved)
|
|
1051
|
+
return api.store.session_socket(moved.id)
|
|
1052
|
+
|
|
1053
|
+
def _now_id(self, session_id: str) -> str:
|
|
1054
|
+
"""The id a session has now: the session it moved to, after every move this
|
|
1055
|
+
client followed."""
|
|
1056
|
+
while session_id in self._moved_ids:
|
|
1057
|
+
session_id = self._moved_ids[session_id]
|
|
1058
|
+
return session_id
|
|
1059
|
+
|
|
1060
|
+
@contextlib.contextmanager
|
|
1061
|
+
def _catalog(self) -> Generator[_run_source.RunSource, None, None]:
|
|
1062
|
+
"""Where runs are read from: the current session, else the account's
|
|
1063
|
+
catalog."""
|
|
1064
|
+
if self._local or self._session_ref is not None:
|
|
1065
|
+
with self._connection() as client:
|
|
1066
|
+
yield client
|
|
1067
|
+
return
|
|
1068
|
+
with self._account_catalog() as source:
|
|
1069
|
+
yield source
|
|
1070
|
+
|
|
1071
|
+
@contextlib.contextmanager
|
|
1072
|
+
def _account_catalog(
|
|
1073
|
+
self,
|
|
1074
|
+
) -> Generator[_cloud_run_source.CloudRunSource, None, None]:
|
|
1075
|
+
"""The account's catalog of runs, whichever session is current."""
|
|
1076
|
+
api_client, creds = self._account()
|
|
1077
|
+
source = _cloud_run_source.CloudRunSource(api_client, creds)
|
|
1078
|
+
try:
|
|
1079
|
+
with _translated():
|
|
1080
|
+
yield source
|
|
1081
|
+
finally:
|
|
1082
|
+
source.close()
|
|
1083
|
+
|
|
1084
|
+
@contextlib.contextmanager
|
|
1085
|
+
def _copies(self) -> Generator[_ipc.Client, None, None]:
|
|
1086
|
+
"""A connection to the service of the session whose disk holds the
|
|
1087
|
+
copies."""
|
|
1088
|
+
if self._local:
|
|
1089
|
+
raise InvalidArgument(
|
|
1090
|
+
"The service on this machine keeps every run on its disk,"
|
|
1091
|
+
" so it has no copies to preload or offload."
|
|
1092
|
+
)
|
|
1093
|
+
with self._connection() as client:
|
|
1094
|
+
yield client
|
|
1095
|
+
|
|
1096
|
+
def _disk_id(self, name: str) -> str:
|
|
1097
|
+
"""The id of the account's disk `name`."""
|
|
1098
|
+
client, creds = self._account()
|
|
1099
|
+
with _translated():
|
|
1100
|
+
row = api.auth.authed(creds, lambda t: client.disk_get(t, name))
|
|
1101
|
+
return str(row["disk_id"])
|
|
1102
|
+
|
|
1103
|
+
def _name(self, source: _run_source.RunSource, run_id: str) -> str:
|
|
1104
|
+
"""What a run is called, read from where runs are read."""
|
|
1105
|
+
if isinstance(source, _ipc.Client):
|
|
1106
|
+
return _machines.display_name(source, run_id)
|
|
1107
|
+
return _machines.catalog_name(source, run_id)
|
|
1108
|
+
|
|
1109
|
+
def _ops(
|
|
1110
|
+
self,
|
|
1111
|
+
run_id: Optional[str],
|
|
1112
|
+
*,
|
|
1113
|
+
op_id: Optional[int],
|
|
1114
|
+
states: Optional[builtins.list[str]],
|
|
1115
|
+
types: Optional[builtins.list[str]],
|
|
1116
|
+
since: "_when.When | None",
|
|
1117
|
+
until: "_when.When | None",
|
|
1118
|
+
limit: Optional[int],
|
|
1119
|
+
offset: int,
|
|
1120
|
+
watch: bool,
|
|
1121
|
+
timeout: Optional[float],
|
|
1122
|
+
) -> "builtins.list[results.Op] | Generator[builtins.list[results.Op], None, None]":
|
|
1123
|
+
filters = _args.op_filters(op_id, states, types, _machines.OP_TYPE_CHOICES)
|
|
1124
|
+
|
|
1125
|
+
def read() -> builtins.list[results.Op]:
|
|
1126
|
+
with self._connection() as client, _translated():
|
|
1127
|
+
rows = _machines.list_ops(
|
|
1128
|
+
client,
|
|
1129
|
+
run_id=run_id,
|
|
1130
|
+
op_id=op_id,
|
|
1131
|
+
states=filters.states,
|
|
1132
|
+
op_types=filters.types,
|
|
1133
|
+
since=_when.unix_seconds(since),
|
|
1134
|
+
until=_when.unix_seconds(until),
|
|
1135
|
+
limit=limit,
|
|
1136
|
+
offset=offset,
|
|
1137
|
+
)
|
|
1138
|
+
return [results.op(dataclasses.asdict(row)) for row in rows]
|
|
1139
|
+
|
|
1140
|
+
return _args.watched(read, _args.same, watch, timeout)
|
|
1141
|
+
|
|
1142
|
+
def _branches(
|
|
1143
|
+
self,
|
|
1144
|
+
run_id: Optional[str],
|
|
1145
|
+
*,
|
|
1146
|
+
running: bool,
|
|
1147
|
+
depth: Optional[int],
|
|
1148
|
+
since: "_when.When | None",
|
|
1149
|
+
until: "_when.When | None",
|
|
1150
|
+
sort: str,
|
|
1151
|
+
reverse: bool,
|
|
1152
|
+
limit: Optional[int],
|
|
1153
|
+
offset: int,
|
|
1154
|
+
watch: bool,
|
|
1155
|
+
timeout: Optional[float],
|
|
1156
|
+
) -> "builtins.list[Run] | Generator[builtins.list[Run], None, None]":
|
|
1157
|
+
_args.check_choice("sort", sort, _sorting.BRANCH_SORT_COLS)
|
|
1158
|
+
|
|
1159
|
+
def read() -> builtins.list[tuple[str, str, str]]:
|
|
1160
|
+
"""Each branch's id, name and execution: a branch starting or
|
|
1161
|
+
stopping is a change."""
|
|
1162
|
+
with self._catalog() as source, _translated():
|
|
1163
|
+
listed = source.branch_list(scope_run_id=run_id)
|
|
1164
|
+
liveness = _machines.catalog_liveness(source, run_id)
|
|
1165
|
+
page, _ = _branches.page_branches(
|
|
1166
|
+
listed,
|
|
1167
|
+
running=running,
|
|
1168
|
+
depth=depth,
|
|
1169
|
+
liveness=liveness,
|
|
1170
|
+
since=_when.unix_seconds(since),
|
|
1171
|
+
until=_when.unix_seconds(until),
|
|
1172
|
+
sort=sort,
|
|
1173
|
+
reverse=reverse,
|
|
1174
|
+
limit=limit,
|
|
1175
|
+
offset=offset,
|
|
1176
|
+
)
|
|
1177
|
+
return [
|
|
1178
|
+
(
|
|
1179
|
+
b.run_id,
|
|
1180
|
+
b.name,
|
|
1181
|
+
liveness[b.run_id].execution if b.run_id in liveness else "",
|
|
1182
|
+
)
|
|
1183
|
+
for b in page
|
|
1184
|
+
]
|
|
1185
|
+
|
|
1186
|
+
def present(rows: builtins.list[tuple[str, str, str]]) -> builtins.list[Run]:
|
|
1187
|
+
return [Run(self, run_id, name) for run_id, name, _ in rows]
|
|
1188
|
+
|
|
1189
|
+
return _args.watched(read, present, watch, timeout)
|
|
1190
|
+
|
|
1191
|
+
def _marks(self, run_id: str) -> builtins.list[results.Mark]:
|
|
1192
|
+
with self._catalog() as source, _translated():
|
|
1193
|
+
rows = source.list_marks(run_id)
|
|
1194
|
+
wanted = {marked for _, marked in rows}
|
|
1195
|
+
names = {
|
|
1196
|
+
row.run_id: row.name
|
|
1197
|
+
for row in source.runs_query()
|
|
1198
|
+
if row.run_id in wanted
|
|
1199
|
+
}
|
|
1200
|
+
return [
|
|
1201
|
+
results.Mark(name=word, run_id=marked, run_name=names.get(marked, ""))
|
|
1202
|
+
for word, marked in rows
|
|
1203
|
+
]
|
|
1204
|
+
|
|
1205
|
+
|
|
1206
|
+
class Runs:
|
|
1207
|
+
"""The account's runs, as `Client.runs`; not created directly.
|
|
1208
|
+
|
|
1209
|
+
Call it to list runs, as `machine runs` does, and index it by name, id,
|
|
1210
|
+
unique id prefix or `%mark` to get one.
|
|
1211
|
+
|
|
1212
|
+
Example:
|
|
1213
|
+
>>> run = client.runs["app"]
|
|
1214
|
+
>>> [r.name for r in client.runs(states=["running"])]
|
|
1215
|
+
['app', 'app/try-1']
|
|
1216
|
+
"""
|
|
1217
|
+
|
|
1218
|
+
def __init__(self, client: Client) -> None:
|
|
1219
|
+
self._client = client
|
|
1220
|
+
|
|
1221
|
+
@overload
|
|
1222
|
+
def __call__(
|
|
1223
|
+
self,
|
|
1224
|
+
root: "Run | str | None" = ...,
|
|
1225
|
+
*,
|
|
1226
|
+
states: Optional[builtins.list[str]] = ...,
|
|
1227
|
+
running: bool = ...,
|
|
1228
|
+
roots: bool = ...,
|
|
1229
|
+
labels: Optional[dict[str, str]] = ...,
|
|
1230
|
+
flags: Optional[dict[str, int]] = ...,
|
|
1231
|
+
disk: "Disk | str | None" = ...,
|
|
1232
|
+
parents: Optional[str] = ...,
|
|
1233
|
+
synced: Optional[bool] = ...,
|
|
1234
|
+
since: "_when.When | None" = ...,
|
|
1235
|
+
until: "_when.When | None" = ...,
|
|
1236
|
+
sort: str = ...,
|
|
1237
|
+
reverse: bool = ...,
|
|
1238
|
+
limit: Optional[int] = ...,
|
|
1239
|
+
offset: int = ...,
|
|
1240
|
+
size: bool = ...,
|
|
1241
|
+
watch: Literal[False] = ...,
|
|
1242
|
+
timeout: Optional[float] = ...,
|
|
1243
|
+
) -> builtins.list[Run]: ...
|
|
1244
|
+
|
|
1245
|
+
@overload
|
|
1246
|
+
def __call__(
|
|
1247
|
+
self,
|
|
1248
|
+
root: "Run | str | None" = ...,
|
|
1249
|
+
*,
|
|
1250
|
+
states: Optional[builtins.list[str]] = ...,
|
|
1251
|
+
running: bool = ...,
|
|
1252
|
+
roots: bool = ...,
|
|
1253
|
+
labels: Optional[dict[str, str]] = ...,
|
|
1254
|
+
flags: Optional[dict[str, int]] = ...,
|
|
1255
|
+
disk: "Disk | str | None" = ...,
|
|
1256
|
+
parents: Optional[str] = ...,
|
|
1257
|
+
synced: Optional[bool] = ...,
|
|
1258
|
+
since: "_when.When | None" = ...,
|
|
1259
|
+
until: "_when.When | None" = ...,
|
|
1260
|
+
sort: str = ...,
|
|
1261
|
+
reverse: bool = ...,
|
|
1262
|
+
limit: Optional[int] = ...,
|
|
1263
|
+
offset: int = ...,
|
|
1264
|
+
size: bool = ...,
|
|
1265
|
+
watch: Literal[True],
|
|
1266
|
+
timeout: Optional[float] = ...,
|
|
1267
|
+
) -> Generator[builtins.list[Run], None, None]: ...
|
|
1268
|
+
|
|
1269
|
+
def __call__(
|
|
1270
|
+
self,
|
|
1271
|
+
root: "Run | str | None" = None,
|
|
1272
|
+
*,
|
|
1273
|
+
states: Optional[builtins.list[str]] = None,
|
|
1274
|
+
running: bool = False,
|
|
1275
|
+
roots: bool = False,
|
|
1276
|
+
labels: Optional[dict[str, str]] = None,
|
|
1277
|
+
flags: Optional[dict[str, int]] = None,
|
|
1278
|
+
disk: "Disk | str | None" = None,
|
|
1279
|
+
parents: Optional[str] = None,
|
|
1280
|
+
synced: Optional[bool] = None,
|
|
1281
|
+
since: "_when.When | None" = None,
|
|
1282
|
+
until: "_when.When | None" = None,
|
|
1283
|
+
sort: str = "created",
|
|
1284
|
+
reverse: bool = False,
|
|
1285
|
+
limit: Optional[int] = None,
|
|
1286
|
+
offset: int = 0,
|
|
1287
|
+
size: bool = False,
|
|
1288
|
+
watch: bool = False,
|
|
1289
|
+
timeout: Optional[float] = None,
|
|
1290
|
+
) -> "builtins.list[Run] | Generator[builtins.list[Run], None, None]":
|
|
1291
|
+
"""List runs, newest first, as `machine runs` does.
|
|
1292
|
+
|
|
1293
|
+
Every run and branch is returned, or those matching every filter
|
|
1294
|
+
given. The list is read from the catalog, so stopped runs are listed
|
|
1295
|
+
too, and with no session it answers from the account's catalog.
|
|
1296
|
+
Each run's `listing` holds its row. Every matching run is returned
|
|
1297
|
+
unless `limit` is given.
|
|
1298
|
+
|
|
1299
|
+
With `watch`, it yields the list at once, then again each time it
|
|
1300
|
+
changes: a run created or deleted, or one changing state. Leave the
|
|
1301
|
+
loop, or pass `timeout`, to stop watching.
|
|
1302
|
+
|
|
1303
|
+
Example:
|
|
1304
|
+
>>> client.runs(labels={"suite": "nightly"}, states=["^running"])
|
|
1305
|
+
>>> client.runs("app", sort="name")
|
|
1306
|
+
>>> [(r.listing.running, r.listing.runs) for r in client.runs(roots=True)]
|
|
1307
|
+
[(1, 3), (0, 1)]
|
|
1308
|
+
|
|
1309
|
+
Args:
|
|
1310
|
+
root (Run or str, optional): List one tree, by its root or any
|
|
1311
|
+
run in it.
|
|
1312
|
+
states (list[str], optional): Keep these states: `running`,
|
|
1313
|
+
`paused`, `halted`, `recovering`, `read-only`, `stopped`,
|
|
1314
|
+
`killed`, `terminated`, `died`, `failed`; `^state` leaves one
|
|
1315
|
+
out.
|
|
1316
|
+
running (bool, optional): Keep running runs, as
|
|
1317
|
+
`states=["running"]`. Defaults to False.
|
|
1318
|
+
roots (bool, optional): One run per tree, its root, with the
|
|
1319
|
+
tree's counts in `listing.running` and `listing.runs`.
|
|
1320
|
+
Defaults to False.
|
|
1321
|
+
labels (dict[str, str], optional): Keep runs with every one of
|
|
1322
|
+
these labels.
|
|
1323
|
+
flags (dict[str, int], optional): Keep runs whose flags match: a
|
|
1324
|
+
flag name, such as `rng` or `daemon`, to 1 for set or 0 for
|
|
1325
|
+
unset.
|
|
1326
|
+
disk (str, optional): Keep runs that boot this disk, by name
|
|
1327
|
+
or id.
|
|
1328
|
+
parents (str, optional): Keep runs that read their data parents
|
|
1329
|
+
`local` or `remote`.
|
|
1330
|
+
synced (bool, optional): True keeps runs published to shared
|
|
1331
|
+
storage, False those with nothing published.
|
|
1332
|
+
since (datetime or int or str, optional): Keep those created at
|
|
1333
|
+
or after this time: a datetime, Unix seconds, a date or an
|
|
1334
|
+
age such as `7d`.
|
|
1335
|
+
until (datetime or int or str, optional): Keep those created at
|
|
1336
|
+
or before this time.
|
|
1337
|
+
sort (str, optional): `created`, newest first, or `name`,
|
|
1338
|
+
`cpus`, `memory`, `rng-seed` or `limit`, lowest first.
|
|
1339
|
+
Defaults to `created`.
|
|
1340
|
+
reverse (bool, optional): Turn the order round. Defaults to
|
|
1341
|
+
False.
|
|
1342
|
+
limit (int, optional): Return at most this many.
|
|
1343
|
+
offset (int, optional): Skip this many first. Defaults to 0.
|
|
1344
|
+
size (bool, optional): Measure each run's size on the session,
|
|
1345
|
+
into `listing.disk_bytes` and `listing.ram_bytes`; needs a
|
|
1346
|
+
session. Defaults to False.
|
|
1347
|
+
watch (bool, optional): Yield the list each time it changes.
|
|
1348
|
+
Defaults to False.
|
|
1349
|
+
timeout (float, optional): With `watch`, end the generator after
|
|
1350
|
+
this many seconds, without raising; None watches until the
|
|
1351
|
+
caller leaves the loop.
|
|
1352
|
+
|
|
1353
|
+
Returns:
|
|
1354
|
+
list[Run]: The runs; with `watch`, an iterator of such lists.
|
|
1355
|
+
|
|
1356
|
+
Raises:
|
|
1357
|
+
InvalidArgument: A filter, sort key or time is not valid.
|
|
1358
|
+
"""
|
|
1359
|
+
terms = list(states or [])
|
|
1360
|
+
if running:
|
|
1361
|
+
terms.append("running")
|
|
1362
|
+
for term in terms:
|
|
1363
|
+
_args.check_choice(
|
|
1364
|
+
"state", term.removeprefix("^"), _machines.CATALOG_STATE_CHOICES
|
|
1365
|
+
)
|
|
1366
|
+
for flag in flags or {}:
|
|
1367
|
+
_args.check_choice("flag", flag, _machines.CATALOG_FLAG_NAMES)
|
|
1368
|
+
_args.check_choice("parents", parents, _PARENTS)
|
|
1369
|
+
_args.check_choice("sort", sort, _sorting.CATALOG_SORT_COLS)
|
|
1370
|
+
columns: dict[str, object] = {}
|
|
1371
|
+
if parents is not None:
|
|
1372
|
+
columns["parents"] = parents
|
|
1373
|
+
if synced is not None:
|
|
1374
|
+
columns["durable_through"] = synced
|
|
1375
|
+
query = _machines.CatalogQuery(
|
|
1376
|
+
states=_machines.resolve_states(terms or None),
|
|
1377
|
+
disk=None if disk is None else _args.disk_ref(disk),
|
|
1378
|
+
filters=columns or None,
|
|
1379
|
+
flag_filters=dict(flags) if flags else None,
|
|
1380
|
+
label_filters=(
|
|
1381
|
+
{_machines.to_wire_label_name(k): v for k, v in labels.items()}
|
|
1382
|
+
if labels
|
|
1383
|
+
else None
|
|
1384
|
+
),
|
|
1385
|
+
since=_when.unix_seconds(since),
|
|
1386
|
+
until=_when.unix_seconds(until),
|
|
1387
|
+
sort=_sorting.CATALOG_SORT_FIELDS[sort],
|
|
1388
|
+
reverse=_sorting.descending(sort, reverse),
|
|
1389
|
+
limit=limit,
|
|
1390
|
+
offset=offset,
|
|
1391
|
+
usage=size,
|
|
1392
|
+
)
|
|
1393
|
+
catalog = self._client._connection if size else self._client._catalog
|
|
1394
|
+
|
|
1395
|
+
def read() -> Sequence[_machines.RunRootRow | _machines.RunTreeRow]:
|
|
1396
|
+
with catalog() as source, _translated():
|
|
1397
|
+
if roots and root is None:
|
|
1398
|
+
return _machines.catalog_roots(source, query)
|
|
1399
|
+
scope = None
|
|
1400
|
+
if root is not None:
|
|
1401
|
+
ref = root.id if isinstance(root, Run) else root
|
|
1402
|
+
scope = _machines.tree_root(source, source.resolve_run_ref(ref))
|
|
1403
|
+
return _machines.catalog_runs(source, query, root=scope)
|
|
1404
|
+
|
|
1405
|
+
def present(
|
|
1406
|
+
rows: Sequence[_machines.RunRootRow | _machines.RunTreeRow],
|
|
1407
|
+
) -> builtins.list[Run]:
|
|
1408
|
+
return [
|
|
1409
|
+
Run(self._client, row.run_id, row.name, results.listing(row))
|
|
1410
|
+
for row in rows
|
|
1411
|
+
]
|
|
1412
|
+
|
|
1413
|
+
return _args.watched(read, present, watch, timeout)
|
|
1414
|
+
|
|
1415
|
+
def get(self, ref: str) -> Run:
|
|
1416
|
+
"""Get a run by name, as every command that takes a run resolves
|
|
1417
|
+
one; `runs[ref]` does the same.
|
|
1418
|
+
|
|
1419
|
+
Args:
|
|
1420
|
+
ref (str): The run's name, a branch's as `root/branch`, its id, a
|
|
1421
|
+
prefix of the id only it has, or `%mark`.
|
|
1422
|
+
|
|
1423
|
+
Returns:
|
|
1424
|
+
Run: The run.
|
|
1425
|
+
|
|
1426
|
+
Raises:
|
|
1427
|
+
NotFound: No run matches.
|
|
1428
|
+
InvalidArgument: The prefix matches more than one run.
|
|
1429
|
+
"""
|
|
1430
|
+
with self._client._catalog() as source, _translated():
|
|
1431
|
+
run_id = source.resolve_run_ref(ref)
|
|
1432
|
+
name = self._client._name(source, run_id)
|
|
1433
|
+
return Run(self._client, run_id, name)
|
|
1434
|
+
|
|
1435
|
+
def __getitem__(self, ref: str) -> Run:
|
|
1436
|
+
return self.get(ref)
|
|
1437
|
+
|
|
1438
|
+
|
|
1439
|
+
class Marks:
|
|
1440
|
+
"""The marks of every tree, as `Client.marks`; not created directly.
|
|
1441
|
+
|
|
1442
|
+
`Run.marks` lists one tree's, and adds and removes a run's.
|
|
1443
|
+
|
|
1444
|
+
Example:
|
|
1445
|
+
>>> [(m.name, m.run_name) for m in client.marks.list()]
|
|
1446
|
+
[('found-race', 'app/try-3')]
|
|
1447
|
+
"""
|
|
1448
|
+
|
|
1449
|
+
def __init__(self, client: Client) -> None:
|
|
1450
|
+
self._client = client
|
|
1451
|
+
|
|
1452
|
+
def list(self) -> builtins.list[results.Mark]:
|
|
1453
|
+
"""List the marks of every tree, as `machine marks list` does.
|
|
1454
|
+
|
|
1455
|
+
Answers with no session, from the account's catalog.
|
|
1456
|
+
|
|
1457
|
+
Returns:
|
|
1458
|
+
list[Mark]: Each mark and the run that carries it.
|
|
1459
|
+
"""
|
|
1460
|
+
return self._client._marks("")
|
|
1461
|
+
|
|
1462
|
+
|
|
1463
|
+
class Storage:
|
|
1464
|
+
"""What the account stores, as `Client.storage`; not created directly.
|
|
1465
|
+
|
|
1466
|
+
Runs and disks live in the account's shared storage; deleting them
|
|
1467
|
+
frees their space once a prune reclaims it.
|
|
1468
|
+
|
|
1469
|
+
Example:
|
|
1470
|
+
>>> client.storage.show()["used_bytes"]
|
|
1471
|
+
52428800
|
|
1472
|
+
>>> client.storage.prune()
|
|
1473
|
+
"""
|
|
1474
|
+
|
|
1475
|
+
def __init__(self, client: Client) -> None:
|
|
1476
|
+
self._client = client
|
|
1477
|
+
|
|
1478
|
+
def show(self, *, refresh: bool = False) -> results.StorageInfo:
|
|
1479
|
+
"""Read what the account stores and its allowance, as `machine
|
|
1480
|
+
storage show` does.
|
|
1481
|
+
|
|
1482
|
+
The sizes come from the last measurement, which the document dates.
|
|
1483
|
+
`refresh` starts a new measurement; `ops` shows it running.
|
|
1484
|
+
|
|
1485
|
+
Args:
|
|
1486
|
+
refresh (bool, optional): Measure again instead of reading the
|
|
1487
|
+
last measurement. Defaults to False.
|
|
1488
|
+
|
|
1489
|
+
Returns:
|
|
1490
|
+
StorageInfo: The sizes, the allowance,
|
|
1491
|
+
and under `prune` the current or last prune.
|
|
1492
|
+
|
|
1493
|
+
Raises:
|
|
1494
|
+
NotSignedIn: There are no credentials.
|
|
1495
|
+
"""
|
|
1496
|
+
client, creds = self._client._account()
|
|
1497
|
+
with _translated():
|
|
1498
|
+
used = api.auth.authed(creds, lambda t: client.storage(t, refresh=refresh))
|
|
1499
|
+
running = api.auth.authed(creds, client.prune_status)
|
|
1500
|
+
return results.storage({**used, "prune": running})
|
|
1501
|
+
|
|
1502
|
+
@overload
|
|
1503
|
+
def ops(
|
|
1504
|
+
self,
|
|
1505
|
+
*,
|
|
1506
|
+
op_id: Optional[int] = ...,
|
|
1507
|
+
states: Optional[builtins.list[str]] = ...,
|
|
1508
|
+
types: Optional[builtins.list[str]] = ...,
|
|
1509
|
+
since: "_when.When | None" = ...,
|
|
1510
|
+
until: "_when.When | None" = ...,
|
|
1511
|
+
limit: Optional[int] = ...,
|
|
1512
|
+
offset: int = ...,
|
|
1513
|
+
watch: Literal[False] = ...,
|
|
1514
|
+
timeout: Optional[float] = ...,
|
|
1515
|
+
) -> builtins.list[results.StorageOp]: ...
|
|
1516
|
+
|
|
1517
|
+
@overload
|
|
1518
|
+
def ops(
|
|
1519
|
+
self,
|
|
1520
|
+
*,
|
|
1521
|
+
op_id: Optional[int] = ...,
|
|
1522
|
+
states: Optional[builtins.list[str]] = ...,
|
|
1523
|
+
types: Optional[builtins.list[str]] = ...,
|
|
1524
|
+
since: "_when.When | None" = ...,
|
|
1525
|
+
until: "_when.When | None" = ...,
|
|
1526
|
+
limit: Optional[int] = ...,
|
|
1527
|
+
offset: int = ...,
|
|
1528
|
+
watch: Literal[True],
|
|
1529
|
+
timeout: Optional[float] = ...,
|
|
1530
|
+
) -> Generator[builtins.list[results.StorageOp], None, None]: ...
|
|
1531
|
+
|
|
1532
|
+
def ops(
|
|
1533
|
+
self,
|
|
1534
|
+
*,
|
|
1535
|
+
op_id: Optional[int] = None,
|
|
1536
|
+
states: Optional[builtins.list[str]] = None,
|
|
1537
|
+
types: Optional[builtins.list[str]] = None,
|
|
1538
|
+
since: "_when.When | None" = None,
|
|
1539
|
+
until: "_when.When | None" = None,
|
|
1540
|
+
limit: Optional[int] = None,
|
|
1541
|
+
offset: int = 0,
|
|
1542
|
+
watch: bool = False,
|
|
1543
|
+
timeout: Optional[float] = None,
|
|
1544
|
+
) -> "builtins.list[results.StorageOp] | Generator[builtins.list[results.StorageOp], None, None]":
|
|
1545
|
+
"""List the account's storage operations, newest first, as `machine
|
|
1546
|
+
storage ops` does.
|
|
1547
|
+
|
|
1548
|
+
A storage operation is a measurement, a prune or a check. The list is
|
|
1549
|
+
every operation in every state; `states` keeps some. Every matching
|
|
1550
|
+
operation is returned unless `limit` is given.
|
|
1551
|
+
With `watch`, it yields the list at once, then again each time it
|
|
1552
|
+
changes, such as a prune finishing.
|
|
1553
|
+
|
|
1554
|
+
Example:
|
|
1555
|
+
>>> client.storage.ops(states=["failed"], since="1d")
|
|
1556
|
+
[{'op_id': '42', 'op_type': 'prune', 'state': 'failed', ...}]
|
|
1557
|
+
|
|
1558
|
+
Args:
|
|
1559
|
+
op_id (int, optional): Return this operation alone, whatever its
|
|
1560
|
+
state or type.
|
|
1561
|
+
states (list[str], optional): Keep these states: `pending`,
|
|
1562
|
+
`running`, `done`, `failed`; `^state` leaves one out.
|
|
1563
|
+
Defaults to every state.
|
|
1564
|
+
types (list[str], optional): Keep these types: `measure`,
|
|
1565
|
+
`prune`, `verify`, `check`; `^type` leaves one out. Defaults
|
|
1566
|
+
to every type.
|
|
1567
|
+
since (datetime or int or str, optional): Keep those started at
|
|
1568
|
+
or after this time: a datetime, Unix seconds, a date or an
|
|
1569
|
+
age such as `7d`.
|
|
1570
|
+
until (datetime or int or str, optional): Keep those started at
|
|
1571
|
+
or before this time.
|
|
1572
|
+
limit (int, optional): Return at most this many.
|
|
1573
|
+
offset (int, optional): Skip this many first. Defaults to 0.
|
|
1574
|
+
watch (bool, optional): Yield the list each time it changes.
|
|
1575
|
+
Defaults to False.
|
|
1576
|
+
timeout (float, optional): With `watch`, end the generator after
|
|
1577
|
+
this many seconds, without raising; None watches until the
|
|
1578
|
+
caller leaves the loop.
|
|
1579
|
+
|
|
1580
|
+
Returns:
|
|
1581
|
+
list[StorageOp]: One record per operation: its id,
|
|
1582
|
+
type, state, start time and error; with `watch`, an iterator of
|
|
1583
|
+
such lists.
|
|
1584
|
+
|
|
1585
|
+
Raises:
|
|
1586
|
+
InvalidArgument: A state, type or time is not valid.
|
|
1587
|
+
NotSignedIn: There are no credentials.
|
|
1588
|
+
"""
|
|
1589
|
+
filters = _args.op_filters(
|
|
1590
|
+
op_id,
|
|
1591
|
+
states,
|
|
1592
|
+
types,
|
|
1593
|
+
_machines.STORAGE_OP_TYPE_CHOICES,
|
|
1594
|
+
_machines.STORAGE_OP_STATE_CHOICES,
|
|
1595
|
+
)
|
|
1596
|
+
|
|
1597
|
+
def read() -> builtins.list[results.StorageOp]:
|
|
1598
|
+
with self._client._account_catalog() as source:
|
|
1599
|
+
rows = _machines.list_storage_ops(
|
|
1600
|
+
source,
|
|
1601
|
+
op_id=op_id,
|
|
1602
|
+
states=filters.states,
|
|
1603
|
+
op_types=filters.types,
|
|
1604
|
+
since=_when.unix_seconds(since),
|
|
1605
|
+
until=_when.unix_seconds(until),
|
|
1606
|
+
limit=limit,
|
|
1607
|
+
offset=offset,
|
|
1608
|
+
)
|
|
1609
|
+
return [results.storage_op(dataclasses.asdict(row)) for row in rows]
|
|
1610
|
+
|
|
1611
|
+
return _args.watched(read, _args.same, watch, timeout)
|
|
1612
|
+
|
|
1613
|
+
def prune(self, *, status: bool = False) -> Optional[results.Prune]:
|
|
1614
|
+
"""Start reclaiming the space deleted runs and disks still hold, as
|
|
1615
|
+
`machine storage prune` does.
|
|
1616
|
+
|
|
1617
|
+
The prune runs on the service after this returns; `status` and `ops`
|
|
1618
|
+
show it. It needs the account's storage to itself, so it is refused
|
|
1619
|
+
while any session is running: `sessions.delete(all=True)` first.
|
|
1620
|
+
|
|
1621
|
+
Args:
|
|
1622
|
+
status (bool, optional): Return the current or last prune
|
|
1623
|
+
instead of starting one. Defaults to False.
|
|
1624
|
+
|
|
1625
|
+
Returns:
|
|
1626
|
+
Prune: The prune; with `status`,
|
|
1627
|
+
None when there has been none.
|
|
1628
|
+
|
|
1629
|
+
Raises:
|
|
1630
|
+
Conflict: A session is running.
|
|
1631
|
+
NotSignedIn: There are no credentials.
|
|
1632
|
+
"""
|
|
1633
|
+
client, creds = self._client._account()
|
|
1634
|
+
with _translated():
|
|
1635
|
+
if status:
|
|
1636
|
+
row: Optional[dict[str, Any]] = api.auth.authed(
|
|
1637
|
+
creds, client.prune_status
|
|
1638
|
+
)
|
|
1639
|
+
return None if row is None else results.prune(row)
|
|
1640
|
+
started: dict[str, Any] = api.auth.authed(creds, client.prune_start)
|
|
1641
|
+
return results.prune(started)
|
|
1642
|
+
|
|
1643
|
+
def check(
|
|
1644
|
+
self,
|
|
1645
|
+
*,
|
|
1646
|
+
wait: bool = True,
|
|
1647
|
+
timeout: Optional[float] = None,
|
|
1648
|
+
status: bool = False,
|
|
1649
|
+
) -> Optional[results.StorageCheck]:
|
|
1650
|
+
"""Read every object of the account's storage back and check it
|
|
1651
|
+
against its name, as `machine storage check` does.
|
|
1652
|
+
|
|
1653
|
+
Nothing is written and nothing is locked, so it runs beside sessions
|
|
1654
|
+
and beside a prune. The check runs on the service: it goes on when a
|
|
1655
|
+
wait here ends, and `ops` lists it. One runs at a time, and asking
|
|
1656
|
+
while one runs follows that one.
|
|
1657
|
+
|
|
1658
|
+
Example:
|
|
1659
|
+
>>> found = client.storage.check()
|
|
1660
|
+
>>> (found.state, found.failed, found.unreadable)
|
|
1661
|
+
('done', 0, [])
|
|
1662
|
+
|
|
1663
|
+
Args:
|
|
1664
|
+
wait (bool, optional): Return once the check has ended. Defaults
|
|
1665
|
+
to True.
|
|
1666
|
+
timeout (float, optional): Seconds to wait; None waits without a
|
|
1667
|
+
limit.
|
|
1668
|
+
status (bool, optional): Return the current or last check
|
|
1669
|
+
instead of starting one. Defaults to False.
|
|
1670
|
+
|
|
1671
|
+
Returns:
|
|
1672
|
+
StorageCheck: The check, with `state` `failed` and `unreadable`
|
|
1673
|
+
filled in when an object could not be read; with `status`, None
|
|
1674
|
+
when there has been none.
|
|
1675
|
+
|
|
1676
|
+
Raises:
|
|
1677
|
+
Conflict: The account has no storage yet.
|
|
1678
|
+
Timeout: The check had not ended within `timeout`. It goes on.
|
|
1679
|
+
NotSignedIn: There are no credentials.
|
|
1680
|
+
"""
|
|
1681
|
+
client, creds = self._client._account()
|
|
1682
|
+
with _translated():
|
|
1683
|
+
if status:
|
|
1684
|
+
last: Optional[dict[str, Any]] = api.auth.authed(
|
|
1685
|
+
creds, client.storage_check
|
|
1686
|
+
)
|
|
1687
|
+
return None if last is None else results.storage_check(last)
|
|
1688
|
+
row: dict[str, Any] = api.auth.authed(creds, client.storage_check_start)
|
|
1689
|
+
if wait:
|
|
1690
|
+
row = _storage_ops.wait_check(
|
|
1691
|
+
lambda: api.auth.authed(creds, client.storage_check),
|
|
1692
|
+
row,
|
|
1693
|
+
timeout_s=timeout,
|
|
1694
|
+
)
|
|
1695
|
+
return results.storage_check(row)
|
|
1696
|
+
|
|
1697
|
+
|
|
1698
|
+
class Delete:
|
|
1699
|
+
"""Deleting runs, as `Client.delete`; not created directly.
|
|
1700
|
+
|
|
1701
|
+
Call it to delete every run in the current session, or every run that
|
|
1702
|
+
boots a disk, as `machine delete -a` and `machine delete --disk` do.
|
|
1703
|
+
`cleanup` resumes deletes that were interrupted. `Run.delete` deletes
|
|
1704
|
+
one run and its branches.
|
|
1705
|
+
|
|
1706
|
+
Example:
|
|
1707
|
+
>>> client.delete(disk="app", dry_run=True)
|
|
1708
|
+
[]
|
|
1709
|
+
>>> client.delete.cleanup().recovered
|
|
1710
|
+
0
|
|
1711
|
+
"""
|
|
1712
|
+
|
|
1713
|
+
def __init__(self, client: "Client") -> None:
|
|
1714
|
+
self._client = client
|
|
1715
|
+
|
|
1716
|
+
def __call__(
|
|
1717
|
+
self,
|
|
1718
|
+
*,
|
|
1719
|
+
all: bool = False,
|
|
1720
|
+
disk: "Disk | str | None" = None,
|
|
1721
|
+
stop: bool = False,
|
|
1722
|
+
force: bool = False,
|
|
1723
|
+
dry_run: bool = False,
|
|
1724
|
+
timeout: Optional[float] = None,
|
|
1725
|
+
) -> builtins.list["Run"]:
|
|
1726
|
+
"""Delete every run in the current session, or every run that boots
|
|
1727
|
+
a disk, as `machine delete -a` and `machine delete --disk` do.
|
|
1728
|
+
|
|
1729
|
+
`timeout` bounds the whole call. Run out part way, the run it
|
|
1730
|
+
stopped at goes on being deleted and the rest were not started.
|
|
1731
|
+
|
|
1732
|
+
`Run.delete` deletes one run and its branches; `cleanup` resumes
|
|
1733
|
+
deletes that were interrupted.
|
|
1734
|
+
|
|
1735
|
+
Example:
|
|
1736
|
+
>>> [r.name for r in client.delete(disk="app", dry_run=True)]
|
|
1737
|
+
['app', 'app/try-1']
|
|
1738
|
+
>>> client.delete(all=True, stop=True)
|
|
1739
|
+
|
|
1740
|
+
Args:
|
|
1741
|
+
all (bool, optional): Delete every run in the session. Defaults
|
|
1742
|
+
to False.
|
|
1743
|
+
disk (str, optional): Delete every run that boots this disk,
|
|
1744
|
+
by name or id, whichever session it is in.
|
|
1745
|
+
stop (bool, optional): Kill running machines first. Defaults to
|
|
1746
|
+
False.
|
|
1747
|
+
force (bool, optional): As `stop`, and delete a run whose machine
|
|
1748
|
+
is already gone or was replaced. Defaults to False.
|
|
1749
|
+
dry_run (bool, optional): List what would be deleted or
|
|
1750
|
+
recovered, acting on nothing. Defaults to False.
|
|
1751
|
+
timeout (float, optional): Seconds to wait for every run to be
|
|
1752
|
+
gone; None waits without a limit.
|
|
1753
|
+
|
|
1754
|
+
Returns:
|
|
1755
|
+
list[Run]: The runs deleted, or with `dry_run` those that would
|
|
1756
|
+
be.
|
|
1757
|
+
|
|
1758
|
+
Raises:
|
|
1759
|
+
InvalidArgument: Neither `all` nor `disk` is given, both are,
|
|
1760
|
+
or the client has no session.
|
|
1761
|
+
NotFound: No run boots `disk`.
|
|
1762
|
+
ApiError: Some runs could not be deleted; the message names the
|
|
1763
|
+
first.
|
|
1764
|
+
"""
|
|
1765
|
+
if all == (disk is not None):
|
|
1766
|
+
raise InvalidArgument(
|
|
1767
|
+
"Pass all=True or disk=, not both."
|
|
1768
|
+
if all
|
|
1769
|
+
else "Delete one run with run.delete(), or pass all=True or disk=."
|
|
1770
|
+
)
|
|
1771
|
+
stop = stop or force
|
|
1772
|
+
deadline = None if timeout is None else time.monotonic() + timeout
|
|
1773
|
+
rows = []
|
|
1774
|
+
if disk is not None:
|
|
1775
|
+
# Which runs boot a disk is the account's question: it holds the
|
|
1776
|
+
# disks, and a session's service has none.
|
|
1777
|
+
if self._client._local:
|
|
1778
|
+
raise InvalidArgument(
|
|
1779
|
+
"disk= needs a session: the local service has no account disks."
|
|
1780
|
+
)
|
|
1781
|
+
with self._client._account_catalog() as source, _translated():
|
|
1782
|
+
rows = _lifecycle.booting(source, _args.disk_ref(disk))
|
|
1783
|
+
with self._client._connection() as client, _translated():
|
|
1784
|
+
names = _names(client)
|
|
1785
|
+
if disk is not None:
|
|
1786
|
+
done, errors = _lifecycle.delete_by_disk(
|
|
1787
|
+
client,
|
|
1788
|
+
rows,
|
|
1789
|
+
force=stop,
|
|
1790
|
+
force_override=force,
|
|
1791
|
+
dry_run=dry_run,
|
|
1792
|
+
deadline=deadline,
|
|
1793
|
+
)
|
|
1794
|
+
else:
|
|
1795
|
+
done, errors = _lifecycle.delete_all(
|
|
1796
|
+
client,
|
|
1797
|
+
force=stop,
|
|
1798
|
+
force_override=force,
|
|
1799
|
+
dry_run=dry_run,
|
|
1800
|
+
deadline=deadline,
|
|
1801
|
+
)
|
|
1802
|
+
if isinstance(done, _machines.DeleteDryRunResult):
|
|
1803
|
+
return [
|
|
1804
|
+
Run(self._client, run_id, name) for name, run_id in done.would_delete
|
|
1805
|
+
]
|
|
1806
|
+
_raise_first(errors, f"Deleted {done.deleted}")
|
|
1807
|
+
return [
|
|
1808
|
+
Run(self._client, run_id, names.get(run_id, "")) for run_id in done.run_ids
|
|
1809
|
+
]
|
|
1810
|
+
|
|
1811
|
+
def cleanup(
|
|
1812
|
+
self, *, full_scan: bool = False, dry_run: bool = False
|
|
1813
|
+
) -> results.Recovery:
|
|
1814
|
+
"""Resume deletes that were interrupted, as `machine delete --cleanup -a`
|
|
1815
|
+
does.
|
|
1816
|
+
|
|
1817
|
+
A delete that stopped part way leaves a marker; this finishes the
|
|
1818
|
+
work each one names.
|
|
1819
|
+
|
|
1820
|
+
Args:
|
|
1821
|
+
full_scan (bool, optional): Scan every branch instead of the
|
|
1822
|
+
recorded roots, for a marker the index lost. Defaults to
|
|
1823
|
+
False.
|
|
1824
|
+
dry_run (bool, optional): List what would be recovered, doing
|
|
1825
|
+
nothing. Defaults to False.
|
|
1826
|
+
|
|
1827
|
+
Returns:
|
|
1828
|
+
Recovery: What it finished, what it left, and what went wrong.
|
|
1829
|
+
|
|
1830
|
+
Raises:
|
|
1831
|
+
InvalidArgument: The client has no session.
|
|
1832
|
+
"""
|
|
1833
|
+
with self._client._connection() as client, _translated():
|
|
1834
|
+
return results.recovery(
|
|
1835
|
+
dataclasses.asdict(
|
|
1836
|
+
_machines.delete_cleanup(
|
|
1837
|
+
client, full_scan=full_scan, dry_run=dry_run
|
|
1838
|
+
)
|
|
1839
|
+
)
|
|
1840
|
+
)
|
|
1841
|
+
|
|
1842
|
+
|
|
1843
|
+
class Preload:
|
|
1844
|
+
"""Copies onto the current session's disk, as `Client.preload`; not
|
|
1845
|
+
created directly.
|
|
1846
|
+
|
|
1847
|
+
A run is kept in shared storage and copied to a session's disk when it
|
|
1848
|
+
starts there; preloading copies it ahead of time, so starting it does
|
|
1849
|
+
not wait for the download. Call it with `all=True` to copy everything
|
|
1850
|
+
the session's runs need, as `machine preload -a` does. `Run.preload`
|
|
1851
|
+
copies one run.
|
|
1852
|
+
|
|
1853
|
+
Example:
|
|
1854
|
+
>>> client.preload(all=True)
|
|
1855
|
+
>>> client.preload.disk("app")
|
|
1856
|
+
"""
|
|
1857
|
+
|
|
1858
|
+
def __init__(self, client: Client) -> None:
|
|
1859
|
+
self._client = client
|
|
1860
|
+
|
|
1861
|
+
def __call__(self, *, all: bool, timeout: Optional[float] = None) -> None:
|
|
1862
|
+
"""Copy every run folder and boot disk the session's runs need, as
|
|
1863
|
+
`machine preload -a` does.
|
|
1864
|
+
|
|
1865
|
+
Args:
|
|
1866
|
+
all (bool): Copy everything; required, as `-a` is.
|
|
1867
|
+
timeout (float, optional): Seconds to wait for the copies; None
|
|
1868
|
+
waits without a limit.
|
|
1869
|
+
|
|
1870
|
+
Raises:
|
|
1871
|
+
InvalidArgument: `all` is False, or the client has no session.
|
|
1872
|
+
Timeout: The copies did not finish within `timeout`.
|
|
1873
|
+
"""
|
|
1874
|
+
_require_all(all, "Preload one run with run.preload().")
|
|
1875
|
+
with self._client._copies() as client, _translated():
|
|
1876
|
+
_machines.preload_all_both(client, timeout_s=timeout)
|
|
1877
|
+
|
|
1878
|
+
def run(self, *, all: bool, timeout: Optional[float] = None) -> None:
|
|
1879
|
+
"""Copy every data parent the session's runs need, as `machine
|
|
1880
|
+
preload run -a` does. `Run.preload` copies one run.
|
|
1881
|
+
|
|
1882
|
+
Args:
|
|
1883
|
+
all (bool): Copy every one; required, as `-a` is.
|
|
1884
|
+
timeout (float, optional): Seconds to wait for the copies; None
|
|
1885
|
+
waits without a limit.
|
|
1886
|
+
|
|
1887
|
+
Raises:
|
|
1888
|
+
InvalidArgument: `all` is False, or the client has no session.
|
|
1889
|
+
Timeout: The copies did not finish within `timeout`.
|
|
1890
|
+
"""
|
|
1891
|
+
_require_all(all, "Preload one run with run.preload().")
|
|
1892
|
+
with self._client._copies() as client, _translated():
|
|
1893
|
+
_machines.preload_all(client, timeout_s=timeout)
|
|
1894
|
+
|
|
1895
|
+
def disk(
|
|
1896
|
+
self,
|
|
1897
|
+
disk: "Disk | str | None" = None,
|
|
1898
|
+
*,
|
|
1899
|
+
all: bool = False,
|
|
1900
|
+
timeout: Optional[float] = None,
|
|
1901
|
+
) -> None:
|
|
1902
|
+
"""Copy a disk onto the session, as `machine preload disk` does, so
|
|
1903
|
+
creating a run from it does not wait for the download.
|
|
1904
|
+
|
|
1905
|
+
Args:
|
|
1906
|
+
disk (str, optional): The disk's name or id.
|
|
1907
|
+
all (bool, optional): Copy every boot disk the session's runs
|
|
1908
|
+
need instead. Defaults to False.
|
|
1909
|
+
timeout (float, optional): Seconds to wait for the copy; None
|
|
1910
|
+
waits without a limit.
|
|
1911
|
+
|
|
1912
|
+
Raises:
|
|
1913
|
+
InvalidArgument: Neither or both of `disk` and `all` are given,
|
|
1914
|
+
or the client has no session.
|
|
1915
|
+
NotFound: No disk has this name.
|
|
1916
|
+
Timeout: The copy did not finish within `timeout`.
|
|
1917
|
+
"""
|
|
1918
|
+
with self._client._copies() as client, _translated():
|
|
1919
|
+
if _one_or_all(disk, all) is None:
|
|
1920
|
+
_machines.preload_all(client, timeout_s=timeout, disks=True)
|
|
1921
|
+
else:
|
|
1922
|
+
digest = self._client._disk_id(str(disk))
|
|
1923
|
+
_machines.preload_disk(client, digest, timeout_s=timeout)
|
|
1924
|
+
|
|
1925
|
+
|
|
1926
|
+
class Offload:
|
|
1927
|
+
"""Copies dropped from the current session, as `Client.offload`;
|
|
1928
|
+
not created directly.
|
|
1929
|
+
|
|
1930
|
+
Offloading frees the session's storage; the runs and disks stay in shared
|
|
1931
|
+
storage and are copied back when a run needs them. Call it with no
|
|
1932
|
+
arguments to list what can be dropped, as `machine offload` does, or
|
|
1933
|
+
with `all=True` to drop all of it. `Run.offload` drops one run.
|
|
1934
|
+
|
|
1935
|
+
Example:
|
|
1936
|
+
>>> client.offload()["runs"]
|
|
1937
|
+
[{'run_id': 'whAK...', 'name': 'app', 'disk_bytes': 52428800, ...}]
|
|
1938
|
+
>>> client.offload(all=True)
|
|
1939
|
+
"""
|
|
1940
|
+
|
|
1941
|
+
def __init__(self, client: Client) -> None:
|
|
1942
|
+
self._client = client
|
|
1943
|
+
|
|
1944
|
+
def __call__(
|
|
1945
|
+
self,
|
|
1946
|
+
*,
|
|
1947
|
+
all: bool,
|
|
1948
|
+
runs: bool = False,
|
|
1949
|
+
disks: bool = False,
|
|
1950
|
+
timeout: Optional[float] = None,
|
|
1951
|
+
) -> None:
|
|
1952
|
+
"""Drop every copy not in use, as `machine offload -a` does.
|
|
1953
|
+
|
|
1954
|
+
`list` reads what can be dropped; this drops it. `Run.offload` drops
|
|
1955
|
+
one run, and `disk` one disk.
|
|
1956
|
+
|
|
1957
|
+
Example:
|
|
1958
|
+
>>> client.offload(all=True)
|
|
1959
|
+
>>> [c.name for c in client.offload.list().runs]
|
|
1960
|
+
[]
|
|
1961
|
+
|
|
1962
|
+
Args:
|
|
1963
|
+
all (bool): Drop every one; required, as `-a` is.
|
|
1964
|
+
runs (bool, optional): Only the session's run folders. Defaults
|
|
1965
|
+
to False.
|
|
1966
|
+
disks (bool, optional): Only the session's disk files.
|
|
1967
|
+
Defaults to False.
|
|
1968
|
+
timeout (float, optional): Seconds to wait; None waits without a
|
|
1969
|
+
limit.
|
|
1970
|
+
|
|
1971
|
+
Raises:
|
|
1972
|
+
InvalidArgument: `all` is False, or the client has no session.
|
|
1973
|
+
"""
|
|
1974
|
+
_require_all(all, "Read what can be dropped with client.offload.list().")
|
|
1975
|
+
both = runs == disks
|
|
1976
|
+
with self._client._copies() as client, _translated():
|
|
1977
|
+
if both:
|
|
1978
|
+
_machines.offload_all_both(client, timeout_s=timeout)
|
|
1979
|
+
else:
|
|
1980
|
+
_machines.offload_all(client, timeout_s=timeout, disks=disks)
|
|
1981
|
+
|
|
1982
|
+
def list(self, *, runs: bool = False, disks: bool = False) -> results.Copies:
|
|
1983
|
+
"""List the session's copies that can be dropped, as `machine
|
|
1984
|
+
offload list` does.
|
|
1985
|
+
|
|
1986
|
+
Only copies not in use are listed; the rest are counted.
|
|
1987
|
+
|
|
1988
|
+
Args:
|
|
1989
|
+
runs (bool, optional): Only the session's run folders. Defaults
|
|
1990
|
+
to False.
|
|
1991
|
+
disks (bool, optional): Only the session's disk files.
|
|
1992
|
+
Defaults to False.
|
|
1993
|
+
|
|
1994
|
+
Returns:
|
|
1995
|
+
Copies: The run folders and the disk files it holds.
|
|
1996
|
+
|
|
1997
|
+
Raises:
|
|
1998
|
+
InvalidArgument: The client has no session.
|
|
1999
|
+
"""
|
|
2000
|
+
both = runs == disks
|
|
2001
|
+
with self._client._copies() as client, _translated():
|
|
2002
|
+
return results.copies(
|
|
2003
|
+
_machines.cache_list(
|
|
2004
|
+
client, runs=runs or both, disks=disks or both
|
|
2005
|
+
).to_json()
|
|
2006
|
+
)
|
|
2007
|
+
|
|
2008
|
+
def run(self, *, all: bool, timeout: Optional[float] = None) -> None:
|
|
2009
|
+
"""Drop every run folder not in use, as `machine offload run -a`
|
|
2010
|
+
does. `Run.offload` drops one run.
|
|
2011
|
+
|
|
2012
|
+
Args:
|
|
2013
|
+
all (bool): Drop every one; required, as `-a` is.
|
|
2014
|
+
timeout (float, optional): Seconds to wait; None waits without a
|
|
2015
|
+
limit.
|
|
2016
|
+
|
|
2017
|
+
Raises:
|
|
2018
|
+
InvalidArgument: `all` is False, or the client has no session.
|
|
2019
|
+
"""
|
|
2020
|
+
_require_all(all, "Offload one run with run.offload().")
|
|
2021
|
+
with self._client._copies() as client, _translated():
|
|
2022
|
+
_machines.offload_all(client, timeout_s=timeout)
|
|
2023
|
+
|
|
2024
|
+
def disk(
|
|
2025
|
+
self,
|
|
2026
|
+
disk: "Disk | str | None" = None,
|
|
2027
|
+
*,
|
|
2028
|
+
all: bool = False,
|
|
2029
|
+
timeout: Optional[float] = None,
|
|
2030
|
+
) -> None:
|
|
2031
|
+
"""Drop the session's copy of a disk, as `machine offload disk`
|
|
2032
|
+
does. The disk stays in the account.
|
|
2033
|
+
|
|
2034
|
+
Args:
|
|
2035
|
+
disk (str, optional): The disk's name or id.
|
|
2036
|
+
all (bool, optional): Drop every disk not in use instead.
|
|
2037
|
+
Defaults to False.
|
|
2038
|
+
timeout (float, optional): Seconds to wait; None waits without a
|
|
2039
|
+
limit.
|
|
2040
|
+
|
|
2041
|
+
Raises:
|
|
2042
|
+
InvalidArgument: Neither or both of `disk` and `all` are given,
|
|
2043
|
+
or the client has no session.
|
|
2044
|
+
NotFound: No disk has this name.
|
|
2045
|
+
Conflict: A running machine uses the copy.
|
|
2046
|
+
"""
|
|
2047
|
+
with self._client._copies() as client, _translated():
|
|
2048
|
+
if _one_or_all(disk, all) is None:
|
|
2049
|
+
_machines.offload_all(client, timeout_s=timeout, disks=True)
|
|
2050
|
+
else:
|
|
2051
|
+
digest = self._client._disk_id(str(disk))
|
|
2052
|
+
_machines.offload_disk(client, digest, timeout_s=timeout)
|
|
2053
|
+
|
|
2054
|
+
|
|
2055
|
+
def _require_all(all: bool, hint: str) -> None:
|
|
2056
|
+
"""InvalidArgument unless `all`, the only form a many-run command has."""
|
|
2057
|
+
if not all:
|
|
2058
|
+
raise InvalidArgument(f"Pass all=True. {hint}")
|
|
2059
|
+
|
|
2060
|
+
|
|
2061
|
+
def _one_or_all(name: "Disk | str | None", all: bool) -> Optional[str]:
|
|
2062
|
+
"""The disk named, or None for `all`; InvalidArgument unless exactly one
|
|
2063
|
+
is given. A `Disk` names itself, as a run and a session do."""
|
|
2064
|
+
if (name is None) == (not all):
|
|
2065
|
+
raise InvalidArgument("Name a disk, or pass all=True.")
|
|
2066
|
+
return None if name is None else _args.disk_ref(name)
|
|
2067
|
+
|
|
2068
|
+
|
|
2069
|
+
def _names(client: _ipc.Client) -> dict[str, str]:
|
|
2070
|
+
"""Every run's name by id, read before a many-run command removes them."""
|
|
2071
|
+
return {b.run_id: b.name for b in client.branch_list()}
|
|
2072
|
+
|
|
2073
|
+
|
|
2074
|
+
def _raise_first(errors: builtins.list[str], done: str) -> None:
|
|
2075
|
+
"""ApiError naming the first of `errors`, when there are any."""
|
|
2076
|
+
if errors:
|
|
2077
|
+
raise ApiError(f"{done}, {len(errors)} failed; first {errors[0]}")
|