manifold-cli 0.0.39__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- manifold_cli/__init__.py +3 -0
- manifold_cli/__main__.py +4 -0
- manifold_cli/bridge.py +70 -0
- manifold_cli/cli.py +54 -0
- manifold_cli/clients/__init__.py +0 -0
- manifold_cli/clients/auth_flow.py +169 -0
- manifold_cli/clients/credentials.py +53 -0
- manifold_cli/clients/docker_engine.py +43 -0
- manifold_cli/clients/image.py +130 -0
- manifold_cli/clients/manifold.py +1288 -0
- manifold_cli/clients/runner_home.py +165 -0
- manifold_cli/clients/workos.py +99 -0
- manifold_cli/commands/__init__.py +0 -0
- manifold_cli/commands/auth.py +101 -0
- manifold_cli/commands/benchmark.py +539 -0
- manifold_cli/commands/organization.py +154 -0
- manifold_cli/commands/policy.py +749 -0
- manifold_cli/commands/run.py +337 -0
- manifold_cli/commands/run_log.py +291 -0
- manifold_cli/commands/runner.py +852 -0
- manifold_cli/commands/table.py +101 -0
- manifold_cli/commands/token.py +144 -0
- manifold_cli/dependencies.py +116 -0
- manifold_cli/models/__init__.py +0 -0
- manifold_cli/models/auth.py +49 -0
- manifold_cli/models/benchmark.py +87 -0
- manifold_cli/models/catalog.py +189 -0
- manifold_cli/models/credentials.py +19 -0
- manifold_cli/models/errors.py +121 -0
- manifold_cli/models/log.py +114 -0
- manifold_cli/models/observability.py +121 -0
- manifold_cli/models/organization.py +19 -0
- manifold_cli/models/pagination.py +59 -0
- manifold_cli/models/policy.py +49 -0
- manifold_cli/models/run.py +381 -0
- manifold_cli/models/runner.py +186 -0
- manifold_cli/models/runner_home.py +69 -0
- manifold_cli/models/task_queue.py +40 -0
- manifold_cli/models/token.py +39 -0
- manifold_cli/replay/__init__.py +68 -0
- manifold_cli/replay/log.py +814 -0
- manifold_cli/runner/__init__.py +72 -0
- manifold_cli/runner/api_retry.py +36 -0
- manifold_cli/runner/benchmark/__init__.py +800 -0
- manifold_cli/runner/benchmark/live_view.py +248 -0
- manifold_cli/runner/benchmark/preassigned_scheduling.py +325 -0
- manifold_cli/runner/benchmark/recording.py +1208 -0
- manifold_cli/runner/benchmark/scheduling.py +256 -0
- manifold_cli/runner/benchmark/task_queue_scheduling.py +717 -0
- manifold_cli/runner/benchmark/task_server.py +123 -0
- manifold_cli/runner/benchmark/video.py +124 -0
- manifold_cli/runner/benchmark_steps.py +178 -0
- manifold_cli/runner/cancellation.py +22 -0
- manifold_cli/runner/capture.py +282 -0
- manifold_cli/runner/container.py +370 -0
- manifold_cli/runner/current_dispatch.py +25 -0
- manifold_cli/runner/daemon.py +520 -0
- manifold_cli/runner/dispatch.py +38 -0
- manifold_cli/runner/gpu.py +182 -0
- manifold_cli/runner/heartbeat.py +202 -0
- manifold_cli/runner/loopback.py +102 -0
- manifold_cli/runner/loopback_bench.py +183 -0
- manifold_cli/runner/loopback_policy.py +39 -0
- manifold_cli/runner/output.py +218 -0
- manifold_cli/runner/pairing_acceptance.py +57 -0
- manifold_cli/runner/plan.py +369 -0
- manifold_cli/runner/policy.py +350 -0
- manifold_cli/runner/policy_steps.py +164 -0
- manifold_cli/runner/preparation.py +47 -0
- manifold_cli/runner/process.py +219 -0
- manifold_cli/runner/ram.py +17 -0
- manifold_cli/runner/resource_usage.py +265 -0
- manifold_cli/runner/served.py +218 -0
- manifold_cli/runner/service.py +120 -0
- manifold_cli/runner/side_failure.py +68 -0
- manifold_cli/runner/signature.py +98 -0
- manifold_cli/runner/source.py +66 -0
- manifold_cli/runner/stages.py +241 -0
- manifold_cli/runner/step_timing.py +229 -0
- manifold_cli/runner/worker.py +534 -0
- manifold_cli/runner/workload.py +401 -0
- manifold_cli/services/__init__.py +0 -0
- manifold_cli/services/auth.py +89 -0
- manifold_cli/services/benchmark.py +357 -0
- manifold_cli/services/organization.py +22 -0
- manifold_cli/services/policy.py +234 -0
- manifold_cli/services/run.py +304 -0
- manifold_cli/services/runner.py +207 -0
- manifold_cli/services/runner_daemon.py +146 -0
- manifold_cli/services/token.py +52 -0
- manifold_cli/settings.py +30 -0
- manifold_cli/transport/__init__.py +31 -0
- manifold_cli/transport/diagnostics.py +72 -0
- manifold_cli/transport/interfaces.py +69 -0
- manifold_cli/transport/iroh.py +317 -0
- manifold_cli/transport/registry.py +34 -0
- manifold_cli/transport/tcp.py +146 -0
- manifold_cli-0.0.39.dist-info/METADATA +406 -0
- manifold_cli-0.0.39.dist-info/RECORD +102 -0
- manifold_cli-0.0.39.dist-info/WHEEL +4 -0
- manifold_cli-0.0.39.dist-info/entry_points.txt +2 -0
- manifold_cli-0.0.39.dist-info/licenses/LICENSE +373 -0
|
@@ -0,0 +1,401 @@
|
|
|
1
|
+
"""Pairing one worker with the transport leg on its own side of a run.
|
|
2
|
+
|
|
3
|
+
`serve_policy` and `run_benchmark` are the two halves of a container run. Each
|
|
4
|
+
owns the worker on its own machine: it starts it, establishes that it is ready,
|
|
5
|
+
and only then involves the other side. Neither worker addresses the other -- both
|
|
6
|
+
speak plain local TCP to the bridge these functions run in-process.
|
|
7
|
+
|
|
8
|
+
The two sides sequence differently, because a policy container is dialled and a
|
|
9
|
+
benchmark container dials:
|
|
10
|
+
|
|
11
|
+
- A policy container listens, so its readiness is the port it opens, and the
|
|
12
|
+
transport listener stays shut until that port answers.
|
|
13
|
+
- A benchmark container dials out, so the local listener comes first and the
|
|
14
|
+
container's dial into it is the readiness signal.
|
|
15
|
+
|
|
16
|
+
Nothing calls these yet: the command that claims a side and picks its plan lands
|
|
17
|
+
in a later slice.
|
|
18
|
+
|
|
19
|
+
NOTE: a claim is not the signal to call either half. The two sides of a run are
|
|
20
|
+
claimed separately, and a claim only records the assigned runner -- the worker
|
|
21
|
+
starts once both sides are assigned, so the caller waits for the pairing rather
|
|
22
|
+
than acting on its own claim.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
import asyncio
|
|
26
|
+
import contextlib
|
|
27
|
+
from collections.abc import Callable
|
|
28
|
+
from dataclasses import dataclass
|
|
29
|
+
|
|
30
|
+
from rich.markup import escape
|
|
31
|
+
|
|
32
|
+
from manifold_cli.bridge import run_bridge
|
|
33
|
+
from manifold_cli.models.observability import BenchmarkStep, PolicyStep
|
|
34
|
+
from manifold_cli.runner.api_retry import retry_transient_api_failures
|
|
35
|
+
from manifold_cli.runner.benchmark_steps import RoundTripTimingConnector
|
|
36
|
+
from manifold_cli.runner.capture import LineSink, WorkerCapture
|
|
37
|
+
from manifold_cli.runner.pairing_acceptance import PairingAcceptanceConnector
|
|
38
|
+
from manifold_cli.runner.plan import (
|
|
39
|
+
PlanError,
|
|
40
|
+
WireBinding,
|
|
41
|
+
WorkerPlan,
|
|
42
|
+
published_wire_from,
|
|
43
|
+
)
|
|
44
|
+
from manifold_cli.runner.policy_steps import StepTimingConnector
|
|
45
|
+
from manifold_cli.runner.process import ProcessHandle, Spawner
|
|
46
|
+
from manifold_cli.runner.stages import SideStages
|
|
47
|
+
from manifold_cli.runner.step_timing import StepBuffer
|
|
48
|
+
from manifold_cli.runner.worker import (
|
|
49
|
+
STDERR_TAIL,
|
|
50
|
+
WorkerExit,
|
|
51
|
+
WorkerStartError,
|
|
52
|
+
classify_exit,
|
|
53
|
+
running_worker,
|
|
54
|
+
with_tail,
|
|
55
|
+
)
|
|
56
|
+
from manifold_cli.transport import (
|
|
57
|
+
DEFAULT_TRANSPORTS,
|
|
58
|
+
ByteStream,
|
|
59
|
+
Connector,
|
|
60
|
+
Listener,
|
|
61
|
+
TcpConnector,
|
|
62
|
+
TcpTransport,
|
|
63
|
+
TransportError,
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
UPSTREAM_RETRY_SECONDS = 0.5
|
|
67
|
+
|
|
68
|
+
# Which worker the policy side's output belongs to. That side runs one, so its
|
|
69
|
+
# lines are always worker 0's, but the runner writes each benchmark worker's
|
|
70
|
+
# lines under the index of that worker's shard.
|
|
71
|
+
SOLE_WORKER_INDEX = 0
|
|
72
|
+
|
|
73
|
+
# How long `plan.published_wire` gets. It asks docker for a port an already
|
|
74
|
+
# running container has bound, which is a local answer docker holds already.
|
|
75
|
+
PUBLISHED_WIRE_TIMEOUT_SECONDS = 10.0
|
|
76
|
+
|
|
77
|
+
# How often `bridge_to_local_policy` checks whether the policy process exited.
|
|
78
|
+
EXIT_POLL_SECONDS = 0.5
|
|
79
|
+
|
|
80
|
+
Log = Callable[[str], None]
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
async def serve_policy(
|
|
84
|
+
plan: WorkerPlan,
|
|
85
|
+
target: str,
|
|
86
|
+
*,
|
|
87
|
+
wire: WireBinding,
|
|
88
|
+
spawner: Spawner,
|
|
89
|
+
stages: SideStages,
|
|
90
|
+
on_listening: Callable[[str], None],
|
|
91
|
+
policy_steps: StepBuffer[PolicyStep],
|
|
92
|
+
on_accepted: Callable[[], None],
|
|
93
|
+
sink: LineSink | None = None,
|
|
94
|
+
) -> WorkerExit:
|
|
95
|
+
"""Start `plan`'s policy worker on `wire` and bridge `target` to it.
|
|
96
|
+
|
|
97
|
+
The bridge adds each policy step it times to `policy_steps`, and calls
|
|
98
|
+
`on_accepted` each time the policy accepts the benchmark's signature.
|
|
99
|
+
|
|
100
|
+
`on_listening` is handed the address the benchmark side dials as soon as the
|
|
101
|
+
bridge behind it is up, and handed it again after a transient API failure:
|
|
102
|
+
publishing the same address twice replaces it with itself. It is what
|
|
103
|
+
reaches the server, which is the only way the far side learns where to dial:
|
|
104
|
+
any other failure from it therefore ends the pairing, because an address the
|
|
105
|
+
server never recorded is one nothing will ever dial. It is called on a
|
|
106
|
+
thread, because the call it makes is a synchronous one over the network, and
|
|
107
|
+
the listener has to be accepting for the whole of it.
|
|
108
|
+
|
|
109
|
+
Returns how the worker ended, and raises the transport leg's failure where
|
|
110
|
+
that is what ended the pairing. A worker that never becomes ready raises
|
|
111
|
+
`WorkerStartError`, having been torn down.
|
|
112
|
+
|
|
113
|
+
NOTE: the listener on `target` opens only once the worker is ready. Opening
|
|
114
|
+
it earlier and not accepting is not a gate -- `listen()` hands the kernel a
|
|
115
|
+
backlog, and the kernel completes a dialer's handshake without userspace, so
|
|
116
|
+
the benchmark side would connect and then talk into silence.
|
|
117
|
+
"""
|
|
118
|
+
transport, rest = DEFAULT_TRANSPORTS.resolve(target)
|
|
119
|
+
captured = WorkerCapture(worker_index=SOLE_WORKER_INDEX, sink=sink)
|
|
120
|
+
async with running_worker(
|
|
121
|
+
plan, spawner=spawner, captured=captured, stages=stages
|
|
122
|
+
) as process:
|
|
123
|
+
bound = await _bound_wire(
|
|
124
|
+
plan, wire, process, spawner=spawner, captured=captured
|
|
125
|
+
)
|
|
126
|
+
listener = await transport.open_listener(rest)
|
|
127
|
+
bridge_task = asyncio.create_task(
|
|
128
|
+
run_bridge(
|
|
129
|
+
listener,
|
|
130
|
+
StepTimingConnector(
|
|
131
|
+
PairingAcceptanceConnector(
|
|
132
|
+
TcpConnector(bound.host, bound.port), on_accepted
|
|
133
|
+
),
|
|
134
|
+
policy_steps,
|
|
135
|
+
),
|
|
136
|
+
)
|
|
137
|
+
)
|
|
138
|
+
try:
|
|
139
|
+
await retry_transient_api_failures(
|
|
140
|
+
lambda: asyncio.to_thread(on_listening, listener.target)
|
|
141
|
+
)
|
|
142
|
+
stages.ready(f"Dial it at: [bold]{escape(listener.target)}[/bold]")
|
|
143
|
+
code = await _wait_worker_or_bridge(process, bridge_task)
|
|
144
|
+
# Inside the worker's context, where the container it started still
|
|
145
|
+
# exists to be asked how it ended.
|
|
146
|
+
return await classify_exit(plan, code, spawner=spawner, captured=captured)
|
|
147
|
+
finally:
|
|
148
|
+
await _stop_bridge(bridge_task, listener)
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
async def bridge_to_local_policy(
|
|
152
|
+
wire: WireBinding,
|
|
153
|
+
target: str,
|
|
154
|
+
*,
|
|
155
|
+
exited: Callable[[], bool],
|
|
156
|
+
stages: SideStages,
|
|
157
|
+
on_listening: Callable[[str], None],
|
|
158
|
+
policy_steps: StepBuffer[PolicyStep],
|
|
159
|
+
on_accepted: Callable[[], None],
|
|
160
|
+
) -> None:
|
|
161
|
+
"""Bridge `target` to a policy that already listens on `wire`.
|
|
162
|
+
|
|
163
|
+
`manifold policy serve` starts the policy before any run, so nothing here
|
|
164
|
+
starts or stops a process. `on_listening`, `policy_steps` and `on_accepted`
|
|
165
|
+
work as they do in `serve_policy`.
|
|
166
|
+
|
|
167
|
+
Returns once `exited` says that the policy process has exited. Otherwise it
|
|
168
|
+
runs until cancelled, which is how a run ends when the benchmark side
|
|
169
|
+
completes. Raises the transport failure when the bridge fails.
|
|
170
|
+
"""
|
|
171
|
+
transport, rest = DEFAULT_TRANSPORTS.resolve(target)
|
|
172
|
+
listener = await transport.open_listener(rest)
|
|
173
|
+
bridge_task = asyncio.create_task(
|
|
174
|
+
run_bridge(
|
|
175
|
+
listener,
|
|
176
|
+
StepTimingConnector(
|
|
177
|
+
PairingAcceptanceConnector(
|
|
178
|
+
TcpConnector(wire.host, wire.port), on_accepted
|
|
179
|
+
),
|
|
180
|
+
policy_steps,
|
|
181
|
+
),
|
|
182
|
+
)
|
|
183
|
+
)
|
|
184
|
+
try:
|
|
185
|
+
await retry_transient_api_failures(
|
|
186
|
+
lambda: asyncio.to_thread(on_listening, listener.target)
|
|
187
|
+
)
|
|
188
|
+
stages.ready(f"Dial it at: [bold]{escape(listener.target)}[/bold]")
|
|
189
|
+
while not exited():
|
|
190
|
+
await asyncio.wait({bridge_task}, timeout=EXIT_POLL_SECONDS)
|
|
191
|
+
if bridge_task.done():
|
|
192
|
+
exc = bridge_task.exception()
|
|
193
|
+
if exc is not None:
|
|
194
|
+
raise exc
|
|
195
|
+
raise TransportError(
|
|
196
|
+
"the transport bridge stopped before the policy did"
|
|
197
|
+
)
|
|
198
|
+
finally:
|
|
199
|
+
await _stop_bridge(bridge_task, listener)
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
async def run_benchmark(
|
|
203
|
+
plan: WorkerPlan,
|
|
204
|
+
target: str,
|
|
205
|
+
*,
|
|
206
|
+
wire: WireBinding,
|
|
207
|
+
spawner: Spawner,
|
|
208
|
+
log: Log,
|
|
209
|
+
upstream_budget_s: float,
|
|
210
|
+
worker_index: int,
|
|
211
|
+
benchmark_steps: StepBuffer[BenchmarkStep] | None,
|
|
212
|
+
on_accepted: Callable[[], None],
|
|
213
|
+
sink: LineSink | None = None,
|
|
214
|
+
) -> WorkerExit:
|
|
215
|
+
"""Bridge `wire` to `target` and run `plan`'s benchmark worker against it.
|
|
216
|
+
|
|
217
|
+
The listener on `wire` opens before the worker starts, because the worker
|
|
218
|
+
dialling into it is what says the worker is up: a client binds no port for a
|
|
219
|
+
probe to check.
|
|
220
|
+
|
|
221
|
+
`worker_index` is the index of the shard that this worker runs, and the
|
|
222
|
+
runner writes the worker's output lines under that index. A shard is one
|
|
223
|
+
slice of the side's episodes rather than a run of its own, so the caller
|
|
224
|
+
states the index.
|
|
225
|
+
|
|
226
|
+
`upstream_budget_s` is how long a refused dial of `target` stays normal: the
|
|
227
|
+
`startup_budget_s` of the plan on the far side, whose bridge holds its
|
|
228
|
+
listener shut until its own worker answers. It is the policy plan's budget
|
|
229
|
+
and not this one's, so the caller passes it in.
|
|
230
|
+
|
|
231
|
+
The bridge adds each step it times to `benchmark_steps`. Pass None when the
|
|
232
|
+
policy side's bridge cannot send step times. The bridge then does not time
|
|
233
|
+
steps.
|
|
234
|
+
|
|
235
|
+
The bridge calls `on_accepted` each time the policy accepts the benchmark's
|
|
236
|
+
signature.
|
|
237
|
+
|
|
238
|
+
Returns how the worker ended, and raises the transport leg's failure where
|
|
239
|
+
that is what ended the pairing.
|
|
240
|
+
"""
|
|
241
|
+
transport, rest = DEFAULT_TRANSPORTS.resolve(target)
|
|
242
|
+
# Built before anything is bound: a target the transport cannot parse must
|
|
243
|
+
# not leave a listener behind on its way out.
|
|
244
|
+
upstream = _RetryingConnector(
|
|
245
|
+
transport.build_connector(rest), timeout_s=upstream_budget_s
|
|
246
|
+
)
|
|
247
|
+
connector = PairingAcceptanceConnector(
|
|
248
|
+
(
|
|
249
|
+
upstream
|
|
250
|
+
if benchmark_steps is None
|
|
251
|
+
else RoundTripTimingConnector(upstream, benchmark_steps)
|
|
252
|
+
),
|
|
253
|
+
on_accepted,
|
|
254
|
+
)
|
|
255
|
+
listener = await TcpTransport().open_listener(wire.address)
|
|
256
|
+
captured = WorkerCapture(worker_index=worker_index, sink=sink)
|
|
257
|
+
bridge_task = asyncio.create_task(run_bridge(listener, connector))
|
|
258
|
+
try:
|
|
259
|
+
log(f"Bridging [bold]{wire.address}[/bold] to [bold]{escape(target)}[/bold].")
|
|
260
|
+
async with running_worker(plan, spawner=spawner, captured=captured) as process:
|
|
261
|
+
code = await _wait_worker_or_bridge(process, bridge_task)
|
|
262
|
+
return await classify_exit(plan, code, spawner=spawner, captured=captured)
|
|
263
|
+
finally:
|
|
264
|
+
await _stop_bridge(bridge_task, listener)
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
async def _bound_wire(
|
|
268
|
+
plan: WorkerPlan,
|
|
269
|
+
wire: WireBinding,
|
|
270
|
+
process: ProcessHandle,
|
|
271
|
+
*,
|
|
272
|
+
spawner: Spawner,
|
|
273
|
+
captured: WorkerCapture,
|
|
274
|
+
) -> WireBinding:
|
|
275
|
+
"""The address the worker's port actually ended up on.
|
|
276
|
+
|
|
277
|
+
`wire` for a plan that named the port itself. Where the plan left the port
|
|
278
|
+
to docker, `plan.published_wire` reads back the one docker chose, so it
|
|
279
|
+
runs only once the worker is up and something has bound it.
|
|
280
|
+
|
|
281
|
+
Raises WorkerStartError where the command fails, and PlanError where its
|
|
282
|
+
output names no address. Where the worker has exited by then, both give way
|
|
283
|
+
to its own exit and last output: docker answers for the container it finds,
|
|
284
|
+
which publishes nothing and says nothing about why it stopped.
|
|
285
|
+
"""
|
|
286
|
+
if plan.published_wire is None:
|
|
287
|
+
return wire
|
|
288
|
+
try:
|
|
289
|
+
output = await _published_wire_output(plan.published_wire, spawner=spawner)
|
|
290
|
+
return published_wire_from(output)
|
|
291
|
+
except (WorkerStartError, PlanError) as exc:
|
|
292
|
+
if process.returncode is None:
|
|
293
|
+
raise
|
|
294
|
+
ended = await classify_exit(
|
|
295
|
+
plan, process.returncode, spawner=spawner, captured=captured
|
|
296
|
+
)
|
|
297
|
+
raise WorkerStartError(
|
|
298
|
+
with_tail(
|
|
299
|
+
f"The worker {ended.cause} before it published its port.",
|
|
300
|
+
ended.log_tail,
|
|
301
|
+
)
|
|
302
|
+
) from exc
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
async def _published_wire_output(argv: tuple[str, ...], *, spawner: Spawner) -> str:
|
|
306
|
+
"""What `plan.published_wire` printed, raising WorkerStartError if it failed."""
|
|
307
|
+
try:
|
|
308
|
+
result = await spawner.run(argv, timeout_s=PUBLISHED_WIRE_TIMEOUT_SECONDS)
|
|
309
|
+
except TimeoutError as exc:
|
|
310
|
+
raise WorkerStartError(
|
|
311
|
+
"Reading the worker's published port did not finish within "
|
|
312
|
+
f"{PUBLISHED_WIRE_TIMEOUT_SECONDS:g}s."
|
|
313
|
+
) from exc
|
|
314
|
+
except OSError as exc:
|
|
315
|
+
raise WorkerStartError(
|
|
316
|
+
f"Reading the worker's published port could not run: {exc}"
|
|
317
|
+
) from exc
|
|
318
|
+
if result.exit_code != 0:
|
|
319
|
+
raise WorkerStartError(
|
|
320
|
+
f"Reading the worker's published port exited {result.exit_code}: "
|
|
321
|
+
f"{result.stderr.strip()[-STDERR_TAIL:]}"
|
|
322
|
+
)
|
|
323
|
+
return result.stdout
|
|
324
|
+
|
|
325
|
+
|
|
326
|
+
@dataclass
|
|
327
|
+
class _RetryingConnector:
|
|
328
|
+
"""Dials `upstream` until it answers, or `timeout_s` is up.
|
|
329
|
+
|
|
330
|
+
A refused dial is the normal early state of a pairing rather than an error:
|
|
331
|
+
the policy bridge on the far side holds its listener shut until its own
|
|
332
|
+
container is ready.
|
|
333
|
+
|
|
334
|
+
Only the first dial gets that budget. Once the far side has answered, a
|
|
335
|
+
refusal means the far side has gone away, and spending the budget again
|
|
336
|
+
would hold the bridge's accept loop for the whole of it without accepting
|
|
337
|
+
anything further.
|
|
338
|
+
"""
|
|
339
|
+
|
|
340
|
+
upstream: Connector
|
|
341
|
+
timeout_s: float
|
|
342
|
+
has_connected: bool = False
|
|
343
|
+
|
|
344
|
+
async def connect(self) -> ByteStream:
|
|
345
|
+
if self.has_connected:
|
|
346
|
+
return await self.upstream.connect()
|
|
347
|
+
loop = asyncio.get_running_loop()
|
|
348
|
+
deadline = loop.time() + self.timeout_s
|
|
349
|
+
while True:
|
|
350
|
+
try:
|
|
351
|
+
stream = await self.upstream.connect()
|
|
352
|
+
except (OSError, TransportError):
|
|
353
|
+
remaining = deadline - loop.time()
|
|
354
|
+
if remaining <= 0:
|
|
355
|
+
raise
|
|
356
|
+
await asyncio.sleep(min(UPSTREAM_RETRY_SECONDS, remaining))
|
|
357
|
+
else:
|
|
358
|
+
self.has_connected = True
|
|
359
|
+
return stream
|
|
360
|
+
|
|
361
|
+
|
|
362
|
+
async def _wait_worker_or_bridge(
|
|
363
|
+
process: ProcessHandle, bridge_task: asyncio.Task[None]
|
|
364
|
+
) -> int:
|
|
365
|
+
"""The worker's exit code, or the transport leg's failure if it ended first.
|
|
366
|
+
|
|
367
|
+
A bridge that dies leaves the worker talking to nothing, so it ends the wait
|
|
368
|
+
rather than being discovered whenever the worker happens to exit.
|
|
369
|
+
"""
|
|
370
|
+
waiter = asyncio.ensure_future(process.wait())
|
|
371
|
+
try:
|
|
372
|
+
await asyncio.wait({waiter, bridge_task}, return_when=asyncio.FIRST_COMPLETED)
|
|
373
|
+
if waiter.done():
|
|
374
|
+
return await waiter
|
|
375
|
+
exc = bridge_task.exception()
|
|
376
|
+
if exc is not None:
|
|
377
|
+
raise exc
|
|
378
|
+
raise TransportError("the transport bridge stopped before the worker did")
|
|
379
|
+
finally:
|
|
380
|
+
if not waiter.done():
|
|
381
|
+
waiter.cancel()
|
|
382
|
+
with contextlib.suppress(asyncio.CancelledError):
|
|
383
|
+
await waiter
|
|
384
|
+
|
|
385
|
+
|
|
386
|
+
async def _stop_bridge(bridge_task: asyncio.Task[None], listener: Listener) -> None:
|
|
387
|
+
"""Tear the bridge down, swallowing whatever it ends with.
|
|
388
|
+
|
|
389
|
+
Whatever it failed with was already raised by the wait above, and a bridge
|
|
390
|
+
still running has nothing to report.
|
|
391
|
+
|
|
392
|
+
NOTE: the listener is closed here as well as by `run_bridge`, which closes
|
|
393
|
+
it on its own way out. A task cancelled before it has run a first step never
|
|
394
|
+
enters the body holding that cleanup; the socket it opened stays open, and
|
|
395
|
+
the only report of it is a `ResourceWarning` raised far from here.
|
|
396
|
+
"""
|
|
397
|
+
bridge_task.cancel()
|
|
398
|
+
with contextlib.suppress(asyncio.CancelledError, TransportError, OSError):
|
|
399
|
+
await bridge_task
|
|
400
|
+
with contextlib.suppress(TransportError, OSError):
|
|
401
|
+
await listener.close()
|
|
File without changes
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
"""Auth use case — WorkOS device-authorization flow (RFC 8628).
|
|
2
|
+
|
|
3
|
+
Owns login/logout/status end-to-end so command layers never touch
|
|
4
|
+
transports directly. The polling state machine (interval, slow_down,
|
|
5
|
+
expires_in) lives here because it is a use-case decision, not a transport one.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import time
|
|
9
|
+
from collections.abc import Callable
|
|
10
|
+
from datetime import timedelta
|
|
11
|
+
|
|
12
|
+
from manifold_cli.clients.credentials import CredentialsClient
|
|
13
|
+
from manifold_cli.clients.manifold import ManifoldApiClient
|
|
14
|
+
from manifold_cli.clients.workos import WorkOSDeviceClient
|
|
15
|
+
from manifold_cli.models.auth import (
|
|
16
|
+
AuthState,
|
|
17
|
+
AuthStatus,
|
|
18
|
+
DeviceAuthorization,
|
|
19
|
+
PollPending,
|
|
20
|
+
TokenResponse,
|
|
21
|
+
)
|
|
22
|
+
from manifold_cli.models.credentials import Credentials
|
|
23
|
+
from manifold_cli.models.errors import (
|
|
24
|
+
NotLoggedInError,
|
|
25
|
+
SessionExpiredError,
|
|
26
|
+
WorkOSAuthError,
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class AuthService:
|
|
31
|
+
_SLOW_DOWN_INCREMENT = timedelta(seconds=5)
|
|
32
|
+
|
|
33
|
+
def __init__(
|
|
34
|
+
self,
|
|
35
|
+
manifold_client: ManifoldApiClient,
|
|
36
|
+
workos_client: WorkOSDeviceClient,
|
|
37
|
+
credentials_client: CredentialsClient,
|
|
38
|
+
manifold_api_url: str,
|
|
39
|
+
):
|
|
40
|
+
self._manifold_client = manifold_client
|
|
41
|
+
self._workos_client = workos_client
|
|
42
|
+
self._credentials_client = credentials_client
|
|
43
|
+
self._manifold_api_url = manifold_api_url
|
|
44
|
+
|
|
45
|
+
def login(self, on_device_code: Callable[[DeviceAuthorization], None]) -> None:
|
|
46
|
+
device_auth = self._workos_client.start_device_authorization()
|
|
47
|
+
on_device_code(device_auth)
|
|
48
|
+
token_response = self._poll_until_done(device_auth)
|
|
49
|
+
config = self._manifold_client.get_cli_config()
|
|
50
|
+
self._credentials_client.save(
|
|
51
|
+
Credentials(
|
|
52
|
+
api_url=self._manifold_api_url,
|
|
53
|
+
workos_client_id=config.workos_client_id,
|
|
54
|
+
access_token=token_response.access_token,
|
|
55
|
+
refresh_token=token_response.refresh_token,
|
|
56
|
+
)
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
def logout(self) -> None:
|
|
60
|
+
self._credentials_client.clear()
|
|
61
|
+
|
|
62
|
+
def status(self) -> AuthStatus:
|
|
63
|
+
# NotLoggedInError and SessionExpiredError surface from the transport's
|
|
64
|
+
# auth flow while making the request; a valid session returns the user.
|
|
65
|
+
try:
|
|
66
|
+
user = self._manifold_client.get_me()
|
|
67
|
+
except NotLoggedInError:
|
|
68
|
+
return AuthStatus(state=AuthState.LOGGED_OUT)
|
|
69
|
+
except SessionExpiredError:
|
|
70
|
+
return AuthStatus(state=AuthState.SESSION_EXPIRED)
|
|
71
|
+
return AuthStatus(state=AuthState.LOGGED_IN, user=user)
|
|
72
|
+
|
|
73
|
+
def _poll_until_done(
|
|
74
|
+
self,
|
|
75
|
+
device_auth: DeviceAuthorization,
|
|
76
|
+
) -> TokenResponse:
|
|
77
|
+
deadline = time.monotonic() + device_auth.expires_in
|
|
78
|
+
interval_s = device_auth.interval
|
|
79
|
+
while True:
|
|
80
|
+
if time.monotonic() >= deadline:
|
|
81
|
+
raise WorkOSAuthError(
|
|
82
|
+
"expired_token", "device authorization expired before approval"
|
|
83
|
+
)
|
|
84
|
+
result = self._workos_client.poll_token(device_auth.device_code)
|
|
85
|
+
if isinstance(result, TokenResponse):
|
|
86
|
+
return result
|
|
87
|
+
if result is PollPending.SLOW_DOWN:
|
|
88
|
+
interval_s += self._SLOW_DOWN_INCREMENT.total_seconds()
|
|
89
|
+
time.sleep(interval_s)
|