letify 2.1.3__tar.gz → 2.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {letify-2.1.3 → letify-2.3.0}/PKG-INFO +1 -1
- {letify-2.1.3 → letify-2.3.0}/letify/__init__.py +1 -1
- {letify-2.1.3 → letify-2.3.0}/letify/launcher.py +1 -1
- {letify-2.1.3 → letify-2.3.0}/letify/protocol/worker.py +67 -4
- {letify-2.1.3 → letify-2.3.0}/letify/providers/base.py +49 -0
- {letify-2.1.3 → letify-2.3.0}/letify/providers/colab.py +2 -0
- {letify-2.1.3 → letify-2.3.0}/letify/providers/kaggle.py +182 -3
- {letify-2.1.3 → letify-2.3.0}/letify/runtime/bootstrap.py +37 -0
- {letify-2.1.3 → letify-2.3.0}/letify/runtime/session.py +60 -0
- {letify-2.1.3 → letify-2.3.0}/letify/store/pathdata.py +30 -4
- {letify-2.1.3 → letify-2.3.0}/letify/transport/link.py +3 -0
- {letify-2.1.3 → letify-2.3.0}/letify/transport/pipeline.py +18 -5
- {letify-2.1.3 → letify-2.3.0}/letify-core/Cargo.lock +3 -3
- {letify-2.1.3 → letify-2.3.0}/letify-core/Cargo.toml +1 -1
- {letify-2.1.3 → letify-2.3.0}/letify-ext/package-lock.json +2 -2
- {letify-2.1.3 → letify-2.3.0}/letify-ext/package.json +1 -1
- {letify-2.1.3 → letify-2.3.0}/pyproject.toml +1 -1
- {letify-2.1.3 → letify-2.3.0}/tests/conftest.py +11 -0
- {letify-2.1.3 → letify-2.3.0}/tests/test_core.py +52 -0
- {letify-2.1.3 → letify-2.3.0}/tests/test_elice.py +5 -1
- {letify-2.1.3 → letify-2.3.0}/tests/test_kaggle.py +112 -4
- {letify-2.1.3 → letify-2.3.0}/tests/test_kaggle_link.py +74 -0
- {letify-2.1.3 → letify-2.3.0}/tests/test_parallel_placement.py +39 -5
- {letify-2.1.3 → letify-2.3.0}/tests/test_runtime.py +69 -1
- {letify-2.1.3 → letify-2.3.0}/tests/test_transport.py +25 -0
- {letify-2.1.3 → letify-2.3.0}/.github/workflows/ci.yml +0 -0
- {letify-2.1.3 → letify-2.3.0}/.github/workflows/docs.yml +0 -0
- {letify-2.1.3 → letify-2.3.0}/.github/workflows/publish.yml +0 -0
- {letify-2.1.3 → letify-2.3.0}/.gitignore +0 -0
- {letify-2.1.3 → letify-2.3.0}/.researchtree +0 -0
- {letify-2.1.3 → letify-2.3.0}/LICENSE +0 -0
- {letify-2.1.3 → letify-2.3.0}/README.md +0 -0
- {letify-2.1.3 → letify-2.3.0}/docs/guide/01-getting-started.md +0 -0
- {letify-2.1.3 → letify-2.3.0}/docs/guide/02-providers.md +0 -0
- {letify-2.1.3 → letify-2.3.0}/docs/guide/03-execution-modes.md +0 -0
- {letify-2.1.3 → letify-2.3.0}/docs/guide/04-environments-and-data.md +0 -0
- {letify-2.1.3 → letify-2.3.0}/docs/guide/05-concurrency.md +0 -0
- {letify-2.1.3 → letify-2.3.0}/docs/guide/06-cost.md +0 -0
- {letify-2.1.3 → letify-2.3.0}/docs/guide/07-troubleshooting.md +0 -0
- {letify-2.1.3 → letify-2.3.0}/docs/guide/08-releasing.md +0 -0
- {letify-2.1.3 → letify-2.3.0}/docs/guide/README.md +0 -0
- {letify-2.1.3 → letify-2.3.0}/docs/guide/ko/01-/354/213/234/354/236/221/355/225/230/352/270/260.md" +0 -0
- {letify-2.1.3 → letify-2.3.0}/docs/guide/ko/02-/355/224/204/353/241/234/353/260/224/354/235/264/353/215/224.md" +0 -0
- {letify-2.1.3 → letify-2.3.0}/docs/guide/ko/03-/354/213/244/355/226/211-/353/260/251/354/213/235.md" +0 -0
- {letify-2.1.3 → letify-2.3.0}/docs/guide/ko/04-/355/231/230/352/262/275/352/263/274-/353/215/260/354/235/264/355/204/260.md" +0 -0
- {letify-2.1.3 → letify-2.3.0}/docs/guide/ko/05-/353/217/231/354/213/234-/354/213/244/355/226/211.md" +0 -0
- {letify-2.1.3 → letify-2.3.0}/docs/guide/ko/06-/353/271/204/354/232/251.md" +0 -0
- {letify-2.1.3 → letify-2.3.0}/docs/guide/ko/07-/353/254/270/354/240/234-/355/225/264/352/262/260.md" +0 -0
- {letify-2.1.3 → letify-2.3.0}/docs/guide/ko/08-/353/246/264/353/246/254/354/212/244.md" +0 -0
- {letify-2.1.3 → letify-2.3.0}/docs/guide/ko/README.md +0 -0
- {letify-2.1.3 → letify-2.3.0}/docs/locales/README_ko.md +0 -0
- {letify-2.1.3 → letify-2.3.0}/docs/mkdocs.yml +0 -0
- {letify-2.1.3 → letify-2.3.0}/examples/README.md +0 -0
- {letify-2.1.3 → letify-2.3.0}/examples/nvfp4-lora/.letify.example +0 -0
- {letify-2.1.3 → letify-2.3.0}/examples/nvfp4-lora/00_smoke.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/examples/nvfp4-lora/01_sweep.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/examples/nvfp4-lora/02_resume.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/examples/nvfp4-lora/03_watch.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/examples/nvfp4-lora/README.md +0 -0
- {letify-2.1.3 → letify-2.3.0}/examples/nvfp4-lora/README_ko.md +0 -0
- {letify-2.1.3 → letify-2.3.0}/examples/nvfp4-lora/common.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/examples/nvfp4-lora/recipe.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/cli.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/config/__init__.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/config/inventory.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/config/login.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/config/schema.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/config/secrets.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/config/writer.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/declare/__init__.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/declare/cache.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/declare/env.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/declare/function.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/declare/instance.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/errors.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/install.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/paths.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/protocol/__init__.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/protocol/codec.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/protocol/driver.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/protocol/handle.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/protocol/wire.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/providers/__init__.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/providers/colab_files.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/providers/colab_keepalive.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/providers/elice.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/providers/kaggle_adapter.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/providers/local.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/providers/modal.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/providers/modal_adapter.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/providers/naming.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/providers/shell.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/providers/tunnel.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/providers/usage.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/remoting/__init__.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/remoting/capability.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/remoting/device/__init__.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/remoting/device/client.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/remoting/device/cuda.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/remoting/device/executor.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/remoting/device/frames.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/remoting/device/guard.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/remoting/device/tensor.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/remoting/device/trace.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/remoting/device/value.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/remoting/loader.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/remoting/probe.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/render.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/runtime/__init__.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/runtime/channel.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/runtime/lease.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/runtime/pool.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/runtime/telemetry.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/store/__init__.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/store/backends/__init__.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/store/backends/filesystem.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/store/backends/google_auth.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/store/backends/layout.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/store/backends/objects.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/store/cas.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/store/sendorder.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/store/volume.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/stubs.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/tools.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/transport/__init__.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/transport/agent.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/transport/announce.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/transport/nat.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/transport/portmap.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/transport/probe.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/transport/quic.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/transport/rendezvous.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/transport/setup.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/transport/sshopts.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify/transport/strategies.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify-core/.gitignore +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify-core/build.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify-core/letify-agent/Cargo.toml +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify-core/letify-agent/src/driver.rs +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify-core/letify-agent/src/main.rs +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify-core/letify-driver/Cargo.toml +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify-core/letify-driver/src/client.rs +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify-core/letify-driver/src/exports.rs +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify-core/letify-driver/src/image.rs +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify-core/letify-driver/src/lib.rs +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify-core/letify-driver/src/status.rs +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify-core/letify-driver/src/table.rs +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify-core/letify-quic/Cargo.toml +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify-core/letify-quic/src/main.rs +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify-core/letify-wire/Cargo.toml +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify-core/letify-wire/src/lib.rs +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify-core/letify-wire/tests/throughput.rs +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify-core/rust-toolchain.toml +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify-ext/.vscodeignore +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify-ext/LICENSE +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify-ext/README.md +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify-ext/media/icon.svg +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify-ext/src/cli.ts +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify-ext/src/extension.ts +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify-ext/src/history.ts +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify-ext/src/model.ts +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify-ext/src/poll.ts +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify-ext/src/render.ts +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify-ext/test/cli.test.ts +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify-ext/test/manifest.test.ts +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify-ext/test/model.test.ts +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify-ext/test/regressions.test.ts +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify-ext/tsconfig.json +0 -0
- {letify-2.1.3 → letify-2.3.0}/letify_providers.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/scripts/sync-main.sh +0 -0
- {letify-2.1.3 → letify-2.3.0}/scripts/version.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/tests/fake_kaggle_adapter.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/tests/fake_modal_adapter.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/tests/test_cli.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/tests/test_cli_output.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/tests/test_colab_keepalive.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/tests/test_config.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/tests/test_data.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/tests/test_data_streaming.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/tests/test_declare.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/tests/test_device.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/tests/test_editor_sessions.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/tests/test_frames.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/tests/test_install.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/tests/test_kaggle_cookie.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/tests/test_link_log.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/tests/test_notebook.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/tests/test_paths.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/tests/test_portmap.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/tests/test_protocol.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/tests/test_providers.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/tests/test_quic.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/tests/test_remoting.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/tests/test_render.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/tests/test_ship.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/tests/test_store.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/tests/test_strategies.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/tests/test_stubs.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/tests/test_tunnel_setup.py +0 -0
- {letify-2.1.3 → letify-2.3.0}/tests/test_version.py +0 -0
|
@@ -64,6 +64,21 @@ widen_pipe(_FRAME_FD)
|
|
|
64
64
|
_SENDER = (TextSender if _LETIFY_TEXT_FRAMES else Sender)(fd_writer(_FRAME_FD))
|
|
65
65
|
|
|
66
66
|
|
|
67
|
+
#: Written to a captured descriptor to find out where its reader has got to. A pipe is
|
|
68
|
+
#: read in order, so every byte written before this has been framed once it is reached.
|
|
69
|
+
#: It is never forwarded. Spec "Worker output".
|
|
70
|
+
_DRAIN_TOKEN = b"\x1a\x1aLETIFY-DRAIN-7f3a9c4e\x1a\x1a"
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _held_back(data):
|
|
74
|
+
"""How many bytes at the end of ``data`` could be the start of a split token."""
|
|
75
|
+
most = min(len(_DRAIN_TOKEN) - 1, len(data))
|
|
76
|
+
for size in range(most, 0, -1):
|
|
77
|
+
if _DRAIN_TOKEN.startswith(data[len(data) - size:]):
|
|
78
|
+
return size
|
|
79
|
+
return 0
|
|
80
|
+
|
|
81
|
+
|
|
67
82
|
class _Pump(threading.Thread):
|
|
68
83
|
"""Reads one captured descriptor and sends what it reads as output frames."""
|
|
69
84
|
|
|
@@ -71,10 +86,23 @@ class _Pump(threading.Thread):
|
|
|
71
86
|
threading.Thread.__init__(self, daemon=True)
|
|
72
87
|
self.kind = kind
|
|
73
88
|
self.busy = False
|
|
89
|
+
self.fd = fd
|
|
90
|
+
#: Bytes withheld because they may be the start of a token split across two reads.
|
|
91
|
+
self.tail = b""
|
|
92
|
+
self.reached = threading.Event()
|
|
74
93
|
self.read_fd, write_fd = os.pipe()
|
|
75
94
|
os.dup2(write_fd, fd)
|
|
76
95
|
os.close(write_fd)
|
|
77
96
|
|
|
97
|
+
def send(self, data):
|
|
98
|
+
if not data:
|
|
99
|
+
return
|
|
100
|
+
self.busy = True
|
|
101
|
+
try:
|
|
102
|
+
_SENDER.frame(self.kind, 0, data)
|
|
103
|
+
finally:
|
|
104
|
+
self.busy = False
|
|
105
|
+
|
|
78
106
|
def run(self):
|
|
79
107
|
while True:
|
|
80
108
|
try:
|
|
@@ -83,16 +111,48 @@ class _Pump(threading.Thread):
|
|
|
83
111
|
break
|
|
84
112
|
if not data:
|
|
85
113
|
break
|
|
86
|
-
self.
|
|
114
|
+
data = self.tail + data
|
|
115
|
+
self.tail = b""
|
|
87
116
|
try:
|
|
88
|
-
|
|
117
|
+
while True:
|
|
118
|
+
at = data.find(_DRAIN_TOKEN)
|
|
119
|
+
if at < 0:
|
|
120
|
+
break
|
|
121
|
+
self.send(data[:at])
|
|
122
|
+
data = data[at + len(_DRAIN_TOKEN):]
|
|
123
|
+
self.reached.set()
|
|
124
|
+
keep = _held_back(data)
|
|
125
|
+
if keep:
|
|
126
|
+
self.tail = data[len(data) - keep:]
|
|
127
|
+
data = data[: len(data) - keep]
|
|
128
|
+
self.send(data)
|
|
89
129
|
except OSError:
|
|
90
130
|
break
|
|
91
|
-
finally:
|
|
92
|
-
self.busy = False
|
|
93
131
|
os.close(self.read_fd)
|
|
94
132
|
|
|
95
133
|
|
|
134
|
+
def _drain_output(timeout=2.0):
|
|
135
|
+
"""Wait until everything written to the captured descriptors has been framed.
|
|
136
|
+
|
|
137
|
+
Spec "Worker output": a call's reply waits for the output its body made, so the
|
|
138
|
+
caller's own next print comes after it. Waiting on the reading thread's state is not
|
|
139
|
+
enough, because it has read bytes it has not yet framed; the token goes through the
|
|
140
|
+
pipe behind those bytes instead. It gives up rather than lose a result.
|
|
141
|
+
"""
|
|
142
|
+
for stream in (sys.stdout, sys.stderr):
|
|
143
|
+
try:
|
|
144
|
+
stream.flush()
|
|
145
|
+
except Exception:
|
|
146
|
+
pass
|
|
147
|
+
for pump in _PUMPS:
|
|
148
|
+
pump.reached.clear()
|
|
149
|
+
try:
|
|
150
|
+
os.write(pump.fd, _DRAIN_TOKEN)
|
|
151
|
+
except OSError:
|
|
152
|
+
continue
|
|
153
|
+
pump.reached.wait(timeout)
|
|
154
|
+
|
|
155
|
+
|
|
96
156
|
_PUMPS = [_Pump(1, STDOUT), _Pump(2, STDERR)]
|
|
97
157
|
for _pump in _PUMPS:
|
|
98
158
|
_pump.start()
|
|
@@ -2066,6 +2126,9 @@ def _run(stream, request, settle):
|
|
|
2066
2126
|
request = None
|
|
2067
2127
|
if settle:
|
|
2068
2128
|
_settle()
|
|
2129
|
+
if name == "call":
|
|
2130
|
+
# Spec "Worker output": the body's output is framed before its result goes out.
|
|
2131
|
+
_drain_output()
|
|
2069
2132
|
_reply(stream, outcome)
|
|
2070
2133
|
|
|
2071
2134
|
|
|
@@ -273,6 +273,17 @@ class Provider(abc.ABC):
|
|
|
273
273
|
value = self.config.option("transfer_streams")
|
|
274
274
|
return max(1, int(value)) if isinstance(value, (int, float)) else STREAMS
|
|
275
275
|
|
|
276
|
+
@property
|
|
277
|
+
def transfer_connection_cost_s(self) -> float:
|
|
278
|
+
"""Seconds one more transfer connection takes to open.
|
|
279
|
+
|
|
280
|
+
Spec "Several connections at once": zero where the provider dials an address, so
|
|
281
|
+
the extra connections are free and only the round trip decides. A provider reached
|
|
282
|
+
by a hole punch overrides it, because each stream punches again.
|
|
283
|
+
"""
|
|
284
|
+
value = self.config.option("transfer_connection_cost_s")
|
|
285
|
+
return max(0.0, float(value)) if isinstance(value, (int, float)) else 0.0
|
|
286
|
+
|
|
276
287
|
@property
|
|
277
288
|
def transfer_parallel_rtt_ms(self) -> int:
|
|
278
289
|
"""The round trip above which a blob is split, from ``transfer_parallel_rtt_ms``.
|
|
@@ -343,10 +354,48 @@ class Provider(abc.ABC):
|
|
|
343
354
|
return table[name]
|
|
344
355
|
if name.upper() in table:
|
|
345
356
|
return table[name.upper()]
|
|
357
|
+
retired = self.retired_reason(name)
|
|
358
|
+
if retired:
|
|
359
|
+
raise UnknownInstance(f"{self.alias} does not offer {name!r}. {retired}")
|
|
346
360
|
raise UnknownInstance(
|
|
347
361
|
f"{self.alias} does not offer {name!r}. Available: {', '.join(sorted(table))}"
|
|
348
362
|
)
|
|
349
363
|
|
|
364
|
+
#: Whether this provider can be asked what sessions the account already holds. A
|
|
365
|
+
#: launcher reads it rather than naming provider kinds. Spec "Kaggle", finding the
|
|
366
|
+
#: notebooks letify owns.
|
|
367
|
+
discovers_sessions: bool = False
|
|
368
|
+
|
|
369
|
+
def sessions(self) -> list[str]:
|
|
370
|
+
"""What the account already holds, for a provider that can be asked.
|
|
371
|
+
|
|
372
|
+
Empty for one that cannot, and empty rather than raising when the answer cannot be
|
|
373
|
+
read: a command asking what exists should not fail because nothing answered.
|
|
374
|
+
"""
|
|
375
|
+
return []
|
|
376
|
+
|
|
377
|
+
def expected_cards(self, instance: Any) -> tuple[str, int] | None:
|
|
378
|
+
"""The card name fragment and count a session for ``instance`` must have, or None.
|
|
379
|
+
|
|
380
|
+
None means the provider gives what it is asked for, so there is nothing to check.
|
|
381
|
+
A provider whose service may answer with another card overrides it. Spec "Kaggle",
|
|
382
|
+
the accelerators.
|
|
383
|
+
"""
|
|
384
|
+
return None
|
|
385
|
+
|
|
386
|
+
@property
|
|
387
|
+
def any_accelerator(self) -> bool:
|
|
388
|
+
"""Whether a card other than the one asked for is accepted, from ``any_accelerator``."""
|
|
389
|
+
return self.config.option("any_accelerator") is True
|
|
390
|
+
|
|
391
|
+
def retired_reason(self, name: str) -> str | None:
|
|
392
|
+
"""Why an accelerator this provider once offered is gone, or None.
|
|
393
|
+
|
|
394
|
+
A name the provider's service answers with a different card belongs here rather
|
|
395
|
+
than in the inventory, because a declaration states where a function runs.
|
|
396
|
+
"""
|
|
397
|
+
return None
|
|
398
|
+
|
|
350
399
|
def device(self, name: str) -> Instance:
|
|
351
400
|
"""Look up an accelerator by name."""
|
|
352
401
|
return getattr(self, name)
|
|
@@ -32,6 +32,7 @@ import urllib.parse
|
|
|
32
32
|
import urllib.request
|
|
33
33
|
from collections.abc import Mapping
|
|
34
34
|
from datetime import UTC, datetime
|
|
35
|
+
from pathlib import Path
|
|
35
36
|
from typing import TYPE_CHECKING, Any
|
|
36
37
|
|
|
37
38
|
from .. import tools
|
|
@@ -67,13 +68,20 @@ if TYPE_CHECKING:
|
|
|
67
68
|
from ..runtime.session import Runtime
|
|
68
69
|
|
|
69
70
|
#: GPU session shapes, with memory per card, device count and the internal API name.
|
|
70
|
-
#: P100 remains available through the API although the web UI no longer lists it.
|
|
71
71
|
#: CPU is the empty compute, so it carries no accelerator name.
|
|
72
72
|
GPUS = {
|
|
73
|
-
"
|
|
74
|
-
"T4": {"vram_gb": 16, "devices": 2, "accelerator": "NVIDIA_TESLA_T4"},
|
|
73
|
+
"T4": {"vram_gb": 16, "devices": 2, "accelerator": "NVIDIA_TESLA_T4", "model": "T4"},
|
|
75
74
|
}
|
|
76
75
|
TPUS = ("TPU_V3_8",)
|
|
76
|
+
#: Cards Kaggle no longer serves, and what it does instead. Offering one would let a
|
|
77
|
+
#: declaration say P100 and run on something else: spec "Kaggle", the accelerators.
|
|
78
|
+
RETIRED = {
|
|
79
|
+
"P100": (
|
|
80
|
+
"Kaggle retired the Tesla P100 on 2026-09-15 and switches a notebook that asks "
|
|
81
|
+
"for it to two T4s, so the card and the card count would both differ from the "
|
|
82
|
+
"declaration. Use T4, which is two cards of 16 GB"
|
|
83
|
+
),
|
|
84
|
+
}
|
|
77
85
|
|
|
78
86
|
#: The internal Kaggle service surface the web app uses, authenticated by the session cookie.
|
|
79
87
|
KAGGLE_INTERNAL = "https://www.kaggle.com/api/i/"
|
|
@@ -504,6 +512,9 @@ def live_session_url(
|
|
|
504
512
|
403 on it is the account's own answer, not a stale notebook's.
|
|
505
513
|
"""
|
|
506
514
|
kernel, slug = new_notebook(alias, cookie)
|
|
515
|
+
# Spec "Kaggle", finding the notebooks letify owns: recorded before anything can fail,
|
|
516
|
+
# so a run that dies next still leaves a trail to its notebook.
|
|
517
|
+
record_notebook(alias, slug)
|
|
507
518
|
run = start_run(cookie, kernel, accelerator)
|
|
508
519
|
id_token = firebase_id_token(cookie)
|
|
509
520
|
webtier = webtier_session(cookie, id_token, run)
|
|
@@ -554,6 +565,116 @@ def read_notebook_owner(alias: str) -> str | None:
|
|
|
554
565
|
return owner or None
|
|
555
566
|
|
|
556
567
|
|
|
568
|
+
def notebook_record(alias: str) -> Path:
|
|
569
|
+
"""Where the refs of the notebooks letify created on this account are kept."""
|
|
570
|
+
return account_directory(alias) / "notebooks"
|
|
571
|
+
|
|
572
|
+
|
|
573
|
+
def recorded_notebooks(alias: str) -> list[str]:
|
|
574
|
+
"""The refs letify believes it created, in the order it created them."""
|
|
575
|
+
try:
|
|
576
|
+
text = notebook_record(alias).read_text(encoding="utf-8")
|
|
577
|
+
except OSError:
|
|
578
|
+
return []
|
|
579
|
+
seen: list[str] = []
|
|
580
|
+
for line in text.splitlines():
|
|
581
|
+
ref = line.strip()
|
|
582
|
+
if ref and ref not in seen:
|
|
583
|
+
seen.append(ref)
|
|
584
|
+
return seen
|
|
585
|
+
|
|
586
|
+
|
|
587
|
+
def record_notebook(alias: str, slug: str | None) -> None:
|
|
588
|
+
"""Note that letify created this notebook, so a later process can still find it.
|
|
589
|
+
|
|
590
|
+
Spec "Kaggle", finding the notebooks letify owns: Kaggle names the notebook itself, so
|
|
591
|
+
the record is the only thing that tells letify's from the user's own. Never raises; a
|
|
592
|
+
notebook letify cannot record is one it may fail to clean up, not a failed run.
|
|
593
|
+
"""
|
|
594
|
+
username = read_notebook_owner(alias)
|
|
595
|
+
if not slug or username is None:
|
|
596
|
+
return
|
|
597
|
+
try:
|
|
598
|
+
path = notebook_record(alias)
|
|
599
|
+
path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
600
|
+
with path.open("a", encoding="utf-8") as handle:
|
|
601
|
+
handle.write(f"{username}/{slug}\n")
|
|
602
|
+
except OSError:
|
|
603
|
+
pass
|
|
604
|
+
|
|
605
|
+
|
|
606
|
+
def forget_notebook(alias: str, slug: str | None) -> None:
|
|
607
|
+
"""Drop a notebook from the record, once it is gone from the account."""
|
|
608
|
+
username = read_notebook_owner(alias)
|
|
609
|
+
if not slug or username is None:
|
|
610
|
+
return
|
|
611
|
+
ref = f"{username}/{slug}"
|
|
612
|
+
kept = [line for line in recorded_notebooks(alias) if line != ref]
|
|
613
|
+
try:
|
|
614
|
+
notebook_record(alias).write_text(
|
|
615
|
+
"".join(f"{line}\n" for line in kept), encoding="utf-8"
|
|
616
|
+
)
|
|
617
|
+
except OSError:
|
|
618
|
+
pass
|
|
619
|
+
|
|
620
|
+
|
|
621
|
+
def notebooks_on_account(alias: str) -> list[str] | None:
|
|
622
|
+
"""The refs the account has, through the official CLI, or None when it cannot be asked.
|
|
623
|
+
|
|
624
|
+
None and an empty list mean different things: nothing answered, against the account
|
|
625
|
+
having no notebooks. Never raises.
|
|
626
|
+
"""
|
|
627
|
+
if read_api_token(alias) is None:
|
|
628
|
+
return None
|
|
629
|
+
uv = tools.find_uv()
|
|
630
|
+
if uv is None:
|
|
631
|
+
return None
|
|
632
|
+
import csv
|
|
633
|
+
import io
|
|
634
|
+
import subprocess
|
|
635
|
+
|
|
636
|
+
try:
|
|
637
|
+
result = subprocess.run(
|
|
638
|
+
[*tools.kaggle_cli_command(uv), "kernels", "list", "--mine", "--csv"],
|
|
639
|
+
env=tools.kaggle_cli_environment(alias),
|
|
640
|
+
capture_output=True,
|
|
641
|
+
text=True,
|
|
642
|
+
timeout=REST_TIMEOUT,
|
|
643
|
+
)
|
|
644
|
+
except (OSError, subprocess.SubprocessError):
|
|
645
|
+
return None
|
|
646
|
+
if result.returncode != 0:
|
|
647
|
+
return None
|
|
648
|
+
try:
|
|
649
|
+
rows = list(csv.DictReader(io.StringIO(result.stdout or "")))
|
|
650
|
+
except csv.Error:
|
|
651
|
+
return None
|
|
652
|
+
return [(row.get("ref") or "").strip() for row in rows if (row.get("ref") or "").strip()]
|
|
653
|
+
|
|
654
|
+
|
|
655
|
+
def list_notebooks_via_cli(alias: str) -> list[str]:
|
|
656
|
+
"""The recorded notebooks this account still has. Spec "Kaggle", finding them.
|
|
657
|
+
|
|
658
|
+
A ref the record names and the account no longer has was deleted elsewhere, so it is
|
|
659
|
+
dropped from the record rather than reported.
|
|
660
|
+
"""
|
|
661
|
+
recorded = recorded_notebooks(alias)
|
|
662
|
+
if not recorded:
|
|
663
|
+
return []
|
|
664
|
+
present = notebooks_on_account(alias)
|
|
665
|
+
if present is None:
|
|
666
|
+
return []
|
|
667
|
+
live = [ref for ref in recorded if ref in set(present)]
|
|
668
|
+
if live != recorded:
|
|
669
|
+
try:
|
|
670
|
+
notebook_record(alias).write_text(
|
|
671
|
+
"".join(f"{ref}\n" for ref in live), encoding="utf-8"
|
|
672
|
+
)
|
|
673
|
+
except OSError:
|
|
674
|
+
pass
|
|
675
|
+
return live
|
|
676
|
+
|
|
677
|
+
|
|
557
678
|
def delete_notebook_via_cli(alias: str, slug: str | None) -> bool:
|
|
558
679
|
"""Delete the ephemeral notebook through the official CLI, the only way it is deleted.
|
|
559
680
|
|
|
@@ -596,6 +717,7 @@ def delete_notebook_best_effort(alias: str, kernel_id: int, slug: str | None) ->
|
|
|
596
717
|
slug or id, never by its cookie, so the user can delete it by hand on kaggle.com.
|
|
597
718
|
"""
|
|
598
719
|
if delete_notebook_via_cli(alias, slug):
|
|
720
|
+
forget_notebook(alias, slug)
|
|
599
721
|
return
|
|
600
722
|
import sys
|
|
601
723
|
|
|
@@ -1025,6 +1147,24 @@ class Kaggle(Provider):
|
|
|
1025
1147
|
table.update({name: Instance(self, tpu=name) for name in TPUS})
|
|
1026
1148
|
return table
|
|
1027
1149
|
|
|
1150
|
+
discovers_sessions = True
|
|
1151
|
+
|
|
1152
|
+
def sessions(self) -> list[str]:
|
|
1153
|
+
"""The notebooks letify owns on this account, which is where an orphan shows up."""
|
|
1154
|
+
return list_notebooks_via_cli(self.alias)
|
|
1155
|
+
|
|
1156
|
+
def expected_cards(self, instance: Any) -> tuple[str, int] | None:
|
|
1157
|
+
"""What a Kaggle session must have, because Kaggle may answer with another card."""
|
|
1158
|
+
gpu = getattr(instance, "gpu", None)
|
|
1159
|
+
spec = GPUS.get(gpu) if gpu else None
|
|
1160
|
+
if spec is None:
|
|
1161
|
+
return None
|
|
1162
|
+
return str(spec["model"]), int(spec["devices"])
|
|
1163
|
+
|
|
1164
|
+
def retired_reason(self, name: str) -> str | None:
|
|
1165
|
+
"""Why Kaggle no longer serves a card it once did. Spec "Kaggle", the accelerators."""
|
|
1166
|
+
return RETIRED.get(name.upper())
|
|
1167
|
+
|
|
1028
1168
|
def store_backend(self) -> str:
|
|
1029
1169
|
return "filesystem"
|
|
1030
1170
|
|
|
@@ -1207,6 +1347,45 @@ class Kaggle(Provider):
|
|
|
1207
1347
|
name=name,
|
|
1208
1348
|
)
|
|
1209
1349
|
|
|
1350
|
+
@property
|
|
1351
|
+
def transfer_connection_cost_s(self) -> float:
|
|
1352
|
+
"""A punch, which is what one more stream costs here. Measured at about 10 s."""
|
|
1353
|
+
value = self.config.option("transfer_connection_cost_s")
|
|
1354
|
+
return max(0.0, float(value)) if isinstance(value, (int, float)) else 10.0
|
|
1355
|
+
|
|
1356
|
+
def link(self, runtime: Runtime | None = None) -> Any:
|
|
1357
|
+
"""The link carrying this runtime, or None when the bridge is carrying it.
|
|
1358
|
+
|
|
1359
|
+
The placement reads ``rtt_ms`` from it to decide how many streams to use.
|
|
1360
|
+
"""
|
|
1361
|
+
name = getattr(runtime, "name", None)
|
|
1362
|
+
return self._links.get(name) if name else None
|
|
1363
|
+
|
|
1364
|
+
def transfer_channel(self, runtime: Runtime | None) -> Channel:
|
|
1365
|
+
"""One more stream for a placement, on a connection of its own.
|
|
1366
|
+
|
|
1367
|
+
Spec "Several connections at once": the link is asked for its own SSH command,
|
|
1368
|
+
which lands on a local forwarding port of its own and so punches again. Reusing
|
|
1369
|
+
the session's command would multiplex onto the one connection instead, and
|
|
1370
|
+
multiplexed channels do not add up.
|
|
1371
|
+
"""
|
|
1372
|
+
import shlex as _shlex
|
|
1373
|
+
|
|
1374
|
+
from ..protocol.worker import BOOTSTRAP
|
|
1375
|
+
from ..runtime.channel import PersistentChannel
|
|
1376
|
+
|
|
1377
|
+
name = getattr(runtime, "name", None)
|
|
1378
|
+
link = self._links.get(name) if name else None
|
|
1379
|
+
if link is None:
|
|
1380
|
+
raise UnsupportedMode(
|
|
1381
|
+
f"{self.alias} is carried by the kernel bridge for {name}, which is one "
|
|
1382
|
+
f"connection and offers no second stream. A punched session does."
|
|
1383
|
+
)
|
|
1384
|
+
return PersistentChannel(
|
|
1385
|
+
link.ssh_command(f"{self.remote_python} -u -c {_shlex.quote(BOOTSTRAP)}"),
|
|
1386
|
+
name=f"{self.alias}-transfer",
|
|
1387
|
+
)
|
|
1388
|
+
|
|
1210
1389
|
def open_channel(self, runtime: Runtime) -> Channel:
|
|
1211
1390
|
"""Start a session from the cookie and open one worker in one cell of it.
|
|
1212
1391
|
|
|
@@ -130,6 +130,34 @@ def project_dir(workspace_root: str, env: Env) -> str:
|
|
|
130
130
|
return f"{workspace_root.rstrip('/')}/project/{env.key}"
|
|
131
131
|
|
|
132
132
|
|
|
133
|
+
#: Reads the cards a runtime has, without nvidia-smi, which a Kaggle image lacks on PATH.
|
|
134
|
+
#: Spec "Kaggle", the accelerators.
|
|
135
|
+
CARD_SOURCE = (
|
|
136
|
+
"import glob\n"
|
|
137
|
+
"__letify_cards = []\n"
|
|
138
|
+
"for _letify_path in sorted(glob.glob('/proc/driver/nvidia/gpus/*/information')):\n"
|
|
139
|
+
" try:\n"
|
|
140
|
+
" with open(_letify_path) as _letify_handle:\n"
|
|
141
|
+
" for _letify_line in _letify_handle:\n"
|
|
142
|
+
" if _letify_line.startswith('Model:'):\n"
|
|
143
|
+
" __letify_cards.append(_letify_line.split(':', 1)[1].strip())\n"
|
|
144
|
+
" break\n"
|
|
145
|
+
" except OSError:\n"
|
|
146
|
+
" pass\n"
|
|
147
|
+
"__letify_value__ = __letify_cards\n"
|
|
148
|
+
)
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def packages_in(files: dict[str, bytes]) -> int:
|
|
152
|
+
"""How many packages the lock names, for the line the environment build prints.
|
|
153
|
+
|
|
154
|
+
The lock is already read, so counting its ``[[package]]`` entries costs nothing and
|
|
155
|
+
tells the reader whether to expect seconds or minutes. Spec "Environment".
|
|
156
|
+
"""
|
|
157
|
+
lock = files.get("uv.lock") or b""
|
|
158
|
+
return lock.count(b"[[package]]")
|
|
159
|
+
|
|
160
|
+
|
|
133
161
|
def project_files(env: Env) -> dict[str, bytes]:
|
|
134
162
|
"""Read the files a runtime syncs from, refusing what cannot give a matching interpreter.
|
|
135
163
|
|
|
@@ -148,6 +176,15 @@ def project_files(env: Env) -> dict[str, bytes]:
|
|
|
148
176
|
f"pyproject.toml and uv.lock, and {directory.resolve()} has no {name}. "
|
|
149
177
|
f"Run 'uv lock' in the project first"
|
|
150
178
|
)
|
|
179
|
+
lock = files.get("uv.lock") or b""
|
|
180
|
+
if b'name = "letify"' not in lock and b'name = "cloudpickle"' not in lock:
|
|
181
|
+
raise ConfigError(
|
|
182
|
+
f"the worker unpickles a declared body with cloudpickle, and a body that uses "
|
|
183
|
+
f"letify's own names imports letify on the runtime, so "
|
|
184
|
+
f"{(directory / 'uv.lock').resolve()} has to name letify or at least cloudpickle. "
|
|
185
|
+
f"It names neither, and the session would fail there with ModuleNotFoundError. "
|
|
186
|
+
f"Add letify to the project and run 'uv lock' again"
|
|
187
|
+
)
|
|
151
188
|
local = local_python()
|
|
152
189
|
pinned = files.get(".python-version")
|
|
153
190
|
if pinned is not None:
|
|
@@ -130,6 +130,7 @@ class Runtime:
|
|
|
130
130
|
self.worker_pid = int(self.stat()["pid"])
|
|
131
131
|
self.provider.add_worker_pid(self.worker_pid)
|
|
132
132
|
self.restrict_devices()
|
|
133
|
+
self.check_devices()
|
|
133
134
|
if self.provider.needs_lease:
|
|
134
135
|
self.lease = Lease(self)
|
|
135
136
|
self.lease.arm()
|
|
@@ -613,6 +614,19 @@ class Runtime:
|
|
|
613
614
|
|
|
614
615
|
return (environment_volume(self.provider),)
|
|
615
616
|
|
|
617
|
+
def _say(self, message: str) -> None:
|
|
618
|
+
"""One decision line about this runtime, as the transport's lines are written.
|
|
619
|
+
|
|
620
|
+
Spec "Environment": the build says it is happening. Silent when the launcher was
|
|
621
|
+
built with ``announce=False``.
|
|
622
|
+
"""
|
|
623
|
+
from ..transport.announce import printer
|
|
624
|
+
|
|
625
|
+
# On the instance dict, not getattr: a provider's __getattr__ answers accelerator
|
|
626
|
+
# names and raises UnknownInstance for anything else, which no default catches.
|
|
627
|
+
announce = self.provider.__dict__.get("announce", True)
|
|
628
|
+
printer(bool(announce))(f"{self.name}: {message}")
|
|
629
|
+
|
|
616
630
|
def install_env(self) -> None:
|
|
617
631
|
"""Build the project's environment in the session and move the worker onto it.
|
|
618
632
|
|
|
@@ -641,6 +655,7 @@ class Runtime:
|
|
|
641
655
|
digest = volume.cached_env(self.env, self.platform)
|
|
642
656
|
if not digest:
|
|
643
657
|
continue
|
|
658
|
+
self._say(f"restoring the environment, cached as {digest[:12]}")
|
|
644
659
|
archive = f"{where['parent']}/.{digest}.tar.gz"
|
|
645
660
|
volume.materialize(
|
|
646
661
|
self, digest, path=archive, unpack=True, target=where["parent"], links=True
|
|
@@ -658,6 +673,13 @@ class Runtime:
|
|
|
658
673
|
self.env, files, root=root, name=self.name, workspace=workspace,
|
|
659
674
|
archive=bool(archives),
|
|
660
675
|
)
|
|
676
|
+
# Spec "Environment": the longest step of a first session, and uv's own output
|
|
677
|
+
# is captured rather than streamed, so it says it is happening.
|
|
678
|
+
self._say(
|
|
679
|
+
f"building the environment, {bootstrap.packages_in(files)} packages "
|
|
680
|
+
f"from {self.env.lock}"
|
|
681
|
+
)
|
|
682
|
+
began = time.monotonic()
|
|
661
683
|
try:
|
|
662
684
|
self.eval(source, timeout=3600)
|
|
663
685
|
except RemoteError as exc:
|
|
@@ -665,10 +687,48 @@ class Runtime:
|
|
|
665
687
|
raise EnvironmentFailure(
|
|
666
688
|
message.removeprefix("RuntimeError: "), stderr=exc.remote_traceback
|
|
667
689
|
) from exc
|
|
690
|
+
self._say(f"environment built in {time.monotonic() - began:.1f} s")
|
|
668
691
|
self.env_source = "sync"
|
|
669
692
|
self.python = where["python"]
|
|
670
693
|
self.channel.switch_interpreter(where["python"])
|
|
671
694
|
|
|
695
|
+
def check_devices(self) -> None:
|
|
696
|
+
"""Refuse a session whose cards are not the ones the instance asked for.
|
|
697
|
+
|
|
698
|
+
Spec "Kaggle", the accelerators: a provider may answer a request with a different
|
|
699
|
+
card. Run before the environment is built, so a wrong card costs seconds rather
|
|
700
|
+
than the minutes a sync takes.
|
|
701
|
+
"""
|
|
702
|
+
from ..errors import InsufficientDevices
|
|
703
|
+
from . import bootstrap
|
|
704
|
+
|
|
705
|
+
instance = getattr(self, "instance", None)
|
|
706
|
+
expected = self.provider.expected_cards(instance)
|
|
707
|
+
if expected is None:
|
|
708
|
+
return
|
|
709
|
+
fragment, count = expected
|
|
710
|
+
try:
|
|
711
|
+
cards = self.eval(bootstrap.CARD_SOURCE, timeout=120)
|
|
712
|
+
except Exception:
|
|
713
|
+
# Reading /proc is not worth failing a session over when it cannot be read.
|
|
714
|
+
return
|
|
715
|
+
cards = [str(card) for card in cards or ()]
|
|
716
|
+
matched = [card for card in cards if fragment.casefold() in card.casefold()]
|
|
717
|
+
if len(matched) == count and len(cards) == count:
|
|
718
|
+
return
|
|
719
|
+
asked = getattr(instance, "accelerator", None) or fragment
|
|
720
|
+
arrived = ", ".join(cards) if cards else "no GPU at all"
|
|
721
|
+
message = (
|
|
722
|
+
f"{self.name}: {self.provider.alias} was asked for {count} x {asked} and the "
|
|
723
|
+
f"session has {arrived}. A declaration says where a function runs, so the "
|
|
724
|
+
f"session is not used. Set any_accelerator = true on the account to accept "
|
|
725
|
+
f"whatever card it is given"
|
|
726
|
+
)
|
|
727
|
+
if self.provider.any_accelerator:
|
|
728
|
+
self._say(f"card mismatch accepted: asked {count} x {asked}, got {arrived}")
|
|
729
|
+
return
|
|
730
|
+
raise InsufficientDevices(message)
|
|
731
|
+
|
|
672
732
|
def check_interpreter(self) -> None:
|
|
673
733
|
"""Refuse a worker whose Python major.minor differs from this process."""
|
|
674
734
|
from . import bootstrap
|
|
@@ -801,19 +801,38 @@ class Stream:
|
|
|
801
801
|
)
|
|
802
802
|
|
|
803
803
|
|
|
804
|
-
def streams_for(
|
|
804
|
+
def streams_for(
|
|
805
|
+
*,
|
|
806
|
+
streams: int,
|
|
807
|
+
threshold_mib: int,
|
|
808
|
+
rtt_ms: float | None,
|
|
809
|
+
size: int,
|
|
810
|
+
rate_bps: float | None = None,
|
|
811
|
+
cost_s: float = 0.0,
|
|
812
|
+
) -> int:
|
|
805
813
|
"""How many connections this blob is worth splitting across.
|
|
806
814
|
|
|
807
815
|
Spec "Several connections at once": one below the size threshold, because opening
|
|
808
816
|
connections to send a few megabytes costs more than it saves; one on a short link,
|
|
809
|
-
because there a single stream already carries everything the link will carry;
|
|
810
|
-
|
|
817
|
+
because there a single stream already carries everything the link will carry; never
|
|
818
|
+
more streams than there are chunks to send; and, when a connection costs something to
|
|
819
|
+
open, only as many as finish the transfer sooner than one would.
|
|
811
820
|
"""
|
|
812
821
|
if streams <= 1 or size < threshold_mib * (1 << 20):
|
|
813
822
|
return 1
|
|
814
823
|
if rtt_ms is None or rtt_ms < PARALLEL_RTT_MS:
|
|
815
824
|
return 1
|
|
816
|
-
|
|
825
|
+
count = max(1, min(streams, (size + CHUNK - 1) // CHUNK))
|
|
826
|
+
if cost_s > 0 and count > 1:
|
|
827
|
+
if not rate_bps or rate_bps <= 0:
|
|
828
|
+
# Nothing to weigh the cost against, and the cost is real, so do not pay it.
|
|
829
|
+
return 1
|
|
830
|
+
alone = size / rate_bps
|
|
831
|
+
# The connections are opened at the same time, so the cost is paid once.
|
|
832
|
+
together = size / (rate_bps * count) + cost_s
|
|
833
|
+
if together >= alone:
|
|
834
|
+
return 1
|
|
835
|
+
return count
|
|
817
836
|
|
|
818
837
|
|
|
819
838
|
def _streams_for(runtime: Any, size: int) -> int:
|
|
@@ -832,9 +851,16 @@ def _streams_for(runtime: Any, size: int) -> int:
|
|
|
832
851
|
rtt = getattr(link, "rtt_ms", None)
|
|
833
852
|
except Exception:
|
|
834
853
|
rtt = None
|
|
854
|
+
rate = None
|
|
855
|
+
try:
|
|
856
|
+
rate = getattr(link, "upload_bps", None)
|
|
857
|
+
except Exception:
|
|
858
|
+
rate = None
|
|
835
859
|
return streams_for(
|
|
836
860
|
streams=int(getattr(provider, "transfer_streams", STREAMS) or 1),
|
|
837
861
|
threshold_mib=int(getattr(provider, "transfer_parallel_mib", PARALLEL_MIB) or 0),
|
|
862
|
+
rate_bps=rate,
|
|
863
|
+
cost_s=float(getattr(provider, "transfer_connection_cost_s", 0.0) or 0.0),
|
|
838
864
|
rtt_ms=rtt,
|
|
839
865
|
size=size,
|
|
840
866
|
)
|
|
@@ -34,6 +34,9 @@ class Link:
|
|
|
34
34
|
self.rank = rank
|
|
35
35
|
#: The round trip the pipeline measured over this link, when it measured one.
|
|
36
36
|
self.rtt_ms: float | None = None
|
|
37
|
+
#: The probe's upload rate, which the placement weighs a connection's cost
|
|
38
|
+
#: against. Spec "Several connections at once".
|
|
39
|
+
self.upload_bps: float | None = None
|
|
37
40
|
|
|
38
41
|
def probe_stream(self) -> object | None:
|
|
39
42
|
"""A stream the probe can run over, or None when this link cannot carry it."""
|