letify 2.4.1__tar.gz → 2.4.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {letify-2.4.1 → letify-2.4.2}/PKG-INFO +1 -1
- {letify-2.4.1 → letify-2.4.2}/letify/__init__.py +1 -1
- {letify-2.4.1 → letify-2.4.2}/letify/cli.py +62 -0
- {letify-2.4.1 → letify-2.4.2}/letify/install.py +31 -0
- {letify-2.4.1 → letify-2.4.2}/letify/launcher.py +34 -0
- {letify-2.4.1 → letify-2.4.2}/letify/protocol/worker.py +99 -11
- {letify-2.4.1 → letify-2.4.2}/letify/providers/base.py +12 -0
- {letify-2.4.1 → letify-2.4.2}/letify/providers/kaggle.py +182 -21
- {letify-2.4.1 → letify-2.4.2}/letify-core/Cargo.lock +3 -3
- {letify-2.4.1 → letify-2.4.2}/letify-core/Cargo.toml +1 -1
- {letify-2.4.1 → letify-2.4.2}/letify-ext/package-lock.json +2 -2
- {letify-2.4.1 → letify-2.4.2}/letify-ext/package.json +1 -1
- {letify-2.4.1 → letify-2.4.2}/pyproject.toml +1 -1
- {letify-2.4.1 → letify-2.4.2}/tests/conftest.py +3 -0
- {letify-2.4.1 → letify-2.4.2}/tests/test_cli.py +75 -0
- {letify-2.4.1 → letify-2.4.2}/tests/test_data_streaming.py +128 -0
- {letify-2.4.1 → letify-2.4.2}/tests/test_install.py +42 -0
- {letify-2.4.1 → letify-2.4.2}/tests/test_kaggle.py +157 -0
- {letify-2.4.1 → letify-2.4.2}/tests/test_parallel_placement.py +52 -0
- {letify-2.4.1 → letify-2.4.2}/.github/workflows/ci.yml +0 -0
- {letify-2.4.1 → letify-2.4.2}/.github/workflows/docs.yml +0 -0
- {letify-2.4.1 → letify-2.4.2}/.github/workflows/publish.yml +0 -0
- {letify-2.4.1 → letify-2.4.2}/.gitignore +0 -0
- {letify-2.4.1 → letify-2.4.2}/.researchtree +0 -0
- {letify-2.4.1 → letify-2.4.2}/LICENSE +0 -0
- {letify-2.4.1 → letify-2.4.2}/README.md +0 -0
- {letify-2.4.1 → letify-2.4.2}/docs/guide/01-getting-started.md +0 -0
- {letify-2.4.1 → letify-2.4.2}/docs/guide/02-providers.md +0 -0
- {letify-2.4.1 → letify-2.4.2}/docs/guide/03-execution-modes.md +0 -0
- {letify-2.4.1 → letify-2.4.2}/docs/guide/04-environments-and-data.md +0 -0
- {letify-2.4.1 → letify-2.4.2}/docs/guide/05-concurrency.md +0 -0
- {letify-2.4.1 → letify-2.4.2}/docs/guide/06-cost.md +0 -0
- {letify-2.4.1 → letify-2.4.2}/docs/guide/07-troubleshooting.md +0 -0
- {letify-2.4.1 → letify-2.4.2}/docs/guide/08-releasing.md +0 -0
- {letify-2.4.1 → letify-2.4.2}/docs/guide/README.md +0 -0
- {letify-2.4.1 → letify-2.4.2}/docs/guide/ko/01-/354/213/234/354/236/221/355/225/230/352/270/260.md" +0 -0
- {letify-2.4.1 → letify-2.4.2}/docs/guide/ko/02-/355/224/204/353/241/234/353/260/224/354/235/264/353/215/224.md" +0 -0
- {letify-2.4.1 → letify-2.4.2}/docs/guide/ko/03-/354/213/244/355/226/211-/353/260/251/354/213/235.md" +0 -0
- {letify-2.4.1 → letify-2.4.2}/docs/guide/ko/04-/355/231/230/352/262/275/352/263/274-/353/215/260/354/235/264/355/204/260.md" +0 -0
- {letify-2.4.1 → letify-2.4.2}/docs/guide/ko/05-/353/217/231/354/213/234-/354/213/244/355/226/211.md" +0 -0
- {letify-2.4.1 → letify-2.4.2}/docs/guide/ko/06-/353/271/204/354/232/251.md" +0 -0
- {letify-2.4.1 → letify-2.4.2}/docs/guide/ko/07-/353/254/270/354/240/234-/355/225/264/352/262/260.md" +0 -0
- {letify-2.4.1 → letify-2.4.2}/docs/guide/ko/08-/353/246/264/353/246/254/354/212/244.md" +0 -0
- {letify-2.4.1 → letify-2.4.2}/docs/guide/ko/README.md +0 -0
- {letify-2.4.1 → letify-2.4.2}/docs/locales/README_ko.md +0 -0
- {letify-2.4.1 → letify-2.4.2}/docs/mkdocs.yml +0 -0
- {letify-2.4.1 → letify-2.4.2}/examples/README.md +0 -0
- {letify-2.4.1 → letify-2.4.2}/examples/nvfp4-lora/.letify.example +0 -0
- {letify-2.4.1 → letify-2.4.2}/examples/nvfp4-lora/00_smoke.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/examples/nvfp4-lora/01_sweep.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/examples/nvfp4-lora/02_resume.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/examples/nvfp4-lora/03_watch.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/examples/nvfp4-lora/README.md +0 -0
- {letify-2.4.1 → letify-2.4.2}/examples/nvfp4-lora/README_ko.md +0 -0
- {letify-2.4.1 → letify-2.4.2}/examples/nvfp4-lora/common.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/examples/nvfp4-lora/recipe.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/config/__init__.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/config/inventory.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/config/login.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/config/schema.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/config/secrets.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/config/writer.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/declare/__init__.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/declare/cache.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/declare/env.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/declare/function.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/declare/instance.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/errors.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/paths.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/protocol/__init__.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/protocol/codec.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/protocol/driver.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/protocol/handle.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/protocol/wire.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/providers/__init__.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/providers/colab.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/providers/colab_files.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/providers/colab_keepalive.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/providers/elice.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/providers/kaggle_adapter.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/providers/local.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/providers/modal.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/providers/modal_adapter.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/providers/naming.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/providers/shell.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/providers/tunnel.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/providers/usage.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/remoting/__init__.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/remoting/capability.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/remoting/device/__init__.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/remoting/device/client.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/remoting/device/cuda.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/remoting/device/executor.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/remoting/device/frames.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/remoting/device/guard.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/remoting/device/tensor.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/remoting/device/trace.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/remoting/device/value.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/remoting/loader.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/remoting/probe.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/render.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/runtime/__init__.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/runtime/bootstrap.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/runtime/channel.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/runtime/lease.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/runtime/pool.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/runtime/session.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/runtime/telemetry.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/store/__init__.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/store/backends/__init__.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/store/backends/filesystem.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/store/backends/google_auth.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/store/backends/layout.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/store/backends/objects.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/store/cas.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/store/pathdata.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/store/sendorder.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/store/volume.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/stubs.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/tools.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/transport/__init__.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/transport/agent.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/transport/announce.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/transport/link.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/transport/nat.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/transport/pipeline.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/transport/portmap.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/transport/probe.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/transport/quic.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/transport/rendezvous.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/transport/setup.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/transport/sshopts.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify/transport/strategies.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify-core/.gitignore +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify-core/build.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify-core/letify-agent/Cargo.toml +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify-core/letify-agent/src/driver.rs +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify-core/letify-agent/src/main.rs +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify-core/letify-driver/Cargo.toml +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify-core/letify-driver/src/client.rs +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify-core/letify-driver/src/exports.rs +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify-core/letify-driver/src/image.rs +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify-core/letify-driver/src/lib.rs +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify-core/letify-driver/src/status.rs +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify-core/letify-driver/src/table.rs +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify-core/letify-quic/Cargo.toml +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify-core/letify-quic/src/main.rs +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify-core/letify-wire/Cargo.toml +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify-core/letify-wire/src/lib.rs +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify-core/letify-wire/tests/throughput.rs +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify-core/rust-toolchain.toml +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify-ext/.vscodeignore +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify-ext/LICENSE +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify-ext/README.md +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify-ext/media/icon.svg +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify-ext/src/cli.ts +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify-ext/src/extension.ts +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify-ext/src/history.ts +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify-ext/src/model.ts +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify-ext/src/poll.ts +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify-ext/src/render.ts +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify-ext/test/cli.test.ts +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify-ext/test/manifest.test.ts +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify-ext/test/model.test.ts +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify-ext/test/regressions.test.ts +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify-ext/tsconfig.json +0 -0
- {letify-2.4.1 → letify-2.4.2}/letify_providers.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/scripts/sync-main.sh +0 -0
- {letify-2.4.1 → letify-2.4.2}/scripts/version.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/tests/fake_kaggle_adapter.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/tests/fake_modal_adapter.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/tests/test_cli_output.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/tests/test_colab_keepalive.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/tests/test_config.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/tests/test_core.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/tests/test_data.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/tests/test_declare.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/tests/test_device.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/tests/test_editor_sessions.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/tests/test_elice.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/tests/test_frames.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/tests/test_kaggle_cookie.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/tests/test_kaggle_link.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/tests/test_link_log.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/tests/test_notebook.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/tests/test_paths.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/tests/test_portmap.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/tests/test_protocol.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/tests/test_providers.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/tests/test_quic.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/tests/test_remoting.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/tests/test_render.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/tests/test_runtime.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/tests/test_ship.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/tests/test_store.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/tests/test_strategies.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/tests/test_stubs.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/tests/test_transport.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/tests/test_tunnel_setup.py +0 -0
- {letify-2.4.1 → letify-2.4.2}/tests/test_version.py +0 -0
|
@@ -40,6 +40,16 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
40
40
|
sessions_parser.add_argument(
|
|
41
41
|
"--json", action="store_true", help="print the records unformatted"
|
|
42
42
|
)
|
|
43
|
+
stop = sub.add_parser("stop", help="stop what a dead process left behind")
|
|
44
|
+
stop.add_argument(
|
|
45
|
+
"--orphans",
|
|
46
|
+
action="store_true",
|
|
47
|
+
help="delete the sessions whose creating process is gone",
|
|
48
|
+
)
|
|
49
|
+
stop.add_argument("--alias", help="only this account")
|
|
50
|
+
stop.add_argument("-y", "--yes", action="store_true", help="do not ask")
|
|
51
|
+
stop.add_argument("--json", action="store_true", help="print the records unformatted")
|
|
52
|
+
|
|
43
53
|
sub.add_parser("stubs", help="write the provider types an editor completes")
|
|
44
54
|
|
|
45
55
|
usage = sub.add_parser("usage", help="show what each account has left")
|
|
@@ -523,6 +533,25 @@ def _say(kind: str, message: str) -> None:
|
|
|
523
533
|
print(f"{render.mark(kind, _out())} {message}")
|
|
524
534
|
|
|
525
535
|
|
|
536
|
+
def _detail(row: dict) -> str:
|
|
537
|
+
"""The reason a stop failed, when there is one to add."""
|
|
538
|
+
return f": {row['error']}" if row.get("error") else ""
|
|
539
|
+
|
|
540
|
+
|
|
541
|
+
def _confirmed(question: str) -> bool:
|
|
542
|
+
"""Ask before deleting. Anything but yes is no, and no terminal is no."""
|
|
543
|
+
import sys as _sys
|
|
544
|
+
|
|
545
|
+
if not _sys.stdin.isatty():
|
|
546
|
+
print(f"{question} not a terminal, so nothing is assumed. Pass -y to delete")
|
|
547
|
+
return False
|
|
548
|
+
try:
|
|
549
|
+
answer = input(f"{question} [y/N] ").strip().lower()
|
|
550
|
+
except EOFError:
|
|
551
|
+
return False
|
|
552
|
+
return answer in ("y", "yes")
|
|
553
|
+
|
|
554
|
+
|
|
526
555
|
def _json(value: object) -> int:
|
|
527
556
|
print(json.dumps(value, indent=2))
|
|
528
557
|
return 0
|
|
@@ -679,6 +708,39 @@ def _dispatch(args: argparse.Namespace) -> int:
|
|
|
679
708
|
print(f"{row['alias']} {detail}")
|
|
680
709
|
return 0
|
|
681
710
|
|
|
711
|
+
if args.command == "stop":
|
|
712
|
+
if not args.orphans:
|
|
713
|
+
# A bare `stop` reads as ending everything, so it ends nothing.
|
|
714
|
+
print("nothing was stopped. Pass --orphans to delete what a dead process left")
|
|
715
|
+
return 0
|
|
716
|
+
rows = let.orphans(args.alias)
|
|
717
|
+
found = [(row["alias"], ref) for row in rows for ref in row.get("orphans", ())]
|
|
718
|
+
unavailable = [row for row in rows if row.get("unavailable")]
|
|
719
|
+
if args.json and not found:
|
|
720
|
+
return _json(rows)
|
|
721
|
+
for row in unavailable:
|
|
722
|
+
print(f"{row['alias']} {row['unavailable']}")
|
|
723
|
+
if not found:
|
|
724
|
+
print("no orphaned sessions")
|
|
725
|
+
return 0
|
|
726
|
+
print(f"{len(found)} orphaned session(s) to delete:")
|
|
727
|
+
for alias, ref in found:
|
|
728
|
+
print(f" {alias} {ref}")
|
|
729
|
+
if not args.yes and not _confirmed("Delete them?"):
|
|
730
|
+
print("nothing was stopped")
|
|
731
|
+
return 0
|
|
732
|
+
done = let.stop_orphans(args.alias)
|
|
733
|
+
if args.json:
|
|
734
|
+
return _json(done)
|
|
735
|
+
failed = 0
|
|
736
|
+
for row in done:
|
|
737
|
+
if row.get("unavailable"):
|
|
738
|
+
continue
|
|
739
|
+
mark = "stopped" if row.get("stopped") else f"failed{_detail(row)}"
|
|
740
|
+
failed += 0 if row.get("stopped") else 1
|
|
741
|
+
print(f"{row['alias']} {row['ref']} {mark}")
|
|
742
|
+
return 1 if failed else 0
|
|
743
|
+
|
|
682
744
|
if args.command == "status":
|
|
683
745
|
status = let.status()
|
|
684
746
|
if args.json:
|
|
@@ -37,6 +37,10 @@ from .transport.setup import TAILCAT_VERSION
|
|
|
37
37
|
|
|
38
38
|
#: Seconds to wait for a release server.
|
|
39
39
|
DOWNLOAD_TIMEOUT = 60.0
|
|
40
|
+
#: Seconds to wait for the connection alone. A host a network blocks has to fail in
|
|
41
|
+
#: seconds, not after the read timeout, and twice that when a release redirects.
|
|
42
|
+
#: Spec "Automatic install".
|
|
43
|
+
CONNECT_TIMEOUT = 10.0
|
|
40
44
|
|
|
41
45
|
#: Bytes read from a release server at a time, so progress can be reported.
|
|
42
46
|
DOWNLOAD_CHUNK = 64 << 10
|
|
@@ -352,8 +356,35 @@ def _isatty(stream) -> bool:
|
|
|
352
356
|
return False
|
|
353
357
|
|
|
354
358
|
|
|
359
|
+
def _reachable(url: str) -> None:
|
|
360
|
+
"""Refuse a download whose host does not answer, before the request is made.
|
|
361
|
+
|
|
362
|
+
Spec "Automatic install": the error names the host so a blocked network is told what
|
|
363
|
+
to open or to configure around, rather than being told the tool is broken.
|
|
364
|
+
"""
|
|
365
|
+
import socket
|
|
366
|
+
import urllib.parse
|
|
367
|
+
|
|
368
|
+
parsed = urllib.parse.urlsplit(url)
|
|
369
|
+
host = parsed.hostname
|
|
370
|
+
if not host:
|
|
371
|
+
return
|
|
372
|
+
port = parsed.port or (443 if parsed.scheme == "https" else 80)
|
|
373
|
+
try:
|
|
374
|
+
socket.create_connection((host, port), timeout=CONNECT_TIMEOUT).close()
|
|
375
|
+
except OSError as exc:
|
|
376
|
+
raise InstallError(
|
|
377
|
+
f"could not reach {host}:{port} within {CONNECT_TIMEOUT:.0f} s ({exc}), so "
|
|
378
|
+
f"{url} was not downloaded. The host, not letify, is what is unreachable: a "
|
|
379
|
+
f"network that allows github.com can still block the host its releases "
|
|
380
|
+
f"redirect to. Fetch the archive another way and name the binary with the "
|
|
381
|
+
f"account's tailcat_binary or eci_binary"
|
|
382
|
+
) from None
|
|
383
|
+
|
|
384
|
+
|
|
355
385
|
def _download(url: str, *, label: str | None = None, stream=None) -> bytes:
|
|
356
386
|
"""Fetch ``url``; with ``label``, report progress on ``stream`` (standard error)."""
|
|
387
|
+
_reachable(url)
|
|
357
388
|
try:
|
|
358
389
|
with urllib.request.urlopen(url, timeout=DOWNLOAD_TIMEOUT) as response:
|
|
359
390
|
if label is None:
|
|
@@ -499,6 +499,40 @@ class Launcher:
|
|
|
499
499
|
rows.append({"alias": alias, "sessions": [], "unavailable": str(exc)})
|
|
500
500
|
return rows
|
|
501
501
|
|
|
502
|
+
def orphans(self, alias: str | None = None) -> list[dict[str, Any]]:
|
|
503
|
+
"""What a dead process left behind, per account.
|
|
504
|
+
|
|
505
|
+
Spec "Kaggle", stopping what a dead process left behind.
|
|
506
|
+
"""
|
|
507
|
+
rows: list[dict[str, Any]] = []
|
|
508
|
+
for name in self.config.order:
|
|
509
|
+
if alias is not None and name != alias:
|
|
510
|
+
continue
|
|
511
|
+
try:
|
|
512
|
+
provider = self.provider(name)
|
|
513
|
+
rows.append({"alias": name, "kind": provider.kind, "orphans": provider.orphans()})
|
|
514
|
+
except LetifyError as exc:
|
|
515
|
+
rows.append({"alias": name, "orphans": [], "unavailable": str(exc)})
|
|
516
|
+
return rows
|
|
517
|
+
|
|
518
|
+
def stop_orphans(self, alias: str | None = None) -> list[dict[str, Any]]:
|
|
519
|
+
"""Delete what ``orphans`` reported, one at a time, reporting each outcome."""
|
|
520
|
+
done: list[dict[str, Any]] = []
|
|
521
|
+
for row in self.orphans(alias):
|
|
522
|
+
if row.get("unavailable"):
|
|
523
|
+
done.append(row)
|
|
524
|
+
continue
|
|
525
|
+
provider = self.provider(row["alias"])
|
|
526
|
+
for ref in row["orphans"]:
|
|
527
|
+
try:
|
|
528
|
+
gone = provider.stop_orphan(ref)
|
|
529
|
+
done.append({"alias": row["alias"], "ref": ref, "stopped": gone})
|
|
530
|
+
except LetifyError as exc:
|
|
531
|
+
done.append(
|
|
532
|
+
{"alias": row["alias"], "ref": ref, "stopped": False, "error": str(exc)}
|
|
533
|
+
)
|
|
534
|
+
return done
|
|
535
|
+
|
|
502
536
|
def status(self) -> dict[str, Any]:
|
|
503
537
|
"""What is running right now, and what it is costing.
|
|
504
538
|
|
|
@@ -953,9 +953,12 @@ def _data_register(data):
|
|
|
953
953
|
# A call still waiting for bytes needs the patch to answer from the manifest,
|
|
954
954
|
# and a call that placed a link needs it for copy on write. Spec "Copy on write".
|
|
955
955
|
_data_install_patch()
|
|
956
|
-
if state["pending"]:
|
|
957
|
-
#
|
|
958
|
-
#
|
|
956
|
+
if state["pending"] or state["linked"]:
|
|
957
|
+
# A call still waiting for bytes needs the manifest on disk for the listings,
|
|
958
|
+
# and a call that placed a link needs it so a process the body starts can
|
|
959
|
+
# convert that link before writing through it: the patch a fresh process
|
|
960
|
+
# installs reads the layout from the manifest and installs nothing without it.
|
|
961
|
+
# Spec "Copy on write". Inside the lock: the data thread moves a path out of pending and
|
|
959
962
|
# into placed as each blob completes, and the manifest is written from both maps.
|
|
960
963
|
_data_write_manifest(state)
|
|
961
964
|
_data_observe(state)
|
|
@@ -1416,7 +1419,7 @@ def _data_write_manifest(state):
|
|
|
1416
1419
|
"""Write the call's manifest beside its files, for a process started fresh.
|
|
1417
1420
|
|
|
1418
1421
|
Spec "What the body sees before a file arrives": a spawned interpreter installs the
|
|
1419
|
-
same patch from ``
|
|
1422
|
+
same patch from ``sitecustomize.py`` and reads the layout from here, because it has
|
|
1420
1423
|
no channel of its own to ask on.
|
|
1421
1424
|
"""
|
|
1422
1425
|
import json
|
|
@@ -1435,10 +1438,11 @@ def _data_write_manifest(state):
|
|
|
1435
1438
|
directory = os.path.join(os.path.dirname(state["blobs"]), "pending")
|
|
1436
1439
|
try:
|
|
1437
1440
|
os.makedirs(directory, exist_ok=True)
|
|
1438
|
-
|
|
1441
|
+
# sitecustomize, not a .pth: site.py runs a .pth only in a directory it adds
|
|
1442
|
+
# through addsitedir, and a PYTHONPATH entry is not one, so a .pth there never
|
|
1443
|
+
# runs. Spec "What the body sees before a file arrives".
|
|
1444
|
+
with open(os.path.join(directory, "sitecustomize.py"), "w") as handle:
|
|
1439
1445
|
handle.write(_PENDING_MODULE)
|
|
1440
|
-
with open(os.path.join(directory, "letify_pending.pth"), "w") as handle:
|
|
1441
|
-
handle.write("import letify_pending\n")
|
|
1442
1446
|
except OSError:
|
|
1443
1447
|
return
|
|
1444
1448
|
existing = os.environ.get("PYTHONPATH", "")
|
|
@@ -1457,6 +1461,8 @@ import builtins
|
|
|
1457
1461
|
import io
|
|
1458
1462
|
import json
|
|
1459
1463
|
import os
|
|
1464
|
+
import shutil
|
|
1465
|
+
import sys
|
|
1460
1466
|
import time
|
|
1461
1467
|
|
|
1462
1468
|
|
|
@@ -1474,13 +1480,25 @@ def _install():
|
|
|
1474
1480
|
real_open, real_io_open = builtins.open, io.open
|
|
1475
1481
|
real_listdir, real_scandir = os.listdir, os.scandir
|
|
1476
1482
|
real_stat = os.stat
|
|
1483
|
+
real_os_open = os.open
|
|
1484
|
+
write_flags = os.O_WRONLY | os.O_RDWR | os.O_APPEND | os.O_TRUNC
|
|
1485
|
+
|
|
1486
|
+
# Whether the file exists, asked through the real stat. os.path.exists goes through
|
|
1487
|
+
# os.stat, and os.stat is what this patch replaces, so asking it here would call the
|
|
1488
|
+
# patch again and recurse without bottom.
|
|
1489
|
+
def there(text):
|
|
1490
|
+
try:
|
|
1491
|
+
real_stat(text)
|
|
1492
|
+
except OSError:
|
|
1493
|
+
return False
|
|
1494
|
+
return True
|
|
1477
1495
|
|
|
1478
1496
|
def missing(path):
|
|
1479
1497
|
try:
|
|
1480
1498
|
text = os.path.abspath(os.fspath(path))
|
|
1481
1499
|
except TypeError:
|
|
1482
1500
|
return None
|
|
1483
|
-
if not text.startswith(call_dir) or
|
|
1501
|
+
if not text.startswith(call_dir) or there(text):
|
|
1484
1502
|
return None
|
|
1485
1503
|
return text if text in files else None
|
|
1486
1504
|
|
|
@@ -1489,19 +1507,64 @@ def _install():
|
|
|
1489
1507
|
if text is None:
|
|
1490
1508
|
return
|
|
1491
1509
|
deadline = time.monotonic() + wait
|
|
1492
|
-
while not
|
|
1510
|
+
while not there(text):
|
|
1493
1511
|
if time.monotonic() >= deadline:
|
|
1494
1512
|
raise RuntimeError("letify: %s did not arrive within %.0f s" % (text, wait))
|
|
1495
1513
|
time.sleep(0.02)
|
|
1496
1514
|
|
|
1515
|
+
# A placed file is a hard link to the cache blob, so a write through it writes into
|
|
1516
|
+
# the cache. The worker converts the link first and this has to as well; it cannot read
|
|
1517
|
+
# the worker's placed map, so a link count above one under the call directory is what
|
|
1518
|
+
# it goes on. Spec "What the body sees before a file arrives".
|
|
1519
|
+
def convert(path):
|
|
1520
|
+
try:
|
|
1521
|
+
text = os.path.abspath(os.fspath(path))
|
|
1522
|
+
except TypeError:
|
|
1523
|
+
return
|
|
1524
|
+
if not text.startswith(call_dir):
|
|
1525
|
+
return
|
|
1526
|
+
try:
|
|
1527
|
+
if real_stat(text).st_nlink < 2:
|
|
1528
|
+
return
|
|
1529
|
+
except OSError:
|
|
1530
|
+
return
|
|
1531
|
+
partial = '%s.letify-placing.%d' % (text, os.getpid())
|
|
1532
|
+
try:
|
|
1533
|
+
shutil.copyfile(text, partial)
|
|
1534
|
+
os.replace(partial, text)
|
|
1535
|
+
except OSError:
|
|
1536
|
+
try:
|
|
1537
|
+
os.remove(partial)
|
|
1538
|
+
except OSError:
|
|
1539
|
+
pass
|
|
1540
|
+
|
|
1541
|
+
def writes(mode):
|
|
1542
|
+
return any(letter in mode for letter in ('w', 'a', 'x', '+'))
|
|
1543
|
+
|
|
1544
|
+
def ready(path, writing):
|
|
1545
|
+
wait_for(path)
|
|
1546
|
+
if writing:
|
|
1547
|
+
convert(path)
|
|
1548
|
+
|
|
1497
1549
|
def opened(file, *args, **kwargs):
|
|
1498
|
-
|
|
1550
|
+
mode = kwargs['mode'] if 'mode' in kwargs else (args[0] if args else 'r')
|
|
1551
|
+
ready(file, writes(mode) if isinstance(mode, str) else False)
|
|
1499
1552
|
return real_open(file, *args, **kwargs)
|
|
1500
1553
|
|
|
1501
1554
|
def io_opened(file, *args, **kwargs):
|
|
1502
|
-
|
|
1555
|
+
mode = kwargs['mode'] if 'mode' in kwargs else (args[0] if args else 'r')
|
|
1556
|
+
ready(file, writes(mode) if isinstance(mode, str) else False)
|
|
1503
1557
|
return real_io_open(file, *args, **kwargs)
|
|
1504
1558
|
|
|
1559
|
+
def os_opened(path, *args, **kwargs):
|
|
1560
|
+
flags = kwargs['flags'] if 'flags' in kwargs else (args[0] if args else 0)
|
|
1561
|
+
try:
|
|
1562
|
+
writing = bool(flags & write_flags)
|
|
1563
|
+
except TypeError:
|
|
1564
|
+
writing = False
|
|
1565
|
+
ready(path, writing)
|
|
1566
|
+
return real_os_open(path, *args, **kwargs)
|
|
1567
|
+
|
|
1505
1568
|
def names(path):
|
|
1506
1569
|
try:
|
|
1507
1570
|
text = os.path.abspath(os.fspath(path))
|
|
@@ -1534,12 +1597,37 @@ def _install():
|
|
|
1534
1597
|
|
|
1535
1598
|
builtins.open = opened
|
|
1536
1599
|
io.open = io_opened
|
|
1600
|
+
os.open = os_opened
|
|
1537
1601
|
os.listdir = listed
|
|
1538
1602
|
os.stat = stated
|
|
1539
1603
|
os.scandir = real_scandir
|
|
1540
1604
|
|
|
1541
1605
|
|
|
1606
|
+
# letify's directory goes first on PYTHONPATH, so it shadows a sitecustomize the user
|
|
1607
|
+
# installed. That one still has to run.
|
|
1608
|
+
def _chain():
|
|
1609
|
+
import importlib.util
|
|
1610
|
+
|
|
1611
|
+
here = os.path.dirname(os.path.abspath(__file__))
|
|
1612
|
+
for entry in sys.path:
|
|
1613
|
+
if not entry:
|
|
1614
|
+
continue
|
|
1615
|
+
try:
|
|
1616
|
+
if os.path.abspath(entry) == here:
|
|
1617
|
+
continue
|
|
1618
|
+
candidate = os.path.join(entry, 'sitecustomize.py')
|
|
1619
|
+
if not os.path.isfile(candidate):
|
|
1620
|
+
continue
|
|
1621
|
+
spec = importlib.util.spec_from_file_location('letify_other_sitecustomize', candidate)
|
|
1622
|
+
module = importlib.util.module_from_spec(spec)
|
|
1623
|
+
spec.loader.exec_module(module)
|
|
1624
|
+
except Exception:
|
|
1625
|
+
pass
|
|
1626
|
+
return
|
|
1627
|
+
|
|
1628
|
+
|
|
1542
1629
|
_install()
|
|
1630
|
+
_chain()
|
|
1543
1631
|
"""
|
|
1544
1632
|
|
|
1545
1633
|
|
|
@@ -374,6 +374,18 @@ class Provider(abc.ABC):
|
|
|
374
374
|
"""
|
|
375
375
|
return []
|
|
376
376
|
|
|
377
|
+
def orphans(self) -> list[str]:
|
|
378
|
+
"""What a dead process left behind on this account, for a provider that can tell.
|
|
379
|
+
|
|
380
|
+
Spec "Kaggle", stopping what a dead process left behind. Empty for a provider that
|
|
381
|
+
cannot, so `letify stop --orphans` has nothing to delete there.
|
|
382
|
+
"""
|
|
383
|
+
return []
|
|
384
|
+
|
|
385
|
+
def stop_orphan(self, ref: str) -> bool:
|
|
386
|
+
"""Delete one thing `orphans` reported, and report whether it is gone."""
|
|
387
|
+
return False
|
|
388
|
+
|
|
377
389
|
def expected_cards(self, instance: Any) -> tuple[str, int] | None:
|
|
378
390
|
"""The card name fragment and count a session for ``instance`` must have, or None.
|
|
379
391
|
|
|
@@ -24,6 +24,7 @@ import base64
|
|
|
24
24
|
import binascii
|
|
25
25
|
import json
|
|
26
26
|
import math
|
|
27
|
+
import os
|
|
27
28
|
import re
|
|
28
29
|
import textwrap
|
|
29
30
|
import time
|
|
@@ -384,7 +385,9 @@ def new_notebook(alias: str, cookie: str) -> tuple[int, str | None]:
|
|
|
384
385
|
return int(kernel), (str(slug) if slug else None)
|
|
385
386
|
|
|
386
387
|
|
|
387
|
-
def start_run(
|
|
388
|
+
def start_run(
|
|
389
|
+
cookie: str, kernel_id: int, accelerator: str | None, alias: str | None = None
|
|
390
|
+
) -> int:
|
|
388
391
|
"""Start an interactive session on the notebook and return its run id.
|
|
389
392
|
|
|
390
393
|
``CommitAndRun`` is what the editor's Run does: it commits the notebook body and starts
|
|
@@ -419,7 +422,10 @@ def start_run(cookie: str, kernel_id: int, accelerator: str | None) -> int:
|
|
|
419
422
|
)
|
|
420
423
|
run = reply.get("kernelRunId")
|
|
421
424
|
if run is None:
|
|
422
|
-
|
|
425
|
+
# Spec "Kaggle", not making sessions faster than Kaggle allows: the refusal has no
|
|
426
|
+
# message of its own, so letify names the cause it can see.
|
|
427
|
+
detail = refusal_detail(alias) if alias else ""
|
|
428
|
+
raise RuntimeFailure(f"Kaggle did not start a session run.{detail}")
|
|
423
429
|
return int(run)
|
|
424
430
|
|
|
425
431
|
|
|
@@ -518,7 +524,7 @@ def live_session_url(
|
|
|
518
524
|
# Spec "Kaggle", finding the notebooks letify owns: recorded before anything can fail,
|
|
519
525
|
# so a run that dies next still leaves a trail to its notebook.
|
|
520
526
|
record_notebook(alias, slug)
|
|
521
|
-
run = start_run(cookie, kernel, accelerator)
|
|
527
|
+
run = start_run(cookie, kernel, accelerator, alias)
|
|
522
528
|
id_token = firebase_id_token(cookie)
|
|
523
529
|
webtier = webtier_session(cookie, id_token, run)
|
|
524
530
|
deadline = time.monotonic() + SESSION_START_TIMEOUT
|
|
@@ -573,20 +579,120 @@ def notebook_record(alias: str) -> Path:
|
|
|
573
579
|
return account_directory(alias) / "notebooks"
|
|
574
580
|
|
|
575
581
|
|
|
576
|
-
|
|
577
|
-
|
|
582
|
+
#: Seconds to leave between sessions on one account, a guess rather than a measurement.
|
|
583
|
+
#: What is measured is one refusal a minute after the previous session. Spec "Kaggle", not
|
|
584
|
+
#: making sessions faster than Kaggle allows.
|
|
585
|
+
MIN_SESSION_INTERVAL_S = 120.0
|
|
586
|
+
|
|
587
|
+
|
|
588
|
+
def recorded_entries(alias: str) -> list[tuple[str, int | None, float | None]]:
|
|
589
|
+
"""Each ref letify created, the pid that made it and when, in order.
|
|
590
|
+
|
|
591
|
+
The pid and the time are None for a line an older letify wrote, which recorded
|
|
592
|
+
neither.
|
|
593
|
+
"""
|
|
578
594
|
try:
|
|
579
595
|
text = notebook_record(alias).read_text(encoding="utf-8")
|
|
580
596
|
except OSError:
|
|
581
597
|
return []
|
|
582
|
-
seen: list[str] = []
|
|
598
|
+
seen: list[tuple[str, int | None, float | None]] = []
|
|
599
|
+
known: set[str] = set()
|
|
583
600
|
for line in text.splitlines():
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
601
|
+
fields = line.strip().split("\t")
|
|
602
|
+
ref = fields[0] if fields else ""
|
|
603
|
+
if not ref or ref in known:
|
|
604
|
+
continue
|
|
605
|
+
known.add(ref)
|
|
606
|
+
owner: int | None = None
|
|
607
|
+
created: float | None = None
|
|
608
|
+
if len(fields) > 1 and fields[1]:
|
|
609
|
+
try:
|
|
610
|
+
owner = int(fields[1])
|
|
611
|
+
except ValueError:
|
|
612
|
+
owner = None
|
|
613
|
+
if len(fields) > 2 and fields[2]:
|
|
614
|
+
try:
|
|
615
|
+
created = float(fields[2])
|
|
616
|
+
except ValueError:
|
|
617
|
+
created = None
|
|
618
|
+
seen.append((ref, owner, created))
|
|
587
619
|
return seen
|
|
588
620
|
|
|
589
621
|
|
|
622
|
+
def newest_creation(alias: str) -> float | None:
|
|
623
|
+
"""When the newest recorded notebook on this account was created, or None."""
|
|
624
|
+
times = [created for _ref, _owner, created in recorded_entries(alias) if created is not None]
|
|
625
|
+
return max(times) if times else None
|
|
626
|
+
|
|
627
|
+
|
|
628
|
+
def wait_before_session(alias: str, interval: float) -> float:
|
|
629
|
+
"""Seconds still to wait before another session may be created on this account.
|
|
630
|
+
|
|
631
|
+
Spec "Kaggle", not making sessions faster than Kaggle allows: Kaggle refuses a run
|
|
632
|
+
made too soon after the last, and only waiting recovers it.
|
|
633
|
+
"""
|
|
634
|
+
if interval <= 0:
|
|
635
|
+
return 0.0
|
|
636
|
+
newest = newest_creation(alias)
|
|
637
|
+
if newest is None:
|
|
638
|
+
return 0.0
|
|
639
|
+
return max(0.0, interval - (time.time() - newest))
|
|
640
|
+
|
|
641
|
+
|
|
642
|
+
def refusal_detail(alias: str) -> str:
|
|
643
|
+
"""What to add to a refusal, naming how long ago the previous session was made.
|
|
644
|
+
|
|
645
|
+
This needs no threshold to be useful: it says what happened, so a reader can tell this
|
|
646
|
+
cause from any other. Empty when nothing is recorded to compare against.
|
|
647
|
+
"""
|
|
648
|
+
newest = newest_creation(alias)
|
|
649
|
+
if newest is None:
|
|
650
|
+
return ""
|
|
651
|
+
return (
|
|
652
|
+
f" The previous session on this account was created {time.time() - newest:.0f} s ago, "
|
|
653
|
+
f"and Kaggle refuses a run made too soon after the last."
|
|
654
|
+
)
|
|
655
|
+
|
|
656
|
+
|
|
657
|
+
def recorded_notebooks(alias: str) -> list[str]:
|
|
658
|
+
"""The refs letify believes it created, in the order it created them."""
|
|
659
|
+
return [ref for ref, _owner, _created in recorded_entries(alias)]
|
|
660
|
+
|
|
661
|
+
|
|
662
|
+
def _process_lives(pid: int) -> bool:
|
|
663
|
+
"""Whether a process with this id is still running on this machine."""
|
|
664
|
+
try:
|
|
665
|
+
os.kill(pid, 0)
|
|
666
|
+
except ProcessLookupError:
|
|
667
|
+
return False
|
|
668
|
+
except PermissionError:
|
|
669
|
+
# Someone else's process, so it is running and not letify's to judge.
|
|
670
|
+
return True
|
|
671
|
+
except OSError:
|
|
672
|
+
return True
|
|
673
|
+
return True
|
|
674
|
+
|
|
675
|
+
|
|
676
|
+
def orphaned_notebooks(alias: str) -> list[str]:
|
|
677
|
+
"""The recorded refs the account still has whose creating process is gone.
|
|
678
|
+
|
|
679
|
+
Spec "Kaggle", stopping what a dead process left behind: a notebook a live process is
|
|
680
|
+
still using is never reported, so nothing deletes a running session's notebook.
|
|
681
|
+
"""
|
|
682
|
+
entries = recorded_entries(alias)
|
|
683
|
+
if not entries:
|
|
684
|
+
return []
|
|
685
|
+
present = notebooks_on_account(alias)
|
|
686
|
+
if present is None:
|
|
687
|
+
return []
|
|
688
|
+
here = set(present)
|
|
689
|
+
return [
|
|
690
|
+
ref
|
|
691
|
+
for ref, owner, _created in entries
|
|
692
|
+
if ref in here and (owner is None or not _process_lives(owner))
|
|
693
|
+
]
|
|
694
|
+
|
|
695
|
+
|
|
590
696
|
def record_notebook(alias: str, slug: str | None) -> None:
|
|
591
697
|
"""Note that letify created this notebook, so a later process can still find it.
|
|
592
698
|
|
|
@@ -601,7 +707,8 @@ def record_notebook(alias: str, slug: str | None) -> None:
|
|
|
601
707
|
path = notebook_record(alias)
|
|
602
708
|
path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
603
709
|
with path.open("a", encoding="utf-8") as handle:
|
|
604
|
-
|
|
710
|
+
# The pid as well, so a live process's notebook is never taken for an orphan.
|
|
711
|
+
handle.write(f"{username}/{slug}\t{os.getpid()}\t{time.time():.3f}\n")
|
|
605
712
|
except OSError:
|
|
606
713
|
pass
|
|
607
714
|
|
|
@@ -611,16 +718,30 @@ def forget_notebook(alias: str, slug: str | None) -> None:
|
|
|
611
718
|
username = read_notebook_owner(alias)
|
|
612
719
|
if not slug or username is None:
|
|
613
720
|
return
|
|
614
|
-
|
|
615
|
-
|
|
721
|
+
forget_ref(alias, f"{username}/{slug}")
|
|
722
|
+
|
|
723
|
+
|
|
724
|
+
def forget_ref(alias: str, ref: str) -> None:
|
|
725
|
+
"""Drop one ref from the record, keeping the pid of every other line."""
|
|
726
|
+
kept = [row for row in recorded_entries(alias) if row[0] != ref]
|
|
616
727
|
try:
|
|
617
728
|
notebook_record(alias).write_text(
|
|
618
|
-
"".join(
|
|
729
|
+
"".join(_record_line(*row) for row in kept), encoding="utf-8"
|
|
619
730
|
)
|
|
620
731
|
except OSError:
|
|
621
732
|
pass
|
|
622
733
|
|
|
623
734
|
|
|
735
|
+
def _record_line(ref: str, owner: int | None, created: float | None) -> str:
|
|
736
|
+
"""One record line, keeping whichever fields the entry has."""
|
|
737
|
+
fields = [ref]
|
|
738
|
+
if owner is not None:
|
|
739
|
+
fields.append(str(owner))
|
|
740
|
+
if created is not None:
|
|
741
|
+
fields.append(f"{created:.3f}")
|
|
742
|
+
return "\t".join(fields) + "\n"
|
|
743
|
+
|
|
744
|
+
|
|
624
745
|
def notebooks_on_account(alias: str) -> list[str] | None:
|
|
625
746
|
"""The refs the account has, through the official CLI, or None when it cannot be asked.
|
|
626
747
|
|
|
@@ -667,14 +788,11 @@ def list_notebooks_via_cli(alias: str) -> list[str]:
|
|
|
667
788
|
present = notebooks_on_account(alias)
|
|
668
789
|
if present is None:
|
|
669
790
|
return []
|
|
670
|
-
|
|
671
|
-
if
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
)
|
|
676
|
-
except OSError:
|
|
677
|
-
pass
|
|
791
|
+
here = set(present)
|
|
792
|
+
live = [ref for ref in recorded if ref in here]
|
|
793
|
+
for ref in recorded:
|
|
794
|
+
if ref not in here:
|
|
795
|
+
forget_ref(alias, ref)
|
|
678
796
|
return live
|
|
679
797
|
|
|
680
798
|
|
|
@@ -1158,6 +1276,40 @@ class Kaggle(Provider):
|
|
|
1158
1276
|
"""The notebooks letify owns on this account, which is where an orphan shows up."""
|
|
1159
1277
|
return list_notebooks_via_cli(self.alias)
|
|
1160
1278
|
|
|
1279
|
+
def _leave_a_gap(self) -> None:
|
|
1280
|
+
"""Wait out the interval since the last session, rather than be refused.
|
|
1281
|
+
|
|
1282
|
+
Spec "Kaggle", not making sessions faster than Kaggle allows: a refusal costs an
|
|
1283
|
+
account that answers nothing for an unknown time, and waiting costs patience.
|
|
1284
|
+
"""
|
|
1285
|
+
from ..transport.announce import printer
|
|
1286
|
+
|
|
1287
|
+
waiting = wait_before_session(self.alias, self.min_session_interval_s)
|
|
1288
|
+
if waiting <= 0:
|
|
1289
|
+
return
|
|
1290
|
+
printer(bool(self.__dict__.get("announce", True)))(
|
|
1291
|
+
f"{self.alias}: waiting {waiting:.0f} s before another session, because Kaggle "
|
|
1292
|
+
f"refuses one made too soon after the last"
|
|
1293
|
+
)
|
|
1294
|
+
time.sleep(waiting)
|
|
1295
|
+
|
|
1296
|
+
def orphans(self) -> list[str]:
|
|
1297
|
+
"""Notebooks this account still has whose creating process is gone."""
|
|
1298
|
+
return orphaned_notebooks(self.alias)
|
|
1299
|
+
|
|
1300
|
+
def stop_orphan(self, ref: str) -> bool:
|
|
1301
|
+
"""Delete one orphaned notebook, which removes its session with it.
|
|
1302
|
+
|
|
1303
|
+
The run id died with the process that made it, so `cancel_run` cannot reach the
|
|
1304
|
+
run; deleting the notebook is the way in. Spec "Kaggle", stopping what a dead
|
|
1305
|
+
process left behind.
|
|
1306
|
+
"""
|
|
1307
|
+
slug = ref.partition("/")[2] or ref
|
|
1308
|
+
if not delete_notebook_via_cli(self.alias, slug):
|
|
1309
|
+
return False
|
|
1310
|
+
forget_ref(self.alias, ref)
|
|
1311
|
+
return True
|
|
1312
|
+
|
|
1161
1313
|
def expected_cards(self, instance: Any) -> tuple[str, int] | None:
|
|
1162
1314
|
"""What a Kaggle session must have, because Kaggle may answer with another card."""
|
|
1163
1315
|
gpu = getattr(instance, "gpu", None)
|
|
@@ -1361,6 +1513,14 @@ class Kaggle(Provider):
|
|
|
1361
1513
|
name=name,
|
|
1362
1514
|
)
|
|
1363
1515
|
|
|
1516
|
+
@property
|
|
1517
|
+
def min_session_interval_s(self) -> float:
|
|
1518
|
+
"""Seconds to leave between sessions on this account, from the account or the default."""
|
|
1519
|
+
value = self.config.option("min_session_interval_s")
|
|
1520
|
+
if isinstance(value, (int, float)):
|
|
1521
|
+
return max(0.0, float(value))
|
|
1522
|
+
return MIN_SESSION_INTERVAL_S
|
|
1523
|
+
|
|
1364
1524
|
@property
|
|
1365
1525
|
def fallback_mib_per_s(self) -> float:
|
|
1366
1526
|
"""The kernel bridge, measured at 2.0 MiB/s sending 64 MiB on a live account.
|
|
@@ -1423,6 +1583,7 @@ class Kaggle(Provider):
|
|
|
1423
1583
|
f"letify login kaggle {self.alias} --username <owner>"
|
|
1424
1584
|
)
|
|
1425
1585
|
cookie = self._require_live_cookie()
|
|
1586
|
+
self._leave_a_gap()
|
|
1426
1587
|
instance = getattr(runtime, "instance", None)
|
|
1427
1588
|
gpu = getattr(instance, "gpu", None)
|
|
1428
1589
|
accelerator = GPUS[gpu]["accelerator"] if gpu in GPUS else getattr(instance, "tpu", None)
|