letify 2.4.1__tar.gz → 2.4.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {letify-2.4.1 → letify-2.4.3}/PKG-INFO +1 -1
- {letify-2.4.1 → letify-2.4.3}/letify/__init__.py +7 -1
- {letify-2.4.1 → letify-2.4.3}/letify/cli.py +62 -0
- {letify-2.4.1 → letify-2.4.3}/letify/declare/function.py +52 -3
- {letify-2.4.1 → letify-2.4.3}/letify/declare/instance.py +3 -0
- letify-2.4.3/letify/declare/replicas.py +111 -0
- {letify-2.4.1 → letify-2.4.3}/letify/install.py +31 -0
- {letify-2.4.1 → letify-2.4.3}/letify/launcher.py +34 -0
- {letify-2.4.1 → letify-2.4.3}/letify/protocol/worker.py +127 -13
- {letify-2.4.1 → letify-2.4.3}/letify/providers/base.py +12 -0
- {letify-2.4.1 → letify-2.4.3}/letify/providers/kaggle.py +182 -21
- {letify-2.4.1 → letify-2.4.3}/letify/store/pathdata.py +40 -1
- {letify-2.4.1 → letify-2.4.3}/letify-core/Cargo.lock +3 -3
- {letify-2.4.1 → letify-2.4.3}/letify-core/Cargo.toml +1 -1
- {letify-2.4.1 → letify-2.4.3}/letify-ext/package-lock.json +2 -2
- {letify-2.4.1 → letify-2.4.3}/letify-ext/package.json +1 -1
- {letify-2.4.1 → letify-2.4.3}/pyproject.toml +1 -1
- {letify-2.4.1 → letify-2.4.3}/tests/conftest.py +3 -0
- {letify-2.4.1 → letify-2.4.3}/tests/test_cli.py +75 -0
- {letify-2.4.1 → letify-2.4.3}/tests/test_data_streaming.py +267 -0
- letify-2.4.3/tests/test_distributed.py +149 -0
- {letify-2.4.1 → letify-2.4.3}/tests/test_install.py +42 -0
- {letify-2.4.1 → letify-2.4.3}/tests/test_kaggle.py +157 -0
- {letify-2.4.1 → letify-2.4.3}/tests/test_parallel_placement.py +52 -0
- {letify-2.4.1 → letify-2.4.3}/.github/workflows/ci.yml +0 -0
- {letify-2.4.1 → letify-2.4.3}/.github/workflows/docs.yml +0 -0
- {letify-2.4.1 → letify-2.4.3}/.github/workflows/publish.yml +0 -0
- {letify-2.4.1 → letify-2.4.3}/.gitignore +0 -0
- {letify-2.4.1 → letify-2.4.3}/.researchtree +0 -0
- {letify-2.4.1 → letify-2.4.3}/LICENSE +0 -0
- {letify-2.4.1 → letify-2.4.3}/README.md +0 -0
- {letify-2.4.1 → letify-2.4.3}/docs/guide/01-getting-started.md +0 -0
- {letify-2.4.1 → letify-2.4.3}/docs/guide/02-providers.md +0 -0
- {letify-2.4.1 → letify-2.4.3}/docs/guide/03-execution-modes.md +0 -0
- {letify-2.4.1 → letify-2.4.3}/docs/guide/04-environments-and-data.md +0 -0
- {letify-2.4.1 → letify-2.4.3}/docs/guide/05-concurrency.md +0 -0
- {letify-2.4.1 → letify-2.4.3}/docs/guide/06-cost.md +0 -0
- {letify-2.4.1 → letify-2.4.3}/docs/guide/07-troubleshooting.md +0 -0
- {letify-2.4.1 → letify-2.4.3}/docs/guide/08-releasing.md +0 -0
- {letify-2.4.1 → letify-2.4.3}/docs/guide/README.md +0 -0
- {letify-2.4.1 → letify-2.4.3}/docs/guide/ko/01-/354/213/234/354/236/221/355/225/230/352/270/260.md" +0 -0
- {letify-2.4.1 → letify-2.4.3}/docs/guide/ko/02-/355/224/204/353/241/234/353/260/224/354/235/264/353/215/224.md" +0 -0
- {letify-2.4.1 → letify-2.4.3}/docs/guide/ko/03-/354/213/244/355/226/211-/353/260/251/354/213/235.md" +0 -0
- {letify-2.4.1 → letify-2.4.3}/docs/guide/ko/04-/355/231/230/352/262/275/352/263/274-/353/215/260/354/235/264/355/204/260.md" +0 -0
- {letify-2.4.1 → letify-2.4.3}/docs/guide/ko/05-/353/217/231/354/213/234-/354/213/244/355/226/211.md" +0 -0
- {letify-2.4.1 → letify-2.4.3}/docs/guide/ko/06-/353/271/204/354/232/251.md" +0 -0
- {letify-2.4.1 → letify-2.4.3}/docs/guide/ko/07-/353/254/270/354/240/234-/355/225/264/352/262/260.md" +0 -0
- {letify-2.4.1 → letify-2.4.3}/docs/guide/ko/08-/353/246/264/353/246/254/354/212/244.md" +0 -0
- {letify-2.4.1 → letify-2.4.3}/docs/guide/ko/README.md +0 -0
- {letify-2.4.1 → letify-2.4.3}/docs/locales/README_ko.md +0 -0
- {letify-2.4.1 → letify-2.4.3}/docs/mkdocs.yml +0 -0
- {letify-2.4.1 → letify-2.4.3}/examples/README.md +0 -0
- {letify-2.4.1 → letify-2.4.3}/examples/nvfp4-lora/.letify.example +0 -0
- {letify-2.4.1 → letify-2.4.3}/examples/nvfp4-lora/00_smoke.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/examples/nvfp4-lora/01_sweep.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/examples/nvfp4-lora/02_resume.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/examples/nvfp4-lora/03_watch.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/examples/nvfp4-lora/README.md +0 -0
- {letify-2.4.1 → letify-2.4.3}/examples/nvfp4-lora/README_ko.md +0 -0
- {letify-2.4.1 → letify-2.4.3}/examples/nvfp4-lora/common.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/examples/nvfp4-lora/recipe.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/config/__init__.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/config/inventory.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/config/login.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/config/schema.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/config/secrets.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/config/writer.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/declare/__init__.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/declare/cache.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/declare/env.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/errors.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/paths.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/protocol/__init__.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/protocol/codec.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/protocol/driver.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/protocol/handle.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/protocol/wire.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/providers/__init__.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/providers/colab.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/providers/colab_files.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/providers/colab_keepalive.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/providers/elice.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/providers/kaggle_adapter.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/providers/local.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/providers/modal.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/providers/modal_adapter.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/providers/naming.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/providers/shell.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/providers/tunnel.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/providers/usage.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/remoting/__init__.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/remoting/capability.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/remoting/device/__init__.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/remoting/device/client.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/remoting/device/cuda.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/remoting/device/executor.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/remoting/device/frames.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/remoting/device/guard.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/remoting/device/tensor.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/remoting/device/trace.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/remoting/device/value.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/remoting/loader.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/remoting/probe.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/render.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/runtime/__init__.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/runtime/bootstrap.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/runtime/channel.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/runtime/lease.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/runtime/pool.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/runtime/session.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/runtime/telemetry.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/store/__init__.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/store/backends/__init__.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/store/backends/filesystem.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/store/backends/google_auth.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/store/backends/layout.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/store/backends/objects.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/store/cas.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/store/sendorder.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/store/volume.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/stubs.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/tools.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/transport/__init__.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/transport/agent.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/transport/announce.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/transport/link.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/transport/nat.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/transport/pipeline.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/transport/portmap.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/transport/probe.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/transport/quic.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/transport/rendezvous.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/transport/setup.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/transport/sshopts.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify/transport/strategies.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify-core/.gitignore +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify-core/build.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify-core/letify-agent/Cargo.toml +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify-core/letify-agent/src/driver.rs +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify-core/letify-agent/src/main.rs +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify-core/letify-driver/Cargo.toml +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify-core/letify-driver/src/client.rs +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify-core/letify-driver/src/exports.rs +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify-core/letify-driver/src/image.rs +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify-core/letify-driver/src/lib.rs +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify-core/letify-driver/src/status.rs +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify-core/letify-driver/src/table.rs +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify-core/letify-quic/Cargo.toml +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify-core/letify-quic/src/main.rs +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify-core/letify-wire/Cargo.toml +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify-core/letify-wire/src/lib.rs +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify-core/letify-wire/tests/throughput.rs +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify-core/rust-toolchain.toml +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify-ext/.vscodeignore +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify-ext/LICENSE +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify-ext/README.md +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify-ext/media/icon.svg +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify-ext/src/cli.ts +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify-ext/src/extension.ts +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify-ext/src/history.ts +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify-ext/src/model.ts +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify-ext/src/poll.ts +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify-ext/src/render.ts +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify-ext/test/cli.test.ts +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify-ext/test/manifest.test.ts +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify-ext/test/model.test.ts +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify-ext/test/regressions.test.ts +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify-ext/tsconfig.json +0 -0
- {letify-2.4.1 → letify-2.4.3}/letify_providers.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/scripts/sync-main.sh +0 -0
- {letify-2.4.1 → letify-2.4.3}/scripts/version.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/tests/fake_kaggle_adapter.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/tests/fake_modal_adapter.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/tests/test_cli_output.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/tests/test_colab_keepalive.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/tests/test_config.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/tests/test_core.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/tests/test_data.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/tests/test_declare.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/tests/test_device.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/tests/test_editor_sessions.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/tests/test_elice.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/tests/test_frames.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/tests/test_kaggle_cookie.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/tests/test_kaggle_link.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/tests/test_link_log.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/tests/test_notebook.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/tests/test_paths.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/tests/test_portmap.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/tests/test_protocol.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/tests/test_providers.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/tests/test_quic.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/tests/test_remoting.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/tests/test_render.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/tests/test_runtime.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/tests/test_ship.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/tests/test_store.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/tests/test_strategies.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/tests/test_stubs.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/tests/test_transport.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/tests/test_tunnel_setup.py +0 -0
- {letify-2.4.1 → letify-2.4.3}/tests/test_version.py +0 -0
|
@@ -33,6 +33,7 @@ from .declare.cache import session_cache
|
|
|
33
33
|
from .declare.env import Env
|
|
34
34
|
from .declare.function import Function
|
|
35
35
|
from .declare.instance import AnyInstance, Host, Instance
|
|
36
|
+
from .declare.replicas import local_steps, rank, world_size
|
|
36
37
|
from .errors import (
|
|
37
38
|
ConfigError,
|
|
38
39
|
EnvironmentFailure,
|
|
@@ -58,11 +59,12 @@ from .store.volume import Volume
|
|
|
58
59
|
#: ``Final`` so a type checker sees the literal member, which the declaration overloads match on.
|
|
59
60
|
local: Final = Host.local
|
|
60
61
|
remote: Final = Host.remote
|
|
62
|
+
distributed: Final = Host.distributed
|
|
61
63
|
|
|
62
64
|
# The enum class stays importable from letify.declare.instance for letify's own use.
|
|
63
65
|
del Host
|
|
64
66
|
|
|
65
|
-
__version__ = "2.4.
|
|
67
|
+
__version__ = "2.4.3"
|
|
66
68
|
|
|
67
69
|
|
|
68
70
|
def fetch(tensor): # type: ignore[no-untyped-def]
|
|
@@ -101,7 +103,11 @@ __all__ = [
|
|
|
101
103
|
"UnsupportedMode",
|
|
102
104
|
"Volume",
|
|
103
105
|
"__version__",
|
|
106
|
+
"distributed",
|
|
104
107
|
"local",
|
|
108
|
+
"local_steps",
|
|
109
|
+
"rank",
|
|
105
110
|
"remote",
|
|
106
111
|
"session_cache",
|
|
112
|
+
"world_size",
|
|
107
113
|
]
|
|
@@ -40,6 +40,16 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
40
40
|
sessions_parser.add_argument(
|
|
41
41
|
"--json", action="store_true", help="print the records unformatted"
|
|
42
42
|
)
|
|
43
|
+
stop = sub.add_parser("stop", help="stop what a dead process left behind")
|
|
44
|
+
stop.add_argument(
|
|
45
|
+
"--orphans",
|
|
46
|
+
action="store_true",
|
|
47
|
+
help="delete the sessions whose creating process is gone",
|
|
48
|
+
)
|
|
49
|
+
stop.add_argument("--alias", help="only this account")
|
|
50
|
+
stop.add_argument("-y", "--yes", action="store_true", help="do not ask")
|
|
51
|
+
stop.add_argument("--json", action="store_true", help="print the records unformatted")
|
|
52
|
+
|
|
43
53
|
sub.add_parser("stubs", help="write the provider types an editor completes")
|
|
44
54
|
|
|
45
55
|
usage = sub.add_parser("usage", help="show what each account has left")
|
|
@@ -523,6 +533,25 @@ def _say(kind: str, message: str) -> None:
|
|
|
523
533
|
print(f"{render.mark(kind, _out())} {message}")
|
|
524
534
|
|
|
525
535
|
|
|
536
|
+
def _detail(row: dict) -> str:
|
|
537
|
+
"""The reason a stop failed, when there is one to add."""
|
|
538
|
+
return f": {row['error']}" if row.get("error") else ""
|
|
539
|
+
|
|
540
|
+
|
|
541
|
+
def _confirmed(question: str) -> bool:
|
|
542
|
+
"""Ask before deleting. Anything but yes is no, and no terminal is no."""
|
|
543
|
+
import sys as _sys
|
|
544
|
+
|
|
545
|
+
if not _sys.stdin.isatty():
|
|
546
|
+
print(f"{question} not a terminal, so nothing is assumed. Pass -y to delete")
|
|
547
|
+
return False
|
|
548
|
+
try:
|
|
549
|
+
answer = input(f"{question} [y/N] ").strip().lower()
|
|
550
|
+
except EOFError:
|
|
551
|
+
return False
|
|
552
|
+
return answer in ("y", "yes")
|
|
553
|
+
|
|
554
|
+
|
|
526
555
|
def _json(value: object) -> int:
|
|
527
556
|
print(json.dumps(value, indent=2))
|
|
528
557
|
return 0
|
|
@@ -679,6 +708,39 @@ def _dispatch(args: argparse.Namespace) -> int:
|
|
|
679
708
|
print(f"{row['alias']} {detail}")
|
|
680
709
|
return 0
|
|
681
710
|
|
|
711
|
+
if args.command == "stop":
|
|
712
|
+
if not args.orphans:
|
|
713
|
+
# A bare `stop` reads as ending everything, so it ends nothing.
|
|
714
|
+
print("nothing was stopped. Pass --orphans to delete what a dead process left")
|
|
715
|
+
return 0
|
|
716
|
+
rows = let.orphans(args.alias)
|
|
717
|
+
found = [(row["alias"], ref) for row in rows for ref in row.get("orphans", ())]
|
|
718
|
+
unavailable = [row for row in rows if row.get("unavailable")]
|
|
719
|
+
if args.json and not found:
|
|
720
|
+
return _json(rows)
|
|
721
|
+
for row in unavailable:
|
|
722
|
+
print(f"{row['alias']} {row['unavailable']}")
|
|
723
|
+
if not found:
|
|
724
|
+
print("no orphaned sessions")
|
|
725
|
+
return 0
|
|
726
|
+
print(f"{len(found)} orphaned session(s) to delete:")
|
|
727
|
+
for alias, ref in found:
|
|
728
|
+
print(f" {alias} {ref}")
|
|
729
|
+
if not args.yes and not _confirmed("Delete them?"):
|
|
730
|
+
print("nothing was stopped")
|
|
731
|
+
return 0
|
|
732
|
+
done = let.stop_orphans(args.alias)
|
|
733
|
+
if args.json:
|
|
734
|
+
return _json(done)
|
|
735
|
+
failed = 0
|
|
736
|
+
for row in done:
|
|
737
|
+
if row.get("unavailable"):
|
|
738
|
+
continue
|
|
739
|
+
mark = "stopped" if row.get("stopped") else f"failed{_detail(row)}"
|
|
740
|
+
failed += 0 if row.get("stopped") else 1
|
|
741
|
+
print(f"{row['alias']} {row['ref']} {mark}")
|
|
742
|
+
return 1 if failed else 0
|
|
743
|
+
|
|
682
744
|
if args.command == "status":
|
|
683
745
|
status = let.status()
|
|
684
746
|
if args.json:
|
|
@@ -65,7 +65,9 @@ class Function(Generic[R]):
|
|
|
65
65
|
self.data_order = data_order
|
|
66
66
|
self.data_first_wave = data_first_wave
|
|
67
67
|
self.is_async = inspect.iscoroutinefunction(fn)
|
|
68
|
-
|
|
68
|
+
#: Spec "Replicas": one Instance per replica, in the order the list named them.
|
|
69
|
+
self.replicas = _replicas(device, self.host)
|
|
70
|
+
self.device = self._place(self.replicas[0] if self.replicas else device)
|
|
69
71
|
update_wrapper(self, fn)
|
|
70
72
|
|
|
71
73
|
def _place(self, device: Instance | AnyInstance) -> Instance | AnyInstance:
|
|
@@ -136,6 +138,16 @@ class Function(Generic[R]):
|
|
|
136
138
|
|
|
137
139
|
def _run(self, args: tuple, kwargs: dict) -> Any:
|
|
138
140
|
launcher = self.launcher
|
|
141
|
+
if self.host is Host.distributed:
|
|
142
|
+
# Spec "Replicas": the placement is declared and the runner is not written
|
|
143
|
+
# yet. Running the first replica alone would bill one runtime for a
|
|
144
|
+
# declaration that asked for several, without saying so.
|
|
145
|
+
raise UnsupportedMode(
|
|
146
|
+
f"host=letify.distributed declares {len(self.replicas)} replicas and "
|
|
147
|
+
f"letify cannot run them yet, so the call is refused rather than run on "
|
|
148
|
+
f"one of them. Use host=letify.remote with a single device until it is "
|
|
149
|
+
f"finished."
|
|
150
|
+
)
|
|
139
151
|
instance = launcher.resolve(self.device)
|
|
140
152
|
if instance.placement is Host.local:
|
|
141
153
|
return self._run_here(instance, args, kwargs)
|
|
@@ -249,6 +261,42 @@ def _report_deferred(delta: Any) -> None:
|
|
|
249
261
|
)
|
|
250
262
|
|
|
251
263
|
|
|
264
|
+
def _replicas(device: Any, host: Host) -> tuple[Instance, ...]:
|
|
265
|
+
"""The replicas a declaration names, or empty when it names a single device.
|
|
266
|
+
|
|
267
|
+
Spec "Replicas": ``device`` takes a list only under ``host=letify.distributed``, each
|
|
268
|
+
entry is one replica, and two replicas of one instance are refused because that asks
|
|
269
|
+
one account for twice the devices it has.
|
|
270
|
+
"""
|
|
271
|
+
listed = isinstance(device, (list, tuple))
|
|
272
|
+
if listed and host is not Host.distributed:
|
|
273
|
+
raise ValueError(
|
|
274
|
+
f"device is a list of {len(device)} instances, which declares one replica "
|
|
275
|
+
f"each, so it needs host=letify.distributed. Under host={host.value!r} name "
|
|
276
|
+
f"one device."
|
|
277
|
+
)
|
|
278
|
+
if not listed:
|
|
279
|
+
return (device,) if host is Host.distributed else ()
|
|
280
|
+
if not device:
|
|
281
|
+
raise ValueError(
|
|
282
|
+
"device is an empty list, so the declaration names no replica. Name at least "
|
|
283
|
+
"one instance."
|
|
284
|
+
)
|
|
285
|
+
seen: list[str] = []
|
|
286
|
+
for instance in device:
|
|
287
|
+
alias = getattr(getattr(instance, "provider", None), "alias", None)
|
|
288
|
+
name = getattr(instance, "accelerator", None)
|
|
289
|
+
key = f"{alias}.{name}"
|
|
290
|
+
if key in seen:
|
|
291
|
+
raise ValueError(
|
|
292
|
+
f"device names {key} twice, so one account would be asked for twice the "
|
|
293
|
+
f"devices one replica takes. Name each account once, and use a larger "
|
|
294
|
+
f"instance such as {name} * 2 for more devices in one replica."
|
|
295
|
+
)
|
|
296
|
+
seen.append(key)
|
|
297
|
+
return tuple(device)
|
|
298
|
+
|
|
299
|
+
|
|
252
300
|
def _where(host: Host | str | None) -> Host:
|
|
253
301
|
"""Validate the declared host placement, naming both options when it is wrong."""
|
|
254
302
|
if host is None:
|
|
@@ -258,8 +306,9 @@ def _where(host: Host | str | None) -> Host:
|
|
|
258
306
|
except ValueError:
|
|
259
307
|
raise ValueError(
|
|
260
308
|
f"host={host!r} is not a host placement. Use host='local' to keep Python "
|
|
261
|
-
f"here and forward PyTorch operators,
|
|
262
|
-
f"the machine that holds the device
|
|
309
|
+
f"here and forward PyTorch operators, host='remote' to ship the function to "
|
|
310
|
+
f"the machine that holds the device, or host='distributed' to run it once per "
|
|
311
|
+
f"device in a list."
|
|
263
312
|
) from None
|
|
264
313
|
|
|
265
314
|
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
"""Where a replica learns which one it is, and what it reduces with.
|
|
2
|
+
|
|
3
|
+
Owns the per-replica facts a distributed body reads, ``rank`` and ``world_size``, and the
|
|
4
|
+
reduction a body asks for. It does not own the loop; spec "Replicas" says so and
|
|
5
|
+
``docs/INTENT.md`` keeps that as a non-goal for every loop but one.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import math
|
|
11
|
+
import threading
|
|
12
|
+
from typing import Any
|
|
13
|
+
|
|
14
|
+
from ..errors import UnsupportedMode
|
|
15
|
+
|
|
16
|
+
#: Set on the thread running a distributed body, so ``rank`` and ``world_size`` answer.
|
|
17
|
+
_WHO = threading.local()
|
|
18
|
+
|
|
19
|
+
_OUTSIDE = (
|
|
20
|
+
"{name}() answers only inside a call declared with host=letify.distributed. A body "
|
|
21
|
+
"under host=letify.remote has one replica and no rank, and answering 0 and 1 here "
|
|
22
|
+
"would hide that it was asked"
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def rank() -> int:
|
|
27
|
+
"""Which replica this is, 0 upward in the order the ``device`` list names.
|
|
28
|
+
|
|
29
|
+
Spec "Replicas": raises outside a distributed call rather than answering 0.
|
|
30
|
+
"""
|
|
31
|
+
found = getattr(_WHO, "rank", None)
|
|
32
|
+
if found is None:
|
|
33
|
+
raise UnsupportedMode(_OUTSIDE.format(name="rank"))
|
|
34
|
+
return int(found)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def world_size() -> int:
|
|
38
|
+
"""How many replicas started. Spec "Replicas"."""
|
|
39
|
+
found = getattr(_WHO, "world_size", None)
|
|
40
|
+
if found is None:
|
|
41
|
+
raise UnsupportedMode(_OUTSIDE.format(name="world_size"))
|
|
42
|
+
return int(found)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
#: The fraction of a step a reduction may take before it is worth doing less often.
|
|
46
|
+
DEFAULT_SHARE = 0.25
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def steps_for(
|
|
50
|
+
*,
|
|
51
|
+
param_bytes: int,
|
|
52
|
+
step_seconds: float,
|
|
53
|
+
rate_bps: float | None,
|
|
54
|
+
share: float = DEFAULT_SHARE,
|
|
55
|
+
) -> int:
|
|
56
|
+
"""The smallest number of local steps for which a reduction is cheap enough.
|
|
57
|
+
|
|
58
|
+
Spec "How often to synchronize": a reduction moves the parameters once, so it is
|
|
59
|
+
worth it only when rare enough against the step it interrupts. Raises rather than
|
|
60
|
+
guessing when the link reports no rate, because a guess here is a guess about the
|
|
61
|
+
whole run's shape.
|
|
62
|
+
"""
|
|
63
|
+
if not rate_bps or rate_bps <= 0:
|
|
64
|
+
raise UnsupportedMode(
|
|
65
|
+
"local_steps needs the link's measured upload rate and this link reports "
|
|
66
|
+
"none, so there is nothing to divide the parameters by. The rate comes from "
|
|
67
|
+
"the probe, so a link the pipeline never probed, such as a provider "
|
|
68
|
+
"fallback, cannot answer. Pass the count instead of letify.auto"
|
|
69
|
+
)
|
|
70
|
+
if step_seconds <= 0:
|
|
71
|
+
raise ValueError("step_seconds has to be above zero to divide a transfer against")
|
|
72
|
+
allowed = max(share, 0.0) * step_seconds
|
|
73
|
+
if allowed <= 0:
|
|
74
|
+
raise ValueError("share has to be above zero, or no number of steps is ever enough")
|
|
75
|
+
transfer = param_bytes / rate_bps
|
|
76
|
+
return max(1, math.ceil(transfer / allowed))
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def local_steps(param_bytes: int, step_seconds: float, share: float = DEFAULT_SHARE) -> int:
|
|
80
|
+
"""How many steps to take between reductions, from the link letify already measured.
|
|
81
|
+
|
|
82
|
+
Spec "How often to synchronize". Answers for the replica this body runs on, so two
|
|
83
|
+
replicas on links of different speed get different answers.
|
|
84
|
+
"""
|
|
85
|
+
rate = getattr(_WHO, "rate_bps", None)
|
|
86
|
+
return steps_for(
|
|
87
|
+
param_bytes=param_bytes,
|
|
88
|
+
step_seconds=step_seconds,
|
|
89
|
+
rate_bps=rate,
|
|
90
|
+
share=share,
|
|
91
|
+
)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
class _Replica:
|
|
95
|
+
"""Holds the rank for the body's thread, so nothing has to be passed through it."""
|
|
96
|
+
|
|
97
|
+
def __init__(self, rank: int, world_size: int, rate_bps: float | None = None) -> None:
|
|
98
|
+
self.rank = rank
|
|
99
|
+
self.world_size = world_size
|
|
100
|
+
self.rate_bps = rate_bps
|
|
101
|
+
|
|
102
|
+
def __enter__(self) -> _Replica:
|
|
103
|
+
_WHO.rank, _WHO.world_size = self.rank, self.world_size
|
|
104
|
+
_WHO.rate_bps = self.rate_bps
|
|
105
|
+
return self
|
|
106
|
+
|
|
107
|
+
def __exit__(self, *_exc: Any) -> None:
|
|
108
|
+
_WHO.rank = _WHO.world_size = _WHO.rate_bps = None
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
__all__ = ["local_steps", "rank", "steps_for", "world_size"]
|
|
@@ -37,6 +37,10 @@ from .transport.setup import TAILCAT_VERSION
|
|
|
37
37
|
|
|
38
38
|
#: Seconds to wait for a release server.
|
|
39
39
|
DOWNLOAD_TIMEOUT = 60.0
|
|
40
|
+
#: Seconds to wait for the connection alone. A host a network blocks has to fail in
|
|
41
|
+
#: seconds, not after the read timeout, and twice that when a release redirects.
|
|
42
|
+
#: Spec "Automatic install".
|
|
43
|
+
CONNECT_TIMEOUT = 10.0
|
|
40
44
|
|
|
41
45
|
#: Bytes read from a release server at a time, so progress can be reported.
|
|
42
46
|
DOWNLOAD_CHUNK = 64 << 10
|
|
@@ -352,8 +356,35 @@ def _isatty(stream) -> bool:
|
|
|
352
356
|
return False
|
|
353
357
|
|
|
354
358
|
|
|
359
|
+
def _reachable(url: str) -> None:
|
|
360
|
+
"""Refuse a download whose host does not answer, before the request is made.
|
|
361
|
+
|
|
362
|
+
Spec "Automatic install": the error names the host so a blocked network is told what
|
|
363
|
+
to open or to configure around, rather than being told the tool is broken.
|
|
364
|
+
"""
|
|
365
|
+
import socket
|
|
366
|
+
import urllib.parse
|
|
367
|
+
|
|
368
|
+
parsed = urllib.parse.urlsplit(url)
|
|
369
|
+
host = parsed.hostname
|
|
370
|
+
if not host:
|
|
371
|
+
return
|
|
372
|
+
port = parsed.port or (443 if parsed.scheme == "https" else 80)
|
|
373
|
+
try:
|
|
374
|
+
socket.create_connection((host, port), timeout=CONNECT_TIMEOUT).close()
|
|
375
|
+
except OSError as exc:
|
|
376
|
+
raise InstallError(
|
|
377
|
+
f"could not reach {host}:{port} within {CONNECT_TIMEOUT:.0f} s ({exc}), so "
|
|
378
|
+
f"{url} was not downloaded. The host, not letify, is what is unreachable: a "
|
|
379
|
+
f"network that allows github.com can still block the host its releases "
|
|
380
|
+
f"redirect to. Fetch the archive another way and name the binary with the "
|
|
381
|
+
f"account's tailcat_binary or eci_binary"
|
|
382
|
+
) from None
|
|
383
|
+
|
|
384
|
+
|
|
355
385
|
def _download(url: str, *, label: str | None = None, stream=None) -> bytes:
|
|
356
386
|
"""Fetch ``url``; with ``label``, report progress on ``stream`` (standard error)."""
|
|
387
|
+
_reachable(url)
|
|
357
388
|
try:
|
|
358
389
|
with urllib.request.urlopen(url, timeout=DOWNLOAD_TIMEOUT) as response:
|
|
359
390
|
if label is None:
|
|
@@ -499,6 +499,40 @@ class Launcher:
|
|
|
499
499
|
rows.append({"alias": alias, "sessions": [], "unavailable": str(exc)})
|
|
500
500
|
return rows
|
|
501
501
|
|
|
502
|
+
def orphans(self, alias: str | None = None) -> list[dict[str, Any]]:
|
|
503
|
+
"""What a dead process left behind, per account.
|
|
504
|
+
|
|
505
|
+
Spec "Kaggle", stopping what a dead process left behind.
|
|
506
|
+
"""
|
|
507
|
+
rows: list[dict[str, Any]] = []
|
|
508
|
+
for name in self.config.order:
|
|
509
|
+
if alias is not None and name != alias:
|
|
510
|
+
continue
|
|
511
|
+
try:
|
|
512
|
+
provider = self.provider(name)
|
|
513
|
+
rows.append({"alias": name, "kind": provider.kind, "orphans": provider.orphans()})
|
|
514
|
+
except LetifyError as exc:
|
|
515
|
+
rows.append({"alias": name, "orphans": [], "unavailable": str(exc)})
|
|
516
|
+
return rows
|
|
517
|
+
|
|
518
|
+
def stop_orphans(self, alias: str | None = None) -> list[dict[str, Any]]:
|
|
519
|
+
"""Delete what ``orphans`` reported, one at a time, reporting each outcome."""
|
|
520
|
+
done: list[dict[str, Any]] = []
|
|
521
|
+
for row in self.orphans(alias):
|
|
522
|
+
if row.get("unavailable"):
|
|
523
|
+
done.append(row)
|
|
524
|
+
continue
|
|
525
|
+
provider = self.provider(row["alias"])
|
|
526
|
+
for ref in row["orphans"]:
|
|
527
|
+
try:
|
|
528
|
+
gone = provider.stop_orphan(ref)
|
|
529
|
+
done.append({"alias": row["alias"], "ref": ref, "stopped": gone})
|
|
530
|
+
except LetifyError as exc:
|
|
531
|
+
done.append(
|
|
532
|
+
{"alias": row["alias"], "ref": ref, "stopped": False, "error": str(exc)}
|
|
533
|
+
)
|
|
534
|
+
return done
|
|
535
|
+
|
|
502
536
|
def status(self) -> dict[str, Any]:
|
|
503
537
|
"""What is running right now, and what it is costing.
|
|
504
538
|
|
|
@@ -675,12 +675,38 @@ def _data_budget(root, total, budget):
|
|
|
675
675
|
return min(_DATA_DEFAULT_BUDGET, (total + free) // 2)
|
|
676
676
|
|
|
677
677
|
|
|
678
|
+
def _digest_of(path):
|
|
679
|
+
"""The digest a blob path holds, which is its shard directory and its name joined."""
|
|
680
|
+
return os.path.basename(path)
|
|
681
|
+
|
|
682
|
+
|
|
683
|
+
def _data_digests_in_flight():
|
|
684
|
+
"""Every digest a running call lists, so the prune does not take one out from under it.
|
|
685
|
+
|
|
686
|
+
Spec "Runtime data cache budget". Read under the lock the data thread writes the maps
|
|
687
|
+
with, so a call registering while this reads cannot change one mid iteration.
|
|
688
|
+
"""
|
|
689
|
+
wanted = set()
|
|
690
|
+
with _DATA_READY:
|
|
691
|
+
for state in list(_DATA_CALLS.values()):
|
|
692
|
+
wanted.update(state["by_digest"])
|
|
693
|
+
return wanted
|
|
694
|
+
|
|
695
|
+
|
|
678
696
|
def _op_data_evict(request):
|
|
679
|
-
"""Remove least recently used blobs no call links until the cache is within budget.
|
|
697
|
+
"""Remove least recently used blobs no call links until the cache is within budget.
|
|
698
|
+
|
|
699
|
+
Spec "Runtime data cache budget": a digest a running call lists is skipped whatever
|
|
700
|
+
its link count and age, because the call was told the runtime held it and so did not
|
|
701
|
+
send it. ``recent`` is the window a blob no call is using is protected for, carried on
|
|
702
|
+
the request beside the budget.
|
|
703
|
+
"""
|
|
680
704
|
started = time.monotonic()
|
|
681
705
|
root = os.path.expanduser(request["dir"])
|
|
682
706
|
files, total = _data_scan(root)
|
|
683
707
|
budget = _data_budget(root, total, request.get("budget"))
|
|
708
|
+
recent = float(request.get("recent", _DATA_RECENT_S))
|
|
709
|
+
wanted = _data_digests_in_flight()
|
|
684
710
|
removed = freed = 0
|
|
685
711
|
if total > budget:
|
|
686
712
|
now = time.time()
|
|
@@ -688,7 +714,7 @@ def _op_data_evict(request):
|
|
|
688
714
|
for used, path, size, links in files:
|
|
689
715
|
if total <= budget:
|
|
690
716
|
break
|
|
691
|
-
if links > 1 or now - used <
|
|
717
|
+
if links > 1 or now - used < recent or _digest_of(path) in wanted:
|
|
692
718
|
continue
|
|
693
719
|
try:
|
|
694
720
|
os.remove(path)
|
|
@@ -953,9 +979,12 @@ def _data_register(data):
|
|
|
953
979
|
# A call still waiting for bytes needs the patch to answer from the manifest,
|
|
954
980
|
# and a call that placed a link needs it for copy on write. Spec "Copy on write".
|
|
955
981
|
_data_install_patch()
|
|
956
|
-
if state["pending"]:
|
|
957
|
-
#
|
|
958
|
-
#
|
|
982
|
+
if state["pending"] or state["linked"]:
|
|
983
|
+
# A call still waiting for bytes needs the manifest on disk for the listings,
|
|
984
|
+
# and a call that placed a link needs it so a process the body starts can
|
|
985
|
+
# convert that link before writing through it: the patch a fresh process
|
|
986
|
+
# installs reads the layout from the manifest and installs nothing without it.
|
|
987
|
+
# Spec "Copy on write". Inside the lock: the data thread moves a path out of pending and
|
|
959
988
|
# into placed as each blob completes, and the manifest is written from both maps.
|
|
960
989
|
_data_write_manifest(state)
|
|
961
990
|
_data_observe(state)
|
|
@@ -1416,7 +1445,7 @@ def _data_write_manifest(state):
|
|
|
1416
1445
|
"""Write the call's manifest beside its files, for a process started fresh.
|
|
1417
1446
|
|
|
1418
1447
|
Spec "What the body sees before a file arrives": a spawned interpreter installs the
|
|
1419
|
-
same patch from ``
|
|
1448
|
+
same patch from ``sitecustomize.py`` and reads the layout from here, because it has
|
|
1420
1449
|
no channel of its own to ask on.
|
|
1421
1450
|
"""
|
|
1422
1451
|
import json
|
|
@@ -1435,10 +1464,11 @@ def _data_write_manifest(state):
|
|
|
1435
1464
|
directory = os.path.join(os.path.dirname(state["blobs"]), "pending")
|
|
1436
1465
|
try:
|
|
1437
1466
|
os.makedirs(directory, exist_ok=True)
|
|
1438
|
-
|
|
1467
|
+
# sitecustomize, not a .pth: site.py runs a .pth only in a directory it adds
|
|
1468
|
+
# through addsitedir, and a PYTHONPATH entry is not one, so a .pth there never
|
|
1469
|
+
# runs. Spec "What the body sees before a file arrives".
|
|
1470
|
+
with open(os.path.join(directory, "sitecustomize.py"), "w") as handle:
|
|
1439
1471
|
handle.write(_PENDING_MODULE)
|
|
1440
|
-
with open(os.path.join(directory, "letify_pending.pth"), "w") as handle:
|
|
1441
|
-
handle.write("import letify_pending\n")
|
|
1442
1472
|
except OSError:
|
|
1443
1473
|
return
|
|
1444
1474
|
existing = os.environ.get("PYTHONPATH", "")
|
|
@@ -1457,6 +1487,8 @@ import builtins
|
|
|
1457
1487
|
import io
|
|
1458
1488
|
import json
|
|
1459
1489
|
import os
|
|
1490
|
+
import shutil
|
|
1491
|
+
import sys
|
|
1460
1492
|
import time
|
|
1461
1493
|
|
|
1462
1494
|
|
|
@@ -1474,13 +1506,25 @@ def _install():
|
|
|
1474
1506
|
real_open, real_io_open = builtins.open, io.open
|
|
1475
1507
|
real_listdir, real_scandir = os.listdir, os.scandir
|
|
1476
1508
|
real_stat = os.stat
|
|
1509
|
+
real_os_open = os.open
|
|
1510
|
+
write_flags = os.O_WRONLY | os.O_RDWR | os.O_APPEND | os.O_TRUNC
|
|
1511
|
+
|
|
1512
|
+
# Whether the file exists, asked through the real stat. os.path.exists goes through
|
|
1513
|
+
# os.stat, and os.stat is what this patch replaces, so asking it here would call the
|
|
1514
|
+
# patch again and recurse without bottom.
|
|
1515
|
+
def there(text):
|
|
1516
|
+
try:
|
|
1517
|
+
real_stat(text)
|
|
1518
|
+
except OSError:
|
|
1519
|
+
return False
|
|
1520
|
+
return True
|
|
1477
1521
|
|
|
1478
1522
|
def missing(path):
|
|
1479
1523
|
try:
|
|
1480
1524
|
text = os.path.abspath(os.fspath(path))
|
|
1481
1525
|
except TypeError:
|
|
1482
1526
|
return None
|
|
1483
|
-
if not text.startswith(call_dir) or
|
|
1527
|
+
if not text.startswith(call_dir) or there(text):
|
|
1484
1528
|
return None
|
|
1485
1529
|
return text if text in files else None
|
|
1486
1530
|
|
|
@@ -1489,19 +1533,64 @@ def _install():
|
|
|
1489
1533
|
if text is None:
|
|
1490
1534
|
return
|
|
1491
1535
|
deadline = time.monotonic() + wait
|
|
1492
|
-
while not
|
|
1536
|
+
while not there(text):
|
|
1493
1537
|
if time.monotonic() >= deadline:
|
|
1494
1538
|
raise RuntimeError("letify: %s did not arrive within %.0f s" % (text, wait))
|
|
1495
1539
|
time.sleep(0.02)
|
|
1496
1540
|
|
|
1541
|
+
# A placed file is a hard link to the cache blob, so a write through it writes into
|
|
1542
|
+
# the cache. The worker converts the link first and this has to as well; it cannot read
|
|
1543
|
+
# the worker's placed map, so a link count above one under the call directory is what
|
|
1544
|
+
# it goes on. Spec "What the body sees before a file arrives".
|
|
1545
|
+
def convert(path):
|
|
1546
|
+
try:
|
|
1547
|
+
text = os.path.abspath(os.fspath(path))
|
|
1548
|
+
except TypeError:
|
|
1549
|
+
return
|
|
1550
|
+
if not text.startswith(call_dir):
|
|
1551
|
+
return
|
|
1552
|
+
try:
|
|
1553
|
+
if real_stat(text).st_nlink < 2:
|
|
1554
|
+
return
|
|
1555
|
+
except OSError:
|
|
1556
|
+
return
|
|
1557
|
+
partial = '%s.letify-placing.%d' % (text, os.getpid())
|
|
1558
|
+
try:
|
|
1559
|
+
shutil.copyfile(text, partial)
|
|
1560
|
+
os.replace(partial, text)
|
|
1561
|
+
except OSError:
|
|
1562
|
+
try:
|
|
1563
|
+
os.remove(partial)
|
|
1564
|
+
except OSError:
|
|
1565
|
+
pass
|
|
1566
|
+
|
|
1567
|
+
def writes(mode):
|
|
1568
|
+
return any(letter in mode for letter in ('w', 'a', 'x', '+'))
|
|
1569
|
+
|
|
1570
|
+
def ready(path, writing):
|
|
1571
|
+
wait_for(path)
|
|
1572
|
+
if writing:
|
|
1573
|
+
convert(path)
|
|
1574
|
+
|
|
1497
1575
|
def opened(file, *args, **kwargs):
|
|
1498
|
-
|
|
1576
|
+
mode = kwargs['mode'] if 'mode' in kwargs else (args[0] if args else 'r')
|
|
1577
|
+
ready(file, writes(mode) if isinstance(mode, str) else False)
|
|
1499
1578
|
return real_open(file, *args, **kwargs)
|
|
1500
1579
|
|
|
1501
1580
|
def io_opened(file, *args, **kwargs):
|
|
1502
|
-
|
|
1581
|
+
mode = kwargs['mode'] if 'mode' in kwargs else (args[0] if args else 'r')
|
|
1582
|
+
ready(file, writes(mode) if isinstance(mode, str) else False)
|
|
1503
1583
|
return real_io_open(file, *args, **kwargs)
|
|
1504
1584
|
|
|
1585
|
+
def os_opened(path, *args, **kwargs):
|
|
1586
|
+
flags = kwargs['flags'] if 'flags' in kwargs else (args[0] if args else 0)
|
|
1587
|
+
try:
|
|
1588
|
+
writing = bool(flags & write_flags)
|
|
1589
|
+
except TypeError:
|
|
1590
|
+
writing = False
|
|
1591
|
+
ready(path, writing)
|
|
1592
|
+
return real_os_open(path, *args, **kwargs)
|
|
1593
|
+
|
|
1505
1594
|
def names(path):
|
|
1506
1595
|
try:
|
|
1507
1596
|
text = os.path.abspath(os.fspath(path))
|
|
@@ -1534,12 +1623,37 @@ def _install():
|
|
|
1534
1623
|
|
|
1535
1624
|
builtins.open = opened
|
|
1536
1625
|
io.open = io_opened
|
|
1626
|
+
os.open = os_opened
|
|
1537
1627
|
os.listdir = listed
|
|
1538
1628
|
os.stat = stated
|
|
1539
1629
|
os.scandir = real_scandir
|
|
1540
1630
|
|
|
1541
1631
|
|
|
1632
|
+
# letify's directory goes first on PYTHONPATH, so it shadows a sitecustomize the user
|
|
1633
|
+
# installed. That one still has to run.
|
|
1634
|
+
def _chain():
|
|
1635
|
+
import importlib.util
|
|
1636
|
+
|
|
1637
|
+
here = os.path.dirname(os.path.abspath(__file__))
|
|
1638
|
+
for entry in sys.path:
|
|
1639
|
+
if not entry:
|
|
1640
|
+
continue
|
|
1641
|
+
try:
|
|
1642
|
+
if os.path.abspath(entry) == here:
|
|
1643
|
+
continue
|
|
1644
|
+
candidate = os.path.join(entry, 'sitecustomize.py')
|
|
1645
|
+
if not os.path.isfile(candidate):
|
|
1646
|
+
continue
|
|
1647
|
+
spec = importlib.util.spec_from_file_location('letify_other_sitecustomize', candidate)
|
|
1648
|
+
module = importlib.util.module_from_spec(spec)
|
|
1649
|
+
spec.loader.exec_module(module)
|
|
1650
|
+
except Exception:
|
|
1651
|
+
pass
|
|
1652
|
+
return
|
|
1653
|
+
|
|
1654
|
+
|
|
1542
1655
|
_install()
|
|
1656
|
+
_chain()
|
|
1543
1657
|
"""
|
|
1544
1658
|
|
|
1545
1659
|
|
|
@@ -374,6 +374,18 @@ class Provider(abc.ABC):
|
|
|
374
374
|
"""
|
|
375
375
|
return []
|
|
376
376
|
|
|
377
|
+
def orphans(self) -> list[str]:
|
|
378
|
+
"""What a dead process left behind on this account, for a provider that can tell.
|
|
379
|
+
|
|
380
|
+
Spec "Kaggle", stopping what a dead process left behind. Empty for a provider that
|
|
381
|
+
cannot, so `letify stop --orphans` has nothing to delete there.
|
|
382
|
+
"""
|
|
383
|
+
return []
|
|
384
|
+
|
|
385
|
+
def stop_orphan(self, ref: str) -> bool:
|
|
386
|
+
"""Delete one thing `orphans` reported, and report whether it is gone."""
|
|
387
|
+
return False
|
|
388
|
+
|
|
377
389
|
def expected_cards(self, instance: Any) -> tuple[str, int] | None:
|
|
378
390
|
"""The card name fragment and count a session for ``instance`` must have, or None.
|
|
379
391
|
|