modal 1.2.7.dev3__tar.gz → 1.2.7.dev4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/PKG-INFO +1 -1
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/_runtime/container_io_manager.py +1 -3
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/_runtime/gpu_memory_snapshot.py +9 -7
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/client.pyi +2 -2
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal.egg-info/PKG-INFO +1 -1
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal_version/__init__.py +1 -1
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/LICENSE +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/README.md +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/__init__.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/__main__.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/_billing.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/_clustered_functions.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/_clustered_functions.pyi +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/_container_entrypoint.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/_functions.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/_grpc_client.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/_ipython.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/_load_context.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/_location.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/_object.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/_output.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/_partial_function.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/_pty.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/_resolver.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/_resources.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/_runtime/__init__.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/_runtime/asgi.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/_runtime/container_io_manager.pyi +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/_runtime/execution_context.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/_runtime/execution_context.pyi +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/_runtime/telemetry.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/_runtime/user_code_event_loop.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/_runtime/user_code_imports.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/_serialization.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/_traceback.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/_tunnel.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/_tunnel.pyi +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/_type_manager.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/_utils/__init__.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/_utils/app_utils.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/_utils/async_utils.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/_utils/auth_token_manager.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/_utils/blob_utils.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/_utils/bytes_io_segment_payload.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/_utils/deprecation.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/_utils/docker_utils.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/_utils/function_utils.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/_utils/git_utils.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/_utils/grpc_testing.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/_utils/grpc_utils.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/_utils/hash_utils.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/_utils/http_utils.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/_utils/jwt_utils.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/_utils/logger.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/_utils/mount_utils.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/_utils/name_utils.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/_utils/package_utils.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/_utils/pattern_utils.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/_utils/rand_pb_testing.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/_utils/shell_utils.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/_utils/task_command_router_client.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/_utils/time_utils.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/_vendor/__init__.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/_vendor/a2wsgi_wsgi.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/_vendor/cloudpickle.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/_vendor/tblib.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/_watcher.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/app.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/app.pyi +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/billing.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/builder/2023.12.312.txt +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/builder/2023.12.txt +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/builder/2024.04.txt +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/builder/2024.10.txt +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/builder/2025.06.txt +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/builder/PREVIEW.txt +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/builder/README.md +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/builder/base-images.json +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/call_graph.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/cli/__init__.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/cli/_download.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/cli/_traceback.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/cli/app.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/cli/cluster.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/cli/config.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/cli/container.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/cli/dict.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/cli/entry_point.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/cli/environment.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/cli/import_refs.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/cli/launch.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/cli/network_file_system.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/cli/profile.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/cli/programs/__init__.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/cli/programs/run_jupyter.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/cli/programs/vscode.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/cli/queues.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/cli/run.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/cli/secret.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/cli/shell.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/cli/token.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/cli/utils.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/cli/volume.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/client.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/cloud_bucket_mount.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/cloud_bucket_mount.pyi +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/cls.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/cls.pyi +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/config.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/container_process.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/container_process.pyi +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/dict.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/dict.pyi +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/environments.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/environments.pyi +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/exception.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/experimental/__init__.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/experimental/flash.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/experimental/flash.pyi +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/experimental/ipython.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/file_io.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/file_io.pyi +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/file_pattern_matcher.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/functions.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/functions.pyi +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/gpu.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/image.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/image.pyi +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/io_streams.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/io_streams.pyi +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/mount.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/mount.pyi +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/network_file_system.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/network_file_system.pyi +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/object.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/object.pyi +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/output.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/parallel_map.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/parallel_map.pyi +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/partial_function.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/partial_function.pyi +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/proxy.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/proxy.pyi +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/py.typed +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/queue.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/queue.pyi +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/retries.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/runner.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/runner.pyi +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/running_app.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/sandbox.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/sandbox.pyi +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/schedule.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/scheduler_placement.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/secret.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/secret.pyi +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/serving.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/serving.pyi +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/snapshot.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/snapshot.pyi +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/stream_type.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/token_flow.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/token_flow.pyi +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/volume.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal/volume.pyi +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal.egg-info/SOURCES.txt +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal.egg-info/dependency_links.txt +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal.egg-info/entry_points.txt +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal.egg-info/requires.txt +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal.egg-info/top_level.txt +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal_docs/__init__.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal_docs/gen_cli_docs.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal_docs/gen_reference_docs.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal_docs/mdmd/__init__.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal_docs/mdmd/mdmd.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal_docs/mdmd/signatures.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal_proto/__init__.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal_proto/api.proto +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal_proto/api_grpc.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal_proto/api_pb2.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal_proto/api_pb2.pyi +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal_proto/api_pb2_grpc.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal_proto/api_pb2_grpc.pyi +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal_proto/modal_api_grpc.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal_proto/py.typed +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal_proto/task_command_router.proto +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal_proto/task_command_router_grpc.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal_proto/task_command_router_pb2.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal_proto/task_command_router_pb2.pyi +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal_proto/task_command_router_pb2_grpc.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal_proto/task_command_router_pb2_grpc.pyi +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/modal_version/__main__.py +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/pyproject.toml +0 -0
- {modal-1.2.7.dev3 → modal-1.2.7.dev4}/setup.cfg +0 -0
|
@@ -991,12 +991,10 @@ class _ContainerIOManager:
|
|
|
991
991
|
# Busy-wait for restore. `/__modal/restore-state.json` is created
|
|
992
992
|
# by the worker process with updates to the container config.
|
|
993
993
|
restored_path = Path(config.get("restore_state_path"))
|
|
994
|
-
|
|
994
|
+
logger.debug("Waiting for restore")
|
|
995
995
|
while not restored_path.exists():
|
|
996
|
-
logger.debug(f"Waiting for restore (elapsed={time.perf_counter() - start:.3f}s)")
|
|
997
996
|
await asyncio.sleep(0.01)
|
|
998
997
|
continue
|
|
999
|
-
|
|
1000
998
|
logger.debug("Container: restored")
|
|
1001
999
|
|
|
1002
1000
|
# Look for state file and create new client with updated credentials.
|
|
@@ -18,11 +18,14 @@ from modal.config import config, logger
|
|
|
18
18
|
|
|
19
19
|
CUDA_CHECKPOINT_PATH: str = config.get("cuda_checkpoint_path")
|
|
20
20
|
|
|
21
|
-
# Maximum total duration for an entire toggle operation.
|
|
22
|
-
CUDA_CHECKPOINT_TOGGLE_TIMEOUT: float = 5 * 60.0
|
|
23
|
-
|
|
24
21
|
# Maximum total duration for each individual `cuda-checkpoint` invocation.
|
|
25
|
-
CUDA_CHECKPOINT_TIMEOUT: float =
|
|
22
|
+
CUDA_CHECKPOINT_TIMEOUT: float = 3 * 60.0
|
|
23
|
+
|
|
24
|
+
# Number of retries for each individual `cuda-checkpoint --toggle` invocation.
|
|
25
|
+
CUDA_CHECKPOINT_TOGGLE_NUM_RETRIES: int = 3
|
|
26
|
+
|
|
27
|
+
# Maximum total duration for an entire toggle operation.
|
|
28
|
+
CUDA_CHECKPOINT_TOGGLE_TIMEOUT: float = CUDA_CHECKPOINT_TOGGLE_NUM_RETRIES * CUDA_CHECKPOINT_TIMEOUT
|
|
26
29
|
|
|
27
30
|
|
|
28
31
|
class CudaCheckpointState(Enum):
|
|
@@ -58,7 +61,7 @@ class CudaCheckpointProcess:
|
|
|
58
61
|
|
|
59
62
|
start_time = time.monotonic()
|
|
60
63
|
retry_count = 0
|
|
61
|
-
max_retries =
|
|
64
|
+
max_retries = CUDA_CHECKPOINT_TOGGLE_NUM_RETRIES
|
|
62
65
|
|
|
63
66
|
attempts = 0
|
|
64
67
|
while self._should_continue_toggle(
|
|
@@ -201,8 +204,7 @@ class CudaCheckpointSession:
|
|
|
201
204
|
[CUDA_CHECKPOINT_PATH, "--get-state", "--pid", str(pid)],
|
|
202
205
|
capture_output=True,
|
|
203
206
|
text=True,
|
|
204
|
-
|
|
205
|
-
timeout=5,
|
|
207
|
+
timeout=CUDA_CHECKPOINT_TIMEOUT,
|
|
206
208
|
)
|
|
207
209
|
|
|
208
210
|
# If the command succeeds (return code 0), this PID has a CUDA session
|
|
@@ -32,7 +32,7 @@ class _Client:
|
|
|
32
32
|
server_url: str,
|
|
33
33
|
client_type: int,
|
|
34
34
|
credentials: typing.Optional[tuple[str, str]],
|
|
35
|
-
version: str = "1.2.7.
|
|
35
|
+
version: str = "1.2.7.dev4",
|
|
36
36
|
):
|
|
37
37
|
"""mdmd:hidden
|
|
38
38
|
The Modal client object is not intended to be instantiated directly by users.
|
|
@@ -163,7 +163,7 @@ class Client:
|
|
|
163
163
|
server_url: str,
|
|
164
164
|
client_type: int,
|
|
165
165
|
credentials: typing.Optional[tuple[str, str]],
|
|
166
|
-
version: str = "1.2.7.
|
|
166
|
+
version: str = "1.2.7.dev4",
|
|
167
167
|
):
|
|
168
168
|
"""mdmd:hidden
|
|
169
169
|
The Modal client object is not intended to be instantiated directly by users.
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|