coiled 1.120.1.dev8__tar.gz → 1.130.2.dev11__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (105) hide show
  1. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/PKG-INFO +1 -1
  2. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/batch.py +17 -1
  3. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/capture_environment.py +57 -82
  4. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/cli/batch/run.py +27 -25
  5. coiled-1.130.2.dev11/coiled/cli/batch/util.py +28 -0
  6. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/cli/core.py +2 -0
  7. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/cli/curl.py +5 -1
  8. coiled-1.130.2.dev11/coiled/cli/file.py +116 -0
  9. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/cli/hello/hello.py +6 -5
  10. coiled-1.130.2.dev11/coiled/cli/mpi.py +252 -0
  11. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/cli/notebook/notebook.py +10 -0
  12. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/cli/run.py +52 -10
  13. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/cli/setup/aws.py +11 -0
  14. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/cli/setup/azure.py +85 -8
  15. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/context.py +2 -2
  16. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/core.py +21 -2
  17. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/filestore.py +181 -44
  18. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/pypi_conda_map.py +22 -96
  19. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/software.py +18 -3
  20. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/software_utils.py +165 -14
  21. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/types.py +20 -1
  22. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/utils.py +2 -0
  23. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/v2/cluster.py +9 -3
  24. coiled-1.130.2.dev11/coiled/v2/cluster_comms.py +72 -0
  25. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/v2/core.py +14 -1
  26. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/v2/widgets/rich.py +2 -1
  27. coiled-1.120.1.dev8/coiled/cli/file.py +0 -41
  28. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/.gitignore +0 -0
  29. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/LICENSE +0 -0
  30. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/README.md +0 -0
  31. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/__init__.py +0 -0
  32. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/__main__.py +0 -0
  33. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/analytics.py +0 -0
  34. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/auth.py +0 -0
  35. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/cli/__init__.py +0 -0
  36. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/cli/batch/__init__.py +0 -0
  37. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/cli/batch/list.py +0 -0
  38. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/cli/batch/logs.py +0 -0
  39. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/cli/batch/status.py +0 -0
  40. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/cli/batch/wait.py +0 -0
  41. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/cli/cluster/__init__.py +0 -0
  42. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/cli/cluster/azure_logs.py +0 -0
  43. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/cli/cluster/better_logs.py +0 -0
  44. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/cli/cluster/crud.py +0 -0
  45. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/cli/cluster/get_address.py +0 -0
  46. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/cli/cluster/list.py +0 -0
  47. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/cli/cluster/logs.py +0 -0
  48. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/cli/cluster/metrics.py +0 -0
  49. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/cli/cluster/ssh.py +0 -0
  50. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/cli/cluster/utils.py +0 -0
  51. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/cli/config.py +0 -0
  52. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/cli/diagnostics.py +0 -0
  53. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/cli/env.py +0 -0
  54. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/cli/hello/__init__.py +0 -0
  55. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/cli/hello/examples/__init__.py +0 -0
  56. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/cli/hello/examples/exit.py +0 -0
  57. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/cli/hello/examples/hello_world.py +0 -0
  58. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/cli/hello/examples/nyc_parquet.py +0 -0
  59. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/cli/hello/examples/pytorch.py +0 -0
  60. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/cli/hello/examples/xarray_nwm.py +0 -0
  61. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/cli/hello/scripts/fill_ipython.py +0 -0
  62. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/cli/hello/scripts/nyc_parquet.py +0 -0
  63. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/cli/hello/scripts/pytorch.py +0 -0
  64. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/cli/hello/scripts/xarray_nwm.py +0 -0
  65. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/cli/hello/utils.py +0 -0
  66. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/cli/login.py +0 -0
  67. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/cli/notebook/__init__.py +0 -0
  68. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/cli/package_sync.py +0 -0
  69. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/cli/prefect.py +0 -0
  70. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/cli/prefect_serve.py +0 -0
  71. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/cli/setup/__init__.py +0 -0
  72. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/cli/setup/amp.py +0 -0
  73. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/cli/setup/entry.py +0 -0
  74. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/cli/setup/gcp.py +0 -0
  75. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/cli/setup/prometheus.py +0 -0
  76. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/cli/setup/util.py +0 -0
  77. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/cli/sync.py +0 -0
  78. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/cli/utils.py +0 -0
  79. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/cluster.py +0 -0
  80. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/coiled.yaml +0 -0
  81. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/compatibility.py +0 -0
  82. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/config.py +0 -0
  83. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/credentials/__init__.py +0 -0
  84. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/credentials/aws.py +0 -0
  85. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/credentials/google.py +0 -0
  86. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/errors.py +0 -0
  87. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/exceptions.py +0 -0
  88. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/extensions/__init__.py +0 -0
  89. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/extensions/prefect/__init__.py +0 -0
  90. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/extensions/prefect/runners.py +0 -0
  91. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/extensions/prefect/workers.py +0 -0
  92. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/function.py +0 -0
  93. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/plugins.py +0 -0
  94. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/prefect.py +0 -0
  95. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/scan.py +0 -0
  96. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/spans.py +0 -0
  97. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/spark.py +0 -0
  98. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/v2/__init__.py +0 -0
  99. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/v2/cwi_log_link.py +0 -0
  100. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/v2/states.py +0 -0
  101. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/v2/widgets/__init__.py +0 -0
  102. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/v2/widgets/interface.py +0 -0
  103. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/v2/widgets/util.py +0 -0
  104. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/coiled/websockets.py +0 -0
  105. {coiled-1.120.1.dev8 → coiled-1.130.2.dev11}/pyproject.toml +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: coiled
3
- Version: 1.120.1.dev8
3
+ Version: 1.130.2.dev11
4
4
  Summary: Python client for coiled.io dask clusters
5
5
  Project-URL: Homepage, https://coiled.io
6
6
  Maintainer-email: Coiled <info@coiled.io>
@@ -17,6 +17,8 @@ def run(
17
17
  workspace: str | None = None,
18
18
  software: str | None = None,
19
19
  container: str | None = None,
20
+ run_on_host: bool | None = None,
21
+ cluster_kwargs: dict | None = None,
20
22
  env: list | dict | None = None,
21
23
  secret_env: list | dict | None = None,
22
24
  tag: list | dict | None = None,
@@ -46,7 +48,11 @@ def run(
46
48
  package_sync_strict: bool = False,
47
49
  package_sync_conda_extras: list | None = None,
48
50
  package_sync_ignore: list[str] | None = None,
51
+ local_upload_path: str | None = None,
52
+ buffers_to_upload: list[dict] | None = None,
49
53
  host_setup_script: str | None = None,
54
+ host_setup_script_content: str | None = None,
55
+ command_as_script: bool | None = None,
50
56
  ignore_container_entrypoint: bool | None = None,
51
57
  job_timeout: str | None = None,
52
58
  logger=None,
@@ -61,8 +67,12 @@ def run(
61
67
  takes a list of dictionaries, so you can specify multiple environment variables for each task.
62
68
  For example, ``[{"FOO": 1, "BAR": 2}, {"FOO": 3, "BAR": 4}]`` will pass ``FOO=1 BAR=2`` to one task and
63
69
  ``FOO=3 BAR=4`` to another.
70
+ buffers_to_upload
71
+ takes a list of dictionaries, each should have path where file should be written on VM(s)
72
+ relative to working directory, and ``io.BytesIO`` which provides content of file,
73
+ for example ``[{"relative_path": "hello.txt", "buffer": io.BytesIO(b"hello")}]``.
64
74
  """
65
- if isinstance(command, str):
75
+ if isinstance(command, str) and not command.startswith("#!") and not command_as_script:
66
76
  command = shlex.split(command)
67
77
 
68
78
  env = dict_to_key_val_list(env)
@@ -76,6 +86,8 @@ def run(
76
86
  workspace=workspace,
77
87
  software=software,
78
88
  container=container,
89
+ run_on_host=run_on_host,
90
+ cluster_kwargs=cluster_kwargs,
79
91
  env=env,
80
92
  secret_env=secret_env,
81
93
  tag=tag,
@@ -106,7 +118,11 @@ def run(
106
118
  package_sync_strict=package_sync_strict,
107
119
  package_sync_conda_extras=package_sync_conda_extras,
108
120
  package_sync_ignore=package_sync_ignore,
121
+ local_upload_path=local_upload_path,
122
+ buffers_to_upload=buffers_to_upload,
109
123
  host_setup_script=host_setup_script,
124
+ host_setup_script_content=host_setup_script_content,
125
+ command_as_script=command_as_script,
110
126
  ignore_container_entrypoint=ignore_container_entrypoint,
111
127
  job_timeout=job_timeout,
112
128
  logger=logger,
@@ -1,6 +1,4 @@
1
- import asyncio
2
1
  import contextlib
3
- import logging
4
2
  import platform
5
3
  import sys
6
4
  import typing
@@ -16,9 +14,12 @@ from typing_extensions import Literal
16
14
  from coiled.context import track_context
17
15
  from coiled.scan import scan_prefix
18
16
  from coiled.software_utils import (
17
+ ANY_AVAILABLE,
18
+ PYTHON_VERSION,
19
19
  check_pip_happy,
20
20
  create_wheels_for_local_python,
21
21
  create_wheels_for_packages,
22
+ get_lockfile,
22
23
  partition_ignored_packages,
23
24
  partition_local_packages,
24
25
  partition_local_python_code_packages,
@@ -36,10 +37,6 @@ from coiled.v2.core import CloudV2
36
37
  from coiled.v2.widgets.rich import CONSOLE_WIDTH, print_rich_package_table
37
38
  from coiled.v2.widgets.util import simple_progress, use_rich_widget
38
39
 
39
- PYTHON_VERSION = platform.python_version_tuple()
40
- ANY_AVAILABLE = "ANY-AVAILABLE"
41
-
42
-
43
40
  logger = getLogger("coiled.package_sync")
44
41
 
45
42
 
@@ -69,49 +66,53 @@ async def approximate_packages(
69
66
  architecture: ArchitectureTypesEnum = ArchitectureTypesEnum.X86_64,
70
67
  pip_check_errors: Optional[Dict[str, List[str]]] = None,
71
68
  gpu_enabled: bool = False,
69
+ use_uv_installer: bool = True,
70
+ lockfile_path: Optional[Path] = None,
72
71
  ) -> typing.List[ResolvedPackageInfo]:
73
72
  user_conda_installed_python = next((p for p in packages if p["name"] == "python"), None)
74
- user_conda_installed_pip = next(
75
- (i for i, p in enumerate(packages) if p["name"] == "pip" and p["source"] == "conda"),
76
- None,
77
- )
78
- if not user_conda_installed_pip:
79
- # This means pip was installed by pip, or the system
80
- # package manager
81
- # Insert a conda version of pip to be installed first, it will
82
- # then be used to install the users version of pip
83
- pip = next(
84
- (p for p in packages if p["name"] == "pip" and p["source"] == "pip"),
73
+ # Only add pip if we need it
74
+ if not use_uv_installer:
75
+ user_conda_installed_pip = next(
76
+ (i for i, p in enumerate(packages) if p["name"] == "pip" and p["source"] == "conda"),
85
77
  None,
86
78
  )
87
- if not pip:
88
- # insert a modern version and hope it does not introduce conflicts
89
- packages.append({
90
- "name": "pip",
91
- "path": None,
92
- "source": "conda",
93
- "channel_url": "https://conda.anaconda.org/conda-forge/",
94
- "channel": "conda-forge",
95
- "subdir": "noarch",
96
- "conda_name": "pip",
97
- "version": "22.3.1",
98
- "wheel_target": None,
99
- "requested": False,
100
- })
101
- else:
102
- # insert the users pip version and hope it exists on conda-forge
103
- packages.append({
104
- "name": "pip",
105
- "path": None,
106
- "source": "conda",
107
- "channel_url": "https://conda.anaconda.org/conda-forge/",
108
- "channel": "conda-forge",
109
- "subdir": "noarch",
110
- "conda_name": "pip",
111
- "version": pip["version"],
112
- "wheel_target": None,
113
- "requested": True,
114
- })
79
+ if not user_conda_installed_pip:
80
+ # This means pip was installed by pip, or the system
81
+ # package manager
82
+ # Insert a conda version of pip to be installed first, it will
83
+ # then be used to install the users version of pip
84
+ pip = next(
85
+ (p for p in packages if p["name"] == "pip" and p["source"] == "pip"),
86
+ None,
87
+ )
88
+ if not pip:
89
+ # insert a modern version and hope it does not introduce conflicts
90
+ packages.append({
91
+ "name": "pip",
92
+ "path": None,
93
+ "source": "conda",
94
+ "channel_url": "https://conda.anaconda.org/conda-forge/",
95
+ "channel": "conda-forge",
96
+ "subdir": "noarch",
97
+ "conda_name": "pip",
98
+ "version": "22.3.1",
99
+ "wheel_target": None,
100
+ "requested": False,
101
+ })
102
+ else:
103
+ # insert the users pip version and hope it exists on conda-forge
104
+ packages.append({
105
+ "name": "pip",
106
+ "path": None,
107
+ "source": "conda",
108
+ "channel_url": "https://conda.anaconda.org/conda-forge/",
109
+ "channel": "conda-forge",
110
+ "subdir": "noarch",
111
+ "conda_name": "pip",
112
+ "version": pip["version"],
113
+ "wheel_target": None,
114
+ "requested": True,
115
+ })
115
116
  coiled_selected_python = None
116
117
  if not user_conda_installed_python:
117
118
  # insert a special python package
@@ -162,6 +163,8 @@ async def approximate_packages(
162
163
  architecture=architecture,
163
164
  pip_check_errors=pip_check_errors,
164
165
  gpu_enabled=gpu_enabled,
166
+ lockfile_name=lockfile_path.name if lockfile_path else None,
167
+ lockfile_content=lockfile_path.read_text() if lockfile_path else None,
165
168
  )
166
169
  finalized_packages: typing.List[ResolvedPackageInfo] = []
167
170
  finalized_packages.extend(await create_wheels_for_local_python(local_python_code, progress=progress))
@@ -208,6 +211,7 @@ async def create_environment_approximation(
208
211
  progress: Optional[Progress] = None,
209
212
  architecture: ArchitectureTypesEnum = ArchitectureTypesEnum.X86_64,
210
213
  gpu_enabled: bool = False,
214
+ use_uv_installer: bool = True,
211
215
  ) -> typing.List[ResolvedPackageInfo]:
212
216
  packages = await scan_prefix(progress=progress)
213
217
  pip_check_errors = await check_pip_happy(progress)
@@ -237,6 +241,8 @@ async def create_environment_approximation(
237
241
  architecture=architecture,
238
242
  pip_check_errors=pip_check_errors,
239
243
  gpu_enabled=gpu_enabled,
244
+ use_uv_installer=use_uv_installer,
245
+ lockfile_path=get_lockfile(),
240
246
  )
241
247
  return result
242
248
 
@@ -259,7 +265,7 @@ async def scan_and_create(
259
265
  ):
260
266
  use_widget = force_rich_widget or (show_widget and use_rich_widget())
261
267
 
262
- local_env_name = Path(sys.prefix).name
268
+ local_env_name = str(get_lockfile() or Path(sys.prefix).name)
263
269
  if use_widget:
264
270
  progress = Progress(TextColumn("[progress.description]{task.description}"), BarColumn(), TimeElapsedColumn())
265
271
  live = Live(Panel(progress, title=f"[green]Package Sync for {local_env_name}", width=CONSOLE_WIDTH))
@@ -268,6 +274,9 @@ async def scan_and_create(
268
274
  progress = None
269
275
 
270
276
  with live:
277
+ # We do this even with lockfiles because some early checks happen
278
+ # on this endpoint to prevent people getting delayed quota errors
279
+ # TODO: Add a lighter weight endpoint that does just these checks
271
280
  with simple_progress("Fetching latest package priorities", progress):
272
281
  logger.info(f"Resolving your local {local_env_name} Python environment...")
273
282
  async with (
@@ -306,6 +315,7 @@ async def scan_and_create(
306
315
  architecture=architecture,
307
316
  gpu_enabled=gpu_enabled,
308
317
  conda_extras=package_sync_conda_extras,
318
+ use_uv_installer=use_uv_installer,
309
319
  )
310
320
 
311
321
  if not package_sync_only:
@@ -367,6 +377,7 @@ async def scan_and_create(
367
377
  # default region in declarative service create_software_environment
368
378
  region_name=region_name,
369
379
  use_uv_installer=use_uv_installer,
380
+ lockfile_path=get_lockfile(),
370
381
  )
371
382
  if use_widget:
372
383
  print_rich_package_table(packages_with_notes, packages_with_errors)
@@ -427,39 +438,3 @@ If you use pip, venv, uv, pixi, etc. create a new environment and then:
427
438
 
428
439
  See https://docs.coiled.io/user_guide/software/package_sync_best_practices.html
429
440
  for more best practices. If that doesn't solve your issue, please contact support@coiled.io.""")
430
-
431
-
432
- if __name__ == "__main__":
433
- from logging import basicConfig
434
-
435
- basicConfig(level=logging.INFO)
436
-
437
- from rich.console import Console
438
- from rich.table import Table
439
-
440
- async def run():
441
- async with CloudV2(asynchronous=True) as cloud:
442
- return await create_environment_approximation(
443
- cloud=cloud,
444
- priorities={
445
- ("dask", "conda"): PackageLevelEnum.CRITICAL,
446
- ("twisted", "conda"): PackageLevelEnum.IGNORE,
447
- ("graphviz", "conda"): PackageLevelEnum.LOOSE,
448
- ("icu", "conda"): PackageLevelEnum.LOOSE,
449
- },
450
- )
451
-
452
- result = asyncio.run(run())
453
-
454
- table = Table(title="Packages")
455
- keys = ("name", "source", "include", "client_version", "specifier", "error", "note")
456
-
457
- for key in keys:
458
- table.add_column(key)
459
-
460
- for pkg in result:
461
- row_values = [str(pkg.get(key, "")) for key in keys]
462
- table.add_row(*row_values)
463
- console = Console()
464
- console.print(table)
465
- console.print(table)
@@ -10,12 +10,12 @@ import shlex
10
10
 
11
11
  import click
12
12
  import dask.config
13
- import yaml
14
13
  from dask.utils import format_bytes, format_time, parse_timedelta
15
14
  from rich.console import Console
16
15
  from rich.panel import Panel
17
16
 
18
17
  import coiled
18
+ from coiled.cli.batch.util import load_sidecar_spec
19
19
  from coiled.cli.batch.wait import batch_job_wait
20
20
  from coiled.cli.curl import sync_request
21
21
  from coiled.cli.run import dict_from_key_val_list
@@ -196,6 +196,7 @@ def get_kwargs_from_header(f: dict, click_params: list):
196
196
  "default is to use the entrypoint (if any) set on the image."
197
197
  ),
198
198
  )
199
+ @click.option("--run-on-host", default=None, help="Run code directly on host, not inside docker container.")
199
200
  @click.option(
200
201
  "--env",
201
202
  "-e",
@@ -472,6 +473,7 @@ def get_kwargs_from_header(f: dict, click_params: list):
472
473
  "For example, you can specify '30 minutes' or '1 hour'. Default is no timeout."
473
474
  ),
474
475
  )
476
+ @click.option("--dask-container", default=None, type=str)
475
477
  @click.argument("command", nargs=-1, required=True)
476
478
  def batch_run_cli(ctx, **kwargs):
477
479
  """
@@ -501,6 +503,14 @@ def batch_run_cli(ctx, **kwargs):
501
503
 
502
504
  def _batch_run(default_kwargs, logger=None, from_cli=False, **kwargs) -> dict:
503
505
  command = kwargs["command"]
506
+ user_files = []
507
+
508
+ if isinstance(command, str) and (command.startswith("#!") or kwargs.get("command_as_script")):
509
+ user_files.append({
510
+ "path": "script",
511
+ "content": command,
512
+ })
513
+ command = ["script"]
504
514
 
505
515
  # Handle command as string case (e.g. `coiled batch run "python myscript.py"`)
506
516
  if len(command) == 1:
@@ -522,7 +532,6 @@ def _batch_run(default_kwargs, logger=None, from_cli=False, **kwargs) -> dict:
522
532
  # unescape escaped COILED env vars in command
523
533
  command = [part.replace("\\$COILED", "$COILED") for part in command]
524
534
 
525
- user_files = []
526
535
  kwargs_from_header = None
527
536
 
528
537
  # identify implicit files referenced in commands like "python foo.py" or "foo.sh"
@@ -713,8 +722,8 @@ def _batch_run(default_kwargs, logger=None, from_cli=False, **kwargs) -> dict:
713
722
  if user_files_from_content:
714
723
  user_files.extend(user_files_from_content)
715
724
 
716
- host_setup_content = None
717
- if kwargs["host_setup_script"]:
725
+ host_setup_content = kwargs.get("host_setup_script_content")
726
+ if not host_setup_content and kwargs["host_setup_script"]:
718
727
  with open(kwargs["host_setup_script"]) as f:
719
728
  host_setup_content = f.read()
720
729
 
@@ -754,25 +763,11 @@ def _batch_run(default_kwargs, logger=None, from_cli=False, **kwargs) -> dict:
754
763
 
755
764
  batch_job_container = f"{kwargs['container']}!" if kwargs["ignore_container_entrypoint"] else kwargs["container"]
756
765
 
757
- scheduler_sidecars = []
758
- if kwargs.get("scheduler_sidecar_spec"):
759
- with open(kwargs["scheduler_sidecar_spec"]) as f:
760
- if kwargs["scheduler_sidecar_spec"].endswith((".yaml", ".yml")):
761
- sidecar_spec = yaml.safe_load(f)
762
- elif kwargs["scheduler_sidecar_spec"].endswith(".json"):
763
- sidecar_spec = json.load(f)
764
- else:
765
- raise ValueError(f"Unknown format for {kwargs['scheduler_sidecar_spec']}, json or yaml expected.")
766
-
767
- # support either list-like or dict-like
768
- if isinstance(sidecar_spec, list):
769
- scheduler_sidecars = sidecar_spec
770
- if isinstance(sidecar_spec, dict):
771
- scheduler_sidecars = [{"name": key, **val} for key, val in sidecar_spec.items()]
766
+ scheduler_sidecars = load_sidecar_spec(kwargs.get("scheduler_sidecar_spec"))
772
767
 
773
- for sidecar in scheduler_sidecars:
774
- # allow `image` as the key, to match docker compose spec
775
- sidecar["container"] = sidecar.get("container") or sidecar.get("image")
768
+ dask_container = (
769
+ kwargs.get("dask_container") or dask.config.get("coiled.batch.dask-container", None) or "ghcr.io/dask/dask"
770
+ )
776
771
 
777
772
  cluster_kwargs = {
778
773
  "name": kwargs["name"],
@@ -786,7 +781,9 @@ def _batch_run(default_kwargs, logger=None, from_cli=False, **kwargs) -> dict:
786
781
  # if batch job is running in extra container, then we just need a pretty minimal dask container
787
782
  # so for now switch the default in that case to basic dask container
788
783
  # TODO would it be better to use a pre-built senv with our `cloud-env-run` container instead?
789
- "container": "daskdev/dask:latest" if kwargs["container"] and not kwargs["software"] else None,
784
+ "container": dask_container
785
+ if (kwargs["container"] or kwargs.get("run_on_host")) and not kwargs["software"]
786
+ else None,
790
787
  "region": kwargs["region"],
791
788
  "scheduler_options": {
792
789
  "idle_timeout": "520 weeks", # TODO allow job timeout?
@@ -807,6 +804,7 @@ def _batch_run(default_kwargs, logger=None, from_cli=False, **kwargs) -> dict:
807
804
  "package_sync_ignore": kwargs.get("package_sync_ignore"),
808
805
  "allow_cross_zone": True if kwargs["allow_cross_zone"] is None else kwargs["allow_cross_zone"],
809
806
  "scheduler_sidecars": scheduler_sidecars,
807
+ **(kwargs.get("cluster_kwargs") or {}),
810
808
  }
811
809
 
812
810
  # when task will run on scheduler, give it the same VM specs as worker node
@@ -842,6 +840,7 @@ def _batch_run(default_kwargs, logger=None, from_cli=False, **kwargs) -> dict:
842
840
  "pipe_to_files": bool(kwargs.get("pipe_to_files")),
843
841
  "host_setup": host_setup_content,
844
842
  "job_timeout_seconds": parse_timedelta(kwargs["job_timeout"]) if kwargs["job_timeout"] else None,
843
+ "run_in_container": not kwargs.get("run_on_host"),
845
844
  }
846
845
 
847
846
  with coiled.Cloud(workspace=kwargs["workspace"]) as cloud:
@@ -879,6 +878,7 @@ def _batch_run(default_kwargs, logger=None, from_cli=False, **kwargs) -> dict:
879
878
  or kwargs.get("pipe_to_files")
880
879
  or kwargs.get("input_filestore")
881
880
  or kwargs.get("output_filestore")
881
+ or kwargs.get("buffers_to_upload")
882
882
  ):
883
883
  fs_base_name = kwargs["name"] or f"batch-job-{job_id}"
884
884
 
@@ -899,9 +899,11 @@ def _batch_run(default_kwargs, logger=None, from_cli=False, **kwargs) -> dict:
899
899
  {"id": out_fs["id"], "output": True, "path": "/scratch/batch/", "primary": True},
900
900
  ])
901
901
 
902
- if kwargs.get("local_upload_path") or kwargs.get("local_sync_path"):
902
+ if kwargs.get("local_upload_path") or kwargs.get("local_sync_path") or kwargs.get("buffers_to_upload"):
903
903
  upload_to_filestore_with_ui(
904
- fs=in_fs, local_dir=kwargs.get("local_upload_path") or kwargs.get("local_sync_path")
904
+ fs=in_fs,
905
+ local_dir=kwargs.get("local_upload_path") or kwargs.get("local_sync_path"),
906
+ file_buffers=kwargs.get("buffers_to_upload"),
905
907
  )
906
908
 
907
909
  # Run the job on a cluster
@@ -0,0 +1,28 @@
1
+ from __future__ import annotations
2
+
3
+ import json
4
+
5
+ import yaml
6
+
7
+
8
+ def load_sidecar_spec(spec_path: str | None):
9
+ scheduler_sidecars = []
10
+ if spec_path:
11
+ with open(spec_path) as f:
12
+ if spec_path.endswith((".yaml", ".yml")):
13
+ sidecar_spec = yaml.safe_load(f)
14
+ elif spec_path.endswith(".json"):
15
+ sidecar_spec = json.load(f)
16
+ else:
17
+ raise ValueError(f"Unknown format for {spec_path}, json or yaml expected.")
18
+
19
+ # support either list-like or dict-like
20
+ if isinstance(sidecar_spec, list):
21
+ scheduler_sidecars = sidecar_spec
22
+ if isinstance(sidecar_spec, dict):
23
+ scheduler_sidecars = [{"name": key, **val} for key, val in sidecar_spec.items()]
24
+
25
+ for sidecar in scheduler_sidecars:
26
+ # allow `image` as the key, to match docker compose spec
27
+ sidecar["container"] = sidecar.get("container") or sidecar.get("image")
28
+ return scheduler_sidecars
@@ -10,6 +10,7 @@ from .env import env
10
10
  from .file import file_group
11
11
  from .hello import hello
12
12
  from .login import login
13
+ from .mpi import mpi_group
13
14
  from .notebook import notebook_group
14
15
  from .package_sync import package_sync
15
16
  from .prefect import prefect
@@ -42,3 +43,4 @@ cli.add_command(better_logs, "logs")
42
43
  cli.add_command(hello)
43
44
  cli.add_command(hello, "quickstart")
44
45
  cli.add_command(file_group)
46
+ cli.add_command(mpi_group, "mpi")
@@ -1,3 +1,4 @@
1
+ from json import dumps as json_dumps
1
2
  from json import loads as json_loads
2
3
 
3
4
  import click
@@ -24,7 +25,10 @@ def curl(url: str, request, data, json, json_output):
24
25
  url = f"{cloud.server}{url}"
25
26
  response = sync_request(cloud, url, method=request, data=all_data, json=json, json_output=json_output)
26
27
 
27
- print(response)
28
+ if json_output:
29
+ print(json_dumps(response, indent=4))
30
+ else:
31
+ print(response)
28
32
 
29
33
 
30
34
  def sync_request(cloud, url, method, data=None, json: bool = False, json_output: bool = False):
@@ -0,0 +1,116 @@
1
+ import click
2
+
3
+ import coiled
4
+ from coiled.filestore import FilestoreManager
5
+
6
+ from .cluster.utils import find_cluster
7
+ from .utils import CONTEXT_SETTINGS
8
+
9
+
10
+ @click.command(
11
+ context_settings=CONTEXT_SETTINGS,
12
+ )
13
+ @click.argument("cluster", default="", required=False)
14
+ @click.option(
15
+ "--workspace",
16
+ default=None,
17
+ help="Coiled workspace (uses default workspace if not specified).",
18
+ )
19
+ @click.option(
20
+ "--filestore",
21
+ default=None,
22
+ help="Name of filestore (optional).",
23
+ )
24
+ @click.option(
25
+ "--filter",
26
+ "name_includes",
27
+ default=None,
28
+ help="Filter on file paths and/or names to download (optional).",
29
+ )
30
+ @click.option("--into", default=".")
31
+ def download(cluster, workspace, filestore, name_includes, into):
32
+ if filestore:
33
+ filestores = FilestoreManager.get_filestore(name=filestore) or []
34
+ if not filestores:
35
+ print(f"{filestore} filestore not found")
36
+
37
+ for fs in filestores:
38
+ coiled.filestore.download_from_filestore_with_ui(
39
+ fs=fs,
40
+ into=into,
41
+ name_includes=name_includes,
42
+ )
43
+
44
+ else:
45
+ with coiled.Cloud(workspace=workspace) as cloud:
46
+ cluster_info = find_cluster(cloud, cluster)
47
+ cluster_id = cluster_info["id"]
48
+ attachments = FilestoreManager.get_cluster_attachments(cluster_id)
49
+ if not attachments:
50
+ print(f"No filestore found for {cluster_info['name']} ({cluster_info['id']})")
51
+
52
+ # TODO (possible enhancement) if there are multiple output filestores, let user pick which to download
53
+ for attachment in attachments:
54
+ if attachment["output"]:
55
+ coiled.filestore.download_from_filestore_with_ui(
56
+ fs=attachment["filestore"],
57
+ into=into,
58
+ name_includes=name_includes,
59
+ )
60
+
61
+
62
+ @click.command(
63
+ context_settings=CONTEXT_SETTINGS,
64
+ )
65
+ @click.argument("cluster", default="", required=False)
66
+ @click.option(
67
+ "--workspace",
68
+ default=None,
69
+ help="Coiled workspace (uses default workspace if not specified).",
70
+ )
71
+ @click.option(
72
+ "--filestore",
73
+ default=None,
74
+ help="Name of filestore (optional).",
75
+ )
76
+ @click.option(
77
+ "--filter",
78
+ "name_includes",
79
+ default=None,
80
+ help="Filter on file paths and/or names to download (optional).",
81
+ )
82
+ def list_files(cluster, workspace, filestore, name_includes):
83
+ if filestore:
84
+ filestores = FilestoreManager.get_filestore(name=filestore) or []
85
+ if not filestores:
86
+ print(f"{filestore} filestore not found")
87
+
88
+ for fs in filestores:
89
+ coiled.filestore.list_files_ui(
90
+ fs=fs,
91
+ name_includes=name_includes,
92
+ )
93
+
94
+ else:
95
+ with coiled.Cloud(workspace=workspace) as cloud:
96
+ cluster_info = find_cluster(cloud, cluster)
97
+ cluster_id = cluster_info["id"]
98
+ attachments = FilestoreManager.get_cluster_attachments(cluster_id)
99
+ if not attachments:
100
+ print(f"No filestore found for {cluster_info['name']} ({cluster_info['id']})")
101
+
102
+ # TODO (possible enhancement) if there are multiple output filestores, let user pick which to download
103
+ for attachment in attachments:
104
+ if attachment["output"]:
105
+ coiled.filestore.list_files_ui(
106
+ fs=attachment["filestore"],
107
+ name_includes=name_includes,
108
+ )
109
+
110
+
111
+ @click.group(name="file", context_settings=CONTEXT_SETTINGS)
112
+ def file_group(): ...
113
+
114
+
115
+ file_group.add_command(download)
116
+ file_group.add_command(list_files, "list")
@@ -1,8 +1,6 @@
1
1
  from __future__ import annotations
2
2
 
3
3
  import asyncio
4
- import json
5
- import subprocess
6
4
  import sys
7
5
  import time
8
6
 
@@ -18,6 +16,7 @@ import coiled
18
16
  from coiled.scan import scan_prefix
19
17
  from coiled.utils import login_if_required
20
18
 
19
+ from ..curl import sync_request
21
20
  from .examples import examples
22
21
  from .utils import PRIMARY_COLOR, Panel, console, has_macos_system_python, log_interactions
23
22
 
@@ -41,8 +40,10 @@ def needs_login():
41
40
 
42
41
 
43
42
  def get_interactions():
44
- response = subprocess.check_output(["coiled", "curl", "/api/v2/interactions/user-interactions/hello"])
45
- return json.loads(response, strict=False)
43
+ with coiled.Cloud() as cloud:
44
+ return sync_request(
45
+ cloud, url=f"{cloud.server}/api/v2/interactions/user-interactions/hello", method="get", json_output=True
46
+ )
46
47
 
47
48
 
48
49
  def get_already_run_examples():
@@ -294,7 +295,7 @@ Choose any computation you'd like to run:
294
295
  Yee-haw you've done all my examples 🎉
295
296
  Now you can:
296
297
  - Try Coiled in your own use case
297
- - [Talk to us](https://calendly.com/d/cmph-386-cjt/coiled-help-getting-started)
298
+ - [Ask us questions](mailto:support@coiled.io)
298
299
  - Explore the [docs](https://docs.coiled.io?utm_source=coiled-hello&utm_medium=finished) to see all the other things Coiled can do
299
300
  """), # noqa
300
301
  border_style="green",