xmanager-slurm 0.4.0__py3-none-any.whl → 0.4.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of xmanager-slurm might be problematic. Click here for more details.
- xm_slurm/__init__.py +4 -2
- xm_slurm/api.py +1 -1
- xm_slurm/config.py +7 -2
- xm_slurm/constants.py +4 -0
- xm_slurm/contrib/clusters/__init__.py +25 -0
- xm_slurm/dependencies.py +171 -0
- xm_slurm/executables.py +20 -15
- xm_slurm/execution.py +246 -96
- xm_slurm/executors.py +8 -12
- xm_slurm/experiment.py +374 -83
- xm_slurm/experimental/parameter_controller.py +12 -10
- xm_slurm/packaging/{docker/local.py → docker.py} +126 -32
- xm_slurm/packaging/router.py +3 -1
- xm_slurm/packaging/utils.py +4 -28
- xm_slurm/scripts/cli.py +52 -0
- xm_slurm/templates/docker/mamba.Dockerfile +1 -1
- xm_slurm/templates/slurm/fragments/monitor.bash.j2 +5 -0
- xm_slurm/templates/slurm/job-array.bash.j2 +1 -2
- xm_slurm/templates/slurm/job.bash.j2 +4 -3
- xm_slurm/types.py +23 -0
- {xmanager_slurm-0.4.0.dist-info → xmanager_slurm-0.4.1.dist-info}/METADATA +1 -1
- xmanager_slurm-0.4.1.dist-info/RECORD +44 -0
- xmanager_slurm-0.4.1.dist-info/entry_points.txt +2 -0
- xm_slurm/packaging/docker/__init__.py +0 -69
- xm_slurm/packaging/docker/abc.py +0 -112
- xmanager_slurm-0.4.0.dist-info/RECORD +0 -42
- {xmanager_slurm-0.4.0.dist-info → xmanager_slurm-0.4.1.dist-info}/WHEEL +0 -0
- {xmanager_slurm-0.4.0.dist-info → xmanager_slurm-0.4.1.dist-info}/licenses/LICENSE.md +0 -0
xm_slurm/packaging/docker/abc.py
DELETED
|
@@ -1,112 +0,0 @@
|
|
|
1
|
-
import abc
|
|
2
|
-
import collections.abc
|
|
3
|
-
import dataclasses
|
|
4
|
-
import functools
|
|
5
|
-
import os
|
|
6
|
-
from typing import Literal, Mapping, Protocol, Sequence
|
|
7
|
-
|
|
8
|
-
import jinja2 as j2
|
|
9
|
-
from xmanager import xm
|
|
10
|
-
|
|
11
|
-
from xm_slurm.executables import RemoteImage, RemoteRepositoryCredentials
|
|
12
|
-
from xm_slurm.packaging.registry import IndexedContainer
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
class DockerCommandProtocol(Protocol):
|
|
16
|
-
def to_args(self) -> xm.SequentialArgs: ...
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
@dataclasses.dataclass(frozen=True, kw_only=True)
|
|
20
|
-
class DockerBakeCommand(DockerCommandProtocol):
|
|
21
|
-
targets: str | Sequence[str] | None = None
|
|
22
|
-
builder: str | None = None
|
|
23
|
-
files: str | os.PathLike[str] | Sequence[os.PathLike[str] | str] | None = None
|
|
24
|
-
load: bool = False
|
|
25
|
-
cache: bool = True
|
|
26
|
-
print: bool = False
|
|
27
|
-
pull: bool = False
|
|
28
|
-
push: bool = False
|
|
29
|
-
metadata_file: str | os.PathLike[str] | None = None
|
|
30
|
-
progress: Literal["auto", "plain", "tty"] = "auto"
|
|
31
|
-
set: Mapping[str, str] | None = None
|
|
32
|
-
|
|
33
|
-
def to_args(self) -> xm.SequentialArgs:
|
|
34
|
-
files = self.files
|
|
35
|
-
if files is None:
|
|
36
|
-
files = []
|
|
37
|
-
if not isinstance(files, collections.abc.Sequence):
|
|
38
|
-
files = [files]
|
|
39
|
-
|
|
40
|
-
targets = self.targets
|
|
41
|
-
if targets is None:
|
|
42
|
-
targets = []
|
|
43
|
-
elif not isinstance(targets, collections.abc.Sequence):
|
|
44
|
-
targets = [targets]
|
|
45
|
-
|
|
46
|
-
return xm.merge_args(
|
|
47
|
-
["buildx", "bake"],
|
|
48
|
-
[f"--progress={self.progress}"],
|
|
49
|
-
[f"--builder={self.builder}"] if self.builder else [],
|
|
50
|
-
[f"--metadata-file={self.metadata_file}"] if self.metadata_file else [],
|
|
51
|
-
["--print"] if self.print else [],
|
|
52
|
-
["--push"] if self.push else [],
|
|
53
|
-
["--pull"] if self.pull else [],
|
|
54
|
-
["--load"] if self.load else [],
|
|
55
|
-
["--no-cache"] if not self.cache else [],
|
|
56
|
-
[f"--file={file}" for file in files],
|
|
57
|
-
[f"--set={key}={value}" for key, value in self.set.items()] if self.set else [],
|
|
58
|
-
targets,
|
|
59
|
-
)
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
@dataclasses.dataclass(frozen=True, kw_only=True)
|
|
63
|
-
class DockerPullCommand(DockerCommandProtocol):
|
|
64
|
-
image: str
|
|
65
|
-
|
|
66
|
-
def to_args(self) -> xm.SequentialArgs:
|
|
67
|
-
return xm.merge_args(["pull", self.image])
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
@dataclasses.dataclass(frozen=True, kw_only=True)
|
|
71
|
-
class DockerLoginCommand(DockerCommandProtocol):
|
|
72
|
-
server: str
|
|
73
|
-
username: str
|
|
74
|
-
password: str | None = None
|
|
75
|
-
password_stdin: bool = False
|
|
76
|
-
|
|
77
|
-
def __post_init__(self):
|
|
78
|
-
if self.password is None and not self.password_stdin:
|
|
79
|
-
raise ValueError("Either password or password_stdin must be set")
|
|
80
|
-
if self.password is not None and self.password_stdin:
|
|
81
|
-
raise ValueError("Only one of password or password_stdin must be set")
|
|
82
|
-
|
|
83
|
-
def to_args(self) -> xm.SequentialArgs:
|
|
84
|
-
return xm.merge_args(
|
|
85
|
-
["login", "--username", self.username],
|
|
86
|
-
["--password", self.password] if self.password else [],
|
|
87
|
-
["--password-stdin"] if self.password_stdin else [],
|
|
88
|
-
[self.server],
|
|
89
|
-
)
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
@dataclasses.dataclass(frozen=True, kw_only=True)
|
|
93
|
-
class DockerVersionCommand(DockerCommandProtocol):
|
|
94
|
-
def to_args(self) -> xm.SequentialArgs:
|
|
95
|
-
return xm.merge_args(["buildx", "version"])
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
class DockerClient(abc.ABC):
|
|
99
|
-
@functools.cached_property
|
|
100
|
-
def _bake_template(self) -> j2.Template:
|
|
101
|
-
template_loader = j2.PackageLoader("xm_slurm", "templates/docker")
|
|
102
|
-
template_env = j2.Environment(loader=template_loader, trim_blocks=True, lstrip_blocks=False)
|
|
103
|
-
|
|
104
|
-
return template_env.get_template("docker-bake.hcl.j2")
|
|
105
|
-
|
|
106
|
-
@abc.abstractmethod
|
|
107
|
-
def credentials(self, *, hostname: str) -> RemoteRepositoryCredentials | None: ...
|
|
108
|
-
|
|
109
|
-
@abc.abstractmethod
|
|
110
|
-
def bake(
|
|
111
|
-
self, *, targets: Sequence[IndexedContainer[xm.Packageable]]
|
|
112
|
-
) -> list[IndexedContainer[RemoteImage]]: ...
|
|
@@ -1,42 +0,0 @@
|
|
|
1
|
-
xm_slurm/__init__.py,sha256=DARUE67bf_e7WeC2MWxCIxeTJ2SO-yHO8M9m_p4t_I4,1061
|
|
2
|
-
xm_slurm/api.py,sha256=-z_1gYkUUtuGCw-Z36iYLL9lUpAJqUerX2LV8o9fSCY,18503
|
|
3
|
-
xm_slurm/batching.py,sha256=GbKBsNz9w8gIc2fHLZpslC0e4K9YUfLXFHmjduRRCfQ,4385
|
|
4
|
-
xm_slurm/config.py,sha256=ZtHPwTfuFTijaCf6s0HBRjbBKvrHZD3mPR0vxVv2gsU,6610
|
|
5
|
-
xm_slurm/console.py,sha256=UpMqeJ0C8i0pkue1AHnnyyX0bFJ9zZeJ7HBR6yhuA8A,54
|
|
6
|
-
xm_slurm/constants.py,sha256=Jzli9uutHRZebXJwWC_X7leKlycKPDQqSxuk_dNtnQ4,882
|
|
7
|
-
xm_slurm/executables.py,sha256=BZWQsN_srdFqXSCG7OmoeqVNgLgzYncMOL_aEo8CUQA,6142
|
|
8
|
-
xm_slurm/execution.py,sha256=14UDPl5aYwLVF3oPodhy3FrhyqsiI6RzuOAyvr9UBdA,20000
|
|
9
|
-
xm_slurm/executors.py,sha256=vilogTjlxHLfZDms4aYOZWUW8w-2IdxU7xh-9vcW1Y0,4723
|
|
10
|
-
xm_slurm/experiment.py,sha256=AZ0hzJWHEW8mo2FDF5UtGvvpWW_0rAUfO5nKIDAE7oE,31484
|
|
11
|
-
xm_slurm/job_blocks.py,sha256=_F8CKCs5BQFj40a2-mjG71HfacvWoBXBDPDKEaKTbXc,616
|
|
12
|
-
xm_slurm/packageables.py,sha256=YZFTL6UWx9A_zyztTy1THUlj3pW1rA0cBPHJxD1LOJk,12884
|
|
13
|
-
xm_slurm/resources.py,sha256=EaYDATVudrEDPKKdSZoWgfqPiidc6DMjIctmzLQmiH0,5683
|
|
14
|
-
xm_slurm/status.py,sha256=WTWiDHi-ZHtwHRnDP0cGa-27zTSm6LkA-GCKsN-zBgg,6916
|
|
15
|
-
xm_slurm/utils.py,sha256=ESjOkGT7bRSzIeZrUtZplSHP4oaH6VZ92y2woYdcyKM,2239
|
|
16
|
-
xm_slurm/contrib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
17
|
-
xm_slurm/contrib/clusters/__init__.py,sha256=mwppnH0h20E3hB4Zry4GCbKVWxVv7jciRjDkZCdiPwY,2086
|
|
18
|
-
xm_slurm/contrib/clusters/drac.py,sha256=tJeQFWFIpeZ1gD3j6AAJssNoLSiDkB-3lz1_ObnkRhc,5905
|
|
19
|
-
xm_slurm/experimental/parameter_controller.py,sha256=hl9-hScTC5hnCLRv5KEUuThSpw1m7bzGaaCMBrt6vn8,8429
|
|
20
|
-
xm_slurm/packaging/__init__.py,sha256=dh307yLpUT9KN7rJ1e9fYC6hegGKfZcGboUq9nGpDVQ,233
|
|
21
|
-
xm_slurm/packaging/registry.py,sha256=GrdmQg9MgSo38OiqOzMKWSkQyBuyryOfc3zcdgZ4CUE,1148
|
|
22
|
-
xm_slurm/packaging/router.py,sha256=6qjtsy4BoYgSaQzC_pQSHVHeWcphG_xWVsWgW6ALC7U,2033
|
|
23
|
-
xm_slurm/packaging/utils.py,sha256=LKzkQrdBfON96KCwxyraBX9Ylv2I1JTYoRsc0ZYMfGk,4887
|
|
24
|
-
xm_slurm/packaging/docker/__init__.py,sha256=sJuLRKV04QkpiSpVZWTPebkYjgMESSWGKXpkSpPytGU,2230
|
|
25
|
-
xm_slurm/packaging/docker/abc.py,sha256=f8XvUA_FusIpXI45PR5isA2msxM003ycW5mWbAyiKfk,3830
|
|
26
|
-
xm_slurm/packaging/docker/local.py,sha256=dA_TXqsqpO4WST201AObcvwoyQZIC-pJ8DMZLIvISvU,8234
|
|
27
|
-
xm_slurm/scripts/_cloudpickle.py,sha256=dlJYf2SceOuUn8wi-ozuoYAQg71wqD2MUVOUCyOwWIY,647
|
|
28
|
-
xm_slurm/templates/docker/docker-bake.hcl.j2,sha256=ClsFpj91Mr1VfA8L6eqBG3HQz0Z8VenF6mEfmAhQgUo,1498
|
|
29
|
-
xm_slurm/templates/docker/mamba.Dockerfile,sha256=tsWqvfz1kYOl27ro2mRRWTHQPFv4hANFNIvH3Pgk7wY,745
|
|
30
|
-
xm_slurm/templates/docker/python.Dockerfile,sha256=U4b4QVkopckQ0o9jJIE7d_M6TvExEYlYDirNwCoZ7W4,865
|
|
31
|
-
xm_slurm/templates/docker/uv.Dockerfile,sha256=kYD32oUS1jUaARsNV1o6EFnIfLCNh5GMmck27b-5NRU,969
|
|
32
|
-
xm_slurm/templates/slurm/job-array.bash.j2,sha256=d4twfV1PATGQwTIleFBUIGmMAIHH-F7RjBsdfaAIQko,599
|
|
33
|
-
xm_slurm/templates/slurm/job-group.bash.j2,sha256=UkjfBE7jg9mepcUWaHZEAjkiXsIM1j_sLxLzxkteD-Y,1120
|
|
34
|
-
xm_slurm/templates/slurm/job.bash.j2,sha256=EUeq3P2xqTIqlHi2SVhFBT7NL4lUj8okYUa3GnlaIIc,1852
|
|
35
|
-
xm_slurm/templates/slurm/fragments/monitor.bash.j2,sha256=CxtbxOJzd0Un-ApDO6T8JHuKlSv6uwwBFMJPeGjCKnk,1071
|
|
36
|
-
xm_slurm/templates/slurm/fragments/proxy.bash.j2,sha256=VJLglZo-Nvx9R-qe3rHTxr07CylTQ6Z9NwBzvIpAZrA,814
|
|
37
|
-
xm_slurm/templates/slurm/runtimes/apptainer.bash.j2,sha256=dMntzelhs8DqKyIpO9S6wzMfH2PDevmgvyjCW8Xc2dY,3222
|
|
38
|
-
xm_slurm/templates/slurm/runtimes/podman.bash.j2,sha256=xKXYFvQvazMx0PgvmlRXR6eecoiBUl8y52dIzQtWkBE,1469
|
|
39
|
-
xmanager_slurm-0.4.0.dist-info/METADATA,sha256=6XIfRqQ-uIyx1XWp2GwVmIopwxZqUMmr1blLQSXwD-Y,954
|
|
40
|
-
xmanager_slurm-0.4.0.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
|
|
41
|
-
xmanager_slurm-0.4.0.dist-info/licenses/LICENSE.md,sha256=IxstXr3MPHwTJ5jMrByHrQsR1ZAGQ2U_uz_4qzI_15Y,11756
|
|
42
|
-
xmanager_slurm-0.4.0.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|