xpk 0.11.0__py3-none-any.whl → 0.12.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- xpk/commands/cluster.py +10 -11
- xpk/commands/cluster_gcluster.py +2 -1
- xpk/commands/common.py +3 -3
- xpk/commands/info.py +12 -12
- xpk/commands/job.py +12 -10
- xpk/commands/kjob_common.py +2 -1
- xpk/commands/storage.py +1 -1
- xpk/commands/workload.py +12 -6
- xpk/core/blueprint/blueprint_generator.py +7 -7
- xpk/core/blueprint/blueprint_test.py +218 -0
- xpk/core/capacity.py +3 -1
- xpk/core/cluster.py +9 -7
- xpk/core/cluster_private.py +5 -1
- xpk/core/commands.py +3 -3
- xpk/core/config.py +3 -4
- xpk/core/config_test.py +71 -0
- xpk/core/docker_manager.py +1 -1
- xpk/core/docker_resources.py +1 -1
- xpk/core/filestore.py +7 -2
- xpk/core/gcloud_context.py +2 -2
- xpk/core/kjob.py +2 -1
- xpk/core/kueue.py +6 -2
- xpk/core/nap.py +4 -4
- xpk/core/nodepool_test.py +82 -0
- xpk/core/resources.py +1 -7
- xpk/core/storage.py +14 -14
- xpk/core/system_characteristics.py +1 -1
- xpk/core/workload.py +11 -0
- xpk/core/workload_decorators/rdma_decorator.py +3 -2
- xpk/core/workload_decorators/storage_decorator.py +2 -1
- xpk/core/workload_decorators/tcpx_decorator.py +4 -2
- xpk/core/workload_decorators/tcpx_decorator_test.py +267 -0
- xpk/core/workload_decorators/tcpxo_decorator.py +2 -1
- xpk/core/workload_test.py +28 -0
- xpk/main.py +9 -10
- xpk/parser/cluster.py +67 -49
- xpk/parser/common.py +45 -36
- xpk/parser/storage.py +12 -13
- xpk/parser/workload.py +57 -39
- xpk/utils/console.py +2 -1
- {xpk-0.11.0.dist-info → xpk-0.12.0.dist-info}/METADATA +4 -1
- {xpk-0.11.0.dist-info → xpk-0.12.0.dist-info}/RECORD +46 -41
- {xpk-0.11.0.dist-info → xpk-0.12.0.dist-info}/WHEEL +0 -0
- {xpk-0.11.0.dist-info → xpk-0.12.0.dist-info}/entry_points.txt +0 -0
- {xpk-0.11.0.dist-info → xpk-0.12.0.dist-info}/licenses/LICENSE +0 -0
- {xpk-0.11.0.dist-info → xpk-0.12.0.dist-info}/top_level.txt +0 -0
xpk/parser/common.py
CHANGED
|
@@ -15,24 +15,31 @@ limitations under the License.
|
|
|
15
15
|
"""
|
|
16
16
|
|
|
17
17
|
import argparse
|
|
18
|
+
from typing import Protocol, Any
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class ParserOrArgumentGroup(Protocol):
|
|
22
|
+
|
|
23
|
+
def add_argument(self, *args, **kwargs) -> Any:
|
|
24
|
+
...
|
|
18
25
|
|
|
19
26
|
|
|
20
27
|
def add_shared_arguments(
|
|
21
|
-
|
|
28
|
+
custom_parser_or_group: ParserOrArgumentGroup, required=False
|
|
22
29
|
) -> None:
|
|
23
|
-
"""Add shared arguments to the parser.
|
|
30
|
+
"""Add shared arguments to the parser or argument group.
|
|
24
31
|
|
|
25
32
|
Args:
|
|
26
|
-
|
|
33
|
+
custom_parser_or_group: parser or argument group to add shared arguments to.
|
|
27
34
|
"""
|
|
28
|
-
|
|
35
|
+
custom_parser_or_group.add_argument(
|
|
29
36
|
'--project',
|
|
30
37
|
type=str,
|
|
31
38
|
default=None,
|
|
32
39
|
help='GCE project name, defaults to "gcloud config project."',
|
|
33
40
|
required=required,
|
|
34
41
|
)
|
|
35
|
-
|
|
42
|
+
custom_parser_or_group.add_argument(
|
|
36
43
|
'--zone',
|
|
37
44
|
type=str,
|
|
38
45
|
default=None,
|
|
@@ -43,7 +50,7 @@ def add_shared_arguments(
|
|
|
43
50
|
),
|
|
44
51
|
required=required,
|
|
45
52
|
)
|
|
46
|
-
|
|
53
|
+
custom_parser_or_group.add_argument(
|
|
47
54
|
'--dry-run',
|
|
48
55
|
type=bool,
|
|
49
56
|
action=argparse.BooleanOptionalAction,
|
|
@@ -58,14 +65,14 @@ def add_shared_arguments(
|
|
|
58
65
|
|
|
59
66
|
|
|
60
67
|
def add_cluster_arguments(
|
|
61
|
-
|
|
68
|
+
custom_parser_or_group: ParserOrArgumentGroup, required=False
|
|
62
69
|
) -> None:
|
|
63
|
-
"""Add cluster argument to the parser.
|
|
70
|
+
"""Add cluster argument to the parser or argument group.
|
|
64
71
|
|
|
65
72
|
Args:
|
|
66
|
-
|
|
73
|
+
custom_parser_or_group: parser or argument group to add shared arguments to.
|
|
67
74
|
"""
|
|
68
|
-
|
|
75
|
+
custom_parser_or_group.add_argument(
|
|
69
76
|
'--cluster',
|
|
70
77
|
type=str,
|
|
71
78
|
default=None,
|
|
@@ -74,13 +81,15 @@ def add_cluster_arguments(
|
|
|
74
81
|
)
|
|
75
82
|
|
|
76
83
|
|
|
77
|
-
def add_kind_cluster_arguments(
|
|
78
|
-
|
|
84
|
+
def add_kind_cluster_arguments(
|
|
85
|
+
custom_parser_or_group: ParserOrArgumentGroup,
|
|
86
|
+
) -> None:
|
|
87
|
+
"""Add kind cluster arguments to the parser or argument group.
|
|
79
88
|
|
|
80
89
|
Args:
|
|
81
|
-
|
|
90
|
+
custom_parser_or_group: parser or argument group to add shared arguments to.
|
|
82
91
|
"""
|
|
83
|
-
|
|
92
|
+
custom_parser_or_group.add_argument(
|
|
84
93
|
'--kind-cluster',
|
|
85
94
|
type=bool,
|
|
86
95
|
action=argparse.BooleanOptionalAction,
|
|
@@ -89,13 +98,13 @@ def add_kind_cluster_arguments(custom_parser: argparse.ArgumentParser) -> None:
|
|
|
89
98
|
)
|
|
90
99
|
|
|
91
100
|
|
|
92
|
-
def add_global_arguments(
|
|
101
|
+
def add_global_arguments(custom_parser_or_group: ParserOrArgumentGroup):
|
|
93
102
|
"""Add global - no cloud dependent - arguments to the parser.
|
|
94
103
|
|
|
95
104
|
Args:
|
|
96
|
-
|
|
105
|
+
custom_parser_or_group: parser or argument group to add global arguments to.
|
|
97
106
|
"""
|
|
98
|
-
|
|
107
|
+
custom_parser_or_group.add_argument(
|
|
99
108
|
'--dry-run',
|
|
100
109
|
type=bool,
|
|
101
110
|
action=argparse.BooleanOptionalAction,
|
|
@@ -108,20 +117,20 @@ def add_global_arguments(custom_parser: argparse.ArgumentParser):
|
|
|
108
117
|
)
|
|
109
118
|
|
|
110
119
|
|
|
111
|
-
def add_slurm_arguments(
|
|
120
|
+
def add_slurm_arguments(custom_parser_or_group: ParserOrArgumentGroup):
|
|
112
121
|
"""Add Slurm job arguments to the parser.
|
|
113
122
|
|
|
114
123
|
Args:
|
|
115
|
-
|
|
124
|
+
custom_parser_or_group: parser or argument group to add global arguments to.
|
|
116
125
|
"""
|
|
117
|
-
|
|
126
|
+
custom_parser_or_group.add_argument(
|
|
118
127
|
'--ignore-unknown-flags',
|
|
119
128
|
type=bool,
|
|
120
129
|
action=argparse.BooleanOptionalAction,
|
|
121
130
|
default=False,
|
|
122
131
|
help='Ignore all the unsupported flags in the bash script.',
|
|
123
132
|
)
|
|
124
|
-
|
|
133
|
+
custom_parser_or_group.add_argument(
|
|
125
134
|
'-a',
|
|
126
135
|
'--array',
|
|
127
136
|
type=str,
|
|
@@ -137,32 +146,32 @@ def add_slurm_arguments(custom_parser: argparse.ArgumentParser):
|
|
|
137
146
|
' 0. The maximum index value is 2147483647.'
|
|
138
147
|
),
|
|
139
148
|
)
|
|
140
|
-
|
|
149
|
+
custom_parser_or_group.add_argument(
|
|
141
150
|
'-c',
|
|
142
151
|
'--cpus-per-task',
|
|
143
152
|
type=str,
|
|
144
153
|
default=None,
|
|
145
154
|
help='How much cpus a container inside a pod requires.',
|
|
146
155
|
)
|
|
147
|
-
|
|
156
|
+
custom_parser_or_group.add_argument(
|
|
148
157
|
'--gpus-per-task',
|
|
149
158
|
type=str,
|
|
150
159
|
default=None,
|
|
151
160
|
help='How much gpus a container inside a pod requires.',
|
|
152
161
|
)
|
|
153
|
-
|
|
162
|
+
custom_parser_or_group.add_argument(
|
|
154
163
|
'--mem',
|
|
155
164
|
type=str,
|
|
156
165
|
default=None,
|
|
157
166
|
help='How much memory a pod requires.',
|
|
158
167
|
)
|
|
159
|
-
|
|
168
|
+
custom_parser_or_group.add_argument(
|
|
160
169
|
'--mem-per-task',
|
|
161
170
|
type=str,
|
|
162
171
|
default=None,
|
|
163
172
|
help='How much memory a container requires.',
|
|
164
173
|
)
|
|
165
|
-
|
|
174
|
+
custom_parser_or_group.add_argument(
|
|
166
175
|
'--mem-per-cpu',
|
|
167
176
|
type=str,
|
|
168
177
|
default=None,
|
|
@@ -171,7 +180,7 @@ def add_slurm_arguments(custom_parser: argparse.ArgumentParser):
|
|
|
171
180
|
'of requested cpus per task by mem-per-cpu.'
|
|
172
181
|
),
|
|
173
182
|
)
|
|
174
|
-
|
|
183
|
+
custom_parser_or_group.add_argument(
|
|
175
184
|
'--mem-per-gpu',
|
|
176
185
|
type=str,
|
|
177
186
|
default=None,
|
|
@@ -180,21 +189,21 @@ def add_slurm_arguments(custom_parser: argparse.ArgumentParser):
|
|
|
180
189
|
'of requested gpus per task by mem-per-gpu.'
|
|
181
190
|
),
|
|
182
191
|
)
|
|
183
|
-
|
|
192
|
+
custom_parser_or_group.add_argument(
|
|
184
193
|
'-N',
|
|
185
194
|
'--nodes',
|
|
186
195
|
type=int,
|
|
187
196
|
default=None,
|
|
188
197
|
help='Number of pods to be used at a time.',
|
|
189
198
|
)
|
|
190
|
-
|
|
199
|
+
custom_parser_or_group.add_argument(
|
|
191
200
|
'-n',
|
|
192
201
|
'--ntasks',
|
|
193
202
|
type=int,
|
|
194
203
|
default=None,
|
|
195
204
|
help='Number of identical containers inside of a pod, usually 1.',
|
|
196
205
|
)
|
|
197
|
-
|
|
206
|
+
custom_parser_or_group.add_argument(
|
|
198
207
|
'-o',
|
|
199
208
|
'--output',
|
|
200
209
|
type=str,
|
|
@@ -204,7 +213,7 @@ def add_slurm_arguments(custom_parser: argparse.ArgumentParser):
|
|
|
204
213
|
' passed it proceeds to stdout, and is available via kubectl logs.'
|
|
205
214
|
),
|
|
206
215
|
)
|
|
207
|
-
|
|
216
|
+
custom_parser_or_group.add_argument(
|
|
208
217
|
'-e',
|
|
209
218
|
'--error',
|
|
210
219
|
type=str,
|
|
@@ -214,27 +223,27 @@ def add_slurm_arguments(custom_parser: argparse.ArgumentParser):
|
|
|
214
223
|
' proceeds to stdout, and is available via kubectl logs.'
|
|
215
224
|
),
|
|
216
225
|
)
|
|
217
|
-
|
|
226
|
+
custom_parser_or_group.add_argument(
|
|
218
227
|
'--input',
|
|
219
228
|
type=str,
|
|
220
229
|
default=None,
|
|
221
230
|
help='What to pipe into the script.',
|
|
222
231
|
)
|
|
223
|
-
|
|
232
|
+
custom_parser_or_group.add_argument(
|
|
224
233
|
'-J',
|
|
225
234
|
'--job-name',
|
|
226
235
|
type=str,
|
|
227
236
|
default=None,
|
|
228
237
|
help='What is the job name.',
|
|
229
238
|
)
|
|
230
|
-
|
|
239
|
+
custom_parser_or_group.add_argument(
|
|
231
240
|
'-D',
|
|
232
241
|
'--chdir',
|
|
233
242
|
type=str,
|
|
234
243
|
default=None,
|
|
235
244
|
help='Change directory before executing the script.',
|
|
236
245
|
)
|
|
237
|
-
|
|
246
|
+
custom_parser_or_group.add_argument(
|
|
238
247
|
'-t',
|
|
239
248
|
'--time',
|
|
240
249
|
type=str,
|
|
@@ -247,7 +256,7 @@ def add_slurm_arguments(custom_parser: argparse.ArgumentParser):
|
|
|
247
256
|
'and "days-hours:minutes:seconds".'
|
|
248
257
|
),
|
|
249
258
|
)
|
|
250
|
-
|
|
259
|
+
custom_parser_or_group.add_argument(
|
|
251
260
|
'--priority',
|
|
252
261
|
type=str,
|
|
253
262
|
default='medium',
|
xpk/parser/storage.py
CHANGED
|
@@ -28,6 +28,13 @@ from .common import (
|
|
|
28
28
|
add_kind_cluster_arguments,
|
|
29
29
|
add_shared_arguments,
|
|
30
30
|
)
|
|
31
|
+
from typing import Protocol, Any
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class Subcommands(Protocol):
|
|
35
|
+
|
|
36
|
+
def add_parser(self, *args, **kwargs) -> Any:
|
|
37
|
+
...
|
|
31
38
|
|
|
32
39
|
|
|
33
40
|
def set_storage_parser(storage_parser: argparse.ArgumentParser) -> None:
|
|
@@ -47,7 +54,7 @@ def set_storage_parser(storage_parser: argparse.ArgumentParser) -> None:
|
|
|
47
54
|
|
|
48
55
|
|
|
49
56
|
def add_storage_attach_parser(
|
|
50
|
-
storage_subcommands_parser:
|
|
57
|
+
storage_subcommands_parser: Subcommands,
|
|
51
58
|
) -> None:
|
|
52
59
|
|
|
53
60
|
storage_attach_parser: argparse.ArgumentParser = (
|
|
@@ -171,9 +178,7 @@ def add_storage_attach_parser(
|
|
|
171
178
|
add_kind_cluster_arguments(opt_args)
|
|
172
179
|
|
|
173
180
|
|
|
174
|
-
def add_storage_create_parser(
|
|
175
|
-
storage_subcommands_parser: argparse.ArgumentParser,
|
|
176
|
-
) -> None:
|
|
181
|
+
def add_storage_create_parser(storage_subcommands_parser: Subcommands) -> None:
|
|
177
182
|
storage_create_parser: argparse.ArgumentParser = (
|
|
178
183
|
storage_subcommands_parser.add_parser(
|
|
179
184
|
'create', help='create XPK Storage.'
|
|
@@ -272,9 +277,7 @@ def add_storage_create_parser(
|
|
|
272
277
|
add_kind_cluster_arguments(opt_args)
|
|
273
278
|
|
|
274
279
|
|
|
275
|
-
def add_storage_list_parser(
|
|
276
|
-
storage_subcommands_parser: argparse.ArgumentParser,
|
|
277
|
-
):
|
|
280
|
+
def add_storage_list_parser(storage_subcommands_parser: Subcommands) -> None:
|
|
278
281
|
storage_list_parser: argparse.ArgumentParser = (
|
|
279
282
|
storage_subcommands_parser.add_parser('list', help='List XPK Storages.')
|
|
280
283
|
)
|
|
@@ -290,9 +293,7 @@ def add_storage_list_parser(
|
|
|
290
293
|
)
|
|
291
294
|
|
|
292
295
|
|
|
293
|
-
def add_storage_detach_parser(
|
|
294
|
-
storage_subcommands_parser: argparse.ArgumentParser,
|
|
295
|
-
):
|
|
296
|
+
def add_storage_detach_parser(storage_subcommands_parser: Subcommands) -> None:
|
|
296
297
|
storage_detach_parser: argparse.ArgumentParser = (
|
|
297
298
|
storage_subcommands_parser.add_parser(
|
|
298
299
|
'detach', help='Detach XPK Storage.'
|
|
@@ -315,9 +316,7 @@ def add_storage_detach_parser(
|
|
|
315
316
|
add_kind_cluster_arguments(opt_args)
|
|
316
317
|
|
|
317
318
|
|
|
318
|
-
def add_storage_delete_parser(
|
|
319
|
-
storage_subcommands_parser: argparse.ArgumentParser,
|
|
320
|
-
):
|
|
319
|
+
def add_storage_delete_parser(storage_subcommands_parser: Subcommands) -> None:
|
|
321
320
|
storage_delete_parser: argparse.ArgumentParser = (
|
|
322
321
|
storage_subcommands_parser.add_parser(
|
|
323
322
|
'delete', help='Delete XPK Storage.'
|
xpk/parser/workload.py
CHANGED
|
@@ -193,43 +193,6 @@ def set_workload_parsers(workload_parser):
|
|
|
193
193
|
),
|
|
194
194
|
)
|
|
195
195
|
|
|
196
|
-
# Autoprovisioning workload arguments
|
|
197
|
-
workload_create_autoprovisioning_arguments.add_argument(
|
|
198
|
-
'--on-demand',
|
|
199
|
-
action='store_true',
|
|
200
|
-
help=(
|
|
201
|
-
'Sets autoprovisioning to use on-demand resources for the workload'
|
|
202
|
-
' request. See `--reservation` or `--spot` for other capacity types.'
|
|
203
|
-
),
|
|
204
|
-
)
|
|
205
|
-
workload_create_autoprovisioning_arguments.add_argument(
|
|
206
|
-
'--reservation',
|
|
207
|
-
type=str,
|
|
208
|
-
help=(
|
|
209
|
-
'Sets autoprovisioning to use reservation resources for the workload'
|
|
210
|
-
' request. This will attempt to find the provided reservation. See'
|
|
211
|
-
' `--spot`, `--flex` or `--on-demand` for other capacity types.'
|
|
212
|
-
),
|
|
213
|
-
)
|
|
214
|
-
workload_create_autoprovisioning_arguments.add_argument(
|
|
215
|
-
'--spot',
|
|
216
|
-
action='store_true',
|
|
217
|
-
help=(
|
|
218
|
-
'Sets autoprovisioning to use spot resources. See `--reservation`,'
|
|
219
|
-
' `--flex` or `--on-demand` for other capacity types.'
|
|
220
|
-
),
|
|
221
|
-
)
|
|
222
|
-
|
|
223
|
-
workload_create_autoprovisioning_arguments.add_argument(
|
|
224
|
-
'--flex',
|
|
225
|
-
action='store_true',
|
|
226
|
-
help=(
|
|
227
|
-
'Sets autoprovisioning to use flex-start resources. See'
|
|
228
|
-
' `--reservation`, `--spot` or `--on-demand` for other capacity'
|
|
229
|
-
' types.'
|
|
230
|
-
),
|
|
231
|
-
)
|
|
232
|
-
|
|
233
196
|
# "workload create-pathways" command parser.
|
|
234
197
|
workload_create_pathways_parser = workload_subcommands.add_parser(
|
|
235
198
|
'create-pathways', help='Create a new job.'
|
|
@@ -257,6 +220,12 @@ def set_workload_parsers(workload_parser):
|
|
|
257
220
|
'`--base-docker-image` is used by default. Set this argument if the'
|
|
258
221
|
' user wants the docker image to be used directly by the xpk workload.',
|
|
259
222
|
)
|
|
223
|
+
workload_create_pathways_autoprovisioning_arguments = (
|
|
224
|
+
workload_create_pathways_parser.add_argument_group(
|
|
225
|
+
'Optional Autoprovisioning Arguments',
|
|
226
|
+
'Arguments for configuring autoprovisioning.',
|
|
227
|
+
)
|
|
228
|
+
)
|
|
260
229
|
workload_create_pathways_vertex_tensorboard_arguments = (
|
|
261
230
|
workload_create_pathways_parser.add_argument_group(
|
|
262
231
|
'Vertex Tensorboard Arguments',
|
|
@@ -407,6 +376,10 @@ def set_workload_parsers(workload_parser):
|
|
|
407
376
|
workload_vertex_tensorboard_arguments,
|
|
408
377
|
workload_create_pathways_vertex_tensorboard_arguments,
|
|
409
378
|
])
|
|
379
|
+
add_shared_workload_create_autoprovisioning_arguments([
|
|
380
|
+
workload_create_autoprovisioning_arguments,
|
|
381
|
+
workload_create_pathways_autoprovisioning_arguments,
|
|
382
|
+
])
|
|
410
383
|
|
|
411
384
|
# Set defaults for both workload create and workload create-pathways after adding all shared args.
|
|
412
385
|
workload_create_parser.set_defaults(func=workload_create)
|
|
@@ -452,7 +425,7 @@ def set_workload_parsers(workload_parser):
|
|
|
452
425
|
type=str,
|
|
453
426
|
help=(
|
|
454
427
|
'Filters the arguments based on job name. Provide a regex'
|
|
455
|
-
'
|
|
428
|
+
' expression to parse jobs that match the pattern or provide a job'
|
|
456
429
|
' name to delete a single job.'
|
|
457
430
|
),
|
|
458
431
|
)
|
|
@@ -521,7 +494,7 @@ def set_workload_parsers(workload_parser):
|
|
|
521
494
|
type=str,
|
|
522
495
|
help=(
|
|
523
496
|
'Filters the arguments based on job name. Provide a regex'
|
|
524
|
-
'
|
|
497
|
+
' expression to parse jobs that match the pattern or provide a job'
|
|
525
498
|
' name to view a single job.'
|
|
526
499
|
),
|
|
527
500
|
required=False,
|
|
@@ -770,3 +743,48 @@ def add_shared_workload_create_tensorboard_arguments(args_parsers):
|
|
|
770
743
|
'<cluster>-<workload> will be created.'
|
|
771
744
|
),
|
|
772
745
|
)
|
|
746
|
+
|
|
747
|
+
|
|
748
|
+
def add_shared_workload_create_autoprovisioning_arguments(args_parsers):
|
|
749
|
+
"""Add shared autoprovisioning arguments
|
|
750
|
+
|
|
751
|
+
Args:
|
|
752
|
+
List of workload create optional arguments parsers
|
|
753
|
+
"""
|
|
754
|
+
for custom_parser in args_parsers:
|
|
755
|
+
custom_parser.add_argument(
|
|
756
|
+
'--on-demand',
|
|
757
|
+
action='store_true',
|
|
758
|
+
help=(
|
|
759
|
+
'Sets autoprovisioning to use on-demand resources for the workload'
|
|
760
|
+
' request. See `--reservation` or `--spot` for other capacity'
|
|
761
|
+
' types.'
|
|
762
|
+
),
|
|
763
|
+
)
|
|
764
|
+
custom_parser.add_argument(
|
|
765
|
+
'--reservation',
|
|
766
|
+
type=str,
|
|
767
|
+
help=(
|
|
768
|
+
'Sets autoprovisioning to use reservation resources for the'
|
|
769
|
+
' workload request. This will attempt to find the provided'
|
|
770
|
+
' reservation. See `--spot`, `--flex` or `--on-demand` for other'
|
|
771
|
+
' capacity types.'
|
|
772
|
+
),
|
|
773
|
+
)
|
|
774
|
+
custom_parser.add_argument(
|
|
775
|
+
'--spot',
|
|
776
|
+
action='store_true',
|
|
777
|
+
help=(
|
|
778
|
+
'Sets autoprovisioning to use spot resources. See `--reservation`,'
|
|
779
|
+
' `--flex` or `--on-demand` for other capacity types.'
|
|
780
|
+
),
|
|
781
|
+
)
|
|
782
|
+
custom_parser.add_argument(
|
|
783
|
+
'--flex',
|
|
784
|
+
action='store_true',
|
|
785
|
+
help=(
|
|
786
|
+
'Sets autoprovisioning to use flex-start resources. See'
|
|
787
|
+
' `--reservation`, `--spot` or `--on-demand` for other capacity'
|
|
788
|
+
' types.'
|
|
789
|
+
),
|
|
790
|
+
)
|
xpk/utils/console.py
CHANGED
|
@@ -15,6 +15,7 @@ limitations under the License.
|
|
|
15
15
|
"""
|
|
16
16
|
|
|
17
17
|
import sys
|
|
18
|
+
from typing import NoReturn
|
|
18
19
|
|
|
19
20
|
|
|
20
21
|
def xpk_print(*args, **kwargs):
|
|
@@ -29,7 +30,7 @@ def xpk_print(*args, **kwargs):
|
|
|
29
30
|
sys.stdout.flush()
|
|
30
31
|
|
|
31
32
|
|
|
32
|
-
def xpk_exit(error_code):
|
|
33
|
+
def xpk_exit(error_code) -> NoReturn:
|
|
33
34
|
"""Helper function to exit xpk with an associated error code.
|
|
34
35
|
|
|
35
36
|
Args:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: xpk
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.12.0
|
|
4
4
|
Summary: xpk helps Cloud developers to orchestrate training jobs on accelerators on GKE.
|
|
5
5
|
Author-email: XPK team <xpk-code-reviewers@google.com>
|
|
6
6
|
License: Apache-2.0
|
|
@@ -28,6 +28,9 @@ Requires-Dist: pylint>=2.6.0; extra == "dev"
|
|
|
28
28
|
Requires-Dist: pre-commit; extra == "dev"
|
|
29
29
|
Requires-Dist: pytest; extra == "dev"
|
|
30
30
|
Requires-Dist: docker==7.1.0; extra == "dev"
|
|
31
|
+
Requires-Dist: mypy~=1.17; extra == "dev"
|
|
32
|
+
Requires-Dist: types-PyYAML==6.0.2; extra == "dev"
|
|
33
|
+
Requires-Dist: types-docker~=7.1.0.0; extra == "dev"
|
|
31
34
|
Dynamic: license-file
|
|
32
35
|
|
|
33
36
|
<!--
|
|
@@ -1,68 +1,73 @@
|
|
|
1
1
|
xpk/__init__.py,sha256=7mu-VQDQMyxM5To0KOhuYe4y2TYGsEkfV7hXZmUyih4,561
|
|
2
|
-
xpk/main.py,sha256=
|
|
2
|
+
xpk/main.py,sha256=fQJ4eZMgmBbfYoHT1_d98PdxSC6Gn9Ug-2REAwL0kys,2365
|
|
3
3
|
xpk/api/__init__.py,sha256=YPwWBbgLAu7L-YlTVGB2r8ZV4TzypURMRBcehSHHlLY,561
|
|
4
4
|
xpk/api/storage_crd.yaml,sha256=r4WFXnSJJ25EUF-t4Ljfbl-cJoSaiFiZkP8451eTub4,1260
|
|
5
5
|
xpk/commands/__init__.py,sha256=YPwWBbgLAu7L-YlTVGB2r8ZV4TzypURMRBcehSHHlLY,561
|
|
6
6
|
xpk/commands/batch.py,sha256=qzKkXf45brC-Ln3C-lajcwdqCc2BCVbKJe5M06AdnKQ,3806
|
|
7
|
-
xpk/commands/cluster.py,sha256=
|
|
8
|
-
xpk/commands/cluster_gcluster.py,sha256=
|
|
9
|
-
xpk/commands/common.py,sha256=
|
|
7
|
+
xpk/commands/cluster.py,sha256=X8G6vqn9-JW7d906qII4Sf-o1wyHEA_XEC7fLv1SmWM,40336
|
|
8
|
+
xpk/commands/cluster_gcluster.py,sha256=8jJ7nHBbkmaPtsVQ2m_GnLxkS5iNV5sSN61KL0K_uEY,10861
|
|
9
|
+
xpk/commands/common.py,sha256=aeZLbnpwwKNbc9SCl1U3RugVLUpiK1MH67YQrurpGnU,2419
|
|
10
10
|
xpk/commands/config.py,sha256=gFNkf3ibsvZmcPpkpKXe-KJmHO5IKucNwLCXNgKvaDc,836
|
|
11
|
-
xpk/commands/info.py,sha256=
|
|
11
|
+
xpk/commands/info.py,sha256=1orA0u5KCB6fj-smHkuFL1WCH96NGrEiDpRCgPrxUW4,7304
|
|
12
12
|
xpk/commands/inspector.py,sha256=bwbZW-cLtiBw2V0zvoMprHWhgMbAYm0ez0GjjEqeUR8,12097
|
|
13
|
-
xpk/commands/job.py,sha256=
|
|
13
|
+
xpk/commands/job.py,sha256=pZdj0XWXpDQyvDAHf6JrZQl2heZ8sBzoGzB_aqzHc_k,5599
|
|
14
14
|
xpk/commands/kind.py,sha256=Vl3RT47kHCR0ORX9dK37HCiYtbmXJUCIAaq-QEbIclU,7578
|
|
15
|
-
xpk/commands/kjob_common.py,sha256=
|
|
15
|
+
xpk/commands/kjob_common.py,sha256=dtT-R0n50byTmu2Qcni0pqKYobUAHNENBN_4pt0l-KE,1952
|
|
16
16
|
xpk/commands/run.py,sha256=5hYMG0DcdHnFWsJ5gmfX09t6ZPVItt7FFoHO_ED0_Dk,3798
|
|
17
17
|
xpk/commands/shell.py,sha256=5-sKcI2Rbk3aCojnBNtipCwgOrbIDnG4f8ah0KIayY8,4182
|
|
18
|
-
xpk/commands/storage.py,sha256=
|
|
18
|
+
xpk/commands/storage.py,sha256=J3kRam3DDwCbvm6JM9h1Zy7_OV7albzAmHi5VkzqyYg,10612
|
|
19
19
|
xpk/commands/version.py,sha256=CU4mb71r66U28krnPAopC6vBpdK-IGclsy5uNaQcgRY,824
|
|
20
|
-
xpk/commands/workload.py,sha256=
|
|
20
|
+
xpk/commands/workload.py,sha256=00KhL0L5j125gxdVLjKyaqXw74oQs1Cbe6lZXV4-sGg,26699
|
|
21
21
|
xpk/core/__init__.py,sha256=YPwWBbgLAu7L-YlTVGB2r8ZV4TzypURMRBcehSHHlLY,561
|
|
22
|
-
xpk/core/capacity.py,sha256=
|
|
23
|
-
xpk/core/cluster.py,sha256=
|
|
24
|
-
xpk/core/cluster_private.py,sha256=
|
|
25
|
-
xpk/core/commands.py,sha256=
|
|
26
|
-
xpk/core/config.py,sha256=
|
|
22
|
+
xpk/core/capacity.py,sha256=SQzncJSLuI4LLJ2VcnpxcRlTjiBG3e8nM0_QxG1986w,7367
|
|
23
|
+
xpk/core/cluster.py,sha256=8_aci7xmR-poDpbirW0Rlwtv7Q9r0K8dshY6CYlpmTM,28948
|
|
24
|
+
xpk/core/cluster_private.py,sha256=NhjaqBZ5uwn6gtsyh6y9ss7hvKXcaKPmpzJjBGBSvQk,6705
|
|
25
|
+
xpk/core/commands.py,sha256=AOLpvcbJoQzY1LEQ_nrbHb4VX6jjxtV9TB1aphu0peA,10759
|
|
26
|
+
xpk/core/config.py,sha256=cGGpJH1IvRCVuYepDmGeaFy6iLTo-ICC949bC1AHnak,3407
|
|
27
|
+
xpk/core/config_test.py,sha256=v1qfyFRzLkYSQ7Wn4nx1N0dBSOFXidLWDfhkeHDZOVM,1847
|
|
27
28
|
xpk/core/docker_container.py,sha256=GvkCJ2S5UKn8uh3pZhRd3X7iS0-PsQpRO8l7QhywVGc,7604
|
|
28
29
|
xpk/core/docker_image.py,sha256=fEdpLQg1C205mMbLREy48WnhvNv2Nm4KQFX87B2CuiA,6624
|
|
29
|
-
xpk/core/docker_manager.py,sha256=
|
|
30
|
-
xpk/core/docker_resources.py,sha256=
|
|
31
|
-
xpk/core/filestore.py,sha256=
|
|
32
|
-
xpk/core/gcloud_context.py,sha256=
|
|
30
|
+
xpk/core/docker_manager.py,sha256=JBFgyD6O7LKwEHJC7YuSoCDZqrFRtb-LjgWNqkfAbR0,10566
|
|
31
|
+
xpk/core/docker_resources.py,sha256=DLaFi7D72pRUUOMpKoccKf91meVBnWqlHBrJXI31GlY,12718
|
|
32
|
+
xpk/core/filestore.py,sha256=mcuUzsAPARbnrBG4fIGsEoN8NmzjaQ6k0tvIwMtjO9k,8068
|
|
33
|
+
xpk/core/gcloud_context.py,sha256=go0avmBbYx45vk_7W3iwQEphmQUx27oaL6dseyocqLI,5836
|
|
33
34
|
xpk/core/gcluster_manager.py,sha256=JFip2hInFczFP2h5AXa70IPIuTaJ475TG6GxkQjKOI8,6337
|
|
34
35
|
xpk/core/gcsfuse.py,sha256=kg5pgxdTjgiqquuGjev9fXzJPb8oiWPTK6wzCddzheQ,2125
|
|
35
36
|
xpk/core/jobset.py,sha256=L_q67_Gq_XjsZKcnznVlCuk1RGZeDFAGDXdCGxbu1oc,4123
|
|
36
|
-
xpk/core/kjob.py,sha256=
|
|
37
|
-
xpk/core/kueue.py,sha256=
|
|
37
|
+
xpk/core/kjob.py,sha256=g44jUbchPOxaOAI5ue7WX24wgJwDPsITBeKXmSMnJo0,14714
|
|
38
|
+
xpk/core/kueue.py,sha256=ulm_m38rZeVsskj6OgNtfBbMRkLYUUo_uktEd795B9U,15247
|
|
38
39
|
xpk/core/monitoring.py,sha256=v9MvLzNfvJAVby_ehSlPe6PaO0_pf3shkXg5gd-UWm8,4338
|
|
39
40
|
xpk/core/mtc.py,sha256=pO7p3l-EzLFdTE8MdwWV8i0Zu-7epGql_kPoksVofIU,6259
|
|
40
|
-
xpk/core/nap.py,sha256=
|
|
41
|
+
xpk/core/nap.py,sha256=L6hVz1gPp2Hcss-Z_7CDCvX70DNU9LFsIU_ILUQ9npc,12814
|
|
41
42
|
xpk/core/network.py,sha256=hQR5Kab5Q5CYjggriNIhNh78Aq0CCF-vPUQI6BC4Wgw,10537
|
|
42
43
|
xpk/core/nodepool.py,sha256=mHYa51og95hwHIxXuzWEycudfewDogzYULoEwrWWTDE,23029
|
|
44
|
+
xpk/core/nodepool_test.py,sha256=QRpmdyZTPRDE2qCibWeKQgE3Q2WCxXt1Onfv0MK4QZQ,2626
|
|
43
45
|
xpk/core/pathways.py,sha256=g4PUSi6RPqpCPVlziWGj14W7zbdNvkw8mrOq09J3s68,10594
|
|
44
46
|
xpk/core/ray.py,sha256=UxOpIc2enHi1fQ4h3KO8FH8bIyEMtYzGtPoeqJKGG4o,6337
|
|
45
|
-
xpk/core/resources.py,sha256=
|
|
47
|
+
xpk/core/resources.py,sha256=itTBXPmy8mv6D2kQkzX9RGT5otr-cxKdhHdQSZvvm7w,8076
|
|
46
48
|
xpk/core/scheduling.py,sha256=HIagFbVAl7lz5gRyjpzwke1tLN4PGEyOkUrtBdmy5Ok,9157
|
|
47
|
-
xpk/core/storage.py,sha256=
|
|
48
|
-
xpk/core/system_characteristics.py,sha256=
|
|
49
|
+
xpk/core/storage.py,sha256=NILvVAcLNMLmp4wKx_TEKbMMF5X1oL-FrQV46PT0_ds,16902
|
|
50
|
+
xpk/core/system_characteristics.py,sha256=2mtQlUiufK98XUXo0_f1D4d06FRGdUk_VNkaBg48Fcs,15152
|
|
49
51
|
xpk/core/vertex.py,sha256=pD9UBL62xHomuqdNu7xKccfD2KCbjgohMk3AhX-CXSw,3644
|
|
50
|
-
xpk/core/workload.py,sha256=
|
|
52
|
+
xpk/core/workload.py,sha256=WyT-H4XdeDY07glL1ikmcs44kLgLjMuMtclyZTxrmg8,8921
|
|
53
|
+
xpk/core/workload_test.py,sha256=tVTvrwDRXD3O1GCoftgEBWilCYTN74ayP1KRP0vptx0,857
|
|
51
54
|
xpk/core/blueprint/__init__.py,sha256=YPwWBbgLAu7L-YlTVGB2r8ZV4TzypURMRBcehSHHlLY,561
|
|
52
55
|
xpk/core/blueprint/blueprint_definitions.py,sha256=5i331XA-2yP_ALyB6XU5tP2Tf9iHcIX5g0TilxQi8zE,1800
|
|
53
|
-
xpk/core/blueprint/blueprint_generator.py,sha256=
|
|
56
|
+
xpk/core/blueprint/blueprint_generator.py,sha256=Uoz3YKsZKLfIoPI8d3bQ5RNrFFsqBOWb3HBQr3OolHo,38010
|
|
57
|
+
xpk/core/blueprint/blueprint_test.py,sha256=f3dL7KU9n6M8Hdnrg4VI5EEi9GA2dnwm4LDf9n5Wbuw,7410
|
|
54
58
|
xpk/core/remote_state/__init__.py,sha256=PkV8D9WOtlJHH5AIxsQaKeIBcmupT_Ol_bwJgN6G2I8,561
|
|
55
59
|
xpk/core/remote_state/fuse_remote_state.py,sha256=3Dx4ZZd0NFF5-MlqGWHzz8H4bjYiPOWdF_YSEnKUPQ8,3246
|
|
56
60
|
xpk/core/remote_state/remote_state_client.py,sha256=6PcR92Xy_RMjlF4AscanQ1jXNHnewLWGNC2v53jbzD4,1077
|
|
57
61
|
xpk/core/workload_decorators/__init__.py,sha256=YPwWBbgLAu7L-YlTVGB2r8ZV4TzypURMRBcehSHHlLY,561
|
|
58
|
-
xpk/core/workload_decorators/rdma_decorator.py,sha256=
|
|
59
|
-
xpk/core/workload_decorators/storage_decorator.py,sha256=
|
|
60
|
-
xpk/core/workload_decorators/tcpx_decorator.py,sha256=
|
|
61
|
-
xpk/core/workload_decorators/
|
|
62
|
+
xpk/core/workload_decorators/rdma_decorator.py,sha256=isbgPnjdu2AT_Da1nVUIRoGE_qZ7jMDOKCgZOLq5r2A,4006
|
|
63
|
+
xpk/core/workload_decorators/storage_decorator.py,sha256=DDYQVO1OKTLhveDOA4V6b2RWr4n0fbwHdnoFFmW7iaQ,2000
|
|
64
|
+
xpk/core/workload_decorators/tcpx_decorator.py,sha256=m5EgzEHjbcOD13ygY91mQdhwQt4Gr5PyalVkKcHyeV8,5975
|
|
65
|
+
xpk/core/workload_decorators/tcpx_decorator_test.py,sha256=iTBS3X_-VwA2oveNDjscduLtll0VOJyFRCp4xmsjg7w,8515
|
|
66
|
+
xpk/core/workload_decorators/tcpxo_decorator.py,sha256=_nLX7tbnxhnS-xv4Jijd1JOP76V4LpNCfW3Np404Cqw,6537
|
|
62
67
|
xpk/parser/__init__.py,sha256=YPwWBbgLAu7L-YlTVGB2r8ZV4TzypURMRBcehSHHlLY,561
|
|
63
68
|
xpk/parser/batch.py,sha256=mJU-Cp1yTLje59vD-B1IiBcUeD-ZmEsoeB4xhj9cflc,1406
|
|
64
|
-
xpk/parser/cluster.py,sha256=
|
|
65
|
-
xpk/parser/common.py,sha256=
|
|
69
|
+
xpk/parser/cluster.py,sha256=EpC7Kx8o84DRxqrxI1XMklc5raTX7vZSqeZcBZniopk,29131
|
|
70
|
+
xpk/parser/common.py,sha256=N6P6wPuptluNEddh9kpUsaWgxXGADNZLMfKT-P7QkW0,7791
|
|
66
71
|
xpk/parser/config.py,sha256=-XnWx9aFsBW4Uzo_hpOMD2ZQ0bdZLvq1ksv83_5jqSM,1633
|
|
67
72
|
xpk/parser/core.py,sha256=VRJerlS92ufoQbG1mZv7B04DAP4qGkBHa4pRXgcbAs0,4761
|
|
68
73
|
xpk/parser/info.py,sha256=UJohxVVWdt9IgUXoPsrVae2DN1BjAVGWrSN2ajrB8RQ,1860
|
|
@@ -71,14 +76,14 @@ xpk/parser/job.py,sha256=5RdE70rucGfrsn65l7Ho6RmO06mag1S0AO-3saVuXyw,4328
|
|
|
71
76
|
xpk/parser/kind.py,sha256=sgPCqNVrgmFLcOBEbhlaphwVXxMh_opP9ntCq4KPePE,2682
|
|
72
77
|
xpk/parser/run.py,sha256=oi_ksSyJ8Ooffe2EgoV_ecpmXEmNGVotjpIQH-HjufE,1481
|
|
73
78
|
xpk/parser/shell.py,sha256=VC8p-kz9XjJZW9DXZ-rnv41XnRDRpQRFywHpB5j7tfc,1970
|
|
74
|
-
xpk/parser/storage.py,sha256=
|
|
79
|
+
xpk/parser/storage.py,sha256=XNynqulEzTmT8_G6wkeBwfXX0XQ1lsd6BFcx0H6rGfU,9971
|
|
75
80
|
xpk/parser/validators.py,sha256=-NBZelvfwZRzjz-YUCreD8EzMLHll8PZM-d-MVm2PG4,1192
|
|
76
81
|
xpk/parser/version.py,sha256=eJo4PAbbmRQZulgKBs_ytbVgV9zAaaXeNzMMxmgFMVY,769
|
|
77
|
-
xpk/parser/workload.py,sha256=
|
|
82
|
+
xpk/parser/workload.py,sha256=J0Wg4NEO7pOdOwf_8uC9bVzXodBXwkYo1c2gRJVyyak,26068
|
|
78
83
|
xpk/templates/__init__.py,sha256=7mu-VQDQMyxM5To0KOhuYe4y2TYGsEkfV7hXZmUyih4,561
|
|
79
84
|
xpk/templates/storage.yaml,sha256=AykdyMtDnKZF8Y_0BYxoYP03hEIzEk6iNalXAQHgAls,163
|
|
80
85
|
xpk/utils/__init__.py,sha256=YPwWBbgLAu7L-YlTVGB2r8ZV4TzypURMRBcehSHHlLY,561
|
|
81
|
-
xpk/utils/console.py,sha256=
|
|
86
|
+
xpk/utils/console.py,sha256=hRbvtog_VAzuxt5GfwK5GZdd5SWaa7kvWG8zo_qFRQc,1519
|
|
82
87
|
xpk/utils/file.py,sha256=jlv2o4ah9UmWJ7NuOCnTwtMZFLerOATBIMQeQ03-kIw,2142
|
|
83
88
|
xpk/utils/gcs_utils.py,sha256=zg-XSTv4G4TFjeT2bNBm2WLdDXPrOZi0rNv_JdppNg4,4113
|
|
84
89
|
xpk/utils/kubectl.py,sha256=WKB9UhpouPN9G4n2ejRi_PgsYLI0R01gzkS1WGU6mJA,1828
|
|
@@ -87,9 +92,9 @@ xpk/utils/objects.py,sha256=OwMNxB4TGX21qnJPdZo2YBMPMbQPqOtHMh19QhoRNRY,2498
|
|
|
87
92
|
xpk/utils/templates.py,sha256=g8zgR1MxyJmTmzM_wnvH30FmcbgQMC47UQwBtLj8B9k,807
|
|
88
93
|
xpk/utils/validation.py,sha256=bSJApIY0Lk48I4EEQP08ZUvolXt_APpYXVGJXFQ_YLA,2711
|
|
89
94
|
xpk/utils/yaml.py,sha256=j8xuAJ9yAAwnQi6ozwZ-nMnDyDnc3xWkeBZMtSuP4RU,844
|
|
90
|
-
xpk-0.
|
|
91
|
-
xpk-0.
|
|
92
|
-
xpk-0.
|
|
93
|
-
xpk-0.
|
|
94
|
-
xpk-0.
|
|
95
|
-
xpk-0.
|
|
95
|
+
xpk-0.12.0.dist-info/licenses/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
|
|
96
|
+
xpk-0.12.0.dist-info/METADATA,sha256=r_BW6P7rX9LHHQbFSLp_3WwEKfl2VFLYMFnyjMbSGes,71759
|
|
97
|
+
xpk-0.12.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
98
|
+
xpk-0.12.0.dist-info/entry_points.txt,sha256=mzEtiIesFkT1kmcTUVDA1o3uOhiniX6tIz2wmOlMu1M,38
|
|
99
|
+
xpk-0.12.0.dist-info/top_level.txt,sha256=aDe4N0jicmuWExx_6w0TxWQJaEuPSs9BnLU-3aF1GLo,4
|
|
100
|
+
xpk-0.12.0.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|