torchx-nightly 2025.10.16__py3-none-any.whl → 2025.10.18__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of torchx-nightly might be problematic. Click here for more details.
- torchx/runner/api.py +16 -32
- torchx/schedulers/api.py +7 -2
- torchx/specs/api.py +135 -12
- torchx/workspace/api.py +63 -42
- {torchx_nightly-2025.10.16.dist-info → torchx_nightly-2025.10.18.dist-info}/METADATA +4 -4
- {torchx_nightly-2025.10.16.dist-info → torchx_nightly-2025.10.18.dist-info}/RECORD +10 -10
- {torchx_nightly-2025.10.16.dist-info → torchx_nightly-2025.10.18.dist-info}/LICENSE +0 -0
- {torchx_nightly-2025.10.16.dist-info → torchx_nightly-2025.10.18.dist-info}/WHEEL +0 -0
- {torchx_nightly-2025.10.16.dist-info → torchx_nightly-2025.10.18.dist-info}/entry_points.txt +0 -0
- {torchx_nightly-2025.10.16.dist-info → torchx_nightly-2025.10.18.dist-info}/top_level.txt +0 -0
torchx/runner/api.py
CHANGED
|
@@ -427,41 +427,25 @@ class Runner:
|
|
|
427
427
|
sched._pre_build_validate(app, scheduler, resolved_cfg)
|
|
428
428
|
|
|
429
429
|
if isinstance(sched, WorkspaceMixin):
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
# later, torchx added support for the workspace attr in Role
|
|
436
|
-
# for BC, give precedence to the workspace argument over the workspace attr for role[0]
|
|
437
|
-
if role_workspace:
|
|
438
|
-
logger.info(
|
|
439
|
-
f"Using workspace={workspace} over role[{i}].workspace={role_workspace} for role[{i}]={role.name}."
|
|
440
|
-
" To use the role's workspace attr pass: --workspace='' from CLI or workspace=None programmatically." # noqa: B950
|
|
441
|
-
)
|
|
442
|
-
role_workspace = workspace
|
|
443
|
-
|
|
444
|
-
if role_workspace:
|
|
445
|
-
old_img = role.image
|
|
430
|
+
if workspace:
|
|
431
|
+
# NOTE: torchx originally took workspace as a runner arg and only applied the workspace to role[0]
|
|
432
|
+
# later, torchx added support for the workspace attr in Role
|
|
433
|
+
# for BC, give precedence to the workspace argument over the workspace attr for role[0]
|
|
434
|
+
if app.roles[0].workspace:
|
|
446
435
|
logger.info(
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
role, role_workspace, resolved_cfg
|
|
436
|
+
"Overriding role[%d] (%s) workspace to `%s`"
|
|
437
|
+
"To use the role's workspace attr pass: --workspace='' from CLI or workspace=None programmatically.",
|
|
438
|
+
0,
|
|
439
|
+
role.name,
|
|
440
|
+
str(app.roles[0].workspace),
|
|
453
441
|
)
|
|
442
|
+
app.roles[0].workspace = (
|
|
443
|
+
Workspace.from_str(workspace)
|
|
444
|
+
if isinstance(workspace, str)
|
|
445
|
+
else workspace
|
|
446
|
+
)
|
|
454
447
|
|
|
455
|
-
|
|
456
|
-
logger.info(
|
|
457
|
-
f"Built new image `{role.image}` based on original image `{old_img}`"
|
|
458
|
-
f" and changes in workspace `{role_workspace}` for role[{i}]={role.name}."
|
|
459
|
-
)
|
|
460
|
-
else:
|
|
461
|
-
logger.info(
|
|
462
|
-
f"Reusing original image `{old_img}` for role[{i}]={role.name}."
|
|
463
|
-
" Either a patch was built or no changes to workspace was detected."
|
|
464
|
-
)
|
|
448
|
+
sched.build_workspaces(app.roles, resolved_cfg)
|
|
465
449
|
|
|
466
450
|
sched._validate(app, scheduler, resolved_cfg)
|
|
467
451
|
dryrun_info = sched.submit_dryrun(app, resolved_cfg)
|
torchx/schedulers/api.py
CHANGED
|
@@ -131,7 +131,7 @@ class Scheduler(abc.ABC, Generic[T, A, D]):
|
|
|
131
131
|
self,
|
|
132
132
|
app: A,
|
|
133
133
|
cfg: T,
|
|
134
|
-
workspace:
|
|
134
|
+
workspace: str | Workspace | None = None,
|
|
135
135
|
) -> str:
|
|
136
136
|
"""
|
|
137
137
|
Submits the application to be run by the scheduler.
|
|
@@ -145,7 +145,12 @@ class Scheduler(abc.ABC, Generic[T, A, D]):
|
|
|
145
145
|
resolved_cfg = self.run_opts().resolve(cfg)
|
|
146
146
|
if workspace:
|
|
147
147
|
assert isinstance(self, WorkspaceMixin)
|
|
148
|
-
|
|
148
|
+
|
|
149
|
+
if isinstance(workspace, str):
|
|
150
|
+
workspace = Workspace.from_str(workspace)
|
|
151
|
+
|
|
152
|
+
app.roles[0].workspace = workspace
|
|
153
|
+
self.build_workspaces(app.roles, resolved_cfg)
|
|
149
154
|
|
|
150
155
|
# pyre-fixme: submit_dryrun takes Generic type for resolved_cfg
|
|
151
156
|
dryrun_info = self.submit_dryrun(app, resolved_cfg)
|
torchx/specs/api.py
CHANGED
|
@@ -14,10 +14,12 @@ import logging as logger
|
|
|
14
14
|
import os
|
|
15
15
|
import pathlib
|
|
16
16
|
import re
|
|
17
|
+
import shutil
|
|
17
18
|
import typing
|
|
19
|
+
import warnings
|
|
18
20
|
from dataclasses import asdict, dataclass, field
|
|
19
21
|
from datetime import datetime
|
|
20
|
-
from enum import Enum
|
|
22
|
+
from enum import Enum, IntEnum
|
|
21
23
|
from json import JSONDecodeError
|
|
22
24
|
from string import Template
|
|
23
25
|
from typing import (
|
|
@@ -380,6 +382,16 @@ class Workspace:
|
|
|
380
382
|
"""False if no projects mapping. Lets us use workspace object in an if-statement"""
|
|
381
383
|
return bool(self.projects)
|
|
382
384
|
|
|
385
|
+
def __eq__(self, other: object) -> bool:
|
|
386
|
+
if not isinstance(other, Workspace):
|
|
387
|
+
return False
|
|
388
|
+
return self.projects == other.projects
|
|
389
|
+
|
|
390
|
+
def __hash__(self) -> int:
|
|
391
|
+
# makes it possible to use Workspace as the key in the workspace build cache
|
|
392
|
+
# see WorkspaceMixin.caching_build_workspace_and_update_role
|
|
393
|
+
return hash(frozenset(self.projects.items()))
|
|
394
|
+
|
|
383
395
|
def is_unmapped_single_project(self) -> bool:
|
|
384
396
|
"""
|
|
385
397
|
Returns ``True`` if this workspace only has 1 project
|
|
@@ -387,6 +399,39 @@ class Workspace:
|
|
|
387
399
|
"""
|
|
388
400
|
return len(self.projects) == 1 and not next(iter(self.projects.values()))
|
|
389
401
|
|
|
402
|
+
def merge_into(self, outdir: str | pathlib.Path) -> None:
|
|
403
|
+
"""
|
|
404
|
+
Copies each project dir of this workspace into the specified ``outdir``.
|
|
405
|
+
Each project dir is copied into ``{outdir}/{target}`` where ``target`` is
|
|
406
|
+
the target mapping of the project dir.
|
|
407
|
+
|
|
408
|
+
For example:
|
|
409
|
+
|
|
410
|
+
.. code-block:: python
|
|
411
|
+
from os.path import expanduser
|
|
412
|
+
|
|
413
|
+
workspace = Workspace(
|
|
414
|
+
projects={
|
|
415
|
+
expanduser("~/workspace/torch"): "torch",
|
|
416
|
+
expanduser("~/workspace/my_project": "")
|
|
417
|
+
}
|
|
418
|
+
)
|
|
419
|
+
workspace.merge_into(expanduser("~/tmp"))
|
|
420
|
+
|
|
421
|
+
Copies:
|
|
422
|
+
|
|
423
|
+
* ``~/workspace/torch/**`` into ``~/tmp/torch/**``
|
|
424
|
+
* ``~/workspace/my_project/**`` into ``~/tmp/**``
|
|
425
|
+
|
|
426
|
+
"""
|
|
427
|
+
|
|
428
|
+
for src, dst in self.projects.items():
|
|
429
|
+
dst_path = pathlib.Path(outdir) / dst
|
|
430
|
+
if pathlib.Path(src).is_file():
|
|
431
|
+
shutil.copy2(src, dst_path)
|
|
432
|
+
else: # src is dir
|
|
433
|
+
shutil.copytree(src, dst_path, dirs_exist_ok=True)
|
|
434
|
+
|
|
390
435
|
@staticmethod
|
|
391
436
|
def from_str(workspace: str | None) -> "Workspace":
|
|
392
437
|
import yaml
|
|
@@ -891,14 +936,42 @@ class runopt:
|
|
|
891
936
|
Represents the metadata about the specific run option
|
|
892
937
|
"""
|
|
893
938
|
|
|
939
|
+
class AutoAlias(IntEnum):
|
|
940
|
+
snake_case = 0x1
|
|
941
|
+
SNAKE_CASE = 0x2
|
|
942
|
+
camelCase = 0x4
|
|
943
|
+
|
|
944
|
+
@staticmethod
|
|
945
|
+
def convert_to_camel_case(alias: str) -> str:
|
|
946
|
+
words = re.split(r"[_\-\s]+|(?<=[a-z])(?=[A-Z])", alias)
|
|
947
|
+
words = [w for w in words if w] # Remove empty strings
|
|
948
|
+
if not words:
|
|
949
|
+
return ""
|
|
950
|
+
return words[0].lower() + "".join(w.capitalize() for w in words[1:])
|
|
951
|
+
|
|
952
|
+
@staticmethod
|
|
953
|
+
def convert_to_snake_case(alias: str) -> str:
|
|
954
|
+
alias = re.sub(r"[-\s]+", "_", alias)
|
|
955
|
+
alias = re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", alias)
|
|
956
|
+
alias = re.sub(r"([A-Z]+)([A-Z][a-z])", r"\1_\2", alias)
|
|
957
|
+
return alias.lower()
|
|
958
|
+
|
|
959
|
+
@staticmethod
|
|
960
|
+
def convert_to_const_case(alias: str) -> str:
|
|
961
|
+
return runopt.AutoAlias.convert_to_snake_case(alias).upper()
|
|
962
|
+
|
|
894
963
|
class alias(str):
|
|
895
964
|
pass
|
|
896
965
|
|
|
966
|
+
class deprecated(str):
|
|
967
|
+
pass
|
|
968
|
+
|
|
897
969
|
default: CfgVal
|
|
898
970
|
opt_type: Type[CfgVal]
|
|
899
971
|
is_required: bool
|
|
900
972
|
help: str
|
|
901
|
-
aliases:
|
|
973
|
+
aliases: set[alias] | None = None
|
|
974
|
+
deprecated_aliases: set[deprecated] | None = None
|
|
902
975
|
|
|
903
976
|
@property
|
|
904
977
|
def is_type_list_of_str(self) -> bool:
|
|
@@ -990,7 +1063,7 @@ class runopts:
|
|
|
990
1063
|
|
|
991
1064
|
def __init__(self) -> None:
|
|
992
1065
|
self._opts: Dict[str, runopt] = {}
|
|
993
|
-
self._alias_to_key: dict[
|
|
1066
|
+
self._alias_to_key: dict[str, str] = {}
|
|
994
1067
|
|
|
995
1068
|
def __iter__(self) -> Iterator[Tuple[str, runopt]]:
|
|
996
1069
|
return self._opts.items().__iter__()
|
|
@@ -1044,12 +1117,24 @@ class runopts:
|
|
|
1044
1117
|
val = resolved_cfg.get(cfg_key)
|
|
1045
1118
|
resolved_name = None
|
|
1046
1119
|
aliases = runopt.aliases or []
|
|
1120
|
+
deprecated_aliases = runopt.deprecated_aliases or []
|
|
1047
1121
|
if val is None:
|
|
1048
1122
|
for alias in aliases:
|
|
1049
1123
|
val = resolved_cfg.get(alias)
|
|
1050
1124
|
if alias in cfg or val is not None:
|
|
1051
1125
|
resolved_name = alias
|
|
1052
1126
|
break
|
|
1127
|
+
for alias in deprecated_aliases:
|
|
1128
|
+
val = resolved_cfg.get(alias)
|
|
1129
|
+
if val is not None:
|
|
1130
|
+
resolved_name = alias
|
|
1131
|
+
use_instead = self._alias_to_key.get(alias)
|
|
1132
|
+
warnings.warn(
|
|
1133
|
+
f"Run option `{alias}` is deprecated, use `{use_instead}` instead",
|
|
1134
|
+
UserWarning,
|
|
1135
|
+
stacklevel=2,
|
|
1136
|
+
)
|
|
1137
|
+
break
|
|
1053
1138
|
else:
|
|
1054
1139
|
resolved_name = cfg_key
|
|
1055
1140
|
for alias in aliases:
|
|
@@ -1172,23 +1257,51 @@ class runopts:
|
|
|
1172
1257
|
cfg[key] = val
|
|
1173
1258
|
return cfg
|
|
1174
1259
|
|
|
1260
|
+
def _generate_aliases(
|
|
1261
|
+
self, auto_alias: int, aliases: set[str]
|
|
1262
|
+
) -> set[runopt.alias]:
|
|
1263
|
+
generated_aliases = set()
|
|
1264
|
+
for alias in aliases:
|
|
1265
|
+
if auto_alias & runopt.AutoAlias.camelCase:
|
|
1266
|
+
generated_aliases.add(runopt.AutoAlias.convert_to_camel_case(alias))
|
|
1267
|
+
if auto_alias & runopt.AutoAlias.snake_case:
|
|
1268
|
+
generated_aliases.add(runopt.AutoAlias.convert_to_snake_case(alias))
|
|
1269
|
+
if auto_alias & runopt.AutoAlias.SNAKE_CASE:
|
|
1270
|
+
generated_aliases.add(runopt.AutoAlias.convert_to_const_case(alias))
|
|
1271
|
+
return generated_aliases
|
|
1272
|
+
|
|
1175
1273
|
def _get_primary_key_and_aliases(
|
|
1176
1274
|
self,
|
|
1177
|
-
cfg_key: list[str] | str,
|
|
1178
|
-
) -> tuple[str,
|
|
1275
|
+
cfg_key: list[str | int] | str,
|
|
1276
|
+
) -> tuple[str, set[runopt.alias], set[runopt.deprecated]]:
|
|
1179
1277
|
"""
|
|
1180
1278
|
Returns the primary key and aliases for the given cfg_key.
|
|
1181
1279
|
"""
|
|
1182
1280
|
if isinstance(cfg_key, str):
|
|
1183
|
-
return cfg_key,
|
|
1281
|
+
return cfg_key, set(), set()
|
|
1184
1282
|
|
|
1185
1283
|
if len(cfg_key) == 0:
|
|
1186
1284
|
raise ValueError("cfg_key must be a non-empty list")
|
|
1285
|
+
|
|
1286
|
+
if isinstance(cfg_key[0], runopt.alias) or isinstance(
|
|
1287
|
+
cfg_key[0], runopt.deprecated
|
|
1288
|
+
):
|
|
1289
|
+
warnings.warn(
|
|
1290
|
+
"The main name of the run option should be the head of the list.",
|
|
1291
|
+
UserWarning,
|
|
1292
|
+
stacklevel=2,
|
|
1293
|
+
)
|
|
1187
1294
|
primary_key = None
|
|
1188
|
-
|
|
1295
|
+
auto_alias = 0x0
|
|
1296
|
+
aliases = set[runopt.alias]()
|
|
1297
|
+
deprecated_aliases = set[runopt.deprecated]()
|
|
1189
1298
|
for name in cfg_key:
|
|
1190
1299
|
if isinstance(name, runopt.alias):
|
|
1191
|
-
aliases.
|
|
1300
|
+
aliases.add(name)
|
|
1301
|
+
elif isinstance(name, runopt.deprecated):
|
|
1302
|
+
deprecated_aliases.add(name)
|
|
1303
|
+
elif isinstance(name, int):
|
|
1304
|
+
auto_alias = auto_alias | name
|
|
1192
1305
|
else:
|
|
1193
1306
|
if primary_key is not None:
|
|
1194
1307
|
raise ValueError(
|
|
@@ -1199,11 +1312,17 @@ class runopts:
|
|
|
1199
1312
|
raise ValueError(
|
|
1200
1313
|
"Missing cfg_key. Please provide one other than the aliases."
|
|
1201
1314
|
)
|
|
1202
|
-
|
|
1315
|
+
if auto_alias != 0x0:
|
|
1316
|
+
aliases_to_generate_for = aliases | {primary_key}
|
|
1317
|
+
additional_aliases = self._generate_aliases(
|
|
1318
|
+
auto_alias, aliases_to_generate_for
|
|
1319
|
+
)
|
|
1320
|
+
aliases.update(additional_aliases)
|
|
1321
|
+
return primary_key, aliases, deprecated_aliases
|
|
1203
1322
|
|
|
1204
1323
|
def add(
|
|
1205
1324
|
self,
|
|
1206
|
-
cfg_key: str | list[str],
|
|
1325
|
+
cfg_key: str | list[str | int],
|
|
1207
1326
|
type_: Type[CfgVal],
|
|
1208
1327
|
help: str,
|
|
1209
1328
|
default: CfgVal = None,
|
|
@@ -1214,7 +1333,9 @@ class runopts:
|
|
|
1214
1333
|
value (if any). If the ``default`` is not specified then this option
|
|
1215
1334
|
is a required option.
|
|
1216
1335
|
"""
|
|
1217
|
-
primary_key, aliases = self._get_primary_key_and_aliases(
|
|
1336
|
+
primary_key, aliases, deprecated_aliases = self._get_primary_key_and_aliases(
|
|
1337
|
+
cfg_key
|
|
1338
|
+
)
|
|
1218
1339
|
if required and default is not None:
|
|
1219
1340
|
raise ValueError(
|
|
1220
1341
|
f"Required option: {cfg_key} must not specify default value. Given: {default}"
|
|
@@ -1225,9 +1346,11 @@ class runopts:
|
|
|
1225
1346
|
f"Option: {cfg_key}, must be of type: {type_}."
|
|
1226
1347
|
f" Given: {default} ({type(default).__name__})"
|
|
1227
1348
|
)
|
|
1228
|
-
opt = runopt(default, type_, required, help, aliases)
|
|
1349
|
+
opt = runopt(default, type_, required, help, aliases, deprecated_aliases)
|
|
1229
1350
|
for alias in aliases:
|
|
1230
1351
|
self._alias_to_key[alias] = primary_key
|
|
1352
|
+
for deprecated_alias in deprecated_aliases:
|
|
1353
|
+
self._alias_to_key[deprecated_alias] = primary_key
|
|
1231
1354
|
self._opts[primary_key] = opt
|
|
1232
1355
|
|
|
1233
1356
|
def update(self, other: "runopts") -> None:
|
torchx/workspace/api.py
CHANGED
|
@@ -8,26 +8,17 @@
|
|
|
8
8
|
|
|
9
9
|
import abc
|
|
10
10
|
import fnmatch
|
|
11
|
+
import logging
|
|
11
12
|
import posixpath
|
|
12
|
-
import shutil
|
|
13
13
|
import tempfile
|
|
14
14
|
import warnings
|
|
15
15
|
from dataclasses import dataclass
|
|
16
|
-
from
|
|
17
|
-
from typing import (
|
|
18
|
-
Any,
|
|
19
|
-
Dict,
|
|
20
|
-
Generic,
|
|
21
|
-
Iterable,
|
|
22
|
-
Mapping,
|
|
23
|
-
Tuple,
|
|
24
|
-
TYPE_CHECKING,
|
|
25
|
-
TypeVar,
|
|
26
|
-
Union,
|
|
27
|
-
)
|
|
16
|
+
from typing import Any, Dict, Generic, Iterable, Mapping, Tuple, TYPE_CHECKING, TypeVar
|
|
28
17
|
|
|
29
18
|
from torchx.specs import AppDef, CfgVal, Role, runopts, Workspace
|
|
30
19
|
|
|
20
|
+
logger: logging.Logger = logging.getLogger(__name__)
|
|
21
|
+
|
|
31
22
|
if TYPE_CHECKING:
|
|
32
23
|
from fsspec import AbstractFileSystem
|
|
33
24
|
|
|
@@ -113,45 +104,72 @@ class WorkspaceMixin(abc.ABC, Generic[T]):
|
|
|
113
104
|
"""
|
|
114
105
|
return runopts()
|
|
115
106
|
|
|
116
|
-
def
|
|
107
|
+
def build_workspaces(self, roles: list[Role], cfg: Mapping[str, CfgVal]) -> None:
|
|
108
|
+
"""
|
|
109
|
+
NOTE: this method MUTATES the passed roles!
|
|
110
|
+
|
|
111
|
+
Builds the workspaces (if any) for each role and updates the role to reflect the built workspace.
|
|
112
|
+
Typically ``role.image`` is updated with the newly built image that reflects the local workspace.
|
|
113
|
+
Some workspace implementations may add extra environment variables to make it easier for other
|
|
114
|
+
parts of the program to access the workspace. For example a ``WORKSPACE_DIR`` env var may be added
|
|
115
|
+
to ``role.env`` that scripts can use to refert to the workspace directory in the container.
|
|
116
|
+
"""
|
|
117
|
+
|
|
118
|
+
build_cache: dict[object, object] = {}
|
|
119
|
+
|
|
120
|
+
for i, role in enumerate(roles):
|
|
121
|
+
if role.workspace:
|
|
122
|
+
old_img = role.image
|
|
123
|
+
self.caching_build_workspace_and_update_role(role, cfg, build_cache)
|
|
124
|
+
|
|
125
|
+
if old_img != role.image:
|
|
126
|
+
logger.info(
|
|
127
|
+
"role[%d]=%s updated with new image to include workspace changes",
|
|
128
|
+
i,
|
|
129
|
+
role.name,
|
|
130
|
+
)
|
|
131
|
+
|
|
132
|
+
def caching_build_workspace_and_update_role(
|
|
117
133
|
self,
|
|
118
134
|
role: Role,
|
|
119
|
-
workspace: Union[Workspace, str],
|
|
120
135
|
cfg: Mapping[str, CfgVal],
|
|
136
|
+
build_cache: dict[object, object],
|
|
121
137
|
) -> None:
|
|
122
138
|
"""
|
|
123
|
-
Same as :py:meth:`build_workspace_and_update_role` but
|
|
124
|
-
|
|
125
|
-
|
|
139
|
+
Same as :py:meth:`build_workspace_and_update_role` but takes
|
|
140
|
+
a ``build_cache`` that can be used to cache pointers to build artifacts
|
|
141
|
+
between building workspace for each role.
|
|
126
142
|
|
|
127
|
-
|
|
143
|
+
This is useful when an appdef has multiple roles where the image and workspace
|
|
144
|
+
of the roles are the same but other attributes such as entrypoint or args are different.
|
|
145
|
+
|
|
146
|
+
NOTE: ``build_cache``'s lifetime is within :py:meth:`build_workspace_and_update_roles`
|
|
147
|
+
NOTE: the workspace implementation decides what to cache
|
|
148
|
+
|
|
149
|
+
Workspace subclasses should prefer implementing this method over
|
|
128
150
|
:py:meth:`build_workspace_and_update_role`.
|
|
129
151
|
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
152
|
+
The default implementation of this method simply calls the (deprecated) non-caching
|
|
153
|
+
:py:meth:`build_workspace_and_update_role` and deals with multi-dir workspaces by
|
|
154
|
+
merging them into a single tmpdir before passing it down.
|
|
133
155
|
|
|
134
|
-
Subclasses can override this method to customize multi-project
|
|
135
|
-
workspace building logic.
|
|
136
156
|
"""
|
|
137
|
-
if isinstance(workspace, Workspace):
|
|
138
|
-
if not workspace.is_unmapped_single_project():
|
|
139
|
-
with tempfile.TemporaryDirectory(suffix="torchx_workspace_") as outdir:
|
|
140
|
-
for src, dst in workspace.projects.items():
|
|
141
|
-
dst_path = Path(outdir) / dst
|
|
142
|
-
if Path(src).is_file():
|
|
143
|
-
shutil.copy2(src, dst_path)
|
|
144
|
-
else: # src is dir
|
|
145
|
-
shutil.copytree(src, dst_path, dirs_exist_ok=True)
|
|
146
|
-
|
|
147
|
-
self.build_workspace_and_update_role(role, outdir, cfg)
|
|
148
|
-
return
|
|
149
|
-
else: # single project workspace with no target mapping (treat like a str workspace)
|
|
150
|
-
workspace = str(workspace)
|
|
151
|
-
|
|
152
|
-
self.build_workspace_and_update_role(role, workspace, cfg)
|
|
153
157
|
|
|
154
|
-
|
|
158
|
+
workspace = role.workspace
|
|
159
|
+
|
|
160
|
+
if not workspace:
|
|
161
|
+
return
|
|
162
|
+
|
|
163
|
+
if workspace.is_unmapped_single_project():
|
|
164
|
+
# single-dir workspace with no target map; no need to copy to a tmp dir
|
|
165
|
+
self.build_workspace_and_update_role(role, str(workspace), cfg)
|
|
166
|
+
else:
|
|
167
|
+
# multi-dirs or single-dir with a target map;
|
|
168
|
+
# copy all dirs to a tmp dir and treat the tmp dir as a single-dir workspace
|
|
169
|
+
with tempfile.TemporaryDirectory(suffix="torchx_workspace_") as outdir:
|
|
170
|
+
workspace.merge_into(outdir)
|
|
171
|
+
self.build_workspace_and_update_role(role, outdir, cfg)
|
|
172
|
+
|
|
155
173
|
def build_workspace_and_update_role(
|
|
156
174
|
self,
|
|
157
175
|
role: Role,
|
|
@@ -159,6 +177,9 @@ class WorkspaceMixin(abc.ABC, Generic[T]):
|
|
|
159
177
|
cfg: Mapping[str, CfgVal],
|
|
160
178
|
) -> None:
|
|
161
179
|
"""
|
|
180
|
+
.. note:: DEPRECATED: Workspace subclasses should implement
|
|
181
|
+
:py:meth:`caching_build_workspace_and_update_role` over this method.
|
|
182
|
+
|
|
162
183
|
Builds the specified ``workspace`` with respect to ``img``
|
|
163
184
|
and updates the ``role`` to reflect the built workspace artifacts.
|
|
164
185
|
In the simplest case, this method builds a new image and updates
|
|
@@ -167,7 +188,7 @@ class WorkspaceMixin(abc.ABC, Generic[T]):
|
|
|
167
188
|
|
|
168
189
|
Note: this method mutates the passed ``role``.
|
|
169
190
|
"""
|
|
170
|
-
|
|
191
|
+
raise NotImplementedError("implement `caching_build_workspace_and_update_role`")
|
|
171
192
|
|
|
172
193
|
def dryrun_push_images(self, app: AppDef, cfg: Mapping[str, CfgVal]) -> T:
|
|
173
194
|
"""
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: torchx-nightly
|
|
3
|
-
Version: 2025.10.
|
|
3
|
+
Version: 2025.10.18
|
|
4
4
|
Summary: TorchX SDK and Components
|
|
5
5
|
Home-page: https://github.com/meta-pytorch/torchx
|
|
6
6
|
Author: TorchX Devs
|
|
@@ -47,11 +47,11 @@ Requires-Dist: pytorch-lightning==2.5.0; extra == "dev"
|
|
|
47
47
|
Requires-Dist: tensorboard==2.14.0; extra == "dev"
|
|
48
48
|
Requires-Dist: sagemaker==2.230.0; extra == "dev"
|
|
49
49
|
Requires-Dist: torch-model-archiver>=0.4.2; extra == "dev"
|
|
50
|
-
Requires-Dist: torch
|
|
50
|
+
Requires-Dist: torch; extra == "dev"
|
|
51
51
|
Requires-Dist: torchmetrics==1.6.3; extra == "dev"
|
|
52
52
|
Requires-Dist: torchserve>=0.10.0; extra == "dev"
|
|
53
|
-
Requires-Dist: torchtext
|
|
54
|
-
Requires-Dist: torchvision
|
|
53
|
+
Requires-Dist: torchtext; extra == "dev"
|
|
54
|
+
Requires-Dist: torchvision; extra == "dev"
|
|
55
55
|
Requires-Dist: typing-extensions; extra == "dev"
|
|
56
56
|
Requires-Dist: ts==0.5.1; extra == "dev"
|
|
57
57
|
Requires-Dist: wheel; extra == "dev"
|
|
@@ -48,7 +48,7 @@ torchx/examples/apps/lightning/profiler.py,sha256=SSSihnwjeUTkBoz0E3qn1b-wbkfUIo
|
|
|
48
48
|
torchx/examples/apps/lightning/train.py,sha256=0wvvshGHvZowePB4LfclXwn40X7i9euM0ReETWBcPSo,6253
|
|
49
49
|
torchx/pipelines/__init__.py,sha256=2MbRVk5xwRjg-d2qPemeXpEhDsocMQumPQ53lsesZAI,606
|
|
50
50
|
torchx/runner/__init__.py,sha256=x8Sz7s_tLxPgJgvWIhK4ju9BNZU61uBFywGwDY6CqJs,315
|
|
51
|
-
torchx/runner/api.py,sha256=
|
|
51
|
+
torchx/runner/api.py,sha256=s-Fh9FJoZ9JiEFXyGVF0vdJuXMQ-Wa2v4yLjvWJKBX8,30134
|
|
52
52
|
torchx/runner/config.py,sha256=SaKOB50d79WaMFPWK8CC4as6UaNFaRGhrBkfajq3KC4,18311
|
|
53
53
|
torchx/runner/events/__init__.py,sha256=cMiNjnr4eUNQ2Nxxtu4nsvN5lu56b-a6nJ-ct3i7DQk,5536
|
|
54
54
|
torchx/runner/events/api.py,sha256=bvxKBAYK8LzbrBNaNLgL1x0aivtfANmWo1EMGOrSR8k,2668
|
|
@@ -57,7 +57,7 @@ torchx/runtime/__init__.py,sha256=Wxje2BryzeQneFu5r6P9JJiEKG-_C9W1CcZ_JNrKT6g,59
|
|
|
57
57
|
torchx/runtime/tracking/__init__.py,sha256=dYnAPnrXYREfPXkpHhdOFkcYIODWEbA13PdD-wLQYBo,3055
|
|
58
58
|
torchx/runtime/tracking/api.py,sha256=SmUQyUKZqG3KlAhT7CJOGqRz1O274E4m63wQeOVq3CU,5472
|
|
59
59
|
torchx/schedulers/__init__.py,sha256=FQN9boQM4mwOD3sK9LZ3GBgw-gJ7Vx4MFj6z6ATQIrc,2211
|
|
60
|
-
torchx/schedulers/api.py,sha256=
|
|
60
|
+
torchx/schedulers/api.py,sha256=smoUv1ocfqsBRmesXbz9i1F86zBOixZ8QHxYmI_MzgQ,14649
|
|
61
61
|
torchx/schedulers/aws_batch_scheduler.py,sha256=-HpjNVhSFBDxZo3cebK-3YEguB49dxoaud2gz30cAVM,29437
|
|
62
62
|
torchx/schedulers/aws_sagemaker_scheduler.py,sha256=flN8GumKE2Dz4X_foAt6Jnvt-ZVojWs6pcyrHwB0hz0,20921
|
|
63
63
|
torchx/schedulers/devices.py,sha256=RjVcu22ZRl_9OKtOtmA1A3vNXgu2qD6A9ST0L0Hsg4I,1734
|
|
@@ -70,7 +70,7 @@ torchx/schedulers/lsf_scheduler.py,sha256=YS6Yel8tXJqLPxbcGz95lZG2nCi36AQXdNDyuB
|
|
|
70
70
|
torchx/schedulers/slurm_scheduler.py,sha256=vypGaCZe61bkyNkqRlK4Iwmk_NaAUQi-DsspaWd6BZw,31873
|
|
71
71
|
torchx/schedulers/streams.py,sha256=8_SLezgnWgfv_zXUsJCUM34-h2dtv25NmZuxEwkzmxw,2007
|
|
72
72
|
torchx/specs/__init__.py,sha256=SXS4r_roOkbbAL-p7EY5fl5ou-AG7S9Ck-zKtRBdHOk,6760
|
|
73
|
-
torchx/specs/api.py,sha256=
|
|
73
|
+
torchx/specs/api.py,sha256=6_Q5SOUqfMRqQdnlvem36dBusHpynp4PCa9uTNwvzwo,52630
|
|
74
74
|
torchx/specs/builders.py,sha256=Ye3of4MupJ-da8vLaX6_-nzGo_FRw1BFpYsX6dAZCNk,13730
|
|
75
75
|
torchx/specs/file_linter.py,sha256=z0c4mKJv47BWiPaWCdUM0A8kHwnj4b1s7oTmESuD9Tc,14407
|
|
76
76
|
torchx/specs/finder.py,sha256=gWQNEFrLYqrZoI0gMMhQ70YAC4sxqS0ZFpoWAmcVi44,17438
|
|
@@ -99,12 +99,12 @@ torchx/util/shlex.py,sha256=eXEKu8KC3zIcd8tEy9_s8Ds5oma8BORr-0VGWNpG2dk,463
|
|
|
99
99
|
torchx/util/strings.py,sha256=7Ef1loz2IYMrzeJ6Lewywi5cBIc3X3g7lSPbT1Tn_z4,664
|
|
100
100
|
torchx/util/types.py,sha256=E9dxAWQnsJkIDuHtg-poeOJ4etucSI_xP_Z5kNJX8uI,9229
|
|
101
101
|
torchx/workspace/__init__.py,sha256=FqN8AN4VhR1C_SBY10MggQvNZmyanbbuPuE-JCjkyUY,798
|
|
102
|
-
torchx/workspace/api.py,sha256=
|
|
102
|
+
torchx/workspace/api.py,sha256=UESQ4qgxXjsb6Y1wP9OGv2ixaFgaTs3SqghmNuOJIZM,10235
|
|
103
103
|
torchx/workspace/dir_workspace.py,sha256=npNW_IjUZm_yS5r-8hrRkH46ndDd9a_eApT64m1S1T4,2268
|
|
104
104
|
torchx/workspace/docker_workspace.py,sha256=PFu2KQNVC-0p2aKJ-W_BKA9ZOmXdCY2ABEkCExp3udQ,10269
|
|
105
|
-
torchx_nightly-2025.10.
|
|
106
|
-
torchx_nightly-2025.10.
|
|
107
|
-
torchx_nightly-2025.10.
|
|
108
|
-
torchx_nightly-2025.10.
|
|
109
|
-
torchx_nightly-2025.10.
|
|
110
|
-
torchx_nightly-2025.10.
|
|
105
|
+
torchx_nightly-2025.10.18.dist-info/LICENSE,sha256=WVHfXhFC0Ia8LTKt_nJVYobdqTJVg_4J3Crrfm2A8KQ,1721
|
|
106
|
+
torchx_nightly-2025.10.18.dist-info/METADATA,sha256=uFXR5-5aPu-A24rIhwXi5QhcJUfQEWX0Vz5Ohb66JcA,5046
|
|
107
|
+
torchx_nightly-2025.10.18.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
|
|
108
|
+
torchx_nightly-2025.10.18.dist-info/entry_points.txt,sha256=T328AMXeKI3JZnnxfkEew2ZcMN1oQDtkXjMz7lkV-P4,169
|
|
109
|
+
torchx_nightly-2025.10.18.dist-info/top_level.txt,sha256=pxew3bc2gsiViS0zADs0jb6kC5v8o_Yy_85fhHj_J1A,7
|
|
110
|
+
torchx_nightly-2025.10.18.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
{torchx_nightly-2025.10.16.dist-info → torchx_nightly-2025.10.18.dist-info}/entry_points.txt
RENAMED
|
File without changes
|
|
File without changes
|