yellowdog-python-examples 7.19.7__tar.gz → 7.20.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/PKG-INFO +9 -3
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/README.md +3 -2
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/pyproject.toml +7 -1
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/requirements.txt +1 -1
- yellowdog_python_examples-7.20.0/yellowdog_cli/__init__.py +1 -0
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/compare.py +147 -167
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/create.py +4 -0
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/list.py +27 -37
- yellowdog_python_examples-7.20.0/yellowdog_cli/resize.py +140 -0
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/show.py +6 -1
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/submit.py +2 -1
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/terminate.py +27 -31
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/utils/args.py +3 -10
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/utils/cloudwizard_aws.py +1 -0
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/utils/entity_utils.py +73 -5
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/utils/items.py +2 -0
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/utils/printing.py +24 -16
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/utils/property_names.py +3 -1
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/utils/settings.py +3 -0
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/utils/submit_utils.py +31 -2
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_python_examples.egg-info/PKG-INFO +9 -3
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_python_examples.egg-info/requires.txt +1 -1
- yellowdog_python_examples-7.19.7/yellowdog_cli/__init__.py +0 -1
- yellowdog_python_examples-7.19.7/yellowdog_cli/resize.py +0 -144
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/LICENSE +0 -0
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/PYPI_README.md +0 -0
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/setup.cfg +0 -0
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/tests/test_create_remove.py +0 -0
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/tests/test_demos.py +0 -0
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/tests/test_dryruns.py +0 -0
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/tests/test_entrypoints.py +0 -0
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/tests/test_gui.py +0 -0
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/tests/test_list.py +0 -0
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/tests/test_objects.py +0 -0
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/tests/test_variable_processing.py +0 -0
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/abort.py +0 -0
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/admin.py +0 -0
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/boost.py +0 -0
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/cancel.py +0 -0
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/cloudwizard.py +0 -0
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/delete.py +0 -0
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/download.py +0 -0
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/follow.py +0 -0
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/format_json.py +0 -0
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/hold.py +0 -0
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/instantiate.py +0 -0
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/jsonnet2json.py +0 -0
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/provision.py +0 -0
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/remove.py +0 -0
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/shutdown.py +0 -0
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/start.py +0 -0
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/upload.py +0 -0
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/utils/__init__.py +0 -0
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/utils/check_imports.py +0 -0
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/utils/cloudwizard_aws_types.py +0 -0
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/utils/cloudwizard_azure.py +0 -0
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/utils/cloudwizard_common.py +0 -0
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/utils/cloudwizard_gcp.py +0 -0
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/utils/compact_json.py +0 -0
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/utils/config_types.py +0 -0
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/utils/csv_data.py +0 -0
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/utils/follow_utils.py +0 -0
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/utils/interactive.py +0 -0
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/utils/load_config.py +0 -0
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/utils/load_resources.py +0 -0
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/utils/misc_utils.py +0 -0
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/utils/provision_utils.py +0 -0
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/utils/rich_console_input_fixed.py +0 -0
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/utils/start_hold_common.py +0 -0
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/utils/type_check.py +0 -0
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/utils/upload_utils.py +0 -0
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/utils/validate_properties.py +0 -0
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/utils/variables.py +0 -0
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/utils/wrapper.py +0 -0
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/utils/ydid_utils.py +0 -0
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/version.py +0 -0
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_python_examples.egg-info/SOURCES.txt +0 -0
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_python_examples.egg-info/dependency_links.txt +0 -0
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_python_examples.egg-info/entry_points.txt +0 -0
- {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_python_examples.egg-info/top_level.txt +0 -0
|
@@ -1,13 +1,19 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: yellowdog-python-examples
|
|
3
|
-
Version: 7.
|
|
3
|
+
Version: 7.20.0
|
|
4
4
|
Summary: Python CLI commands using the YellowDog Python SDK
|
|
5
5
|
Author-email: YellowDog Limited <support@yellowdog.co>
|
|
6
|
+
License-Expression: Apache-2.0
|
|
6
7
|
Project-URL: Homepage, https://github.com/yellowdog/python-examples
|
|
7
8
|
Classifier: Programming Language :: Python
|
|
8
9
|
Classifier: Programming Language :: Python :: 3
|
|
9
10
|
Classifier: Programming Language :: Python :: 3.7
|
|
10
|
-
Classifier:
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
11
17
|
Classifier: Operating System :: OS Independent
|
|
12
18
|
Classifier: Development Status :: 4 - Beta
|
|
13
19
|
Requires-Python: >=3.7
|
|
@@ -20,7 +26,7 @@ Requires-Dist: requests
|
|
|
20
26
|
Requires-Dist: rich==13.9.4
|
|
21
27
|
Requires-Dist: tabulate>=0.9.0
|
|
22
28
|
Requires-Dist: toml
|
|
23
|
-
Requires-Dist: yellowdog-sdk>=
|
|
29
|
+
Requires-Dist: yellowdog-sdk>=11.2.0
|
|
24
30
|
Provides-Extra: jsonnet
|
|
25
31
|
Requires-Dist: jsonnet; extra == "jsonnet"
|
|
26
32
|
Provides-Extra: cloudwizard
|
|
@@ -602,7 +602,8 @@ All properties are optional except for **`taskType`** (or **`taskTypes`**).
|
|
|
602
602
|
| `completedTaskTtl` | The time (in minutes) to live for completed Tasks. If set, Tasks that have been completed for longer than this period will be deleted. E.g.: `10.0`. | Yes | Yes | Yes | |
|
|
603
603
|
| `csvFile` | The name of the CSV file used to derive Task data. An alternative to `csvFiles` that can be used when there's only a single CSV file. E.g. `"file.csv"`. | Yes | | | |
|
|
604
604
|
| `csvFiles` | A list of CSV files used to derive Task data. E.g. `["file.csv", "file_2.csv:2]`. | Yes | | | |
|
|
605
|
-
| `
|
|
605
|
+
| `dependencies` | The names of other Task Groups within the same Work Requirement that must be successfully completed before the Task Group is started. E.g. `["task_group_1", "task_group_2"]`. | | | Yes | |
|
|
606
|
+
| `dependentOn` | [Deprecated in favour of `dependencies`] The name of another Task Group within the same Work Requirement that must be successfully completed before the Task Group is started. E.g. `"task_group_1"`. | | | Yes | |
|
|
606
607
|
| `dockerEnvironment` | The environment to be passed to a Docker container. Only used by the `docker` Task Type. E.g., JSON: `{"VAR_1": "abc"}`, TOML: `{VAR_1 = "abc", VAR_2 = "def"}`. | Yes | Yes | Yes | Yes |
|
|
607
608
|
| `dockerOptions` | Additional options to be passed to the docker run command. Only used by the `docker` Task Type. E.g.,`["--runtime nvidia, "--gpus all"]`. | Yes | Yes | Yes | Yes |
|
|
608
609
|
| `dockerPassword` | The password for the Docker container registry; only used by the `docker` Task Type. E,g., `"my_password"`. | Yes | Yes | Yes | Yes |
|
|
@@ -1017,7 +1018,7 @@ Showing all possible properties at the Task Group level:
|
|
|
1017
1018
|
},
|
|
1018
1019
|
{
|
|
1019
1020
|
"name": "second-task-group",
|
|
1020
|
-
"
|
|
1021
|
+
"dependencies": ["first-task-group"],
|
|
1021
1022
|
"tasks": [
|
|
1022
1023
|
{}
|
|
1023
1024
|
]
|
|
@@ -9,12 +9,18 @@
|
|
|
9
9
|
"Programming Language :: Python",
|
|
10
10
|
"Programming Language :: Python :: 3",
|
|
11
11
|
"Programming Language :: Python :: 3.7",
|
|
12
|
-
"
|
|
12
|
+
"Programming Language :: Python :: 3.8",
|
|
13
|
+
"Programming Language :: Python :: 3.9",
|
|
14
|
+
"Programming Language :: Python :: 3.10",
|
|
15
|
+
"Programming Language :: Python :: 3.11",
|
|
16
|
+
"Programming Language :: Python :: 3.12",
|
|
17
|
+
"Programming Language :: Python :: 3.13",
|
|
13
18
|
"Operating System :: OS Independent",
|
|
14
19
|
"Development Status :: 4 - Beta",
|
|
15
20
|
]
|
|
16
21
|
requires-python = ">=3.7"
|
|
17
22
|
dynamic = ["version", "dependencies"]
|
|
23
|
+
license = "Apache-2.0"
|
|
18
24
|
|
|
19
25
|
[[project.authors]]
|
|
20
26
|
name = "YellowDog Limited"
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "7.20.0"
|
{yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/compare.py
RENAMED
|
@@ -1,21 +1,24 @@
|
|
|
1
1
|
#!/usr/bin/env python3
|
|
2
2
|
|
|
3
3
|
"""
|
|
4
|
-
A script to compare work requirements and task groups with worker pools,
|
|
5
|
-
and check for matches.
|
|
4
|
+
A script to compare work requirements and task groups with provisioned worker pools,
|
|
5
|
+
and to check for matches.
|
|
6
6
|
"""
|
|
7
7
|
|
|
8
8
|
from dataclasses import dataclass
|
|
9
9
|
from enum import Enum
|
|
10
10
|
from functools import cache
|
|
11
|
-
from typing import List, Optional
|
|
11
|
+
from typing import List, Optional, Set
|
|
12
12
|
|
|
13
13
|
from tabulate import tabulate
|
|
14
14
|
from yellowdog_client.model import (
|
|
15
|
+
ComputeRequirement,
|
|
16
|
+
ComputeSource,
|
|
15
17
|
DoubleRange,
|
|
16
18
|
Node,
|
|
17
19
|
NodeSearch,
|
|
18
20
|
NodeStatus,
|
|
21
|
+
ProvisionedWorkerPool,
|
|
19
22
|
TaskGroup,
|
|
20
23
|
WorkerPool,
|
|
21
24
|
WorkRequirement,
|
|
@@ -33,14 +36,19 @@ from yellowdog_cli.utils.ydid_utils import YDIDType, get_ydid_type
|
|
|
33
36
|
|
|
34
37
|
NONE_STRING = "NONE"
|
|
35
38
|
EMPTY_STRING = ""
|
|
36
|
-
UNKNOWN_STRING = "NOT
|
|
39
|
+
UNKNOWN_STRING = "NOT CURRENTLY KNOWN"
|
|
40
|
+
|
|
41
|
+
AWS = "AWS"
|
|
42
|
+
AZURE = "AZURE"
|
|
43
|
+
GCE = "GCE"
|
|
44
|
+
GOOGLE = "GOOGLE"
|
|
45
|
+
OCI = "OCI"
|
|
37
46
|
|
|
38
47
|
|
|
39
48
|
class MatchType(Enum):
|
|
40
49
|
YES = "YES" # Definite match to the worker pool (so far)
|
|
41
50
|
NO = "NO" # Definite non-match to the worker pool
|
|
42
51
|
MAYBE = "MAYBE (no Nodes available)" # Possible match to the worker pool; no nodes available
|
|
43
|
-
PARTIAL = "PARTIAL (one or more Nodes match, but not all)" # Some nodes in the worker pool match
|
|
44
52
|
|
|
45
53
|
|
|
46
54
|
@dataclass
|
|
@@ -85,14 +93,14 @@ class MatchReport:
|
|
|
85
93
|
self._ram = ram
|
|
86
94
|
self._vcpus = vcpus
|
|
87
95
|
self._property_match_list = [
|
|
88
|
-
self._worker_tags,
|
|
89
|
-
self._namespaces,
|
|
90
96
|
self._instance_types,
|
|
97
|
+
self._namespaces,
|
|
91
98
|
self._providers,
|
|
92
99
|
self._ram,
|
|
93
100
|
self._regions,
|
|
94
101
|
self._task_types,
|
|
95
102
|
self._vcpus,
|
|
103
|
+
self._worker_tags,
|
|
96
104
|
]
|
|
97
105
|
|
|
98
106
|
@cache
|
|
@@ -106,12 +114,6 @@ class MatchReport:
|
|
|
106
114
|
elif any(p.match == MatchType.NO for p in self._property_match_list):
|
|
107
115
|
return MatchType.NO
|
|
108
116
|
|
|
109
|
-
elif all(
|
|
110
|
-
p.match == MatchType.YES or p.match == MatchType.PARTIAL
|
|
111
|
-
for p in self._property_match_list
|
|
112
|
-
):
|
|
113
|
-
return MatchType.PARTIAL
|
|
114
|
-
|
|
115
117
|
elif all(
|
|
116
118
|
p.match == MatchType.YES or p.match == MatchType.MAYBE
|
|
117
119
|
for p in self._property_match_list
|
|
@@ -119,7 +121,7 @@ class MatchReport:
|
|
|
119
121
|
return MatchType.MAYBE
|
|
120
122
|
|
|
121
123
|
# Shouldn't get here
|
|
122
|
-
print_warning("Unable to calculate YES/
|
|
124
|
+
print_warning("Unable to calculate YES/MAYBE/NO summary")
|
|
123
125
|
return MatchType.NO
|
|
124
126
|
|
|
125
127
|
def print_detailed_report(self):
|
|
@@ -130,12 +132,10 @@ class MatchReport:
|
|
|
130
132
|
match_str = "MATCHING"
|
|
131
133
|
elif self.summary() == MatchType.MAYBE:
|
|
132
134
|
match_str = "MAYBE MATCHING"
|
|
133
|
-
elif self.summary() == MatchType.PARTIAL:
|
|
134
|
-
match_str = "PARTIALLY MATCHING"
|
|
135
135
|
else:
|
|
136
136
|
match_str = "NON-MATCHING"
|
|
137
137
|
print_log(
|
|
138
|
-
f"Detailed comparison report for {match_str} ({self.worker_pool_status})
|
|
138
|
+
f"Detailed comparison report for {match_str} ({self.worker_pool_status}) Worker Pool "
|
|
139
139
|
f"'{self.worker_pool_name}' ({self.worker_pool_id})",
|
|
140
140
|
override_quiet=True,
|
|
141
141
|
)
|
|
@@ -144,7 +144,7 @@ class MatchReport:
|
|
|
144
144
|
header_row = [
|
|
145
145
|
"Property",
|
|
146
146
|
"Task Group Run Specification",
|
|
147
|
-
"Worker Pool
|
|
147
|
+
"Worker Pool",
|
|
148
148
|
"Match Status",
|
|
149
149
|
]
|
|
150
150
|
table_rows = []
|
|
@@ -179,7 +179,7 @@ class WorkerPools:
|
|
|
179
179
|
Populates once for each run of the script.
|
|
180
180
|
"""
|
|
181
181
|
|
|
182
|
-
def __init__(self, worker_pools: List[
|
|
182
|
+
def __init__(self, worker_pools: List[ProvisionedWorkerPool]):
|
|
183
183
|
self._populated = False
|
|
184
184
|
self._worker_pools = worker_pools
|
|
185
185
|
|
|
@@ -215,9 +215,59 @@ class WorkerPools:
|
|
|
215
215
|
vcpus=self._match_vcpus(task_group, worker_pool),
|
|
216
216
|
)
|
|
217
217
|
|
|
218
|
+
def _get_providers(self, worker_pool: ProvisionedWorkerPool) -> Set[str]:
|
|
219
|
+
return {
|
|
220
|
+
self._get_provider_from_source(source)
|
|
221
|
+
for source in self._get_cr_from_wp(worker_pool).provisionStrategy.sources
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
def _get_regions(self, worker_pool: ProvisionedWorkerPool) -> Set[str]:
|
|
225
|
+
return {
|
|
226
|
+
source.region
|
|
227
|
+
for source in self._get_cr_from_wp(worker_pool).provisionStrategy.sources
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
def _get_instance_types(self, worker_pool: ProvisionedWorkerPool) -> Set[str]:
|
|
231
|
+
instance_types = set()
|
|
232
|
+
for source in self._get_cr_from_wp(worker_pool).provisionStrategy.sources:
|
|
233
|
+
provider = self._get_provider_from_source(source)
|
|
234
|
+
if provider == AWS:
|
|
235
|
+
instance_types.add(source.instanceType)
|
|
236
|
+
try: # Only for Fleet sources
|
|
237
|
+
for override in source.instanceOverrides:
|
|
238
|
+
instance_types.add(override.instanceType)
|
|
239
|
+
except:
|
|
240
|
+
pass
|
|
241
|
+
# ToDo: Add similar checks for the fleet equivalents
|
|
242
|
+
elif provider == AZURE:
|
|
243
|
+
instance_types.add(source.vmSize)
|
|
244
|
+
elif provider == GOOGLE:
|
|
245
|
+
instance_types.add(source.machineType)
|
|
246
|
+
elif provider == OCI:
|
|
247
|
+
instance_types.add(source.shape)
|
|
248
|
+
return instance_types
|
|
249
|
+
|
|
250
|
+
@staticmethod
|
|
251
|
+
def _get_cr_from_wp(worker_pool: ProvisionedWorkerPool) -> ComputeRequirement:
|
|
252
|
+
return CLIENT.compute_client.get_compute_requirement_by_id(
|
|
253
|
+
worker_pool.computeRequirementId
|
|
254
|
+
)
|
|
255
|
+
|
|
256
|
+
@staticmethod
|
|
257
|
+
def _get_provider_from_source(source: ComputeSource) -> Optional[str]:
|
|
258
|
+
if AWS.lower() in source.type.lower():
|
|
259
|
+
return AWS
|
|
260
|
+
elif AZURE.lower() in source.type.lower():
|
|
261
|
+
return AZURE
|
|
262
|
+
elif GCE.lower() in source.type.lower():
|
|
263
|
+
return GOOGLE
|
|
264
|
+
elif OCI.lower() in source.type.lower():
|
|
265
|
+
return OCI
|
|
266
|
+
return None
|
|
267
|
+
|
|
218
268
|
@staticmethod
|
|
219
269
|
def _match_worker_tags(
|
|
220
|
-
task_group: TaskGroup, worker_pool:
|
|
270
|
+
task_group: TaskGroup, worker_pool: ProvisionedWorkerPool
|
|
221
271
|
) -> PropertyMatch:
|
|
222
272
|
return PropertyMatch(
|
|
223
273
|
property_name="Worker Tag(s)",
|
|
@@ -244,63 +294,44 @@ class WorkerPools:
|
|
|
244
294
|
)
|
|
245
295
|
|
|
246
296
|
def _match_instance_types(
|
|
247
|
-
self, task_group: TaskGroup, worker_pool:
|
|
297
|
+
self, task_group: TaskGroup, worker_pool: ProvisionedWorkerPool
|
|
248
298
|
) -> PropertyMatch:
|
|
249
299
|
runspec_instance_types = (
|
|
250
300
|
set()
|
|
251
301
|
if task_group.runSpecification.instanceTypes is None
|
|
252
302
|
else set(task_group.runSpecification.instanceTypes)
|
|
253
303
|
)
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
node.details.instanceType
|
|
257
|
-
for node in nodes
|
|
258
|
-
if node.details.instanceType != ""
|
|
259
|
-
}
|
|
304
|
+
|
|
305
|
+
worker_pool_instance_types = self._get_instance_types(worker_pool)
|
|
260
306
|
worker_pool_values = (
|
|
261
|
-
|
|
262
|
-
if len(
|
|
263
|
-
else
|
|
264
|
-
", ".join(node_instance_types)
|
|
265
|
-
if len(node_instance_types) > 0
|
|
266
|
-
else NONE_STRING
|
|
267
|
-
)
|
|
307
|
+
", ".join(sorted(list(worker_pool_instance_types)))
|
|
308
|
+
if len(worker_pool_instance_types) > 0
|
|
309
|
+
else NONE_STRING
|
|
268
310
|
)
|
|
269
311
|
|
|
270
|
-
# Calculate match
|
|
271
|
-
|
|
272
|
-
if
|
|
312
|
+
# Calculate match: the instance types in the worker pool must be
|
|
313
|
+
# a subset of those in the run specification
|
|
314
|
+
if (
|
|
315
|
+
len(runspec_instance_types) == 0
|
|
316
|
+
or worker_pool_instance_types <= runspec_instance_types
|
|
317
|
+
):
|
|
273
318
|
match_type = MatchType.YES
|
|
274
|
-
elif len(nodes) == 0:
|
|
275
|
-
match_type = MatchType.MAYBE
|
|
276
319
|
else:
|
|
277
|
-
|
|
278
|
-
if node.details.instanceType in runspec_instance_types:
|
|
279
|
-
matching_nodes_counter += 1
|
|
280
|
-
if matching_nodes_counter == 0:
|
|
281
|
-
match_type = MatchType.NO
|
|
282
|
-
elif matching_nodes_counter < len(nodes):
|
|
283
|
-
match_type = MatchType.PARTIAL
|
|
284
|
-
else:
|
|
285
|
-
match_type = MatchType.YES
|
|
320
|
+
match_type = MatchType.NO
|
|
286
321
|
|
|
287
322
|
return PropertyMatch(
|
|
288
323
|
property_name="Instance Type(s)",
|
|
289
324
|
task_group_values=(
|
|
290
325
|
EMPTY_STRING
|
|
291
326
|
if task_group.runSpecification.instanceTypes is None
|
|
292
|
-
else ", ".join(task_group.runSpecification.instanceTypes)
|
|
327
|
+
else ", ".join(sorted(task_group.runSpecification.instanceTypes))
|
|
293
328
|
),
|
|
294
329
|
worker_pool_values=worker_pool_values,
|
|
295
330
|
match=match_type,
|
|
296
|
-
match_count=(
|
|
297
|
-
matching_nodes_counter if len(runspec_instance_types) > 0 else None
|
|
298
|
-
),
|
|
299
|
-
total_nodes=len(nodes) if len(runspec_instance_types) > 0 else None,
|
|
300
331
|
)
|
|
301
332
|
|
|
302
333
|
def _match_task_types(
|
|
303
|
-
self, task_group: TaskGroup, worker_pool:
|
|
334
|
+
self, task_group: TaskGroup, worker_pool: ProvisionedWorkerPool
|
|
304
335
|
) -> PropertyMatch:
|
|
305
336
|
runspec_task_types = (
|
|
306
337
|
set()
|
|
@@ -308,149 +339,108 @@ class WorkerPools:
|
|
|
308
339
|
else set(task_group.runSpecification.taskTypes)
|
|
309
340
|
)
|
|
310
341
|
nodes = self._get_all_nodes_in_worker_pool(worker_pool)
|
|
311
|
-
|
|
312
|
-
[
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
for task_type in node.details.supportedTaskTypes
|
|
316
|
-
]
|
|
317
|
-
)
|
|
342
|
+
if len(nodes) > 0:
|
|
343
|
+
node_task_types = set(nodes[0].details.supportedTaskTypes)
|
|
344
|
+
else:
|
|
345
|
+
node_task_types = set()
|
|
318
346
|
|
|
319
347
|
worker_pool_values = (
|
|
320
348
|
UNKNOWN_STRING
|
|
321
349
|
if len(nodes) == 0
|
|
322
350
|
else (
|
|
323
|
-
", ".join(
|
|
351
|
+
", ".join(sorted(list(node_task_types)))
|
|
352
|
+
if len(node_task_types) > 0
|
|
353
|
+
else NONE_STRING
|
|
324
354
|
)
|
|
325
355
|
)
|
|
326
356
|
|
|
327
|
-
# Calculate match
|
|
328
|
-
|
|
357
|
+
# Calculate match: the task types in the worker pool must include
|
|
358
|
+
# all of those in the run specification. The scheduler calculates
|
|
359
|
+
# this based on what the first node reports, but we have to take
|
|
360
|
+
# a node that possibly is not the first.
|
|
329
361
|
if len(nodes) == 0:
|
|
330
362
|
match_type = MatchType.MAYBE
|
|
363
|
+
elif runspec_task_types <= node_task_types:
|
|
364
|
+
match_type = MatchType.YES
|
|
331
365
|
else:
|
|
332
|
-
|
|
333
|
-
if runspec_task_types <= set(node.details.supportedTaskTypes):
|
|
334
|
-
matching_node_counter += 1
|
|
335
|
-
if matching_node_counter == 0:
|
|
336
|
-
match_type = MatchType.NO
|
|
337
|
-
elif matching_node_counter < len(nodes):
|
|
338
|
-
match_type = MatchType.PARTIAL
|
|
339
|
-
else:
|
|
340
|
-
match_type = MatchType.YES
|
|
366
|
+
match_type = MatchType.NO
|
|
341
367
|
|
|
342
368
|
return PropertyMatch(
|
|
343
369
|
property_name="Task Type(s)",
|
|
344
370
|
task_group_values=(
|
|
345
371
|
NONE_STRING
|
|
346
372
|
if task_group.runSpecification.taskTypes is None
|
|
347
|
-
else ", ".join(task_group.runSpecification.taskTypes)
|
|
373
|
+
else ", ".join(sorted(task_group.runSpecification.taskTypes))
|
|
348
374
|
),
|
|
349
375
|
worker_pool_values=worker_pool_values,
|
|
350
376
|
match=match_type,
|
|
351
|
-
match_count=matching_node_counter,
|
|
352
|
-
total_nodes=len(nodes),
|
|
353
377
|
)
|
|
354
378
|
|
|
355
379
|
def _match_providers(
|
|
356
|
-
self, task_group: TaskGroup, worker_pool:
|
|
380
|
+
self, task_group: TaskGroup, worker_pool: ProvisionedWorkerPool
|
|
357
381
|
) -> PropertyMatch:
|
|
358
382
|
runspec_providers = (
|
|
359
383
|
set()
|
|
360
384
|
if task_group.runSpecification.providers is None
|
|
361
|
-
else
|
|
385
|
+
else {provider.value for provider in task_group.runSpecification.providers}
|
|
362
386
|
)
|
|
363
|
-
|
|
364
|
-
node_providers = {node.details.provider.value for node in nodes}
|
|
387
|
+
worker_pool_providers = self._get_providers(worker_pool)
|
|
365
388
|
|
|
366
|
-
# Calculate match
|
|
367
|
-
|
|
368
|
-
if len(runspec_providers) == 0:
|
|
389
|
+
# Calculate match: the providers in the worker pool must be
|
|
390
|
+
# a subset of those in the run specification
|
|
391
|
+
if len(runspec_providers) == 0 or worker_pool_providers <= runspec_providers:
|
|
369
392
|
match_type = MatchType.YES
|
|
370
|
-
elif len(nodes) == 0:
|
|
371
|
-
match_type = MatchType.MAYBE
|
|
372
393
|
else:
|
|
373
|
-
|
|
374
|
-
if node.details.provider in runspec_providers:
|
|
375
|
-
matching_node_counter += 1
|
|
376
|
-
if matching_node_counter == 0:
|
|
377
|
-
match_type = MatchType.NO
|
|
378
|
-
elif matching_node_counter < len(nodes):
|
|
379
|
-
match_type = MatchType.PARTIAL
|
|
380
|
-
else:
|
|
381
|
-
match_type = MatchType.YES
|
|
394
|
+
match_type = MatchType.NO
|
|
382
395
|
|
|
383
396
|
return PropertyMatch(
|
|
384
397
|
property_name="Provider(s)",
|
|
385
398
|
task_group_values=(
|
|
386
399
|
EMPTY_STRING
|
|
387
400
|
if task_group.runSpecification.providers is None
|
|
388
|
-
else ", ".join(
|
|
389
|
-
|
|
390
|
-
worker_pool_values=(
|
|
391
|
-
UNKNOWN_STRING
|
|
392
|
-
if len(nodes) == 0
|
|
393
|
-
else (
|
|
394
|
-
EMPTY_STRING
|
|
395
|
-
if len(node_providers) == 0
|
|
396
|
-
else ", ".join(node_providers)
|
|
401
|
+
else ", ".join(
|
|
402
|
+
sorted([x.value for x in task_group.runSpecification.providers])
|
|
397
403
|
)
|
|
398
404
|
),
|
|
405
|
+
worker_pool_values=", ".join(sorted(list(worker_pool_providers))),
|
|
399
406
|
match=match_type,
|
|
400
|
-
match_count=matching_node_counter if len(runspec_providers) > 0 else None,
|
|
401
|
-
total_nodes=len(nodes) if len(runspec_providers) > 0 else None,
|
|
402
407
|
)
|
|
403
408
|
|
|
404
409
|
def _match_regions(
|
|
405
|
-
self, task_group: TaskGroup, worker_pool:
|
|
410
|
+
self, task_group: TaskGroup, worker_pool: ProvisionedWorkerPool
|
|
406
411
|
) -> PropertyMatch:
|
|
407
412
|
runspec_regions = (
|
|
408
413
|
set()
|
|
409
414
|
if task_group.runSpecification.regions is None
|
|
410
415
|
else set(task_group.runSpecification.regions)
|
|
411
416
|
)
|
|
412
|
-
|
|
413
|
-
node_regions = {
|
|
414
|
-
node.details.region for node in nodes if node.details.region != ""
|
|
415
|
-
}
|
|
417
|
+
worker_pool_regions = self._get_regions(worker_pool)
|
|
416
418
|
|
|
417
|
-
# Calculate match
|
|
418
|
-
|
|
419
|
-
if len(runspec_regions) == 0:
|
|
419
|
+
# Calculate match: the regions in the worker pool must be
|
|
420
|
+
# a subset of those in the run specification
|
|
421
|
+
if len(runspec_regions) == 0 or worker_pool_regions <= runspec_regions:
|
|
420
422
|
match_type = MatchType.YES
|
|
421
|
-
elif len(nodes) == 0:
|
|
422
|
-
match_type = MatchType.MAYBE
|
|
423
423
|
else:
|
|
424
|
-
|
|
425
|
-
if node.details.region in runspec_regions:
|
|
426
|
-
matching_node_counter += 1
|
|
427
|
-
if matching_node_counter == 0:
|
|
428
|
-
match_type = MatchType.NO
|
|
429
|
-
elif matching_node_counter < len(nodes):
|
|
430
|
-
match_type = MatchType.PARTIAL
|
|
431
|
-
else:
|
|
432
|
-
match_type = MatchType.YES
|
|
424
|
+
match_type = MatchType.NO
|
|
433
425
|
|
|
434
426
|
return PropertyMatch(
|
|
435
427
|
property_name="Region(s)",
|
|
436
428
|
task_group_values=(
|
|
437
429
|
EMPTY_STRING
|
|
438
430
|
if task_group.runSpecification.regions is None
|
|
439
|
-
else ", ".join(task_group.runSpecification.regions)
|
|
431
|
+
else ", ".join(sorted(task_group.runSpecification.regions))
|
|
440
432
|
),
|
|
441
433
|
worker_pool_values=(
|
|
442
|
-
|
|
443
|
-
if len(
|
|
444
|
-
else
|
|
434
|
+
", ".join(sorted(list(worker_pool_regions)))
|
|
435
|
+
if len(worker_pool_regions) > 0
|
|
436
|
+
else NONE_STRING
|
|
445
437
|
),
|
|
446
438
|
match=match_type,
|
|
447
|
-
match_count=matching_node_counter if len(runspec_regions) > 0 else None,
|
|
448
|
-
total_nodes=len(nodes) if len(runspec_regions) > 0 else None,
|
|
449
439
|
)
|
|
450
440
|
|
|
451
441
|
@staticmethod
|
|
452
442
|
def _match_namespaces(
|
|
453
|
-
task_group: TaskGroup, worker_pool:
|
|
443
|
+
task_group: TaskGroup, worker_pool: ProvisionedWorkerPool
|
|
454
444
|
) -> PropertyMatch:
|
|
455
445
|
return PropertyMatch(
|
|
456
446
|
property_name="Namespace(s)",
|
|
@@ -471,29 +461,28 @@ class WorkerPools:
|
|
|
471
461
|
)
|
|
472
462
|
|
|
473
463
|
def _match_ram(
|
|
474
|
-
self, task_group: TaskGroup, worker_pool:
|
|
464
|
+
self, task_group: TaskGroup, worker_pool: ProvisionedWorkerPool
|
|
475
465
|
) -> PropertyMatch:
|
|
476
466
|
|
|
477
467
|
nodes = self._get_all_nodes_in_worker_pool(worker_pool)
|
|
478
468
|
nodes_ram = {node.details.ram for node in nodes}
|
|
479
469
|
|
|
480
470
|
# Calculate match
|
|
481
|
-
matching_node_counter = 0
|
|
482
471
|
if task_group.runSpecification.ram is None:
|
|
483
472
|
match_type = MatchType.YES
|
|
484
473
|
elif len(nodes) == 0:
|
|
485
474
|
match_type = MatchType.MAYBE
|
|
486
475
|
else:
|
|
487
476
|
for node in nodes:
|
|
488
|
-
if self._check_in_range(
|
|
477
|
+
if not self._check_in_range(
|
|
489
478
|
node.details.ram, task_group.runSpecification.ram
|
|
490
479
|
):
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
match_type = MatchType.PARTIAL
|
|
480
|
+
# If ANY nodes fail to match, the worker
|
|
481
|
+
# pool is not considered a match
|
|
482
|
+
match_type = MatchType.NO
|
|
483
|
+
break
|
|
496
484
|
else:
|
|
485
|
+
# All current nodes match
|
|
497
486
|
match_type = MatchType.YES
|
|
498
487
|
|
|
499
488
|
return PropertyMatch(
|
|
@@ -509,38 +498,31 @@ class WorkerPools:
|
|
|
509
498
|
else ", ".join([str(node_ram) for node_ram in nodes_ram])
|
|
510
499
|
),
|
|
511
500
|
match=match_type,
|
|
512
|
-
match_count=(
|
|
513
|
-
None
|
|
514
|
-
if task_group.runSpecification.ram is None
|
|
515
|
-
else matching_node_counter
|
|
516
|
-
),
|
|
517
|
-
total_nodes=None if task_group.runSpecification.ram is None else len(nodes),
|
|
518
501
|
)
|
|
519
502
|
|
|
520
503
|
def _match_vcpus(
|
|
521
|
-
self, task_group: TaskGroup, worker_pool:
|
|
504
|
+
self, task_group: TaskGroup, worker_pool: ProvisionedWorkerPool
|
|
522
505
|
) -> PropertyMatch:
|
|
523
506
|
|
|
524
507
|
nodes = self._get_all_nodes_in_worker_pool(worker_pool)
|
|
525
508
|
nodes_vcpus = {node.details.vcpus for node in nodes}
|
|
526
509
|
|
|
527
510
|
# Calculate match
|
|
528
|
-
matching_node_counter = 0
|
|
529
511
|
if task_group.runSpecification.vcpus is None:
|
|
530
512
|
match_type = MatchType.YES
|
|
531
513
|
elif len(nodes) == 0:
|
|
532
514
|
match_type = MatchType.MAYBE
|
|
533
515
|
else:
|
|
534
516
|
for node in nodes:
|
|
535
|
-
if self._check_in_range(
|
|
517
|
+
if not self._check_in_range(
|
|
536
518
|
node.details.vcpus, task_group.runSpecification.vcpus
|
|
537
519
|
):
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
match_type = MatchType.PARTIAL
|
|
520
|
+
# If ANY nodes fail to match, the worker
|
|
521
|
+
# pool is not considered a match
|
|
522
|
+
match_type = MatchType.NO
|
|
523
|
+
break
|
|
543
524
|
else:
|
|
525
|
+
# All current nodes match
|
|
544
526
|
match_type = MatchType.YES
|
|
545
527
|
|
|
546
528
|
return PropertyMatch(
|
|
@@ -556,14 +538,6 @@ class WorkerPools:
|
|
|
556
538
|
else ", ".join([str(node_vcpus) for node_vcpus in nodes_vcpus])
|
|
557
539
|
),
|
|
558
540
|
match=match_type,
|
|
559
|
-
match_count=(
|
|
560
|
-
None
|
|
561
|
-
if task_group.runSpecification.vcpus is None
|
|
562
|
-
else matching_node_counter
|
|
563
|
-
),
|
|
564
|
-
total_nodes=(
|
|
565
|
-
None if task_group.runSpecification.vcpus is None else len(nodes)
|
|
566
|
-
),
|
|
567
541
|
)
|
|
568
542
|
|
|
569
543
|
@staticmethod
|
|
@@ -636,9 +610,9 @@ def _get_work_requirement_by_id(work_requirement_id) -> WorkRequirement:
|
|
|
636
610
|
)
|
|
637
611
|
|
|
638
612
|
|
|
639
|
-
def
|
|
613
|
+
def _get_provisioned_worker_pool_by_id(worker_pool_id) -> ProvisionedWorkerPool:
|
|
640
614
|
try:
|
|
641
|
-
|
|
615
|
+
worker_pool = get_worker_pool_by_id(CLIENT, worker_pool_id)
|
|
642
616
|
except Exception as e:
|
|
643
617
|
if "404" in str(e):
|
|
644
618
|
raise Exception(f"Work Pool ID '{worker_pool_id}' not found")
|
|
@@ -647,6 +621,14 @@ def _get_worker_pool_by_id(worker_pool_id) -> WorkerPool:
|
|
|
647
621
|
f"Unable to obtain Worker Pool details for '{worker_pool_id}': {e}"
|
|
648
622
|
)
|
|
649
623
|
|
|
624
|
+
if isinstance(worker_pool, ProvisionedWorkerPool):
|
|
625
|
+
return worker_pool
|
|
626
|
+
else:
|
|
627
|
+
raise Exception(
|
|
628
|
+
f"Worker Pool ID '{worker_pool_id}' is not a Provisioned Worker Pool; "
|
|
629
|
+
"Configured Worker Pools are not supported by 'yd-compare'"
|
|
630
|
+
)
|
|
631
|
+
|
|
650
632
|
|
|
651
633
|
def _compare_task_group(task_group: TaskGroup, worker_pools: WorkerPools):
|
|
652
634
|
"""
|
|
@@ -700,13 +682,13 @@ def _compare_task_group(task_group: TaskGroup, worker_pools: WorkerPools):
|
|
|
700
682
|
def main():
|
|
701
683
|
|
|
702
684
|
# Worker pools
|
|
703
|
-
wp_list: List[
|
|
685
|
+
wp_list: List[ProvisionedWorkerPool] = []
|
|
704
686
|
for wp_id in ARGS_PARSER.worker_pool_ids:
|
|
705
687
|
if get_ydid_type(wp_id) != YDIDType.WORKER_POOL:
|
|
706
688
|
raise Exception(
|
|
707
689
|
f"Not a YellowDog Worker Pool ID: '{ARGS_PARSER.wr_or_tg_id}'"
|
|
708
690
|
)
|
|
709
|
-
wp_list.append(
|
|
691
|
+
wp_list.append(_get_provisioned_worker_pool_by_id(wp_id))
|
|
710
692
|
worker_pools = WorkerPools(wp_list)
|
|
711
693
|
|
|
712
694
|
# Task group
|
|
@@ -723,8 +705,6 @@ def main():
|
|
|
723
705
|
f"({work_requirement.id})",
|
|
724
706
|
override_quiet=True,
|
|
725
707
|
)
|
|
726
|
-
if ARGS_PARSER.running_nodes_only:
|
|
727
|
-
print_log("Comparing RUNNING nodes in the worker pool(s) only")
|
|
728
708
|
for task_group in work_requirement.taskGroups:
|
|
729
709
|
_compare_task_group(task_group, worker_pools)
|
|
730
710
|
|