yellowdog-python-examples 7.19.7__tar.gz → 7.20.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/PKG-INFO +9 -3
  2. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/README.md +3 -2
  3. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/pyproject.toml +7 -1
  4. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/requirements.txt +1 -1
  5. yellowdog_python_examples-7.20.0/yellowdog_cli/__init__.py +1 -0
  6. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/compare.py +147 -167
  7. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/create.py +4 -0
  8. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/list.py +27 -37
  9. yellowdog_python_examples-7.20.0/yellowdog_cli/resize.py +140 -0
  10. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/show.py +6 -1
  11. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/submit.py +2 -1
  12. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/terminate.py +27 -31
  13. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/utils/args.py +3 -10
  14. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/utils/cloudwizard_aws.py +1 -0
  15. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/utils/entity_utils.py +73 -5
  16. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/utils/items.py +2 -0
  17. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/utils/printing.py +24 -16
  18. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/utils/property_names.py +3 -1
  19. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/utils/settings.py +3 -0
  20. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/utils/submit_utils.py +31 -2
  21. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_python_examples.egg-info/PKG-INFO +9 -3
  22. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_python_examples.egg-info/requires.txt +1 -1
  23. yellowdog_python_examples-7.19.7/yellowdog_cli/__init__.py +0 -1
  24. yellowdog_python_examples-7.19.7/yellowdog_cli/resize.py +0 -144
  25. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/LICENSE +0 -0
  26. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/PYPI_README.md +0 -0
  27. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/setup.cfg +0 -0
  28. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/tests/test_create_remove.py +0 -0
  29. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/tests/test_demos.py +0 -0
  30. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/tests/test_dryruns.py +0 -0
  31. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/tests/test_entrypoints.py +0 -0
  32. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/tests/test_gui.py +0 -0
  33. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/tests/test_list.py +0 -0
  34. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/tests/test_objects.py +0 -0
  35. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/tests/test_variable_processing.py +0 -0
  36. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/abort.py +0 -0
  37. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/admin.py +0 -0
  38. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/boost.py +0 -0
  39. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/cancel.py +0 -0
  40. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/cloudwizard.py +0 -0
  41. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/delete.py +0 -0
  42. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/download.py +0 -0
  43. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/follow.py +0 -0
  44. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/format_json.py +0 -0
  45. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/hold.py +0 -0
  46. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/instantiate.py +0 -0
  47. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/jsonnet2json.py +0 -0
  48. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/provision.py +0 -0
  49. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/remove.py +0 -0
  50. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/shutdown.py +0 -0
  51. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/start.py +0 -0
  52. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/upload.py +0 -0
  53. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/utils/__init__.py +0 -0
  54. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/utils/check_imports.py +0 -0
  55. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/utils/cloudwizard_aws_types.py +0 -0
  56. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/utils/cloudwizard_azure.py +0 -0
  57. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/utils/cloudwizard_common.py +0 -0
  58. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/utils/cloudwizard_gcp.py +0 -0
  59. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/utils/compact_json.py +0 -0
  60. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/utils/config_types.py +0 -0
  61. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/utils/csv_data.py +0 -0
  62. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/utils/follow_utils.py +0 -0
  63. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/utils/interactive.py +0 -0
  64. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/utils/load_config.py +0 -0
  65. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/utils/load_resources.py +0 -0
  66. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/utils/misc_utils.py +0 -0
  67. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/utils/provision_utils.py +0 -0
  68. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/utils/rich_console_input_fixed.py +0 -0
  69. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/utils/start_hold_common.py +0 -0
  70. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/utils/type_check.py +0 -0
  71. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/utils/upload_utils.py +0 -0
  72. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/utils/validate_properties.py +0 -0
  73. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/utils/variables.py +0 -0
  74. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/utils/wrapper.py +0 -0
  75. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/utils/ydid_utils.py +0 -0
  76. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_cli/version.py +0 -0
  77. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_python_examples.egg-info/SOURCES.txt +0 -0
  78. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_python_examples.egg-info/dependency_links.txt +0 -0
  79. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_python_examples.egg-info/entry_points.txt +0 -0
  80. {yellowdog_python_examples-7.19.7 → yellowdog_python_examples-7.20.0}/yellowdog_python_examples.egg-info/top_level.txt +0 -0
@@ -1,13 +1,19 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: yellowdog-python-examples
3
- Version: 7.19.7
3
+ Version: 7.20.0
4
4
  Summary: Python CLI commands using the YellowDog Python SDK
5
5
  Author-email: YellowDog Limited <support@yellowdog.co>
6
+ License-Expression: Apache-2.0
6
7
  Project-URL: Homepage, https://github.com/yellowdog/python-examples
7
8
  Classifier: Programming Language :: Python
8
9
  Classifier: Programming Language :: Python :: 3
9
10
  Classifier: Programming Language :: Python :: 3.7
10
- Classifier: License :: OSI Approved :: Apache Software License
11
+ Classifier: Programming Language :: Python :: 3.8
12
+ Classifier: Programming Language :: Python :: 3.9
13
+ Classifier: Programming Language :: Python :: 3.10
14
+ Classifier: Programming Language :: Python :: 3.11
15
+ Classifier: Programming Language :: Python :: 3.12
16
+ Classifier: Programming Language :: Python :: 3.13
11
17
  Classifier: Operating System :: OS Independent
12
18
  Classifier: Development Status :: 4 - Beta
13
19
  Requires-Python: >=3.7
@@ -20,7 +26,7 @@ Requires-Dist: requests
20
26
  Requires-Dist: rich==13.9.4
21
27
  Requires-Dist: tabulate>=0.9.0
22
28
  Requires-Dist: toml
23
- Requires-Dist: yellowdog-sdk>=10.0.0
29
+ Requires-Dist: yellowdog-sdk>=11.2.0
24
30
  Provides-Extra: jsonnet
25
31
  Requires-Dist: jsonnet; extra == "jsonnet"
26
32
  Provides-Extra: cloudwizard
@@ -602,7 +602,8 @@ All properties are optional except for **`taskType`** (or **`taskTypes`**).
602
602
  | `completedTaskTtl` | The time (in minutes) to live for completed Tasks. If set, Tasks that have been completed for longer than this period will be deleted. E.g.: `10.0`. | Yes | Yes | Yes | |
603
603
  | `csvFile` | The name of the CSV file used to derive Task data. An alternative to `csvFiles` that can be used when there's only a single CSV file. E.g. `"file.csv"`. | Yes | | | |
604
604
  | `csvFiles` | A list of CSV files used to derive Task data. E.g. `["file.csv", "file_2.csv:2]`. | Yes | | | |
605
- | `dependentOn` | The name of another Task Group within the same Work Requirement that must be successfully completed before the Task Group is started. E.g. `"task_group_1"`. | | | Yes | |
605
+ | `dependencies` | The names of other Task Groups within the same Work Requirement that must be successfully completed before the Task Group is started. E.g. `["task_group_1", "task_group_2"]`. | | | Yes | |
606
+ | `dependentOn` | [Deprecated in favour of `dependencies`] The name of another Task Group within the same Work Requirement that must be successfully completed before the Task Group is started. E.g. `"task_group_1"`. | | | Yes | |
606
607
  | `dockerEnvironment` | The environment to be passed to a Docker container. Only used by the `docker` Task Type. E.g., JSON: `{"VAR_1": "abc"}`, TOML: `{VAR_1 = "abc", VAR_2 = "def"}`. | Yes | Yes | Yes | Yes |
607
608
  | `dockerOptions` | Additional options to be passed to the docker run command. Only used by the `docker` Task Type. E.g.,`["--runtime nvidia, "--gpus all"]`. | Yes | Yes | Yes | Yes |
608
609
  | `dockerPassword` | The password for the Docker container registry; only used by the `docker` Task Type. E,g., `"my_password"`. | Yes | Yes | Yes | Yes |
@@ -1017,7 +1018,7 @@ Showing all possible properties at the Task Group level:
1017
1018
  },
1018
1019
  {
1019
1020
  "name": "second-task-group",
1020
- "dependentOn": "first-task-group",
1021
+ "dependencies": ["first-task-group"],
1021
1022
  "tasks": [
1022
1023
  {}
1023
1024
  ]
@@ -9,12 +9,18 @@
9
9
  "Programming Language :: Python",
10
10
  "Programming Language :: Python :: 3",
11
11
  "Programming Language :: Python :: 3.7",
12
- "License :: OSI Approved :: Apache Software License",
12
+ "Programming Language :: Python :: 3.8",
13
+ "Programming Language :: Python :: 3.9",
14
+ "Programming Language :: Python :: 3.10",
15
+ "Programming Language :: Python :: 3.11",
16
+ "Programming Language :: Python :: 3.12",
17
+ "Programming Language :: Python :: 3.13",
13
18
  "Operating System :: OS Independent",
14
19
  "Development Status :: 4 - Beta",
15
20
  ]
16
21
  requires-python = ">=3.7"
17
22
  dynamic = ["version", "dependencies"]
23
+ license = "Apache-2.0"
18
24
 
19
25
  [[project.authors]]
20
26
  name = "YellowDog Limited"
@@ -5,4 +5,4 @@ requests
5
5
  rich == 13.9.4
6
6
  tabulate >= 0.9.0
7
7
  toml
8
- yellowdog-sdk >= 10.0.0
8
+ yellowdog-sdk >= 11.2.0
@@ -0,0 +1 @@
1
+ __version__ = "7.20.0"
@@ -1,21 +1,24 @@
1
1
  #!/usr/bin/env python3
2
2
 
3
3
  """
4
- A script to compare work requirements and task groups with worker pools,
5
- and check for matches.
4
+ A script to compare work requirements and task groups with provisioned worker pools,
5
+ and to check for matches.
6
6
  """
7
7
 
8
8
  from dataclasses import dataclass
9
9
  from enum import Enum
10
10
  from functools import cache
11
- from typing import List, Optional
11
+ from typing import List, Optional, Set
12
12
 
13
13
  from tabulate import tabulate
14
14
  from yellowdog_client.model import (
15
+ ComputeRequirement,
16
+ ComputeSource,
15
17
  DoubleRange,
16
18
  Node,
17
19
  NodeSearch,
18
20
  NodeStatus,
21
+ ProvisionedWorkerPool,
19
22
  TaskGroup,
20
23
  WorkerPool,
21
24
  WorkRequirement,
@@ -33,14 +36,19 @@ from yellowdog_cli.utils.ydid_utils import YDIDType, get_ydid_type
33
36
 
34
37
  NONE_STRING = "NONE"
35
38
  EMPTY_STRING = ""
36
- UNKNOWN_STRING = "NOT YET KNOWN"
39
+ UNKNOWN_STRING = "NOT CURRENTLY KNOWN"
40
+
41
+ AWS = "AWS"
42
+ AZURE = "AZURE"
43
+ GCE = "GCE"
44
+ GOOGLE = "GOOGLE"
45
+ OCI = "OCI"
37
46
 
38
47
 
39
48
  class MatchType(Enum):
40
49
  YES = "YES" # Definite match to the worker pool (so far)
41
50
  NO = "NO" # Definite non-match to the worker pool
42
51
  MAYBE = "MAYBE (no Nodes available)" # Possible match to the worker pool; no nodes available
43
- PARTIAL = "PARTIAL (one or more Nodes match, but not all)" # Some nodes in the worker pool match
44
52
 
45
53
 
46
54
  @dataclass
@@ -85,14 +93,14 @@ class MatchReport:
85
93
  self._ram = ram
86
94
  self._vcpus = vcpus
87
95
  self._property_match_list = [
88
- self._worker_tags,
89
- self._namespaces,
90
96
  self._instance_types,
97
+ self._namespaces,
91
98
  self._providers,
92
99
  self._ram,
93
100
  self._regions,
94
101
  self._task_types,
95
102
  self._vcpus,
103
+ self._worker_tags,
96
104
  ]
97
105
 
98
106
  @cache
@@ -106,12 +114,6 @@ class MatchReport:
106
114
  elif any(p.match == MatchType.NO for p in self._property_match_list):
107
115
  return MatchType.NO
108
116
 
109
- elif all(
110
- p.match == MatchType.YES or p.match == MatchType.PARTIAL
111
- for p in self._property_match_list
112
- ):
113
- return MatchType.PARTIAL
114
-
115
117
  elif all(
116
118
  p.match == MatchType.YES or p.match == MatchType.MAYBE
117
119
  for p in self._property_match_list
@@ -119,7 +121,7 @@ class MatchReport:
119
121
  return MatchType.MAYBE
120
122
 
121
123
  # Shouldn't get here
122
- print_warning("Unable to calculate YES/PARTIAL/MAYBE/NO summary")
124
+ print_warning("Unable to calculate YES/MAYBE/NO summary")
123
125
  return MatchType.NO
124
126
 
125
127
  def print_detailed_report(self):
@@ -130,12 +132,10 @@ class MatchReport:
130
132
  match_str = "MATCHING"
131
133
  elif self.summary() == MatchType.MAYBE:
132
134
  match_str = "MAYBE MATCHING"
133
- elif self.summary() == MatchType.PARTIAL:
134
- match_str = "PARTIALLY MATCHING"
135
135
  else:
136
136
  match_str = "NON-MATCHING"
137
137
  print_log(
138
- f"Detailed comparison report for {match_str} ({self.worker_pool_status}) worker pool "
138
+ f"Detailed comparison report for {match_str} ({self.worker_pool_status}) Worker Pool "
139
139
  f"'{self.worker_pool_name}' ({self.worker_pool_id})",
140
140
  override_quiet=True,
141
141
  )
@@ -144,7 +144,7 @@ class MatchReport:
144
144
  header_row = [
145
145
  "Property",
146
146
  "Task Group Run Specification",
147
- "Worker Pool Nodes/Workers",
147
+ "Worker Pool",
148
148
  "Match Status",
149
149
  ]
150
150
  table_rows = []
@@ -179,7 +179,7 @@ class WorkerPools:
179
179
  Populates once for each run of the script.
180
180
  """
181
181
 
182
- def __init__(self, worker_pools: List[WorkerPool]):
182
+ def __init__(self, worker_pools: List[ProvisionedWorkerPool]):
183
183
  self._populated = False
184
184
  self._worker_pools = worker_pools
185
185
 
@@ -215,9 +215,59 @@ class WorkerPools:
215
215
  vcpus=self._match_vcpus(task_group, worker_pool),
216
216
  )
217
217
 
218
+ def _get_providers(self, worker_pool: ProvisionedWorkerPool) -> Set[str]:
219
+ return {
220
+ self._get_provider_from_source(source)
221
+ for source in self._get_cr_from_wp(worker_pool).provisionStrategy.sources
222
+ }
223
+
224
+ def _get_regions(self, worker_pool: ProvisionedWorkerPool) -> Set[str]:
225
+ return {
226
+ source.region
227
+ for source in self._get_cr_from_wp(worker_pool).provisionStrategy.sources
228
+ }
229
+
230
+ def _get_instance_types(self, worker_pool: ProvisionedWorkerPool) -> Set[str]:
231
+ instance_types = set()
232
+ for source in self._get_cr_from_wp(worker_pool).provisionStrategy.sources:
233
+ provider = self._get_provider_from_source(source)
234
+ if provider == AWS:
235
+ instance_types.add(source.instanceType)
236
+ try: # Only for Fleet sources
237
+ for override in source.instanceOverrides:
238
+ instance_types.add(override.instanceType)
239
+ except:
240
+ pass
241
+ # ToDo: Add similar checks for the fleet equivalents
242
+ elif provider == AZURE:
243
+ instance_types.add(source.vmSize)
244
+ elif provider == GOOGLE:
245
+ instance_types.add(source.machineType)
246
+ elif provider == OCI:
247
+ instance_types.add(source.shape)
248
+ return instance_types
249
+
250
+ @staticmethod
251
+ def _get_cr_from_wp(worker_pool: ProvisionedWorkerPool) -> ComputeRequirement:
252
+ return CLIENT.compute_client.get_compute_requirement_by_id(
253
+ worker_pool.computeRequirementId
254
+ )
255
+
256
+ @staticmethod
257
+ def _get_provider_from_source(source: ComputeSource) -> Optional[str]:
258
+ if AWS.lower() in source.type.lower():
259
+ return AWS
260
+ elif AZURE.lower() in source.type.lower():
261
+ return AZURE
262
+ elif GCE.lower() in source.type.lower():
263
+ return GOOGLE
264
+ elif OCI.lower() in source.type.lower():
265
+ return OCI
266
+ return None
267
+
218
268
  @staticmethod
219
269
  def _match_worker_tags(
220
- task_group: TaskGroup, worker_pool: WorkerPool
270
+ task_group: TaskGroup, worker_pool: ProvisionedWorkerPool
221
271
  ) -> PropertyMatch:
222
272
  return PropertyMatch(
223
273
  property_name="Worker Tag(s)",
@@ -244,63 +294,44 @@ class WorkerPools:
244
294
  )
245
295
 
246
296
  def _match_instance_types(
247
- self, task_group: TaskGroup, worker_pool: WorkerPool
297
+ self, task_group: TaskGroup, worker_pool: ProvisionedWorkerPool
248
298
  ) -> PropertyMatch:
249
299
  runspec_instance_types = (
250
300
  set()
251
301
  if task_group.runSpecification.instanceTypes is None
252
302
  else set(task_group.runSpecification.instanceTypes)
253
303
  )
254
- nodes = self._get_all_nodes_in_worker_pool(worker_pool)
255
- node_instance_types = {
256
- node.details.instanceType
257
- for node in nodes
258
- if node.details.instanceType != ""
259
- }
304
+
305
+ worker_pool_instance_types = self._get_instance_types(worker_pool)
260
306
  worker_pool_values = (
261
- EMPTY_STRING
262
- if len(nodes) == 0
263
- else (
264
- ", ".join(node_instance_types)
265
- if len(node_instance_types) > 0
266
- else NONE_STRING
267
- )
307
+ ", ".join(sorted(list(worker_pool_instance_types)))
308
+ if len(worker_pool_instance_types) > 0
309
+ else NONE_STRING
268
310
  )
269
311
 
270
- # Calculate match
271
- matching_nodes_counter = 0
272
- if len(runspec_instance_types) == 0:
312
+ # Calculate match: the instance types in the worker pool must be
313
+ # a subset of those in the run specification
314
+ if (
315
+ len(runspec_instance_types) == 0
316
+ or worker_pool_instance_types <= runspec_instance_types
317
+ ):
273
318
  match_type = MatchType.YES
274
- elif len(nodes) == 0:
275
- match_type = MatchType.MAYBE
276
319
  else:
277
- for node in nodes:
278
- if node.details.instanceType in runspec_instance_types:
279
- matching_nodes_counter += 1
280
- if matching_nodes_counter == 0:
281
- match_type = MatchType.NO
282
- elif matching_nodes_counter < len(nodes):
283
- match_type = MatchType.PARTIAL
284
- else:
285
- match_type = MatchType.YES
320
+ match_type = MatchType.NO
286
321
 
287
322
  return PropertyMatch(
288
323
  property_name="Instance Type(s)",
289
324
  task_group_values=(
290
325
  EMPTY_STRING
291
326
  if task_group.runSpecification.instanceTypes is None
292
- else ", ".join(task_group.runSpecification.instanceTypes)
327
+ else ", ".join(sorted(task_group.runSpecification.instanceTypes))
293
328
  ),
294
329
  worker_pool_values=worker_pool_values,
295
330
  match=match_type,
296
- match_count=(
297
- matching_nodes_counter if len(runspec_instance_types) > 0 else None
298
- ),
299
- total_nodes=len(nodes) if len(runspec_instance_types) > 0 else None,
300
331
  )
301
332
 
302
333
  def _match_task_types(
303
- self, task_group: TaskGroup, worker_pool: WorkerPool
334
+ self, task_group: TaskGroup, worker_pool: ProvisionedWorkerPool
304
335
  ) -> PropertyMatch:
305
336
  runspec_task_types = (
306
337
  set()
@@ -308,149 +339,108 @@ class WorkerPools:
308
339
  else set(task_group.runSpecification.taskTypes)
309
340
  )
310
341
  nodes = self._get_all_nodes_in_worker_pool(worker_pool)
311
- node_task_types = set(
312
- [
313
- task_type
314
- for node in nodes
315
- for task_type in node.details.supportedTaskTypes
316
- ]
317
- )
342
+ if len(nodes) > 0:
343
+ node_task_types = set(nodes[0].details.supportedTaskTypes)
344
+ else:
345
+ node_task_types = set()
318
346
 
319
347
  worker_pool_values = (
320
348
  UNKNOWN_STRING
321
349
  if len(nodes) == 0
322
350
  else (
323
- ", ".join(node_task_types) if len(node_task_types) > 0 else NONE_STRING
351
+ ", ".join(sorted(list(node_task_types)))
352
+ if len(node_task_types) > 0
353
+ else NONE_STRING
324
354
  )
325
355
  )
326
356
 
327
- # Calculate match
328
- matching_node_counter = 0
357
+ # Calculate match: the task types in the worker pool must include
358
+ # all of those in the run specification. The scheduler calculates
359
+ # this based on what the first node reports, but we have to take
360
+ # a node that possibly is not the first.
329
361
  if len(nodes) == 0:
330
362
  match_type = MatchType.MAYBE
363
+ elif runspec_task_types <= node_task_types:
364
+ match_type = MatchType.YES
331
365
  else:
332
- for node in nodes:
333
- if runspec_task_types <= set(node.details.supportedTaskTypes):
334
- matching_node_counter += 1
335
- if matching_node_counter == 0:
336
- match_type = MatchType.NO
337
- elif matching_node_counter < len(nodes):
338
- match_type = MatchType.PARTIAL
339
- else:
340
- match_type = MatchType.YES
366
+ match_type = MatchType.NO
341
367
 
342
368
  return PropertyMatch(
343
369
  property_name="Task Type(s)",
344
370
  task_group_values=(
345
371
  NONE_STRING
346
372
  if task_group.runSpecification.taskTypes is None
347
- else ", ".join(task_group.runSpecification.taskTypes)
373
+ else ", ".join(sorted(task_group.runSpecification.taskTypes))
348
374
  ),
349
375
  worker_pool_values=worker_pool_values,
350
376
  match=match_type,
351
- match_count=matching_node_counter,
352
- total_nodes=len(nodes),
353
377
  )
354
378
 
355
379
  def _match_providers(
356
- self, task_group: TaskGroup, worker_pool: WorkerPool
380
+ self, task_group: TaskGroup, worker_pool: ProvisionedWorkerPool
357
381
  ) -> PropertyMatch:
358
382
  runspec_providers = (
359
383
  set()
360
384
  if task_group.runSpecification.providers is None
361
- else set(task_group.runSpecification.providers)
385
+ else {provider.value for provider in task_group.runSpecification.providers}
362
386
  )
363
- nodes = self._get_all_nodes_in_worker_pool(worker_pool)
364
- node_providers = {node.details.provider.value for node in nodes}
387
+ worker_pool_providers = self._get_providers(worker_pool)
365
388
 
366
- # Calculate match
367
- matching_node_counter = 0
368
- if len(runspec_providers) == 0:
389
+ # Calculate match: the providers in the worker pool must be
390
+ # a subset of those in the run specification
391
+ if len(runspec_providers) == 0 or worker_pool_providers <= runspec_providers:
369
392
  match_type = MatchType.YES
370
- elif len(nodes) == 0:
371
- match_type = MatchType.MAYBE
372
393
  else:
373
- for node in nodes:
374
- if node.details.provider in runspec_providers:
375
- matching_node_counter += 1
376
- if matching_node_counter == 0:
377
- match_type = MatchType.NO
378
- elif matching_node_counter < len(nodes):
379
- match_type = MatchType.PARTIAL
380
- else:
381
- match_type = MatchType.YES
394
+ match_type = MatchType.NO
382
395
 
383
396
  return PropertyMatch(
384
397
  property_name="Provider(s)",
385
398
  task_group_values=(
386
399
  EMPTY_STRING
387
400
  if task_group.runSpecification.providers is None
388
- else ", ".join([x.value for x in task_group.runSpecification.providers])
389
- ),
390
- worker_pool_values=(
391
- UNKNOWN_STRING
392
- if len(nodes) == 0
393
- else (
394
- EMPTY_STRING
395
- if len(node_providers) == 0
396
- else ", ".join(node_providers)
401
+ else ", ".join(
402
+ sorted([x.value for x in task_group.runSpecification.providers])
397
403
  )
398
404
  ),
405
+ worker_pool_values=", ".join(sorted(list(worker_pool_providers))),
399
406
  match=match_type,
400
- match_count=matching_node_counter if len(runspec_providers) > 0 else None,
401
- total_nodes=len(nodes) if len(runspec_providers) > 0 else None,
402
407
  )
403
408
 
404
409
  def _match_regions(
405
- self, task_group: TaskGroup, worker_pool: WorkerPool
410
+ self, task_group: TaskGroup, worker_pool: ProvisionedWorkerPool
406
411
  ) -> PropertyMatch:
407
412
  runspec_regions = (
408
413
  set()
409
414
  if task_group.runSpecification.regions is None
410
415
  else set(task_group.runSpecification.regions)
411
416
  )
412
- nodes = self._get_all_nodes_in_worker_pool(worker_pool)
413
- node_regions = {
414
- node.details.region for node in nodes if node.details.region != ""
415
- }
417
+ worker_pool_regions = self._get_regions(worker_pool)
416
418
 
417
- # Calculate match
418
- matching_node_counter = 0
419
- if len(runspec_regions) == 0:
419
+ # Calculate match: the regions in the worker pool must be
420
+ # a subset of those in the run specification
421
+ if len(runspec_regions) == 0 or worker_pool_regions <= runspec_regions:
420
422
  match_type = MatchType.YES
421
- elif len(nodes) == 0:
422
- match_type = MatchType.MAYBE
423
423
  else:
424
- for node in nodes:
425
- if node.details.region in runspec_regions:
426
- matching_node_counter += 1
427
- if matching_node_counter == 0:
428
- match_type = MatchType.NO
429
- elif matching_node_counter < len(nodes):
430
- match_type = MatchType.PARTIAL
431
- else:
432
- match_type = MatchType.YES
424
+ match_type = MatchType.NO
433
425
 
434
426
  return PropertyMatch(
435
427
  property_name="Region(s)",
436
428
  task_group_values=(
437
429
  EMPTY_STRING
438
430
  if task_group.runSpecification.regions is None
439
- else ", ".join(task_group.runSpecification.regions)
431
+ else ", ".join(sorted(task_group.runSpecification.regions))
440
432
  ),
441
433
  worker_pool_values=(
442
- EMPTY_STRING
443
- if len(nodes) == 0
444
- else ", ".join(node_regions) if len(node_regions) > 0 else NONE_STRING
434
+ ", ".join(sorted(list(worker_pool_regions)))
435
+ if len(worker_pool_regions) > 0
436
+ else NONE_STRING
445
437
  ),
446
438
  match=match_type,
447
- match_count=matching_node_counter if len(runspec_regions) > 0 else None,
448
- total_nodes=len(nodes) if len(runspec_regions) > 0 else None,
449
439
  )
450
440
 
451
441
  @staticmethod
452
442
  def _match_namespaces(
453
- task_group: TaskGroup, worker_pool: WorkerPool
443
+ task_group: TaskGroup, worker_pool: ProvisionedWorkerPool
454
444
  ) -> PropertyMatch:
455
445
  return PropertyMatch(
456
446
  property_name="Namespace(s)",
@@ -471,29 +461,28 @@ class WorkerPools:
471
461
  )
472
462
 
473
463
  def _match_ram(
474
- self, task_group: TaskGroup, worker_pool: WorkerPool
464
+ self, task_group: TaskGroup, worker_pool: ProvisionedWorkerPool
475
465
  ) -> PropertyMatch:
476
466
 
477
467
  nodes = self._get_all_nodes_in_worker_pool(worker_pool)
478
468
  nodes_ram = {node.details.ram for node in nodes}
479
469
 
480
470
  # Calculate match
481
- matching_node_counter = 0
482
471
  if task_group.runSpecification.ram is None:
483
472
  match_type = MatchType.YES
484
473
  elif len(nodes) == 0:
485
474
  match_type = MatchType.MAYBE
486
475
  else:
487
476
  for node in nodes:
488
- if self._check_in_range(
477
+ if not self._check_in_range(
489
478
  node.details.ram, task_group.runSpecification.ram
490
479
  ):
491
- matching_node_counter += 1
492
- if matching_node_counter == 0:
493
- match_type = MatchType.NO
494
- elif matching_node_counter < len(nodes):
495
- match_type = MatchType.PARTIAL
480
+ # If ANY nodes fail to match, the worker
481
+ # pool is not considered a match
482
+ match_type = MatchType.NO
483
+ break
496
484
  else:
485
+ # All current nodes match
497
486
  match_type = MatchType.YES
498
487
 
499
488
  return PropertyMatch(
@@ -509,38 +498,31 @@ class WorkerPools:
509
498
  else ", ".join([str(node_ram) for node_ram in nodes_ram])
510
499
  ),
511
500
  match=match_type,
512
- match_count=(
513
- None
514
- if task_group.runSpecification.ram is None
515
- else matching_node_counter
516
- ),
517
- total_nodes=None if task_group.runSpecification.ram is None else len(nodes),
518
501
  )
519
502
 
520
503
  def _match_vcpus(
521
- self, task_group: TaskGroup, worker_pool: WorkerPool
504
+ self, task_group: TaskGroup, worker_pool: ProvisionedWorkerPool
522
505
  ) -> PropertyMatch:
523
506
 
524
507
  nodes = self._get_all_nodes_in_worker_pool(worker_pool)
525
508
  nodes_vcpus = {node.details.vcpus for node in nodes}
526
509
 
527
510
  # Calculate match
528
- matching_node_counter = 0
529
511
  if task_group.runSpecification.vcpus is None:
530
512
  match_type = MatchType.YES
531
513
  elif len(nodes) == 0:
532
514
  match_type = MatchType.MAYBE
533
515
  else:
534
516
  for node in nodes:
535
- if self._check_in_range(
517
+ if not self._check_in_range(
536
518
  node.details.vcpus, task_group.runSpecification.vcpus
537
519
  ):
538
- matching_node_counter += 1
539
- if matching_node_counter == 0:
540
- match_type = MatchType.NO
541
- elif matching_node_counter < len(nodes):
542
- match_type = MatchType.PARTIAL
520
+ # If ANY nodes fail to match, the worker
521
+ # pool is not considered a match
522
+ match_type = MatchType.NO
523
+ break
543
524
  else:
525
+ # All current nodes match
544
526
  match_type = MatchType.YES
545
527
 
546
528
  return PropertyMatch(
@@ -556,14 +538,6 @@ class WorkerPools:
556
538
  else ", ".join([str(node_vcpus) for node_vcpus in nodes_vcpus])
557
539
  ),
558
540
  match=match_type,
559
- match_count=(
560
- None
561
- if task_group.runSpecification.vcpus is None
562
- else matching_node_counter
563
- ),
564
- total_nodes=(
565
- None if task_group.runSpecification.vcpus is None else len(nodes)
566
- ),
567
541
  )
568
542
 
569
543
  @staticmethod
@@ -636,9 +610,9 @@ def _get_work_requirement_by_id(work_requirement_id) -> WorkRequirement:
636
610
  )
637
611
 
638
612
 
639
- def _get_worker_pool_by_id(worker_pool_id) -> WorkerPool:
613
+ def _get_provisioned_worker_pool_by_id(worker_pool_id) -> ProvisionedWorkerPool:
640
614
  try:
641
- return get_worker_pool_by_id(CLIENT, worker_pool_id)
615
+ worker_pool = get_worker_pool_by_id(CLIENT, worker_pool_id)
642
616
  except Exception as e:
643
617
  if "404" in str(e):
644
618
  raise Exception(f"Work Pool ID '{worker_pool_id}' not found")
@@ -647,6 +621,14 @@ def _get_worker_pool_by_id(worker_pool_id) -> WorkerPool:
647
621
  f"Unable to obtain Worker Pool details for '{worker_pool_id}': {e}"
648
622
  )
649
623
 
624
+ if isinstance(worker_pool, ProvisionedWorkerPool):
625
+ return worker_pool
626
+ else:
627
+ raise Exception(
628
+ f"Worker Pool ID '{worker_pool_id}' is not a Provisioned Worker Pool; "
629
+ "Configured Worker Pools are not supported by 'yd-compare'"
630
+ )
631
+
650
632
 
651
633
  def _compare_task_group(task_group: TaskGroup, worker_pools: WorkerPools):
652
634
  """
@@ -700,13 +682,13 @@ def _compare_task_group(task_group: TaskGroup, worker_pools: WorkerPools):
700
682
  def main():
701
683
 
702
684
  # Worker pools
703
- wp_list: List[WorkerPool] = []
685
+ wp_list: List[ProvisionedWorkerPool] = []
704
686
  for wp_id in ARGS_PARSER.worker_pool_ids:
705
687
  if get_ydid_type(wp_id) != YDIDType.WORKER_POOL:
706
688
  raise Exception(
707
689
  f"Not a YellowDog Worker Pool ID: '{ARGS_PARSER.wr_or_tg_id}'"
708
690
  )
709
- wp_list.append(_get_worker_pool_by_id(wp_id))
691
+ wp_list.append(_get_provisioned_worker_pool_by_id(wp_id))
710
692
  worker_pools = WorkerPools(wp_list)
711
693
 
712
694
  # Task group
@@ -723,8 +705,6 @@ def main():
723
705
  f"({work_requirement.id})",
724
706
  override_quiet=True,
725
707
  )
726
- if ARGS_PARSER.running_nodes_only:
727
- print_log("Comparing RUNNING nodes in the worker pool(s) only")
728
708
  for task_group in work_requirement.taskGroups:
729
709
  _compare_task_group(task_group, worker_pools)
730
710