yellowdog-python-examples 8.2.1__py3-none-any.whl → 8.3.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- yellowdog_cli/__init__.py +1 -1
- yellowdog_cli/abort.py +11 -11
- yellowdog_cli/admin.py +2 -2
- yellowdog_cli/application.py +104 -0
- yellowdog_cli/boost.py +3 -3
- yellowdog_cli/cancel.py +9 -9
- yellowdog_cli/cloudwizard.py +4 -4
- yellowdog_cli/compare.py +6 -6
- yellowdog_cli/create.py +56 -54
- yellowdog_cli/delete.py +10 -10
- yellowdog_cli/download.py +15 -15
- yellowdog_cli/finish.py +9 -9
- yellowdog_cli/instantiate.py +19 -17
- yellowdog_cli/list.py +45 -43
- yellowdog_cli/provision.py +28 -28
- yellowdog_cli/remove.py +28 -26
- yellowdog_cli/resize.py +9 -9
- yellowdog_cli/show.py +29 -27
- yellowdog_cli/shutdown.py +9 -9
- yellowdog_cli/submit.py +29 -29
- yellowdog_cli/terminate.py +8 -8
- yellowdog_cli/upload.py +11 -11
- yellowdog_cli/utils/args.py +2 -0
- yellowdog_cli/utils/cloudwizard_aws.py +32 -32
- yellowdog_cli/utils/cloudwizard_azure.py +27 -27
- yellowdog_cli/utils/cloudwizard_common.py +12 -10
- yellowdog_cli/utils/cloudwizard_gcp.py +8 -8
- yellowdog_cli/utils/csv_data.py +7 -7
- yellowdog_cli/utils/entity_utils.py +85 -20
- yellowdog_cli/utils/follow_utils.py +5 -5
- yellowdog_cli/utils/interactive.py +8 -8
- yellowdog_cli/utils/load_config.py +11 -11
- yellowdog_cli/utils/load_resources.py +4 -4
- yellowdog_cli/utils/misc_utils.py +6 -3
- yellowdog_cli/utils/printing.py +10 -9
- yellowdog_cli/utils/provision_utils.py +2 -2
- yellowdog_cli/utils/settings.py +1 -0
- yellowdog_cli/utils/start_hold_common.py +7 -7
- yellowdog_cli/utils/submit_utils.py +5 -5
- yellowdog_cli/utils/upload_utils.py +3 -3
- yellowdog_cli/utils/variables.py +5 -5
- yellowdog_cli/utils/wrapper.py +32 -53
- {yellowdog_python_examples-8.2.1.dist-info → yellowdog_python_examples-8.3.1.dist-info}/METADATA +3 -2
- yellowdog_python_examples-8.3.1.dist-info/RECORD +65 -0
- {yellowdog_python_examples-8.2.1.dist-info → yellowdog_python_examples-8.3.1.dist-info}/entry_points.txt +1 -0
- yellowdog_python_examples-8.2.1.dist-info/RECORD +0 -64
- {yellowdog_python_examples-8.2.1.dist-info → yellowdog_python_examples-8.3.1.dist-info}/WHEEL +0 -0
- {yellowdog_python_examples-8.2.1.dist-info → yellowdog_python_examples-8.3.1.dist-info}/licenses/LICENSE +0 -0
- {yellowdog_python_examples-8.2.1.dist-info → yellowdog_python_examples-8.3.1.dist-info}/top_level.txt +0 -0
|
@@ -19,7 +19,7 @@ from yellowdog_cli.create import create_resources
|
|
|
19
19
|
from yellowdog_cli.remove import remove_resources
|
|
20
20
|
from yellowdog_cli.utils.cloudwizard_common import CommonCloudConfig
|
|
21
21
|
from yellowdog_cli.utils.interactive import confirmed, select
|
|
22
|
-
from yellowdog_cli.utils.printing import print_error,
|
|
22
|
+
from yellowdog_cli.utils.printing import print_error, print_info, print_warning
|
|
23
23
|
from yellowdog_cli.utils.settings import RN_SOURCE_TEMPLATE, RN_STORAGE_CONFIGURATION
|
|
24
24
|
|
|
25
25
|
RESOURCE_PREFIX = "yellowdog-cloudwizard"
|
|
@@ -114,7 +114,7 @@ class AzureConfig(CommonCloudConfig):
|
|
|
114
114
|
"""
|
|
115
115
|
# Keyring creation has to precede the generation of the Azure
|
|
116
116
|
# storage account name
|
|
117
|
-
|
|
117
|
+
print_info(f"Creating YellowDog Keyring '{YD_KEYRING_NAME}'")
|
|
118
118
|
self._create_keyring(keyring_name=YD_KEYRING_NAME)
|
|
119
119
|
self._all_regions = self._get_regions_list()
|
|
120
120
|
self._create_azure_resources()
|
|
@@ -132,12 +132,12 @@ class AzureConfig(CommonCloudConfig):
|
|
|
132
132
|
"""
|
|
133
133
|
Create the required assets in the Azure account, for use with YellowDog.
|
|
134
134
|
"""
|
|
135
|
-
|
|
136
|
-
|
|
135
|
+
print_info("Creating YellowDog resources in the Azure account")
|
|
136
|
+
print_info(
|
|
137
137
|
"Please select the Azure regions in which to create resource groups and"
|
|
138
138
|
" network resources"
|
|
139
139
|
)
|
|
140
|
-
|
|
140
|
+
print_info(
|
|
141
141
|
"*** Note that only the following region(s) contain"
|
|
142
142
|
f" YellowDog base VM images: {AZURE_YD_IMAGE_REGIONS} ***"
|
|
143
143
|
)
|
|
@@ -156,7 +156,7 @@ class AzureConfig(CommonCloudConfig):
|
|
|
156
156
|
"""
|
|
157
157
|
Remove the Cloud Wizard assets in the Azure account.
|
|
158
158
|
"""
|
|
159
|
-
|
|
159
|
+
print_info("Removing all YellowDog-created resources in the Azure account")
|
|
160
160
|
self._remove_resource_groups()
|
|
161
161
|
|
|
162
162
|
def _create_resource_groups_and_network_resources(self):
|
|
@@ -199,12 +199,12 @@ class AzureConfig(CommonCloudConfig):
|
|
|
199
199
|
rg_result = self._resource_client.resource_groups.create_or_update(
|
|
200
200
|
rg_name, {"location": region}
|
|
201
201
|
)
|
|
202
|
-
|
|
202
|
+
print_info(
|
|
203
203
|
f"Created (or updated) Azure resource group '{rg_result.name}' in"
|
|
204
204
|
f" region '{rg_result.location}'"
|
|
205
205
|
)
|
|
206
206
|
if region == self._storage_region and storage_region_added:
|
|
207
|
-
|
|
207
|
+
print_info(
|
|
208
208
|
f"Note: Resource group '{rg_name}' is automatically created to"
|
|
209
209
|
" contain the Azure storage account"
|
|
210
210
|
)
|
|
@@ -244,7 +244,7 @@ class AzureConfig(CommonCloudConfig):
|
|
|
244
244
|
Remove all YellowDog resource groups. Deletion of a resource group
|
|
245
245
|
also deletes any contained resources.
|
|
246
246
|
"""
|
|
247
|
-
|
|
247
|
+
print_info("Removing YellowDog resource groups")
|
|
248
248
|
resource_groups = self._resource_client.resource_groups.list()
|
|
249
249
|
count = 0
|
|
250
250
|
for resource_group in resource_groups:
|
|
@@ -257,7 +257,7 @@ class AzureConfig(CommonCloudConfig):
|
|
|
257
257
|
self._resource_client.resource_groups.begin_delete(
|
|
258
258
|
resource_group.name
|
|
259
259
|
) # Deletion occurs asynchronously unless '.result()' is added
|
|
260
|
-
|
|
260
|
+
print_info(
|
|
261
261
|
"Requested deletion of Azure resource group"
|
|
262
262
|
f" '{resource_group.name}' and all contained resources"
|
|
263
263
|
" (asynchronous operation)"
|
|
@@ -278,9 +278,9 @@ class AzureConfig(CommonCloudConfig):
|
|
|
278
278
|
continue
|
|
279
279
|
|
|
280
280
|
if count == 0:
|
|
281
|
-
|
|
281
|
+
print_info("No Azure resource groups deleted")
|
|
282
282
|
else:
|
|
283
|
-
|
|
283
|
+
print_info(f"{count} Azure resource group(s) deleted")
|
|
284
284
|
|
|
285
285
|
def _remove_resource_group_by_name(self, rg_name: str):
|
|
286
286
|
"""
|
|
@@ -288,7 +288,7 @@ class AzureConfig(CommonCloudConfig):
|
|
|
288
288
|
"""
|
|
289
289
|
try:
|
|
290
290
|
self._resource_client.resource_groups.begin_delete(rg_name)
|
|
291
|
-
|
|
291
|
+
print_info(f"Requested deletion of Azure resource group '{rg_name}'")
|
|
292
292
|
except:
|
|
293
293
|
print_warning(f"Unable to delete Azure resource group '{rg_name}'")
|
|
294
294
|
|
|
@@ -329,7 +329,7 @@ class AzureConfig(CommonCloudConfig):
|
|
|
329
329
|
"address_space": {"address_prefixes": address_prefixes},
|
|
330
330
|
},
|
|
331
331
|
).wait()
|
|
332
|
-
|
|
332
|
+
print_info(
|
|
333
333
|
f"Created (or updated) Azure virtual network '{vnet_name}' with address"
|
|
334
334
|
f" prefixes {address_prefixes}"
|
|
335
335
|
)
|
|
@@ -352,7 +352,7 @@ class AzureConfig(CommonCloudConfig):
|
|
|
352
352
|
id=security_group_name, location=region
|
|
353
353
|
),
|
|
354
354
|
).result()
|
|
355
|
-
|
|
355
|
+
print_info(
|
|
356
356
|
"Created (or updated) Azure network security group"
|
|
357
357
|
f" '{security_group_name}'"
|
|
358
358
|
)
|
|
@@ -387,7 +387,7 @@ class AzureConfig(CommonCloudConfig):
|
|
|
387
387
|
}
|
|
388
388
|
},
|
|
389
389
|
).result()
|
|
390
|
-
|
|
390
|
+
print_info(
|
|
391
391
|
"Added outbound HTTPS rule to Azure security group"
|
|
392
392
|
f" '{security_group_name}'"
|
|
393
393
|
)
|
|
@@ -422,7 +422,7 @@ class AzureConfig(CommonCloudConfig):
|
|
|
422
422
|
},
|
|
423
423
|
},
|
|
424
424
|
).result()
|
|
425
|
-
|
|
425
|
+
print_info(
|
|
426
426
|
f"Created (or updated) Azure subnet '{subnet_name}' with address prefix"
|
|
427
427
|
f" '{address_prefix}'"
|
|
428
428
|
)
|
|
@@ -439,7 +439,7 @@ class AzureConfig(CommonCloudConfig):
|
|
|
439
439
|
"""
|
|
440
440
|
Create the YellowDog resources and save the resource definition file.
|
|
441
441
|
"""
|
|
442
|
-
|
|
442
|
+
print_info("Creating Azure resources in the YellowDog account")
|
|
443
443
|
|
|
444
444
|
for region in self._created_regions:
|
|
445
445
|
name = f"{YD_RESOURCE_PREFIX}-{region}-ondemand"
|
|
@@ -462,7 +462,7 @@ class AzureConfig(CommonCloudConfig):
|
|
|
462
462
|
return
|
|
463
463
|
|
|
464
464
|
# Create Compute Source Templates
|
|
465
|
-
|
|
465
|
+
print_info("Creating YellowDog Compute Source Templates")
|
|
466
466
|
create_resources(self._source_template_resources)
|
|
467
467
|
|
|
468
468
|
# Create Compute Requirement Templates
|
|
@@ -489,7 +489,7 @@ class AzureConfig(CommonCloudConfig):
|
|
|
489
489
|
)
|
|
490
490
|
|
|
491
491
|
# Create namespace configuration (Keyring/Credential creation must come first)
|
|
492
|
-
|
|
492
|
+
print_info(f"Creating YellowDog Namespace Configuration '{self._namespace}'")
|
|
493
493
|
create_resources(
|
|
494
494
|
[
|
|
495
495
|
self._generate_yd_namespace_configuration(
|
|
@@ -545,7 +545,7 @@ class AzureConfig(CommonCloudConfig):
|
|
|
545
545
|
)
|
|
546
546
|
else:
|
|
547
547
|
try:
|
|
548
|
-
|
|
548
|
+
print_info(
|
|
549
549
|
f"Creating Azure storage account '{self._storage_account_name}'"
|
|
550
550
|
)
|
|
551
551
|
self._storage_client.storage_accounts.begin_create(
|
|
@@ -557,7 +557,7 @@ class AzureConfig(CommonCloudConfig):
|
|
|
557
557
|
"sku": {"name": "Standard_LRS"},
|
|
558
558
|
},
|
|
559
559
|
).result()
|
|
560
|
-
|
|
560
|
+
print_info(
|
|
561
561
|
f"Created Azure storage account '{self._storage_account_name}' in"
|
|
562
562
|
f" region '{self._storage_region}'"
|
|
563
563
|
)
|
|
@@ -593,7 +593,7 @@ class AzureConfig(CommonCloudConfig):
|
|
|
593
593
|
self._storage_client.blob_containers.create(
|
|
594
594
|
resource_group_name, self._storage_account_name, STORAGE_BLOB_NAME, {}
|
|
595
595
|
)
|
|
596
|
-
|
|
596
|
+
print_info(f"Created Azure storage blob '{STORAGE_BLOB_NAME}'")
|
|
597
597
|
except Exception as e:
|
|
598
598
|
print_error(
|
|
599
599
|
f"Unable to create Azure storage blob '{STORAGE_BLOB_NAME}': {e}"
|
|
@@ -735,7 +735,7 @@ class AzureConfig(CommonCloudConfig):
|
|
|
735
735
|
security_group_name = self._generate_security_group_name(selected_region)
|
|
736
736
|
security_rule_name = "ssh-inbound"
|
|
737
737
|
if operation == "add-ssh":
|
|
738
|
-
|
|
738
|
+
print_info(
|
|
739
739
|
"Adding inbound SSH rule to Azure security group"
|
|
740
740
|
f" '{security_group_name}'"
|
|
741
741
|
)
|
|
@@ -757,7 +757,7 @@ class AzureConfig(CommonCloudConfig):
|
|
|
757
757
|
}
|
|
758
758
|
},
|
|
759
759
|
).result()
|
|
760
|
-
|
|
760
|
+
print_info(
|
|
761
761
|
"Added inbound SSH rule to Azure security group"
|
|
762
762
|
f" '{security_group_name}'"
|
|
763
763
|
)
|
|
@@ -768,7 +768,7 @@ class AzureConfig(CommonCloudConfig):
|
|
|
768
768
|
)
|
|
769
769
|
|
|
770
770
|
elif operation == "remove-ssh":
|
|
771
|
-
|
|
771
|
+
print_info(
|
|
772
772
|
"Removing inbound SSH rule from Azure security group"
|
|
773
773
|
f" '{security_group_name}'"
|
|
774
774
|
)
|
|
@@ -778,7 +778,7 @@ class AzureConfig(CommonCloudConfig):
|
|
|
778
778
|
network_security_group_name=security_group_name,
|
|
779
779
|
security_rule_name=security_rule_name,
|
|
780
780
|
).result()
|
|
781
|
-
|
|
781
|
+
print_info(
|
|
782
782
|
"Removed inbound SSH rule from Azure security group"
|
|
783
783
|
f" '{security_group_name}' (if present)"
|
|
784
784
|
)
|
|
@@ -20,7 +20,7 @@ from yellowdog_cli.utils.entity_utils import (
|
|
|
20
20
|
get_compute_source_templates,
|
|
21
21
|
)
|
|
22
22
|
from yellowdog_cli.utils.interactive import confirmed
|
|
23
|
-
from yellowdog_cli.utils.printing import print_error,
|
|
23
|
+
from yellowdog_cli.utils.printing import print_error, print_info, print_warning
|
|
24
24
|
from yellowdog_cli.utils.settings import RN_KEYRING, RN_REQUIREMENT_TEMPLATE
|
|
25
25
|
from yellowdog_cli.utils.variables import process_variable_substitutions_insitu
|
|
26
26
|
|
|
@@ -240,14 +240,16 @@ class CommonCloudConfig(ABC):
|
|
|
240
240
|
f"YellowDog resources definition file '{resources_file}' already"
|
|
241
241
|
" exists; OK to overwrite?"
|
|
242
242
|
):
|
|
243
|
-
|
|
243
|
+
print_info("Not overwriting YellowDog resources definition file")
|
|
244
244
|
return False
|
|
245
245
|
|
|
246
246
|
try:
|
|
247
247
|
with open(resources_file, "w") as f:
|
|
248
248
|
json.dump(resource_list, f, indent=2, cls=CompactJSONEncoder)
|
|
249
249
|
f.write("\n")
|
|
250
|
-
|
|
250
|
+
print_info(
|
|
251
|
+
f"Saved YellowDog resource definitions to '{resources_file}'"
|
|
252
|
+
)
|
|
251
253
|
return True
|
|
252
254
|
except Exception as e:
|
|
253
255
|
print_error(
|
|
@@ -261,7 +263,7 @@ class CommonCloudConfig(ABC):
|
|
|
261
263
|
if confirmed(f"Remove Keyring '{keyring_name}'?"):
|
|
262
264
|
try:
|
|
263
265
|
self._client.keyring_client.delete_keyring_by_name(keyring_name)
|
|
264
|
-
|
|
266
|
+
print_info(f"Removed Keyring '{keyring_name}'")
|
|
265
267
|
except Exception as e:
|
|
266
268
|
if "NotFoundException" in str(e):
|
|
267
269
|
print_warning(f"No Keyring '{keyring_name}' to remove")
|
|
@@ -272,7 +274,7 @@ class CommonCloudConfig(ABC):
|
|
|
272
274
|
"""
|
|
273
275
|
Generate the Compute Requirement Templates
|
|
274
276
|
"""
|
|
275
|
-
|
|
277
|
+
print_info(
|
|
276
278
|
"Creating example Compute Requirement Templates with instance type"
|
|
277
279
|
f" '{self._instance_type}'"
|
|
278
280
|
)
|
|
@@ -320,7 +322,7 @@ class CommonCloudConfig(ABC):
|
|
|
320
322
|
),
|
|
321
323
|
]
|
|
322
324
|
|
|
323
|
-
|
|
325
|
+
print_info("Creating YellowDog Compute Requirement Templates")
|
|
324
326
|
create_resources(
|
|
325
327
|
process_variable_substitutions_insitu(
|
|
326
328
|
deepcopy(self._requirement_template_resources)
|
|
@@ -337,7 +339,7 @@ class CommonCloudConfig(ABC):
|
|
|
337
339
|
keyring, self._keyring_password = create_keyring_via_api(
|
|
338
340
|
keyring_name, keyring_resource["description"]
|
|
339
341
|
)
|
|
340
|
-
|
|
342
|
+
print_info(f"Created YellowDog Keyring '{keyring_name}' ({keyring.id})")
|
|
341
343
|
self._keyring_name = keyring_name
|
|
342
344
|
except Exception as e:
|
|
343
345
|
if "A keyring already exists" in str(e):
|
|
@@ -350,10 +352,10 @@ class CommonCloudConfig(ABC):
|
|
|
350
352
|
Print the details of the Keyring and Keyring password.
|
|
351
353
|
"""
|
|
352
354
|
if self._keyring_password is not None:
|
|
353
|
-
|
|
355
|
+
print_info(
|
|
354
356
|
"In the 'Keyrings' section of the YellowDog Portal, please claim your"
|
|
355
357
|
" Keyring using the name and password below. The password will not be"
|
|
356
358
|
" shown again."
|
|
357
359
|
)
|
|
358
|
-
|
|
359
|
-
|
|
360
|
+
print_info(f"--> Keyring name = '{self._keyring_name}'")
|
|
361
|
+
print_info(f"--> Keyring password = '{self._keyring_password}'")
|
|
@@ -13,7 +13,7 @@ from yellowdog_cli.create import create_resources
|
|
|
13
13
|
from yellowdog_cli.remove import remove_resources
|
|
14
14
|
from yellowdog_cli.utils.cloudwizard_common import CommonCloudConfig
|
|
15
15
|
from yellowdog_cli.utils.interactive import confirmed, select
|
|
16
|
-
from yellowdog_cli.utils.printing import print_error,
|
|
16
|
+
from yellowdog_cli.utils.printing import print_error, print_info, print_warning
|
|
17
17
|
from yellowdog_cli.utils.settings import RN_SOURCE_TEMPLATE, RN_STORAGE_CONFIGURATION
|
|
18
18
|
|
|
19
19
|
YD_KEYRING_NAME = "cloudwizard-gcp"
|
|
@@ -82,7 +82,7 @@ class GCPConfig(CommonCloudConfig):
|
|
|
82
82
|
"""
|
|
83
83
|
Set up YellowDog assets.
|
|
84
84
|
"""
|
|
85
|
-
|
|
85
|
+
print_info(
|
|
86
86
|
"Please select the Google Compute Engine regions for which to create"
|
|
87
87
|
" YellowDog Compute Source Templates"
|
|
88
88
|
)
|
|
@@ -118,7 +118,7 @@ class GCPConfig(CommonCloudConfig):
|
|
|
118
118
|
)
|
|
119
119
|
)
|
|
120
120
|
self._source_names_ondemand.append(name)
|
|
121
|
-
|
|
121
|
+
print_info("Creating YellowDog Compute Source Templates")
|
|
122
122
|
create_resources(self._source_template_resources)
|
|
123
123
|
|
|
124
124
|
# Create Compute Requirement Templates
|
|
@@ -260,7 +260,7 @@ class GCPConfig(CommonCloudConfig):
|
|
|
260
260
|
storage_client.create_bucket(
|
|
261
261
|
bucket_or_name=bucket_name, location=GCP_BUCKET_LOCATION
|
|
262
262
|
)
|
|
263
|
-
|
|
263
|
+
print_info(f"Created Google Storage Bucket '{bucket_name}'")
|
|
264
264
|
except Exception as e:
|
|
265
265
|
if "401" in str(e):
|
|
266
266
|
raise Exception(f"Invalid GCP credentials: {e}")
|
|
@@ -289,7 +289,7 @@ class GCPConfig(CommonCloudConfig):
|
|
|
289
289
|
bucket.delete(force=True)
|
|
290
290
|
except Exception:
|
|
291
291
|
objects = storage_client.list_blobs(bucket_or_name=bucket_name)
|
|
292
|
-
|
|
292
|
+
print_info(
|
|
293
293
|
"Deleting any remaining objects in Google Storage Bucket"
|
|
294
294
|
f" '{bucket_name}'"
|
|
295
295
|
)
|
|
@@ -298,16 +298,16 @@ class GCPConfig(CommonCloudConfig):
|
|
|
298
298
|
obj.delete()
|
|
299
299
|
counter += 1
|
|
300
300
|
if counter > 0:
|
|
301
|
-
|
|
301
|
+
print_info(
|
|
302
302
|
f"Deleted {counter} object(s) from Google Storage Bucket"
|
|
303
303
|
f" '{bucket_name}'"
|
|
304
304
|
)
|
|
305
305
|
else:
|
|
306
|
-
|
|
306
|
+
print_info(
|
|
307
307
|
f"No objects to delete in Google Storage Bucket '{bucket_name}'"
|
|
308
308
|
)
|
|
309
309
|
bucket.delete()
|
|
310
|
-
|
|
310
|
+
print_info(f"Deleted Google Storage Bucket '{bucket_name}'")
|
|
311
311
|
except Exception as e:
|
|
312
312
|
if "401" in str(e):
|
|
313
313
|
raise Exception(f"Invalid GCP credentials: {e}")
|
yellowdog_cli/utils/csv_data.py
CHANGED
|
@@ -15,7 +15,7 @@ from toml import load as toml_load
|
|
|
15
15
|
from yellowdog_cli.utils.args import ARGS_PARSER
|
|
16
16
|
from yellowdog_cli.utils.config_types import ConfigWorkRequirement
|
|
17
17
|
from yellowdog_cli.utils.misc_utils import format_yd_name
|
|
18
|
-
from yellowdog_cli.utils.printing import
|
|
18
|
+
from yellowdog_cli.utils.printing import print_info, print_json
|
|
19
19
|
from yellowdog_cli.utils.property_names import *
|
|
20
20
|
from yellowdog_cli.utils.settings import (
|
|
21
21
|
BOOL_TYPE_TAG,
|
|
@@ -178,7 +178,7 @@ def perform_csv_task_expansion(
|
|
|
178
178
|
Expand a Work Requirement using CSV data.
|
|
179
179
|
"""
|
|
180
180
|
if len(wr_data[TASK_GROUPS]) > len(csv_files):
|
|
181
|
-
|
|
181
|
+
print_info(
|
|
182
182
|
f"Note: Number of Task Groups ({len(wr_data[TASK_GROUPS])}) "
|
|
183
183
|
"in Work Requirement is greater than number of CSV files "
|
|
184
184
|
f"({len(csv_files)})"
|
|
@@ -195,7 +195,7 @@ def perform_csv_task_expansion(
|
|
|
195
195
|
resolved_csv_file = relpath(resolve_filename(files_directory, csv_file))
|
|
196
196
|
|
|
197
197
|
task_group = wr_data[TASK_GROUPS][index]
|
|
198
|
-
|
|
198
|
+
print_info(
|
|
199
199
|
f"Loading CSV Task data for Task Group {index + 1} from:"
|
|
200
200
|
f" '{resolved_csv_file}'"
|
|
201
201
|
)
|
|
@@ -209,7 +209,7 @@ def perform_csv_task_expansion(
|
|
|
209
209
|
task_prototype = task_group[TASKS][0]
|
|
210
210
|
|
|
211
211
|
if not substitions_present(csv_data.var_names, str(task_prototype)):
|
|
212
|
-
|
|
212
|
+
print_info(
|
|
213
213
|
"Warning: No CSV substitutions to apply to Task Group "
|
|
214
214
|
f"{index + 1}; not expanding Task list"
|
|
215
215
|
)
|
|
@@ -223,10 +223,10 @@ def perform_csv_task_expansion(
|
|
|
223
223
|
)
|
|
224
224
|
)
|
|
225
225
|
task_group[TASKS] = generated_task_list
|
|
226
|
-
|
|
226
|
+
print_info(f"Generated {len(generated_task_list)} Task(s) from CSV data")
|
|
227
227
|
|
|
228
228
|
if ARGS_PARSER.process_csv_only:
|
|
229
|
-
|
|
229
|
+
print_info("Displaying CSV substitutions only:")
|
|
230
230
|
print_json(wr_data)
|
|
231
231
|
exit(0)
|
|
232
232
|
|
|
@@ -326,7 +326,7 @@ def get_csv_file_index(
|
|
|
326
326
|
# Invalid Task Group naming?
|
|
327
327
|
split_name = csv_filename.split(":")
|
|
328
328
|
if len(split_name) > 1:
|
|
329
|
-
|
|
329
|
+
print_info(
|
|
330
330
|
f"Warning: Possible invalid Task Group name/number '{split_name[-1:]}'?"
|
|
331
331
|
)
|
|
332
332
|
|
|
@@ -3,7 +3,7 @@ Various utility functions for finding objects, etc.
|
|
|
3
3
|
"""
|
|
4
4
|
|
|
5
5
|
from functools import lru_cache
|
|
6
|
-
from typing import Callable, List, Optional, Tuple, Union
|
|
6
|
+
from typing import Callable, Dict, List, Optional, Tuple, Union
|
|
7
7
|
|
|
8
8
|
from yellowdog_client import PlatformClient
|
|
9
9
|
from yellowdog_client.common import SearchClient
|
|
@@ -11,6 +11,7 @@ from yellowdog_client.model import (
|
|
|
11
11
|
AccountAllowance,
|
|
12
12
|
AllowanceSearch,
|
|
13
13
|
Application,
|
|
14
|
+
ApplicationDetails,
|
|
14
15
|
ApplicationSearch,
|
|
15
16
|
ComputeRequirementStatus,
|
|
16
17
|
ComputeRequirementSummary,
|
|
@@ -22,6 +23,7 @@ from yellowdog_client.model import (
|
|
|
22
23
|
ComputeSourceTemplateSearch,
|
|
23
24
|
ComputeSourceTemplateSummary,
|
|
24
25
|
ExternalUser,
|
|
26
|
+
Group,
|
|
25
27
|
GroupSearch,
|
|
26
28
|
GroupSummary,
|
|
27
29
|
ImageAccess,
|
|
@@ -54,7 +56,7 @@ from yellowdog_client.model import (
|
|
|
54
56
|
|
|
55
57
|
from yellowdog_cli.utils.args import ARGS_PARSER
|
|
56
58
|
from yellowdog_cli.utils.interactive import confirmed, select
|
|
57
|
-
from yellowdog_cli.utils.printing import
|
|
59
|
+
from yellowdog_cli.utils.printing import print_info
|
|
58
60
|
from yellowdog_cli.utils.settings import NAMESPACE_PREFIX_SEPARATOR
|
|
59
61
|
from yellowdog_cli.utils.ydid_utils import YDIDType, get_ydid_type
|
|
60
62
|
|
|
@@ -241,7 +243,7 @@ def find_compute_source_template_id_by_name(
|
|
|
241
243
|
"""
|
|
242
244
|
template_id = _find_id_by_name(name, client, get_compute_source_templates)
|
|
243
245
|
if template_id is not None:
|
|
244
|
-
|
|
246
|
+
print_info(
|
|
245
247
|
f"Replaced Compute Source Template name '{name}' with ID {template_id}"
|
|
246
248
|
)
|
|
247
249
|
return template_id
|
|
@@ -385,7 +387,7 @@ def find_image_name_or_id(
|
|
|
385
387
|
PUBLIC images.
|
|
386
388
|
|
|
387
389
|
Finally, if nothing matches, the original ID is returned. This is
|
|
388
|
-
likely to be a provider
|
|
390
|
+
likely to be a provider-specific string.
|
|
389
391
|
"""
|
|
390
392
|
if image_name_or_id is None:
|
|
391
393
|
return None
|
|
@@ -418,7 +420,7 @@ def find_image_name_or_id(
|
|
|
418
420
|
"""
|
|
419
421
|
if report_substitutions and return_val != original_image_name_or_id:
|
|
420
422
|
msg = f"{return_val}" if is_ydid else f"'{return_val}'"
|
|
421
|
-
|
|
423
|
+
print_info(f"Replaced Images ID '{original_image_name_or_id}' with {msg}")
|
|
422
424
|
return return_val
|
|
423
425
|
|
|
424
426
|
split_name = image_name_or_id.split("/")
|
|
@@ -522,6 +524,7 @@ def find_image_name_or_id(
|
|
|
522
524
|
)
|
|
523
525
|
|
|
524
526
|
# Finally, fall through and return the unchanged, original ID string
|
|
527
|
+
print_info(f"No Images ID substitution possible for '{original_image_name_or_id}'")
|
|
525
528
|
return original_image_name_or_id
|
|
526
529
|
|
|
527
530
|
|
|
@@ -542,12 +545,12 @@ def remove_allowances_matching_description(
|
|
|
542
545
|
]
|
|
543
546
|
|
|
544
547
|
if len(allowances) == 0:
|
|
545
|
-
|
|
548
|
+
print_info(f"Cannot find Allowance matching description '{description}'")
|
|
546
549
|
return 0
|
|
547
550
|
|
|
548
551
|
if len(allowances) > 1:
|
|
549
|
-
|
|
550
|
-
|
|
552
|
+
print_info(f"Multiple Allowances match the description '{description}'")
|
|
553
|
+
print_info("Please select which Allowance(s) to remove")
|
|
551
554
|
allowances = select(
|
|
552
555
|
client=client,
|
|
553
556
|
objects=allowances,
|
|
@@ -559,7 +562,7 @@ def remove_allowances_matching_description(
|
|
|
559
562
|
for allowance in allowances:
|
|
560
563
|
if confirmed(f"Remove Allowance with YellowDog ID {allowance.id}?"):
|
|
561
564
|
client.allowances_client.delete_allowance_by_id(allowance.id)
|
|
562
|
-
|
|
565
|
+
print_info(f"Removed Allowance with YellowDog ID {allowance.id}")
|
|
563
566
|
|
|
564
567
|
return len(allowances)
|
|
565
568
|
|
|
@@ -915,21 +918,71 @@ def get_application_id_by_name(client: PlatformClient, app_name: str) -> Optiona
|
|
|
915
918
|
return None
|
|
916
919
|
|
|
917
920
|
|
|
918
|
-
|
|
921
|
+
@lru_cache
|
|
922
|
+
def get_application_details(client: PlatformClient) -> ApplicationDetails:
|
|
919
923
|
"""
|
|
920
|
-
|
|
924
|
+
Load and cache the Application's details
|
|
921
925
|
"""
|
|
922
|
-
|
|
923
|
-
get_application_id_by_name.cache_clear()
|
|
926
|
+
return client.application_client.get_application_details()
|
|
924
927
|
|
|
925
928
|
|
|
926
|
-
|
|
929
|
+
@lru_cache
|
|
930
|
+
def get_application_group_summaries(
|
|
931
|
+
client: PlatformClient, app_id: str
|
|
932
|
+
) -> List[GroupSummary]:
|
|
927
933
|
"""
|
|
928
|
-
Get the groups to which an application belongs.
|
|
934
|
+
Get the summaries of the groups to which an application belongs.
|
|
929
935
|
"""
|
|
930
936
|
return client.account_client.get_application_groups(app_id).list_all()
|
|
931
937
|
|
|
932
938
|
|
|
939
|
+
@lru_cache
|
|
940
|
+
def get_application_groups(client: PlatformClient, app_id: str) -> List[Group]:
|
|
941
|
+
"""
|
|
942
|
+
Get the groups to which an application belongs.
|
|
943
|
+
"""
|
|
944
|
+
return [
|
|
945
|
+
client.account_client.get_group(group_summary.id)
|
|
946
|
+
for group_summary in get_application_group_summaries(client, app_id)
|
|
947
|
+
]
|
|
948
|
+
|
|
949
|
+
|
|
950
|
+
@lru_cache
|
|
951
|
+
def get_all_roles_and_namespaces_for_application(
|
|
952
|
+
client: PlatformClient, application_id: str
|
|
953
|
+
) -> Dict:
|
|
954
|
+
"""
|
|
955
|
+
Get a list of roles and the namespaces to which they apply, for a given application.
|
|
956
|
+
Returns {role_name: [namespace, ...]}, sorted by role name.
|
|
957
|
+
"""
|
|
958
|
+
# Iterate through groups, roles, accumulate namespaces
|
|
959
|
+
roles = dict()
|
|
960
|
+
for group in get_application_groups(client, application_id):
|
|
961
|
+
for role in group.roles:
|
|
962
|
+
if roles.get(role.role.name) is None:
|
|
963
|
+
roles[role.role.name] = []
|
|
964
|
+
if role.scope.global_:
|
|
965
|
+
roles[role.role.name] += ["GLOBAL"]
|
|
966
|
+
else:
|
|
967
|
+
roles[role.role.name] += [
|
|
968
|
+
namespace.namespace for namespace in role.scope.namespaces
|
|
969
|
+
]
|
|
970
|
+
|
|
971
|
+
return {role: namespaces for role, namespaces in sorted(roles.items())}
|
|
972
|
+
|
|
973
|
+
|
|
974
|
+
def clear_application_caches():
|
|
975
|
+
"""
|
|
976
|
+
Clear the application caches.
|
|
977
|
+
"""
|
|
978
|
+
get_all_applications.cache_clear()
|
|
979
|
+
get_application_id_by_name.cache_clear()
|
|
980
|
+
get_application_details.cache_clear()
|
|
981
|
+
get_application_group_summaries.cache_clear()
|
|
982
|
+
get_application_groups.cache_clear()
|
|
983
|
+
get_all_roles_and_namespaces_for_application.cache_clear()
|
|
984
|
+
|
|
985
|
+
|
|
933
986
|
def get_user_groups(client: PlatformClient, user_id: str) -> List[GroupSummary]:
|
|
934
987
|
"""
|
|
935
988
|
Get the groups to which a user belongs.
|
|
@@ -1013,13 +1066,25 @@ def get_image_family_summaries(
|
|
|
1013
1066
|
"""
|
|
1014
1067
|
Obtain and cache the list of image families.
|
|
1015
1068
|
"""
|
|
1016
|
-
#
|
|
1017
|
-
|
|
1018
|
-
|
|
1069
|
+
# Determine namespace(s) to search
|
|
1070
|
+
if namespace is None:
|
|
1071
|
+
# Attempt to use the namespace(s) that are 'readable' by
|
|
1072
|
+
# this application; does not guarantee IMAGE_READ
|
|
1073
|
+
application_details = get_application_details(client)
|
|
1074
|
+
if application_details.allNamespacesReadable:
|
|
1075
|
+
namespaces = None # Search all namespaces
|
|
1076
|
+
elif application_details.readableNamespaces is not None:
|
|
1077
|
+
namespaces = application_details.readableNamespaces
|
|
1078
|
+
else:
|
|
1079
|
+
namespaces = []
|
|
1080
|
+
else:
|
|
1081
|
+
# Use the supplied namespace
|
|
1082
|
+
namespaces = [namespace]
|
|
1083
|
+
|
|
1019
1084
|
try:
|
|
1020
1085
|
if_search = MachineImageFamilySearch(
|
|
1021
1086
|
familyName=None,
|
|
1022
|
-
namespaces=
|
|
1087
|
+
namespaces=namespaces,
|
|
1023
1088
|
includePublic=True,
|
|
1024
1089
|
)
|
|
1025
1090
|
search_client: SearchClient = client.images_client.get_image_families(if_search)
|
|
@@ -1027,7 +1092,7 @@ def get_image_family_summaries(
|
|
|
1027
1092
|
except Exception as e:
|
|
1028
1093
|
if namespace is not None and "MissingPermissionException" in str(e):
|
|
1029
1094
|
# Caching will prevent this warning appearing multiple times
|
|
1030
|
-
|
|
1095
|
+
print_info(
|
|
1031
1096
|
"Warning: Possible 'IMAGE_READ' permission missing if "
|
|
1032
1097
|
f"'{namespace}' is meant as an Image namespace?"
|
|
1033
1098
|
)
|
|
@@ -14,7 +14,7 @@ from yellowdog_cli.utils.entity_utils import (
|
|
|
14
14
|
from yellowdog_cli.utils.printing import (
|
|
15
15
|
print_error,
|
|
16
16
|
print_event,
|
|
17
|
-
|
|
17
|
+
print_info,
|
|
18
18
|
print_warning,
|
|
19
19
|
)
|
|
20
20
|
from yellowdog_cli.utils.settings import EVENT_STREAM_RETRY_INTERVAL
|
|
@@ -42,13 +42,13 @@ def follow_ids(ydids: List[str], auto_cr: bool = False):
|
|
|
42
42
|
if get_ydid_type(ydid) == YDIDType.WORKER_POOL:
|
|
43
43
|
cr_ydid = get_compute_requirement_id_by_worker_pool_id(CLIENT, ydid)
|
|
44
44
|
if cr_ydid is not None:
|
|
45
|
-
|
|
45
|
+
print_info(
|
|
46
46
|
f"Adding event stream for Compute Requirement '{cr_ydid}'"
|
|
47
47
|
)
|
|
48
48
|
cr_ydids.add(cr_ydid)
|
|
49
49
|
ydids_set = ydids_set.union(cr_ydids)
|
|
50
50
|
|
|
51
|
-
|
|
51
|
+
print_info(f"Following the event stream(s) for {len(ydids_set)} YellowDog ID(s)")
|
|
52
52
|
|
|
53
53
|
threads: List[Thread] = []
|
|
54
54
|
|
|
@@ -77,7 +77,7 @@ def follow_ids(ydids: List[str], auto_cr: bool = False):
|
|
|
77
77
|
thread.join()
|
|
78
78
|
|
|
79
79
|
if len(threads) > 1:
|
|
80
|
-
|
|
80
|
+
print_info("All event streams have concluded")
|
|
81
81
|
|
|
82
82
|
|
|
83
83
|
def follow_events(ydid: str, ydid_type: YDIDType):
|
|
@@ -122,7 +122,7 @@ def follow_events(ydid: str, ydid_type: YDIDType):
|
|
|
122
122
|
print_error(f"Event stream error: {e}")
|
|
123
123
|
break
|
|
124
124
|
|
|
125
|
-
|
|
125
|
+
print_info(f"Event stream concluded for '{ydid}'")
|
|
126
126
|
|
|
127
127
|
|
|
128
128
|
def get_event_url(ydid: str, ydid_type: YDIDType) -> Optional[str]:
|