yellowdog-python-examples 8.1.7__tar.gz → 8.2.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {yellowdog_python_examples-8.1.7/yellowdog_python_examples.egg-info → yellowdog_python_examples-8.2.1}/PKG-INFO +4 -4
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/PYPI_README.md +2 -2
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/README.md +7 -3
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/requirements.txt +1 -1
- yellowdog_python_examples-8.2.1/yellowdog_cli/__init__.py +1 -0
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/abort.py +3 -3
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/cancel.py +2 -2
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/create.py +16 -6
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/finish.py +3 -3
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/list.py +1 -1
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/show.py +1 -1
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/shutdown.py +1 -1
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/submit.py +21 -6
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/terminate.py +1 -1
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/utils/config_types.py +1 -0
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/utils/entity_utils.py +1 -1
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/utils/load_config.py +1 -0
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/utils/property_names.py +3 -1
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/utils/provision_utils.py +0 -1
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1/yellowdog_python_examples.egg-info}/PKG-INFO +4 -4
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_python_examples.egg-info/requires.txt +1 -1
- yellowdog_python_examples-8.1.7/yellowdog_cli/__init__.py +0 -1
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/LICENSE +0 -0
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/pyproject.toml +0 -0
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/setup.cfg +0 -0
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/tests/test_create_remove.py +0 -0
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/tests/test_demos.py +0 -0
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/tests/test_dryruns.py +0 -0
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/tests/test_entrypoints.py +0 -0
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/tests/test_gui.py +0 -0
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/tests/test_list.py +0 -0
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/tests/test_objects.py +0 -0
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/tests/test_variable_processing.py +0 -0
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/admin.py +0 -0
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/boost.py +0 -0
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/cloudwizard.py +0 -0
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/compare.py +0 -0
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/delete.py +0 -0
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/download.py +0 -0
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/follow.py +0 -0
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/format_json.py +0 -0
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/hold.py +0 -0
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/instantiate.py +0 -0
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/jsonnet2json.py +0 -0
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/provision.py +0 -0
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/remove.py +0 -0
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/resize.py +0 -0
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/start.py +0 -0
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/upload.py +0 -0
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/utils/__init__.py +0 -0
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/utils/args.py +0 -0
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/utils/check_imports.py +0 -0
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/utils/cloudwizard_aws.py +0 -0
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/utils/cloudwizard_aws_types.py +0 -0
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/utils/cloudwizard_azure.py +0 -0
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/utils/cloudwizard_common.py +0 -0
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/utils/cloudwizard_gcp.py +0 -0
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/utils/compact_json.py +0 -0
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/utils/csv_data.py +0 -0
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/utils/follow_utils.py +0 -0
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/utils/interactive.py +0 -0
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/utils/items.py +0 -0
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/utils/load_resources.py +0 -0
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/utils/misc_utils.py +0 -0
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/utils/printing.py +0 -0
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/utils/rich_console_input_fixed.py +0 -0
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/utils/settings.py +0 -0
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/utils/start_hold_common.py +0 -0
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/utils/submit_utils.py +0 -0
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/utils/type_check.py +0 -0
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/utils/upload_utils.py +0 -0
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/utils/validate_properties.py +0 -0
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/utils/variables.py +0 -0
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/utils/wrapper.py +0 -0
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/utils/ydid_utils.py +0 -0
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/version.py +0 -0
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_python_examples.egg-info/SOURCES.txt +0 -0
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_python_examples.egg-info/dependency_links.txt +0 -0
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_python_examples.egg-info/entry_points.txt +0 -0
- {yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_python_examples.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: yellowdog-python-examples
|
|
3
|
-
Version: 8.1
|
|
3
|
+
Version: 8.2.1
|
|
4
4
|
Summary: Python CLI commands using the YellowDog Python SDK
|
|
5
5
|
Author-email: YellowDog Limited <support@yellowdog.co>
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -26,7 +26,7 @@ Requires-Dist: requests
|
|
|
26
26
|
Requires-Dist: rich==13.9.4
|
|
27
27
|
Requires-Dist: tabulate>=0.9.0
|
|
28
28
|
Requires-Dist: toml
|
|
29
|
-
Requires-Dist: yellowdog-sdk>=11.
|
|
29
|
+
Requires-Dist: yellowdog-sdk>=11.7.0
|
|
30
30
|
Provides-Extra: jsonnet
|
|
31
31
|
Requires-Dist: jsonnet; extra == "jsonnet"
|
|
32
32
|
Provides-Extra: cloudwizard
|
|
@@ -44,7 +44,7 @@ Dynamic: license-file
|
|
|
44
44
|
|
|
45
45
|
## Overview
|
|
46
46
|
|
|
47
|
-
This is a set of Python CLI commands for interacting with the YellowDog Platform, providing examples of usage of the [YellowDog Python SDK](https://docs.yellowdog.
|
|
47
|
+
This is a set of Python CLI commands for interacting with the YellowDog Platform, providing examples of usage of the [YellowDog Python SDK](https://docs.yellowdog.ai/sdk/python/index.html).
|
|
48
48
|
|
|
49
49
|
The commands support:
|
|
50
50
|
|
|
@@ -68,4 +68,4 @@ The commands support:
|
|
|
68
68
|
- **Terminating** Compute Requirements with the **`yd-terminate`** command
|
|
69
69
|
- **Uploading** files to the YellowDog Object Store with the **`yd-upload`** command
|
|
70
70
|
|
|
71
|
-
Please see the
|
|
71
|
+
Please see the documentation in the [GitHub repository](https://github.com/yellowdog/python-examples) for more details.
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
## Overview
|
|
4
4
|
|
|
5
|
-
This is a set of Python CLI commands for interacting with the YellowDog Platform, providing examples of usage of the [YellowDog Python SDK](https://docs.yellowdog.
|
|
5
|
+
This is a set of Python CLI commands for interacting with the YellowDog Platform, providing examples of usage of the [YellowDog Python SDK](https://docs.yellowdog.ai/sdk/python/index.html).
|
|
6
6
|
|
|
7
7
|
The commands support:
|
|
8
8
|
|
|
@@ -26,4 +26,4 @@ The commands support:
|
|
|
26
26
|
- **Terminating** Compute Requirements with the **`yd-terminate`** command
|
|
27
27
|
- **Uploading** files to the YellowDog Object Store with the **`yd-upload`** command
|
|
28
28
|
|
|
29
|
-
Please see the
|
|
29
|
+
Please see the documentation in the [GitHub repository](https://github.com/yellowdog/python-examples) for more details.
|
|
@@ -140,7 +140,7 @@
|
|
|
140
140
|
|
|
141
141
|
# Overview
|
|
142
142
|
|
|
143
|
-
This repository contains a set of command line utilities for driving the YellowDog Platform, written in Python. The scripts use the **[YellowDog Python SDK](https://docs.yellowdog.
|
|
143
|
+
This repository contains a set of command line utilities for driving the YellowDog Platform, written in Python. The scripts use the **[YellowDog Python SDK](https://docs.yellowdog.ai/sdk/python/index.html)**, the code for which can be found [on GitHub](https://github.com/yellowdog/yellowdog-sdk-python-public).
|
|
144
144
|
|
|
145
145
|
|
|
146
146
|
*(Note: these utilities are intended to be a helpful starting point for experimenting with the YellowDog Platform. They are not assured to be of production quality nor do they represent a standard or recommended method for using YellowDog.)*
|
|
@@ -597,7 +597,7 @@ To specify the file containing the JSON document, either populate the `workRequi
|
|
|
597
597
|
|
|
598
598
|
## Property Inheritance
|
|
599
599
|
|
|
600
|
-
Work Requirement
|
|
600
|
+
Work Requirement specifications can be simplified substantially by the property inheritance features in `yd-submit`. In general, properties that are set at a higher level in the hierarchy are inherited at lower levels, unless explicitly overridden.
|
|
601
601
|
|
|
602
602
|
This means that a property set in the `workRequirement` section of the TOML file can be inherited successively by the Work Requirement, Task Groups, and Tasks in the JSON document (assuming the property is available at each level). Hence, Tasks inherit from Task Groups, which inherit from the Work Requirement in the JSON document, which inherits from the `workRequirement` properties in the TOML file.
|
|
603
603
|
|
|
@@ -614,6 +614,7 @@ All properties are optional except for **`taskType`** (or **`taskTypes`**).
|
|
|
614
614
|
| `addYDEnvironmentVariables` | Automatically add YellowDog environment variables to each Task's environment. | Yes | Yes | Yes | Yes |
|
|
615
615
|
| `alwaysUpload` | Whether to attempt to upload task outputs on failure. Default: `true`. | Yes | Yes | Yes | Yes |
|
|
616
616
|
| `arguments` | The list of arguments to be passed to the Task when it is executed. E.g.: `[1, "Two"]`. | Yes | Yes | Yes | Yes |
|
|
617
|
+
| `batchAllocation` | Enables the batch allocation of tasks to nodes. Boolean, Default: `false`. Requires `exclusiveWorkers = true`. | Yes | Yes | Yes | |
|
|
617
618
|
| `completedTaskTtl` | The time (in minutes) to live for completed Tasks. If set, Tasks that have been completed for longer than this period will be deleted. E.g.: `10.0`. | Yes | Yes | Yes | |
|
|
618
619
|
| `csvFile` | The name of the CSV file used to derive Task data. An alternative to `csvFiles` that can be used when there's only a single CSV file. E.g. `"file.csv"`. | Yes | | | |
|
|
619
620
|
| `csvFiles` | A list of CSV files used to derive Task data. E.g. `["file.csv", "file_2.csv:2]`. | Yes | | | |
|
|
@@ -819,6 +820,7 @@ Here's an example of the `workRequirement` section of a TOML configuration file,
|
|
|
819
820
|
addYDEnvironmentVariables = true
|
|
820
821
|
alwaysUpload = true
|
|
821
822
|
arguments = ["1", "TWO"]
|
|
823
|
+
batchAllocation = false
|
|
822
824
|
completedTaskTtl = 10
|
|
823
825
|
csvFile = "file1.csv"
|
|
824
826
|
csvFiles = ["file1.csv", "file3.csv:3"]
|
|
@@ -894,6 +896,7 @@ Showing all possible properties at the Work Requirement level:
|
|
|
894
896
|
"addYDEnvironmentVariables": true,
|
|
895
897
|
"alwaysUpload": true,
|
|
896
898
|
"arguments": [1, "TWO"],
|
|
899
|
+
"batchAllocation": false,
|
|
897
900
|
"completedTaskTtl": 10,
|
|
898
901
|
"dockerEnvironment": {"MY_DOCKER_VAR": 100},
|
|
899
902
|
"dockerPassword": "myPassword",
|
|
@@ -973,6 +976,7 @@ Showing all possible properties at the Task Group level:
|
|
|
973
976
|
"addYDEnvironmentVariables": true,
|
|
974
977
|
"alwaysUpload": true,
|
|
975
978
|
"arguments": [1, "TWO"],
|
|
979
|
+
"batchAllocation": false,
|
|
976
980
|
"completedTaskTtl": 10,
|
|
977
981
|
"dockerEnvironment": {"MY_DOCKER_VAR": 100},
|
|
978
982
|
"dockerPassword": "myPassword",
|
|
@@ -1992,7 +1996,7 @@ yd-remove --ids ydid:crt:D9C548:2a09093d-c74c-4bde-95d1-c576c6f03b13 ydid:imgfam
|
|
|
1992
1996
|
|
|
1993
1997
|
Resources match on **resource names** and (where applicable) **resource namespaces** rather than on YellowDog IDs. This is done for flexibility and to allow the `yd-create` and `yd-remove` commands to be stateless (i.e., we don't need to keep a local record of the YellowDog IDs of the resources created).
|
|
1994
1998
|
|
|
1995
|
-
However, this means that **caution is required** when updating or removing resources, since resource matching is done using **only** the **namespace/name** of the resource -- i.e., the system-generated `ydid` IDs are not used. This means that a resource with a given name could have been removed and replaced in
|
|
1999
|
+
However, this means that **caution is required** when updating or removing resources, since resource matching is done using **only** the **namespace/name** of the resource -- i.e., the system-generated `ydid` IDs are not used. This means that a resource with a given name could have been removed and replaced in the platform by some other means, and the resource specification(s) would still match it.
|
|
1996
2000
|
|
|
1997
2001
|
## Resource Specification Definitions
|
|
1998
2002
|
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "8.2.1"
|
|
@@ -37,9 +37,9 @@ def main():
|
|
|
37
37
|
return
|
|
38
38
|
|
|
39
39
|
print_log(
|
|
40
|
-
"Finding active Work Requirements in"
|
|
41
|
-
f"namespace '{CONFIG_COMMON.namespace}' with "
|
|
42
|
-
f"'{CONFIG_COMMON.name_tag}'
|
|
40
|
+
"Finding active Work Requirements in "
|
|
41
|
+
f"namespace '{CONFIG_COMMON.namespace}' with tags "
|
|
42
|
+
f"including '{CONFIG_COMMON.name_tag}'"
|
|
43
43
|
)
|
|
44
44
|
|
|
45
45
|
# Abort Tasks is always interactive
|
|
@@ -32,8 +32,8 @@ def main():
|
|
|
32
32
|
|
|
33
33
|
print_log(
|
|
34
34
|
"Cancelling Work Requirements in namespace "
|
|
35
|
-
f"'{CONFIG_COMMON.namespace}' with "
|
|
36
|
-
f"'{CONFIG_COMMON.name_tag}'
|
|
35
|
+
f"'{CONFIG_COMMON.namespace}' with tags "
|
|
36
|
+
f"including '{CONFIG_COMMON.name_tag}'"
|
|
37
37
|
)
|
|
38
38
|
|
|
39
39
|
selected_work_requirement_summaries: List[WorkRequirementSummary] = (
|
|
@@ -7,7 +7,7 @@ A script to create or update YellowDog resources.
|
|
|
7
7
|
from copy import deepcopy
|
|
8
8
|
from dataclasses import dataclass
|
|
9
9
|
from datetime import datetime
|
|
10
|
-
from typing import Dict, List, Optional, Set
|
|
10
|
+
from typing import Dict, List, Optional, Set, Tuple
|
|
11
11
|
|
|
12
12
|
import yellowdog_client.model as model
|
|
13
13
|
from dateparser import parse as date_parse
|
|
@@ -48,6 +48,7 @@ from yellowdog_client.model.exceptions import InvalidRequestException
|
|
|
48
48
|
|
|
49
49
|
from yellowdog_cli.utils.entity_utils import (
|
|
50
50
|
clear_application_caches,
|
|
51
|
+
clear_compute_requirement_template_cache,
|
|
51
52
|
clear_compute_source_template_cache,
|
|
52
53
|
clear_group_caches,
|
|
53
54
|
clear_image_caches,
|
|
@@ -130,6 +131,7 @@ from yellowdog_cli.utils.wrapper import ARGS_PARSER, CLIENT, CONFIG_COMMON, main
|
|
|
130
131
|
from yellowdog_cli.utils.ydid_utils import YDIDType, get_ydid_type
|
|
131
132
|
|
|
132
133
|
CLEAR_CST_CACHE: bool = False # Track whether the CST cache needs to be cleared
|
|
134
|
+
CLEAR_CRT_CACHE: bool = False # Track whether the CRT cache needs to be cleared
|
|
133
135
|
CLEAR_IMAGE_FAMILY_CACHE: bool = (
|
|
134
136
|
False # Track whether the image caches need to be cleared
|
|
135
137
|
)
|
|
@@ -278,7 +280,7 @@ def create_compute_source_template(resource: Dict):
|
|
|
278
280
|
compute_source = CLIENT.compute_client.add_compute_source_template(
|
|
279
281
|
compute_source_template
|
|
280
282
|
)
|
|
281
|
-
print_log(f"Created Compute Source Template '{name}'
|
|
283
|
+
print_log(f"Created Compute Source Template '{name}' ({compute_source.id})")
|
|
282
284
|
else:
|
|
283
285
|
if not confirmed(f"Update existing Compute Source Template '{name}'?"):
|
|
284
286
|
return
|
|
@@ -391,6 +393,8 @@ def create_compute_requirement_template(resource: Dict):
|
|
|
391
393
|
template = CLIENT.compute_client.add_compute_requirement_template(
|
|
392
394
|
compute_template
|
|
393
395
|
)
|
|
396
|
+
global CLEAR_CRT_CACHE
|
|
397
|
+
CLEAR_CRT_CACHE = True
|
|
394
398
|
print_log(f"Created Compute Requirement Template '{name}' ({template.id})")
|
|
395
399
|
if ARGS_PARSER.quiet:
|
|
396
400
|
print(template.id)
|
|
@@ -405,9 +409,7 @@ def create_compute_requirement_template(resource: Dict):
|
|
|
405
409
|
template = CLIENT.compute_client.update_compute_requirement_template(
|
|
406
410
|
compute_template
|
|
407
411
|
)
|
|
408
|
-
print_log(
|
|
409
|
-
f"Updated existing Compute Requirement Template '{name}'" f" ({template.id})"
|
|
410
|
-
)
|
|
412
|
+
print_log(f"Updated existing Compute Requirement Template '{name}' ({template.id})")
|
|
411
413
|
if ARGS_PARSER.quiet:
|
|
412
414
|
print(template.id)
|
|
413
415
|
|
|
@@ -446,7 +448,7 @@ def create_keyring(resource: Dict, show_secrets: bool = False):
|
|
|
446
448
|
print_error(f"Failed to create Keyring '{name}': {e}")
|
|
447
449
|
|
|
448
450
|
|
|
449
|
-
def create_keyring_via_api(name: str, description: str) ->
|
|
451
|
+
def create_keyring_via_api(name: str, description: str) -> Tuple[Keyring, str]:
|
|
450
452
|
"""
|
|
451
453
|
Temporary direct API call to create a Keyring and return the shown-once
|
|
452
454
|
password. The password is not available via the SDK call.
|
|
@@ -731,6 +733,10 @@ def create_allowance(resource: Dict):
|
|
|
731
733
|
template_name_or_id = resource.get(PROP_SOURCE_CREATED_FROM, None)
|
|
732
734
|
if template_name_or_id is not None:
|
|
733
735
|
if get_ydid_type(template_name_or_id) != YDIDType.COMPUTE_SOURCE_TEMPLATE:
|
|
736
|
+
global CLEAR_CST_CACHE
|
|
737
|
+
if CLEAR_CST_CACHE: # Update the CST cache if required
|
|
738
|
+
clear_compute_source_template_cache()
|
|
739
|
+
CLEAR_CST_CACHE = False
|
|
734
740
|
template_id = find_compute_source_template_id_by_name(
|
|
735
741
|
client=CLIENT, name=template_name_or_id
|
|
736
742
|
)
|
|
@@ -752,6 +758,10 @@ def create_allowance(resource: Dict):
|
|
|
752
758
|
get_ydid_type(template_name_or_id)
|
|
753
759
|
!= YDIDType.COMPUTE_REQUIREMENT_TEMPLATE
|
|
754
760
|
):
|
|
761
|
+
global CLEAR_CRT_CACHE
|
|
762
|
+
if CLEAR_CRT_CACHE: # Update the CRT cache if required
|
|
763
|
+
clear_compute_requirement_template_cache()
|
|
764
|
+
CLEAR_CRT_CACHE = False
|
|
755
765
|
template_id = find_compute_requirement_template_id_by_name(
|
|
756
766
|
client=CLIENT, name=template_name_or_id
|
|
757
767
|
)
|
|
@@ -31,8 +31,8 @@ def main():
|
|
|
31
31
|
|
|
32
32
|
print_log(
|
|
33
33
|
"Finishing Work Requirements in namespace "
|
|
34
|
-
f"'{CONFIG_COMMON.namespace}' with "
|
|
35
|
-
f"'{CONFIG_COMMON.name_tag}'
|
|
34
|
+
f"'{CONFIG_COMMON.namespace}' with tags "
|
|
35
|
+
f"including '{CONFIG_COMMON.name_tag}'"
|
|
36
36
|
)
|
|
37
37
|
|
|
38
38
|
selected_work_requirement_summaries: List[WorkRequirementSummary] = (
|
|
@@ -59,7 +59,7 @@ def main():
|
|
|
59
59
|
)
|
|
60
60
|
|
|
61
61
|
if len(selected_work_requirement_summaries) > 0 and confirmed(
|
|
62
|
-
f"Finish {len(selected_work_requirement_summaries)}
|
|
62
|
+
f"Finish {len(selected_work_requirement_summaries)} Work Requirement(s)?"
|
|
63
63
|
):
|
|
64
64
|
for work_summary in selected_work_requirement_summaries:
|
|
65
65
|
if work_summary.status != WorkRequirementStatus.FINISHING:
|
|
@@ -115,7 +115,7 @@ def main():
|
|
|
115
115
|
ARGS_PARSER.details = True
|
|
116
116
|
|
|
117
117
|
if ARGS_PARSER.details and ARGS_PARSER.strip_ids:
|
|
118
|
-
print_log("
|
|
118
|
+
print_log("Stripping YellowDog IDs (etc.) from detailed JSON objects")
|
|
119
119
|
|
|
120
120
|
if ARGS_PARSER.output_file and ARGS_PARSER.details:
|
|
121
121
|
if exists(ARGS_PARSER.output_file):
|
|
@@ -44,7 +44,7 @@ def main():
|
|
|
44
44
|
generate_json_list = len(ARGS_PARSER.yellowdog_ids) > 1 and ARGS_PARSER.quiet
|
|
45
45
|
|
|
46
46
|
if ARGS_PARSER.strip_ids:
|
|
47
|
-
print_log("
|
|
47
|
+
print_log("Stripping YellowDog IDs (etc.) from detailed JSON objects")
|
|
48
48
|
|
|
49
49
|
if generate_json_list:
|
|
50
50
|
print("[")
|
|
@@ -477,6 +477,7 @@ def create_task_group(
|
|
|
477
477
|
if providers_data is None
|
|
478
478
|
else [CloudProvider(provider) for provider in providers_data]
|
|
479
479
|
)
|
|
480
|
+
|
|
480
481
|
task_timeout_minutes: Optional[float] = check_float_or_int(
|
|
481
482
|
task_group_data.get(TASK_TIMEOUT, config_wr.task_timeout)
|
|
482
483
|
)
|
|
@@ -486,6 +487,24 @@ def create_task_group(
|
|
|
486
487
|
else timedelta(minutes=task_timeout_minutes)
|
|
487
488
|
)
|
|
488
489
|
|
|
490
|
+
exclusive_workers = check_bool(
|
|
491
|
+
task_group_data.get(
|
|
492
|
+
EXCLUSIVE_WORKERS,
|
|
493
|
+
wr_data.get(EXCLUSIVE_WORKERS, config_wr.exclusive_workers),
|
|
494
|
+
)
|
|
495
|
+
)
|
|
496
|
+
batch_allocation = check_bool(
|
|
497
|
+
task_group_data.get(
|
|
498
|
+
BATCH_ALLOCATION,
|
|
499
|
+
wr_data.get(BATCH_ALLOCATION, config_wr.batch_allocation),
|
|
500
|
+
)
|
|
501
|
+
)
|
|
502
|
+
if batch_allocation and not exclusive_workers:
|
|
503
|
+
raise Exception(
|
|
504
|
+
f"Property '{EXCLUSIVE_WORKERS}' must be set when "
|
|
505
|
+
f"'{BATCH_ALLOCATION}' is specified"
|
|
506
|
+
)
|
|
507
|
+
|
|
489
508
|
run_specification = RunSpecification(
|
|
490
509
|
taskTypes=task_types,
|
|
491
510
|
maximumTaskRetries=check_int(
|
|
@@ -498,12 +517,8 @@ def create_task_group(
|
|
|
498
517
|
WORKER_TAGS, wr_data.get(WORKER_TAGS, config_wr.worker_tags)
|
|
499
518
|
)
|
|
500
519
|
),
|
|
501
|
-
exclusiveWorkers=
|
|
502
|
-
|
|
503
|
-
EXCLUSIVE_WORKERS,
|
|
504
|
-
wr_data.get(EXCLUSIVE_WORKERS, config_wr.exclusive_workers),
|
|
505
|
-
)
|
|
506
|
-
),
|
|
520
|
+
exclusiveWorkers=exclusive_workers,
|
|
521
|
+
batchAllocation=batch_allocation,
|
|
507
522
|
instanceTypes=check_list(
|
|
508
523
|
task_group_data.get(
|
|
509
524
|
INSTANCE_TYPES, wr_data.get(INSTANCE_TYPES, config_wr.instance_types)
|
|
@@ -23,6 +23,7 @@ class ConfigWorkRequirement:
|
|
|
23
23
|
add_yd_env_vars: bool = False
|
|
24
24
|
always_upload: bool = True
|
|
25
25
|
args: List[str] = field(default_factory=list)
|
|
26
|
+
batch_allocation: Optional[bool] = None
|
|
26
27
|
completed_task_ttl: Optional[float] = None # In minutes
|
|
27
28
|
csv_files: Optional[List[str]] = None
|
|
28
29
|
docker_env: Optional[Dict] = None
|
|
@@ -54,7 +54,7 @@ from yellowdog_client.model import (
|
|
|
54
54
|
|
|
55
55
|
from yellowdog_cli.utils.args import ARGS_PARSER
|
|
56
56
|
from yellowdog_cli.utils.interactive import confirmed, select
|
|
57
|
-
from yellowdog_cli.utils.printing import print_log
|
|
57
|
+
from yellowdog_cli.utils.printing import print_log
|
|
58
58
|
from yellowdog_cli.utils.settings import NAMESPACE_PREFIX_SEPARATOR
|
|
59
59
|
from yellowdog_cli.utils.ydid_utils import YDIDType, get_ydid_type
|
|
60
60
|
|
|
@@ -309,6 +309,7 @@ def load_config_work_requirement() -> ConfigWorkRequirement:
|
|
|
309
309
|
add_yd_env_vars=wr_section.get(ADD_YD_ENV_VARS, False),
|
|
310
310
|
always_upload=wr_section.get(ALWAYS_UPLOAD, True),
|
|
311
311
|
args=wr_section.get(ARGS, []),
|
|
312
|
+
batch_allocation=wr_section.get(BATCH_ALLOCATION, None),
|
|
312
313
|
upload_taskoutput=wr_section.get(UPLOAD_TASKOUTPUT, False),
|
|
313
314
|
completed_task_ttl=wr_section.get(COMPLETED_TASK_TTL, None),
|
|
314
315
|
csv_files=csv_files,
|
|
@@ -5,6 +5,7 @@ Property names.
|
|
|
5
5
|
ADD_YD_ENV_VARS = "addYDEnvironmentVariables"
|
|
6
6
|
ALWAYS_UPLOAD = "alwaysUpload" # Bool
|
|
7
7
|
ARGS = "arguments" # List
|
|
8
|
+
BATCH_ALLOCATION = "batchAllocation" # Bool
|
|
8
9
|
CERTIFICATES = "certificates"
|
|
9
10
|
COMMON_SECTION = "common" # No value
|
|
10
11
|
COMPLETED_TASK_TTL = "completedTaskTtl" # Float
|
|
@@ -37,7 +38,7 @@ IMAGES_ID = "imagesId" # String
|
|
|
37
38
|
IMPORT_COMMON = "importCommon" # String
|
|
38
39
|
INPUTS_OPTIONAL = "inputsOptional" # List of Strings
|
|
39
40
|
INPUTS_REQUIRED = "inputs" # List of Strings
|
|
40
|
-
INSTANCE_TAGS = "instanceTags" #
|
|
41
|
+
INSTANCE_TAGS = "instanceTags" # Dictionary
|
|
41
42
|
INSTANCE_TYPES = "instanceTypes" # List of Strings
|
|
42
43
|
KEY = "key" # String
|
|
43
44
|
LOCAL_PATH = "localPath" # String
|
|
@@ -119,6 +120,7 @@ ALL_KEYS = [
|
|
|
119
120
|
ADD_YD_ENV_VARS,
|
|
120
121
|
ALWAYS_UPLOAD,
|
|
121
122
|
ARGS,
|
|
123
|
+
BATCH_ALLOCATION,
|
|
122
124
|
CERTIFICATES,
|
|
123
125
|
COMMON_SECTION,
|
|
124
126
|
COMPLETED_TASK_TTL,
|
|
@@ -11,7 +11,6 @@ from yellowdog_cli.utils.config_types import ConfigWorkerPool
|
|
|
11
11
|
from yellowdog_cli.utils.entity_utils import (
|
|
12
12
|
find_compute_requirement_template_id_by_name,
|
|
13
13
|
find_image_name_or_id,
|
|
14
|
-
split_namespace_and_name,
|
|
15
14
|
)
|
|
16
15
|
from yellowdog_cli.utils.load_config import CONFIG_FILE_DIR
|
|
17
16
|
from yellowdog_cli.utils.printing import print_log
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: yellowdog-python-examples
|
|
3
|
-
Version: 8.1
|
|
3
|
+
Version: 8.2.1
|
|
4
4
|
Summary: Python CLI commands using the YellowDog Python SDK
|
|
5
5
|
Author-email: YellowDog Limited <support@yellowdog.co>
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -26,7 +26,7 @@ Requires-Dist: requests
|
|
|
26
26
|
Requires-Dist: rich==13.9.4
|
|
27
27
|
Requires-Dist: tabulate>=0.9.0
|
|
28
28
|
Requires-Dist: toml
|
|
29
|
-
Requires-Dist: yellowdog-sdk>=11.
|
|
29
|
+
Requires-Dist: yellowdog-sdk>=11.7.0
|
|
30
30
|
Provides-Extra: jsonnet
|
|
31
31
|
Requires-Dist: jsonnet; extra == "jsonnet"
|
|
32
32
|
Provides-Extra: cloudwizard
|
|
@@ -44,7 +44,7 @@ Dynamic: license-file
|
|
|
44
44
|
|
|
45
45
|
## Overview
|
|
46
46
|
|
|
47
|
-
This is a set of Python CLI commands for interacting with the YellowDog Platform, providing examples of usage of the [YellowDog Python SDK](https://docs.yellowdog.
|
|
47
|
+
This is a set of Python CLI commands for interacting with the YellowDog Platform, providing examples of usage of the [YellowDog Python SDK](https://docs.yellowdog.ai/sdk/python/index.html).
|
|
48
48
|
|
|
49
49
|
The commands support:
|
|
50
50
|
|
|
@@ -68,4 +68,4 @@ The commands support:
|
|
|
68
68
|
- **Terminating** Compute Requirements with the **`yd-terminate`** command
|
|
69
69
|
- **Uploading** files to the YellowDog Object Store with the **`yd-upload`** command
|
|
70
70
|
|
|
71
|
-
Please see the
|
|
71
|
+
Please see the documentation in the [GitHub repository](https://github.com/yellowdog/python-examples) for more details.
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = "8.1.7"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/tests/test_create_remove.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/tests/test_entrypoints.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/cloudwizard.py
RENAMED
|
File without changes
|
{yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/compare.py
RENAMED
|
File without changes
|
|
File without changes
|
{yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/download.py
RENAMED
|
File without changes
|
|
File without changes
|
{yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/format_json.py
RENAMED
|
File without changes
|
|
File without changes
|
{yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/instantiate.py
RENAMED
|
File without changes
|
{yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/jsonnet2json.py
RENAMED
|
File without changes
|
{yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/provision.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/utils/__init__.py
RENAMED
|
File without changes
|
{yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/utils/args.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/utils/csv_data.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/utils/items.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/utils/printing.py
RENAMED
|
File without changes
|
|
File without changes
|
{yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/utils/settings.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/utils/variables.py
RENAMED
|
File without changes
|
{yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/utils/wrapper.py
RENAMED
|
File without changes
|
|
File without changes
|
{yellowdog_python_examples-8.1.7 → yellowdog_python_examples-8.2.1}/yellowdog_cli/version.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|