yellowdog-python-examples 7.8.8__tar.gz → 7.9.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {yellowdog_python_examples-7.8.8/yellowdog_python_examples.egg-info → yellowdog_python_examples-7.9.0}/PKG-INFO +7 -4
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/PYPI_README.md +6 -3
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/README.md +67 -31
- yellowdog_python_examples-7.9.0/yd_commands/__init__.py +1 -0
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/create.py +43 -1
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/load_config.py +7 -1
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/load_resources.py +2 -0
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/remove.py +28 -1
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/settings.py +1 -0
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/shutdown.py +17 -7
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0/yellowdog_python_examples.egg-info}/PKG-INFO +7 -4
- yellowdog_python_examples-7.8.8/yd_commands/__init__.py +0 -1
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/LICENSE +0 -0
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/pyproject.toml +0 -0
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/requirements.txt +0 -0
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/setup.cfg +0 -0
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/setup.py +0 -0
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/tests/test_create_remove.py +0 -0
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/tests/test_demos.py +0 -0
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/tests/test_dryruns.py +0 -0
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/tests/test_entrypoints.py +0 -0
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/tests/test_gui.py +0 -0
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/tests/test_list.py +0 -0
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/tests/test_objects.py +0 -0
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/tests/test_variable_processing.py +0 -0
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/abort.py +0 -0
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/admin.py +0 -0
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/args.py +0 -0
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/boost.py +0 -0
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/cancel.py +0 -0
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/check_imports.py +0 -0
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/cloudwizard.py +0 -0
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/cloudwizard_aws.py +0 -0
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/cloudwizard_aws_types.py +0 -0
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/cloudwizard_azure.py +0 -0
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/cloudwizard_common.py +0 -0
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/cloudwizard_gcp.py +0 -0
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/compact_json.py +0 -0
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/config_types.py +0 -0
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/csv_data.py +0 -0
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/delete.py +0 -0
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/download.py +0 -0
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/follow.py +0 -0
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/follow_utils.py +0 -0
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/format_json.py +0 -0
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/hold.py +0 -0
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/id_utils.py +0 -0
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/instantiate.py +0 -0
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/interactive.py +0 -0
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/items.py +0 -0
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/jsonnet2json.py +0 -0
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/list.py +0 -0
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/object_utilities.py +0 -0
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/printing.py +0 -0
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/property_names.py +0 -0
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/provision.py +0 -0
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/provision_utils.py +0 -0
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/resize.py +0 -0
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/start.py +0 -0
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/start_hold_common.py +0 -0
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/submit.py +0 -0
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/submit_utils.py +0 -0
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/terminate.py +0 -0
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/type_check.py +0 -0
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/upload.py +0 -0
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/upload_utils.py +0 -0
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/utils.py +0 -0
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/validate_properties.py +0 -0
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/variables.py +0 -0
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/version.py +0 -0
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/wrapper.py +0 -0
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yellowdog_python_examples.egg-info/SOURCES.txt +0 -0
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yellowdog_python_examples.egg-info/dependency_links.txt +0 -0
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yellowdog_python_examples.egg-info/entry_points.txt +0 -0
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yellowdog_python_examples.egg-info/requires.txt +0 -0
- {yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yellowdog_python_examples.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: yellowdog-python-examples
|
|
3
|
-
Version: 7.
|
|
3
|
+
Version: 7.9.0
|
|
4
4
|
Summary: Example Python commands using the YellowDog Python SDK
|
|
5
5
|
Home-page: https://github.com/yellowdog/python-examples
|
|
6
6
|
Author: YellowDog Limited
|
|
@@ -43,6 +43,7 @@ The command scripts support:
|
|
|
43
43
|
|
|
44
44
|
- **Provisioning** Worker Pools with the **`yd-provision`** command
|
|
45
45
|
- **Submitting** Work Requirements with the **`yd-submit`** command
|
|
46
|
+
- **Starting** HELD Work Requirements and **Holding** (or pausing) RUNNING Work Requirements with the **`yd-start`** and **`yd-hold`** commands
|
|
46
47
|
- **Uploading** files to the YellowDog Object Store with the **`yd-upload`** command
|
|
47
48
|
- **Instantiating** Compute Requirements with the **`yd-instantiate`** command
|
|
48
49
|
- **Downloading** Results from the YellowDog Object Store with the **`yd-download`** command
|
|
@@ -52,7 +53,9 @@ The command scripts support:
|
|
|
52
53
|
- **Terminating** Compute Requirements with the **`yd-terminate`** command
|
|
53
54
|
- **Deleting** objects in the YellowDog Object Store with the **`yd-delete`** command
|
|
54
55
|
- **Listing** YellowDog items using the **`yd-list`** command
|
|
55
|
-
- **Resizing** Worker Pools and Compute Requirements
|
|
56
|
-
- **
|
|
56
|
+
- **Resizing** Worker Pools and Compute Requirements with the **`yd-resize`** command
|
|
57
|
+
- **Boosting** Allowances with the **`yd-boost`** command
|
|
58
|
+
- **Creating, Updating and Removing** Source Templates, Compute Templates, Keyrings, Credentials, Namespace Storage Configurations, Image Families, Allowances, and Configured Worker Pools with the **`yd-create`** and **`yd-remove`** commands
|
|
59
|
+
- **Following Event Streams** for Work Requirements, Worker Pools and Compute Requirements with the **`yd-follow`** command
|
|
57
60
|
|
|
58
|
-
Please see the [GitHub repository](https://github.com/yellowdog/python-examples) for more details.
|
|
61
|
+
Please see the documenation in the [GitHub repository](https://github.com/yellowdog/python-examples) for more details.
|
|
@@ -8,6 +8,7 @@ The command scripts support:
|
|
|
8
8
|
|
|
9
9
|
- **Provisioning** Worker Pools with the **`yd-provision`** command
|
|
10
10
|
- **Submitting** Work Requirements with the **`yd-submit`** command
|
|
11
|
+
- **Starting** HELD Work Requirements and **Holding** (or pausing) RUNNING Work Requirements with the **`yd-start`** and **`yd-hold`** commands
|
|
11
12
|
- **Uploading** files to the YellowDog Object Store with the **`yd-upload`** command
|
|
12
13
|
- **Instantiating** Compute Requirements with the **`yd-instantiate`** command
|
|
13
14
|
- **Downloading** Results from the YellowDog Object Store with the **`yd-download`** command
|
|
@@ -17,7 +18,9 @@ The command scripts support:
|
|
|
17
18
|
- **Terminating** Compute Requirements with the **`yd-terminate`** command
|
|
18
19
|
- **Deleting** objects in the YellowDog Object Store with the **`yd-delete`** command
|
|
19
20
|
- **Listing** YellowDog items using the **`yd-list`** command
|
|
20
|
-
- **Resizing** Worker Pools and Compute Requirements
|
|
21
|
-
- **
|
|
21
|
+
- **Resizing** Worker Pools and Compute Requirements with the **`yd-resize`** command
|
|
22
|
+
- **Boosting** Allowances with the **`yd-boost`** command
|
|
23
|
+
- **Creating, Updating and Removing** Source Templates, Compute Templates, Keyrings, Credentials, Namespace Storage Configurations, Image Families, Allowances, and Configured Worker Pools with the **`yd-create`** and **`yd-remove`** commands
|
|
24
|
+
- **Following Event Streams** for Work Requirements, Worker Pools and Compute Requirements with the **`yd-follow`** command
|
|
22
25
|
|
|
23
|
-
Please see the [GitHub repository](https://github.com/yellowdog/python-examples) for more details.
|
|
26
|
+
Please see the documenation in the [GitHub repository](https://github.com/yellowdog/python-examples) for more details.
|
|
@@ -87,6 +87,7 @@
|
|
|
87
87
|
* [Namespace Storage Configurations](#namespace-storage-configurations)
|
|
88
88
|
* [Configured Worker Pools](#configured-worker-pools)
|
|
89
89
|
* [Allowances](#allowances)
|
|
90
|
+
* [String Attribute Definitions](#string-attribute-definitions)
|
|
90
91
|
* [Jsonnet Support](#jsonnet-support)
|
|
91
92
|
* [Jsonnet Installation](#jsonnet-installation)
|
|
92
93
|
* [Variable Substitutions in Jsonnet Files](#variable-substitutions-in-jsonnet-files)
|
|
@@ -114,7 +115,7 @@
|
|
|
114
115
|
* [yd-boost](#yd-boost)
|
|
115
116
|
|
|
116
117
|
<!-- Created by https://github.com/ekalinin/github-markdown-toc -->
|
|
117
|
-
<!-- Added by: pwt, at:
|
|
118
|
+
<!-- Added by: pwt, at: Wed May 8 16:37:54 BST 2024 -->
|
|
118
119
|
|
|
119
120
|
<!--te-->
|
|
120
121
|
|
|
@@ -156,6 +157,8 @@ Run any command with the `--help`/`-h` option to discover the command's options.
|
|
|
156
157
|
|
|
157
158
|
# YellowDog Prerequisites
|
|
158
159
|
|
|
160
|
+
(See also the quick start guides for [Provisioned](https://github.com/yellowdog/python-examples-templates/blob/main/quickstart.md) and [Configured](https://github.com/yellowdog/python-examples-templates/blob/main/quickstart-configured.md) Worker Pools.)
|
|
161
|
+
|
|
159
162
|
To submit **Work Requirements** to YellowDog for processing by Configured Worker Pools (on-premise) and/or Provisioned Worker Pools (cloud-provisioned resources), you'll need:
|
|
160
163
|
|
|
161
164
|
1. A YellowDog Platform Account.
|
|
@@ -176,7 +179,7 @@ To set up **Configured Worker Pools**, you'll need:
|
|
|
176
179
|
5. A Configured Worker Pool Token: from the **Workers** tab in the [YellowDog Portal](https://portal.yellowdog.co/#/workers), use the **+Add Configured Worker Pool** button to create a new Worker Pool and generate a token.
|
|
177
180
|
|
|
178
181
|
|
|
179
|
-
6. Obtain the YellowDog Agent and install/configure it on your on-premise systems using the Token above.
|
|
182
|
+
6. Obtain the YellowDog Agent and install/configure it on your on-premise systems using the Token obtained above. See guidance for [Linux](https://github.com/yellowdog/resources/blob/main/agent-install/linux/README.md) and [Windows](https://github.com/yellowdog/resources/blob/main/agent-install/windows/README-CONFIGURED.md).
|
|
180
183
|
|
|
181
184
|
# Script Installation with Pip
|
|
182
185
|
|
|
@@ -391,21 +394,25 @@ Substitutions can also be performed for non-string (number, boolean, array, and
|
|
|
391
394
|
|
|
392
395
|
The following substitutions are automatically created and can be used in any section of the configuration file, or in any JSON specification:
|
|
393
396
|
|
|
394
|
-
| Directive
|
|
395
|
-
|
|
396
|
-
| `{{username}}`
|
|
397
|
-
| `{{date}}`
|
|
398
|
-
| `{{time}}`
|
|
399
|
-
| `{{datetime}}`
|
|
400
|
-
| `{{random}}`
|
|
401
|
-
| `{{namespace}}`
|
|
402
|
-
| `{{tag}}`
|
|
403
|
-
| `{{key}}`
|
|
404
|
-
| `{{secret}}`
|
|
405
|
-
| `{{url}}`
|
|
397
|
+
| Directive | Description | Example of Substitution |
|
|
398
|
+
|:----------------------|:---------------------------------------------------------------|:------------------------|
|
|
399
|
+
| `{{username}}` | The current user's login username, lower case, spaces replaced | jane_smith |
|
|
400
|
+
| `{{date}}` | The current date (UTC): YYYYMMDD | 20221027 |
|
|
401
|
+
| `{{time}}` | The current time (UTC): HHMMSSss | 16302699 |
|
|
402
|
+
| `{{datetime}}` | Concatenation of the date and time, with a '-' separator | 20221027-163026 |
|
|
403
|
+
| `{{random}}` | A random, three digit hexadecimal number (lower case) | a1c |
|
|
404
|
+
| `{{namespace}}` | The `namespace` property. | my_namespace |
|
|
405
|
+
| `{{tag}}` | The `tag` property. | my_tag |
|
|
406
|
+
| `{{key}}` | The application `key` property. | |
|
|
407
|
+
| `{{secret}}` | The application `secret` property. | |
|
|
408
|
+
| `{{url}}` | The Platform `url` property. | |
|
|
409
|
+
| `{{config_dir_abs}}` | The absolute directory path of the configuration file | /yellowdog/workloads |
|
|
410
|
+
| `{{config_dir_name}}` | The immediate containing directory of the configuration file | workloads |
|
|
406
411
|
|
|
407
412
|
For the `date`, `time`, `datetime` and `random` directives, the same values will be used for the duration of a command -- i.e., if `{{time}}` is used within multiple properties, the identical value will be used for each substitution.
|
|
408
413
|
|
|
414
|
+
The `config_dir_` substitutions use the name of the directory containing the nominated TOML configuration file, or the invocation directory if no configuration file is supplied.
|
|
415
|
+
|
|
409
416
|
## User-Defined Variables
|
|
410
417
|
|
|
411
418
|
User-defined variables can be supplied using an option on the command line, or by setting environment variables prefixed with `YD_VAR_`, or by including the directives in the `[common]` section of the TOML configuration file.
|
|
@@ -413,7 +420,7 @@ User-defined variables can be supplied using an option on the command line, or b
|
|
|
413
420
|
1. The **command line** option is `--variable` (or `-v`). For example, `yd-submit -v project_code=pr-213-a -v run_id=1234` will establish two new variables that can be used as `{{project_code}}` and `{{run_id}}`, which will be substituted by `pr-213-a` and `1234` respectively.
|
|
414
421
|
|
|
415
422
|
|
|
416
|
-
2. For **environment variables**, setting the variable `YD_VAR_project_code="pr-213-a"` will create a new variable that can be used as `{{project_code}}`, which will be substituted by `pr-213-a`.
|
|
423
|
+
2. For **environment variables**, setting the variable `YD_VAR_project_code="pr-213-a"` will create a new variable that can be used as `{{project_code}}`, which will be substituted by `pr-213-a`. Note that if running on Windows, all environment variable names are case-insensitive and converted to upper case, so choose upper case variable names only.
|
|
417
424
|
|
|
418
425
|
|
|
419
426
|
3. For **setting within the TOML file**, include a **`variables`** table in the `[common]` section of the file. E.g., `variables = {project_code = "pr-213a", run_id = "1234"}`. Note that this can also use the form:
|
|
@@ -662,7 +669,7 @@ inputs = ["my_bash_script.sh"]
|
|
|
662
669
|
arguments = ["{{wr_name}}/my_bash_script.sh", "1", "2", "3"]
|
|
663
670
|
```
|
|
664
671
|
|
|
665
|
-
In the case of Windows batch (`.bat`) files, a `/c` flag is prepended to the `cmd.exe` argument list to ensure correct execution
|
|
672
|
+
In the case of Windows batch (`.bat`) files, a `/c` flag is prepended to the `cmd.exe` argument list to ensure correct execution behaviour. For example:
|
|
666
673
|
|
|
667
674
|
```toml
|
|
668
675
|
taskType = "cmd" # or "bat"
|
|
@@ -1023,11 +1030,10 @@ A simple example of the JSON output is shown below, showing a Work Requirement w
|
|
|
1023
1030
|
`% yd-submit --dry-run --quiet`
|
|
1024
1031
|
```json
|
|
1025
1032
|
{
|
|
1026
|
-
"
|
|
1027
|
-
"
|
|
1028
|
-
"namespace": "pyexamples",
|
|
1033
|
+
"name": "pyex-bash-pwt2_240424-12051160",
|
|
1034
|
+
"namespace": "pyexamples-pwt2",
|
|
1029
1035
|
"priority": 0,
|
|
1030
|
-
"tag": "pyex-bash",
|
|
1036
|
+
"tag": "pyex-bash-pwt2",
|
|
1031
1037
|
"taskGroups": [
|
|
1032
1038
|
{
|
|
1033
1039
|
"finishIfAllTasksFinished": true,
|
|
@@ -1035,22 +1041,33 @@ A simple example of the JSON output is shown below, showing a Work Requirement w
|
|
|
1035
1041
|
"name": "task_group_1",
|
|
1036
1042
|
"priority": 0,
|
|
1037
1043
|
"runSpecification": {
|
|
1038
|
-
"maximumTaskRetries":
|
|
1044
|
+
"maximumTaskRetries": 5,
|
|
1039
1045
|
"taskTypes": ["bash"],
|
|
1040
|
-
"workerTags": ["pyex-bash"]
|
|
1046
|
+
"workerTags": ["pyex-bash-pwt2-worker", "onpremise-pwt2"]
|
|
1041
1047
|
},
|
|
1048
|
+
"starved": false,
|
|
1049
|
+
"waitingOnDependency": false,
|
|
1042
1050
|
"tasks": [
|
|
1043
1051
|
{
|
|
1044
|
-
"arguments": ["pyex-
|
|
1045
|
-
"environment": {
|
|
1052
|
+
"arguments": ["pyex-bash-pwt2_240424-12051160/sleep_script.sh", 1, 2, 3],
|
|
1053
|
+
"environment": {
|
|
1054
|
+
"TEST_ENV_1": "100",
|
|
1055
|
+
"TEST_ENV_2": "200",
|
|
1056
|
+
"YD_TASK_NAME": "task_1",
|
|
1057
|
+
"YD_TASK_NUMBER": "1",
|
|
1058
|
+
"YD_TASK_GROUP_NAME": "task_group_1",
|
|
1059
|
+
"YD_TASK_GROUP_NUMBER": "1",
|
|
1060
|
+
"YD_WORK_REQUIREMENT_NAME": "pyex-bash-pwt2_240424-12051160",
|
|
1061
|
+
"YD_NAMESPACE": "pyexamples-pwt2"
|
|
1062
|
+
},
|
|
1046
1063
|
"inputs": [
|
|
1047
1064
|
{
|
|
1048
|
-
"objectNamePattern": "pyex-
|
|
1065
|
+
"objectNamePattern": "pyex-bash-pwt2_240424-12051160/sleep_script.sh",
|
|
1049
1066
|
"source": "TASK_NAMESPACE",
|
|
1050
|
-
"verification": "
|
|
1067
|
+
"verification": "VERIFY_AT_START"
|
|
1051
1068
|
}
|
|
1052
1069
|
],
|
|
1053
|
-
"name": "
|
|
1070
|
+
"name": "task_1",
|
|
1054
1071
|
"outputs": [
|
|
1055
1072
|
{"alwaysUpload": true, "required": false, "source": "PROCESS_OUTPUT"}
|
|
1056
1073
|
],
|
|
@@ -1587,13 +1604,13 @@ Here's an example of the `workerPool` section of a TOML configuration file, show
|
|
|
1587
1604
|
templateId = "ydid:crt:D9C548:465a107c-7cea-46e3-9fdd-15116cb92c40"
|
|
1588
1605
|
# Note: only one of 'userData'/'userDataFile'/'userDataFiles' should be set
|
|
1589
1606
|
userData = ""
|
|
1590
|
-
userDataFile = "myuserdata.txt"
|
|
1591
|
-
userDataFiles = ["myuserdata1.txt", "myuserdata2.txt"]
|
|
1607
|
+
# userDataFile = "myuserdata.txt"
|
|
1608
|
+
# userDataFiles = ["myuserdata1.txt", "myuserdata2.txt"]
|
|
1592
1609
|
workerTag = "tag-{{username}}"
|
|
1593
1610
|
# Specify either workersPerNode or workersPerVCPU
|
|
1594
1611
|
workersPerNode = 1
|
|
1595
|
-
workersPerVCPU = 1
|
|
1596
|
-
#
|
|
1612
|
+
# workersPerVCPU = 1
|
|
1613
|
+
# workerPoolData = "worker_pool.json" # Optionally specify worker pool JSON specification
|
|
1597
1614
|
```
|
|
1598
1615
|
|
|
1599
1616
|
## Worker Pool Specification Using JSON Documents
|
|
@@ -1782,6 +1799,7 @@ The commands **yd-create** and **yd-remove** allow the creation, update and remo
|
|
|
1782
1799
|
- Namespace Storage Configurations
|
|
1783
1800
|
- Configured Worker Pools
|
|
1784
1801
|
- Allowances
|
|
1802
|
+
- String Attribute Definitions
|
|
1785
1803
|
|
|
1786
1804
|
## Overview of Operation
|
|
1787
1805
|
|
|
@@ -2145,6 +2163,24 @@ When using `yd-remove`, Allowances are again matched using their `description` p
|
|
|
2145
2163
|
|
|
2146
2164
|
Allowances can be **boosted** (have extra hours added to the Allowance) using the `yd-boost` command.
|
|
2147
2165
|
|
|
2166
|
+
## String Attribute Definitions
|
|
2167
|
+
|
|
2168
|
+
The Attribute Definition example and schema can be found at: https://docs.yellowdog.co/api/?spec=Compute%20API#tag/compute/post/compute/attributes/user. Currently, only **String Attribute Definitions** are supported.
|
|
2169
|
+
|
|
2170
|
+
Example:
|
|
2171
|
+
|
|
2172
|
+
```json
|
|
2173
|
+
{
|
|
2174
|
+
"resource": "StringAttributeDefinition",
|
|
2175
|
+
"name": "user.my-attribute",
|
|
2176
|
+
"title": "My attribute title",
|
|
2177
|
+
"description": "This is a description of my attribute",
|
|
2178
|
+
"options": ["yes", "no", "maybe"]
|
|
2179
|
+
}
|
|
2180
|
+
```
|
|
2181
|
+
|
|
2182
|
+
The `name` and `title` properties are required, while the rest are optional. The `user.` prefix is required when specifying the `name` property.
|
|
2183
|
+
|
|
2148
2184
|
# Jsonnet Support
|
|
2149
2185
|
|
|
2150
2186
|
In all circumstances where JSON files are used by the Python Examples commands, **[Jsonnet](https://jsonnet.org)** files can be used instead. This allows the use of Jsonnet's powerful JSON extensions, including comments, variables, functions, etc.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "7.9.0"
|
|
@@ -10,7 +10,7 @@ from typing import Dict, List, Optional
|
|
|
10
10
|
|
|
11
11
|
import yellowdog_client.model as model
|
|
12
12
|
from dateparser import parse as date_parse
|
|
13
|
-
from requests import post
|
|
13
|
+
from requests import post, put
|
|
14
14
|
from requests.exceptions import HTTPError
|
|
15
15
|
from yellowdog_client.model import (
|
|
16
16
|
AccountAllowance,
|
|
@@ -54,6 +54,7 @@ from yd_commands.settings import (
|
|
|
54
54
|
RN_REQUIREMENT_TEMPLATE,
|
|
55
55
|
RN_SOURCE_TEMPLATE,
|
|
56
56
|
RN_STORAGE_CONFIGURATION,
|
|
57
|
+
RN_STRING_ATTRIBUTE_DEFINITION,
|
|
57
58
|
)
|
|
58
59
|
from yd_commands.wrapper import ARGS_PARSER, CLIENT, CONFIG_COMMON, main_wrapper
|
|
59
60
|
|
|
@@ -118,6 +119,8 @@ def create_resources(
|
|
|
118
119
|
create_configured_worker_pool(resource)
|
|
119
120
|
elif resource_type == RN_ALLOWANCE:
|
|
120
121
|
create_allowance(resource)
|
|
122
|
+
elif resource_type == RN_STRING_ATTRIBUTE_DEFINITION:
|
|
123
|
+
create_string_attribute_definition_via_api(resource)
|
|
121
124
|
else:
|
|
122
125
|
print_error(f"Unknown resource type '{resource_type}'")
|
|
123
126
|
except Exception as e:
|
|
@@ -789,6 +792,45 @@ def _get_model_class(class_name: str):
|
|
|
789
792
|
return getattr(model, class_name)
|
|
790
793
|
|
|
791
794
|
|
|
795
|
+
def create_string_attribute_definition_via_api(resource: Dict):
|
|
796
|
+
"""
|
|
797
|
+
Use the API to create/update user string attribute definitions.
|
|
798
|
+
"""
|
|
799
|
+
try:
|
|
800
|
+
name = resource["name"]
|
|
801
|
+
title = resource["title"]
|
|
802
|
+
except KeyError as e:
|
|
803
|
+
raise Exception(f"Expected property to be defined ({e})")
|
|
804
|
+
|
|
805
|
+
url = f"{CONFIG_COMMON.url}/compute/attributes/user"
|
|
806
|
+
headers = {"Authorization": f"yd-key {CONFIG_COMMON.key}:{CONFIG_COMMON.secret}"}
|
|
807
|
+
payload = {
|
|
808
|
+
"name": name,
|
|
809
|
+
"title": title,
|
|
810
|
+
"type": "co.yellowdog.platform.model.StringAttributeDefinition",
|
|
811
|
+
"description": resource.get("description", ""),
|
|
812
|
+
"options": resource.get("options", []),
|
|
813
|
+
}
|
|
814
|
+
|
|
815
|
+
# Attempt attribute creation
|
|
816
|
+
response = post(url=url, headers=headers, json=payload)
|
|
817
|
+
|
|
818
|
+
if response.status_code == 200:
|
|
819
|
+
print_log(f"Created new String Attribute Definition '{name}'")
|
|
820
|
+
return
|
|
821
|
+
|
|
822
|
+
if "Attribute already exists" in response.text:
|
|
823
|
+
if not confirmed(f"Update existing String Attribute Definition '{name}'?"):
|
|
824
|
+
return
|
|
825
|
+
|
|
826
|
+
response = put(url=url, headers=headers, json=payload)
|
|
827
|
+
if response.status_code == 200:
|
|
828
|
+
print_log(f"Updated existing String Attribute Definition '{name}'")
|
|
829
|
+
return
|
|
830
|
+
|
|
831
|
+
raise Exception(f"HTTP {response.status_code} ({response.text})")
|
|
832
|
+
|
|
833
|
+
|
|
792
834
|
# Entry point
|
|
793
835
|
if __name__ == "__main__":
|
|
794
836
|
main()
|
{yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/load_config.py
RENAMED
|
@@ -4,7 +4,8 @@ Common utility functions, mostly related to loading configuration data.
|
|
|
4
4
|
|
|
5
5
|
import os
|
|
6
6
|
from os import getenv
|
|
7
|
-
from os.path import dirname, join, relpath
|
|
7
|
+
from os.path import abspath, dirname, join, relpath
|
|
8
|
+
from pathlib import Path
|
|
8
9
|
from sys import exit
|
|
9
10
|
from typing import Dict, Optional
|
|
10
11
|
|
|
@@ -49,6 +50,11 @@ CONFIG_FILE = relpath(
|
|
|
49
50
|
|
|
50
51
|
try:
|
|
51
52
|
CONFIG_FILE_DIR = dirname(CONFIG_FILE)
|
|
53
|
+
config_dir_abs = abspath(CONFIG_FILE_DIR)
|
|
54
|
+
config_dir_short = Path(config_dir_abs).parts[-1]
|
|
55
|
+
VARIABLE_SUBSTITUTIONS.update(
|
|
56
|
+
{"config_dir_abs": config_dir_abs, "config_dir_name": config_dir_short}
|
|
57
|
+
)
|
|
52
58
|
print_log(f"Loading configuration data from: '{CONFIG_FILE}'")
|
|
53
59
|
CONFIG_TOML: Dict = load_toml_file_with_variable_substitutions(CONFIG_FILE)
|
|
54
60
|
try:
|
{yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/load_resources.py
RENAMED
|
@@ -16,6 +16,7 @@ from yd_commands.settings import (
|
|
|
16
16
|
RN_REQUIREMENT_TEMPLATE,
|
|
17
17
|
RN_SOURCE_TEMPLATE,
|
|
18
18
|
RN_STORAGE_CONFIGURATION,
|
|
19
|
+
RN_STRING_ATTRIBUTE_DEFINITION,
|
|
19
20
|
)
|
|
20
21
|
from yd_commands.variables import (
|
|
21
22
|
load_json_file_with_variable_substitutions,
|
|
@@ -94,6 +95,7 @@ def _resequence_resources(
|
|
|
94
95
|
RN_KEYRING,
|
|
95
96
|
RN_CREDENTIAL,
|
|
96
97
|
RN_IMAGE_FAMILY,
|
|
98
|
+
RN_STRING_ATTRIBUTE_DEFINITION,
|
|
97
99
|
RN_SOURCE_TEMPLATE,
|
|
98
100
|
RN_REQUIREMENT_TEMPLATE,
|
|
99
101
|
RN_ALLOWANCE,
|
|
@@ -7,6 +7,7 @@ A script to remove YellowDog resources.
|
|
|
7
7
|
from copy import deepcopy
|
|
8
8
|
from typing import Dict, List, Optional
|
|
9
9
|
|
|
10
|
+
from requests import delete
|
|
10
11
|
from requests.exceptions import HTTPError
|
|
11
12
|
from yellowdog_client.model import (
|
|
12
13
|
MachineImage,
|
|
@@ -34,8 +35,9 @@ from yd_commands.settings import (
|
|
|
34
35
|
RN_REQUIREMENT_TEMPLATE,
|
|
35
36
|
RN_SOURCE_TEMPLATE,
|
|
36
37
|
RN_STORAGE_CONFIGURATION,
|
|
38
|
+
RN_STRING_ATTRIBUTE_DEFINITION,
|
|
37
39
|
)
|
|
38
|
-
from yd_commands.wrapper import ARGS_PARSER, CLIENT, main_wrapper
|
|
40
|
+
from yd_commands.wrapper import ARGS_PARSER, CLIENT, CONFIG_COMMON, main_wrapper
|
|
39
41
|
|
|
40
42
|
|
|
41
43
|
@main_wrapper
|
|
@@ -92,6 +94,8 @@ def remove_resources(resources: Optional[List[Dict]] = None):
|
|
|
92
94
|
"alternatively, Allowances can be removed by their "
|
|
93
95
|
"YellowDog IDs (yd-remove --ids)"
|
|
94
96
|
)
|
|
97
|
+
elif resource_type == RN_STRING_ATTRIBUTE_DEFINITION:
|
|
98
|
+
remove_string_attribute_definition_via_api(resource)
|
|
95
99
|
else:
|
|
96
100
|
print_error(f"Unknown resource type '{resource_type}'")
|
|
97
101
|
except Exception as e:
|
|
@@ -407,6 +411,29 @@ def remove_resource_by_id(resource_id: str):
|
|
|
407
411
|
print_error(f"Unable to remove resource with ID {resource_id}: {e}")
|
|
408
412
|
|
|
409
413
|
|
|
414
|
+
def remove_string_attribute_definition_via_api(resource: Dict):
|
|
415
|
+
"""
|
|
416
|
+
Use the API to remove user string attribute definitions.
|
|
417
|
+
"""
|
|
418
|
+
try:
|
|
419
|
+
name = resource["name"]
|
|
420
|
+
except KeyError as e:
|
|
421
|
+
raise Exception(f"Expected property to be defined ({e})")
|
|
422
|
+
|
|
423
|
+
if not confirmed(f"Remove String Attribute Definition '{name}'?"):
|
|
424
|
+
return
|
|
425
|
+
|
|
426
|
+
url = f"{CONFIG_COMMON.url}/compute/attributes/user/{name}"
|
|
427
|
+
headers = {"Authorization": f"yd-key {CONFIG_COMMON.key}:{CONFIG_COMMON.secret}"}
|
|
428
|
+
response = delete(url=url, headers=headers)
|
|
429
|
+
|
|
430
|
+
if response.status_code == 200:
|
|
431
|
+
print_log(f"Removed String Attribute Definition '{name}' (if present)")
|
|
432
|
+
return
|
|
433
|
+
|
|
434
|
+
raise Exception(f"HTTP {response.status_code} ({response.text})")
|
|
435
|
+
|
|
436
|
+
|
|
410
437
|
# Entry point
|
|
411
438
|
if __name__ == "__main__":
|
|
412
439
|
main()
|
|
@@ -32,9 +32,11 @@ def main():
|
|
|
32
32
|
return
|
|
33
33
|
|
|
34
34
|
print_log(
|
|
35
|
-
"Shutting down Worker Pools with Compute Requirements in "
|
|
35
|
+
"Shutting down Provisioned Worker Pools with Compute Requirements in "
|
|
36
36
|
f"namespace '{CONFIG_COMMON.namespace}' and "
|
|
37
|
-
f"tag starting with '{CONFIG_COMMON.name_tag}'"
|
|
37
|
+
f"tag starting with '{CONFIG_COMMON.name_tag}' "
|
|
38
|
+
"(or Configured Worker Pools with names starting with "
|
|
39
|
+
f"'{CONFIG_COMMON.name_tag}')"
|
|
38
40
|
)
|
|
39
41
|
worker_pool_summaries: List[WorkerPoolSummary] = (
|
|
40
42
|
CLIENT.worker_pool_client.find_all_worker_pools()
|
|
@@ -43,11 +45,19 @@ def main():
|
|
|
43
45
|
|
|
44
46
|
selected_worker_pool_summaries: List[WorkerPoolSummary] = []
|
|
45
47
|
for worker_pool_summary in worker_pool_summaries:
|
|
46
|
-
if not
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
48
|
+
if worker_pool_summary.status not in [
|
|
49
|
+
WorkerPoolStatus.TERMINATED,
|
|
50
|
+
WorkerPoolStatus.SHUTDOWN,
|
|
51
|
+
]:
|
|
52
|
+
if "ProvisionedWorkerPool" not in worker_pool_summary.type:
|
|
53
|
+
# Configured Worker Pool: check worker pool name only
|
|
54
|
+
if (
|
|
55
|
+
worker_pool_summary.name is not None
|
|
56
|
+
and worker_pool_summary.name.startswith(CONFIG_COMMON.name_tag)
|
|
57
|
+
):
|
|
58
|
+
selected_worker_pool_summaries.append(worker_pool_summary)
|
|
59
|
+
continue
|
|
60
|
+
|
|
51
61
|
worker_pool: WorkerPool = get_worker_pool_by_id(
|
|
52
62
|
CLIENT, worker_pool_summary.id
|
|
53
63
|
)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: yellowdog-python-examples
|
|
3
|
-
Version: 7.
|
|
3
|
+
Version: 7.9.0
|
|
4
4
|
Summary: Example Python commands using the YellowDog Python SDK
|
|
5
5
|
Home-page: https://github.com/yellowdog/python-examples
|
|
6
6
|
Author: YellowDog Limited
|
|
@@ -43,6 +43,7 @@ The command scripts support:
|
|
|
43
43
|
|
|
44
44
|
- **Provisioning** Worker Pools with the **`yd-provision`** command
|
|
45
45
|
- **Submitting** Work Requirements with the **`yd-submit`** command
|
|
46
|
+
- **Starting** HELD Work Requirements and **Holding** (or pausing) RUNNING Work Requirements with the **`yd-start`** and **`yd-hold`** commands
|
|
46
47
|
- **Uploading** files to the YellowDog Object Store with the **`yd-upload`** command
|
|
47
48
|
- **Instantiating** Compute Requirements with the **`yd-instantiate`** command
|
|
48
49
|
- **Downloading** Results from the YellowDog Object Store with the **`yd-download`** command
|
|
@@ -52,7 +53,9 @@ The command scripts support:
|
|
|
52
53
|
- **Terminating** Compute Requirements with the **`yd-terminate`** command
|
|
53
54
|
- **Deleting** objects in the YellowDog Object Store with the **`yd-delete`** command
|
|
54
55
|
- **Listing** YellowDog items using the **`yd-list`** command
|
|
55
|
-
- **Resizing** Worker Pools and Compute Requirements
|
|
56
|
-
- **
|
|
56
|
+
- **Resizing** Worker Pools and Compute Requirements with the **`yd-resize`** command
|
|
57
|
+
- **Boosting** Allowances with the **`yd-boost`** command
|
|
58
|
+
- **Creating, Updating and Removing** Source Templates, Compute Templates, Keyrings, Credentials, Namespace Storage Configurations, Image Families, Allowances, and Configured Worker Pools with the **`yd-create`** and **`yd-remove`** commands
|
|
59
|
+
- **Following Event Streams** for Work Requirements, Worker Pools and Compute Requirements with the **`yd-follow`** command
|
|
57
60
|
|
|
58
|
-
Please see the [GitHub repository](https://github.com/yellowdog/python-examples) for more details.
|
|
61
|
+
Please see the documenation in the [GitHub repository](https://github.com/yellowdog/python-examples) for more details.
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = "7.8.8"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/tests/test_create_remove.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/tests/test_entrypoints.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/check_imports.py
RENAMED
|
File without changes
|
{yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/cloudwizard.py
RENAMED
|
File without changes
|
{yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/cloudwizard_aws.py
RENAMED
|
File without changes
|
|
File without changes
|
{yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/cloudwizard_azure.py
RENAMED
|
File without changes
|
|
File without changes
|
{yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/cloudwizard_gcp.py
RENAMED
|
File without changes
|
{yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/compact_json.py
RENAMED
|
File without changes
|
{yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/config_types.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/follow_utils.py
RENAMED
|
File without changes
|
{yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/format_json.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/instantiate.py
RENAMED
|
File without changes
|
{yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/interactive.py
RENAMED
|
File without changes
|
|
File without changes
|
{yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/jsonnet2json.py
RENAMED
|
File without changes
|
|
File without changes
|
{yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/object_utilities.py
RENAMED
|
File without changes
|
|
File without changes
|
{yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/property_names.py
RENAMED
|
File without changes
|
{yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/provision.py
RENAMED
|
File without changes
|
{yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/provision_utils.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/start_hold_common.py
RENAMED
|
File without changes
|
|
File without changes
|
{yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/submit_utils.py
RENAMED
|
File without changes
|
{yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/terminate.py
RENAMED
|
File without changes
|
{yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/type_check.py
RENAMED
|
File without changes
|
|
File without changes
|
{yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/upload_utils.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{yellowdog_python_examples-7.8.8 → yellowdog_python_examples-7.9.0}/yd_commands/variables.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|